mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-22 17:19:58 -04:00
Triforce v0.7.0 "Sage Of Courage"
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
This commit is contained in:
@@ -15,6 +15,7 @@
|
||||
#include <span>
|
||||
|
||||
#include "common/alignment.h"
|
||||
#include "../vk/cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU
|
||||
#include "common/div_ceil.h"
|
||||
#include "video_core/texture_cache/format_lookup_table.h"
|
||||
#include "video_core/textures/astc.h"
|
||||
@@ -766,7 +767,7 @@ std::unique_ptr<Image> decode(const TICEntry& tic, const TexInfo& info_in, uint6
|
||||
copy_layers >= 4 && level_size.depth == 1 && (size_t)level_size.width * level_size.height * 4 <= (8u << 20)) {
|
||||
const Extent3D block = AdjustMipBlockSize(tiles, li.block, level, li.num_levels);
|
||||
const u32 stride_alignment = StrideAlignment(tiles, info.block, gob, li.bpp_log2);
|
||||
const unsigned nthreads = std::max(1u, std::min<unsigned>(copy_layers, std::thread::hardware_concurrency() - 1));
|
||||
const unsigned nthreads = std::max(1u, std::min<unsigned>(copy_layers, cpu_count() - 1));
|
||||
std::atomic<u32> next_layer{0};
|
||||
static std::atomic<int64_t> t_swz, t_dec, t_up;
|
||||
auto work = [&] {
|
||||
|
||||
145
gpu/vk/achievement_rows.h
Normal file
145
gpu/vk/achievement_rows.h
Normal file
@@ -0,0 +1,145 @@
|
||||
// (Triforce) Achievements on the menu's side: the list the host names (host/achievements.c; vk_api.h HOS_OPT_ACH_*), which of
|
||||
// them are earned, the one a notice shows, and the Achievements page's choice and pages. vk_menu.cpp draws them.
|
||||
//
|
||||
// The achievements are the host's: it reads the lists, asks the save data and keeps what was earned. The menu is told
|
||||
// each one's name, the line about it, its icon and the day it was earned, a byte a call as the save rows' lines are, and
|
||||
// later each one earned as it is. An icon is drawn, not a picture file: a shape and a tone (host/achievements.h has the
|
||||
// same in the same order, ACH_SHAPES and ACH_TONES; tests/test_achievements.py holds the two together).
|
||||
//
|
||||
// The host's calls (text, add, done, earn) come on its threads, one at a time (its lock); the list is drawn on the
|
||||
// renderer's: names, lines and icons are written before count says they are there and never after, a day before its
|
||||
// achievement's earned says so.
|
||||
//
|
||||
// No ImGui and no Vulkan: tests/test_achievements.py compiles it alone.
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
namespace achievement_rows {
|
||||
constexpr int kSlots = 128; // (host/achievements.h ACH_MAX)
|
||||
constexpr int kName = 41, kAbout = 161, kDay = 16;
|
||||
constexpr int kEarned = 256; // (vk_api.h HOS_ACH_EARNED)
|
||||
enum Shape { kStar, kSword, kGem, kMedal, kNote, kToken, kShield, kShapes };
|
||||
enum Tone { kGold, kGreen, kRed, kBlue, kPurple, kOrange, kYellow, kSilver, kTones };
|
||||
constexpr int shape_of(int icon) { return (icon & 15) < kShapes ? icon & 15 : kStar; }
|
||||
constexpr int tone_of(int icon) { return (icon >> 4 & 15) < kTones ? icon >> 4 & 15 : kGold; }
|
||||
|
||||
struct List {
|
||||
char names[kSlots][kName] = {};
|
||||
char abouts[kSlots][kAbout] = {};
|
||||
char days[kSlots][kDay] = {};
|
||||
unsigned char icons[kSlots] = {};
|
||||
std::atomic<int> earned[kSlots] = {};
|
||||
std::atomic<int> count{0}; // (the achievements shown: 0 until the host's list is whole)
|
||||
std::atomic<int> fresh{-1}; // (the one earned now, until the notice takes it)
|
||||
// the host's side
|
||||
char in[kName + kAbout + kDay + 8] = {};
|
||||
int in_n = 0, added = 0;
|
||||
|
||||
/// HOS_OPT_ACH_TEXT: the next byte of what is being named
|
||||
void text(int byte) {
|
||||
if (byte > 0 && byte < 256 && in_n + 1 < (int)sizeof in) in[in_n++] = (char)byte;
|
||||
}
|
||||
/// `from` copied into a field of `cap` bytes, cut at a whole letter
|
||||
static void put(char* to, int cap, const char* from) {
|
||||
int n = (int)strlen(from);
|
||||
if (n > cap - 1) {
|
||||
n = cap - 1;
|
||||
while (n && ((unsigned char)from[n] & 0xC0) == 0x80) n--;
|
||||
}
|
||||
memcpy(to, from, (size_t)n);
|
||||
to[n] = 0;
|
||||
}
|
||||
/// HOS_OPT_ACH_ADD: the bytes so far are the next achievement's name and, each after a line feed, the line about it and
|
||||
/// the day it was earned; bits: its icon, and kEarned
|
||||
void add(int bits) {
|
||||
in[in_n] = 0;
|
||||
in_n = 0;
|
||||
if (count.load(std::memory_order_acquire) || added >= kSlots) return; // (shown already, or no room left)
|
||||
const int slot = added++;
|
||||
char* about = strchr(in, '\n');
|
||||
char* day = about ? strchr(about + 1, '\n') : nullptr;
|
||||
if (about) *about++ = 0;
|
||||
if (day) *day++ = 0;
|
||||
put(names[slot], kName, in[0] ? in : "(no name)");
|
||||
put(abouts[slot], kAbout, about ? about : "");
|
||||
put(days[slot], kDay, day ? day : "");
|
||||
icons[slot] = (unsigned char)(bits & 255);
|
||||
earned[slot].store((bits & kEarned) != 0, std::memory_order_relaxed);
|
||||
}
|
||||
/// HOS_OPT_ACH_COUNT: the list is whole: shown from now on
|
||||
void done(int n) {
|
||||
in_n = 0;
|
||||
if (count.load(std::memory_order_acquire)) return;
|
||||
count.store(n < 0 ? 0 : n < added ? n : added, std::memory_order_release);
|
||||
}
|
||||
/// HOS_OPT_ACH_EARNED: the achievement at that place was earned now, the bytes so far the day. false: no such place
|
||||
bool earn(int slot) {
|
||||
in[in_n] = 0;
|
||||
in_n = 0;
|
||||
if (slot < 0 || slot >= shown()) return false;
|
||||
if (!earned[slot].load(std::memory_order_relaxed)) put(days[slot], kDay, in);
|
||||
earned[slot].store(1, std::memory_order_release);
|
||||
fresh.store(slot, std::memory_order_release);
|
||||
return true;
|
||||
}
|
||||
int shown() const { return count.load(std::memory_order_acquire); }
|
||||
bool is_earned(int slot) const { return slot >= 0 && slot < kSlots && earned[slot].load(std::memory_order_acquire) != 0; }
|
||||
int earned_count() const {
|
||||
int n = 0;
|
||||
for (int i = 0, end = shown(); i < end; i++) n += is_earned(i);
|
||||
return n;
|
||||
}
|
||||
/// The one earned since the last ask, for the notice; -1 none
|
||||
int take_fresh() { return fresh.load(std::memory_order_relaxed) < 0 ? -1 : fresh.exchange(-1, std::memory_order_acq_rel); }
|
||||
};
|
||||
|
||||
/// The page's cards: `columns` to a row, `capacity` to a page (both 1 at least), n of them
|
||||
struct View {
|
||||
int choice = 0, page = 0;
|
||||
|
||||
static int pages(int n, int capacity) { return n <= 0 ? 1 : (n + (capacity < 1 ? 1 : capacity) - 1) / (capacity < 1 ? 1 : capacity); }
|
||||
/// The choice kept to the list, and the page shown to the pages there are
|
||||
void keep(int n, int capacity) {
|
||||
choice = n <= 0 ? 0 : choice < 0 ? 0 : choice >= n ? n - 1 : choice;
|
||||
const int last = pages(n, capacity) - 1;
|
||||
page = page < 0 ? 0 : page > last ? last : page;
|
||||
}
|
||||
/// The page that shows the choice
|
||||
void reveal(int capacity) { page = choice / (capacity < 1 ? 1 : capacity); }
|
||||
/// The arrows (cards: Left and Right, rows: Up and Down), Page Up and Down (turns), Home and End. true: the choice moved
|
||||
bool move(int cards, int rows, int turns, bool home, bool end, int n, int columns, int capacity) {
|
||||
if (n <= 0) return false;
|
||||
const int was = choice;
|
||||
const int step = cards + rows * (columns < 1 ? 1 : columns) + turns * (capacity < 1 ? 1 : capacity);
|
||||
choice = home ? 0 : end ? n - 1 : choice + step;
|
||||
keep(n, capacity);
|
||||
if (choice == was) return false;
|
||||
reveal(capacity);
|
||||
return true;
|
||||
}
|
||||
/// The wheel, or a page arrow that leaves the choice where it is: the page shown turned. true: it changed
|
||||
bool turn(int by, int n, int capacity) {
|
||||
const int was = page;
|
||||
page += by;
|
||||
keep(n, capacity);
|
||||
return page != was;
|
||||
}
|
||||
/// A click on a card. true: the choice changed
|
||||
bool choose(int slot, int n) {
|
||||
if (slot < 0 || slot >= n || slot == choice) return false;
|
||||
choice = slot;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/// How far a notice has come in, 0 (out of sight) to 1 (in place): in over the first 250 ms of its `shown_ms`, out over
|
||||
/// the last 300
|
||||
inline float notice_in(int64_t age_ms, int64_t shown_ms) {
|
||||
if (age_ms < 0 || age_ms >= shown_ms) return 0;
|
||||
const float in = (float)age_ms / 250.0f, out = (float)(shown_ms - age_ms) / 300.0f;
|
||||
const float k = in < out ? in : out;
|
||||
return k > 1 ? 1 : k;
|
||||
}
|
||||
} // namespace achievement_rows
|
||||
@@ -13,4 +13,12 @@ inline bool all_by_vram(uint64_t allowed) {
|
||||
return allowed && allowed < (8704ull << 20);
|
||||
}
|
||||
|
||||
// (Triforce, 11-22, the performance audit's TEX-12, after Source A's) And every integrated GPU, whatever it is allowed. Its
|
||||
// allowance is a share of the system's RAM (a 16 GB handheld reported 10,361 MB), so by the allowance alone it kept the
|
||||
// fine-block textures of 2 to 64 MB as RGBA8, in RAM the game needs too, inside a texture budget capped at 2,560 MB.
|
||||
// Source A measured the difference on a discrete card (RX_VK_ASTC_BC_ALL 0 against 1, Kokiri Forest: decoded ASTC
|
||||
// 2,496-2,532 MB against 1,579-1,587, the GPU's frame the same); it was not run on an integrated GPU there either.
|
||||
// The shape of tex_detail::by_vram(allowed, integrated). RX_VK_ASTC_BC_ALL=0: the finer ones as RGBA8, as before
|
||||
inline bool all(uint64_t allowed, bool integrated) { return integrated || all_by_vram(allowed); }
|
||||
|
||||
} // namespace astc_bc_policy
|
||||
|
||||
@@ -18,12 +18,28 @@
|
||||
// 65535 / 65535 = 1. Modes 4, 5, 10, 12 and 13 carry alpha, and 2, 3, 7, 11, 14 and 15 are the HDR ones, which take the
|
||||
// decoder's other path: no for all of those, without decoding them (such a block may be opaque all the same).
|
||||
// One partition has its mode in bits 13 to 16. Several partitions share one mode when bits 23 and 24 are zero (the mode is
|
||||
// bits 25 to 28); when they do not, the modes' other bits sit below the weights and are not looked for: no.
|
||||
// bits 25 to 28).
|
||||
// - (Triforce, 11-21, the performance audit's TEX-50, after Source B's) When bits 23 and 24 are not zero each partition has a
|
||||
// mode of its own, read as the decoder reads it: 2, 5 or 8 more bits for 2, 3 or 4 partitions sit right below the weights
|
||||
// (at 128 less the weights' bits less their own count); with bits 25 to 28 under them they are a class bit a partition,
|
||||
// then two mode bits a partition, and a partition's mode is (bits 23 and 24 as a number, less 1, plus its class bit) * 4
|
||||
// plus its two bits. Yes only when every partition's mode is one of 0, 1, 6, 8 and 9. Those bits count in the decoder's
|
||||
// last error too: 96 bits of weights, the 29 read and 5 or 8 more wrap, and the block is an error. Such blocks were
|
||||
// refused unread, and a texture with one of them stayed BC3. parts_too = false (RX_VK_TEX_BC1_PARTS=0): refused, as before.
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
|
||||
namespace astc_opaque {
|
||||
|
||||
/// Whether a block with a mode a partition is looked at (above); false: refused unread, as before. Set once, at the start
|
||||
inline bool parts_too = true;
|
||||
/// How many such blocks were found opaque (the log's)
|
||||
inline std::atomic<uint64_t> parts_passed{0};
|
||||
|
||||
/// Whether the decoder gives both endpoints of a colour endpoint mode the alpha 255: the modes without alpha, low dynamic range
|
||||
constexpr bool opaque_mode(unsigned cem) { return cem == 0 || cem == 1 || cem == 6 || cem == 8 || cem == 9; }
|
||||
|
||||
/// `n` bits (16 at most) of a block from bit `at`, the lowest first, as the decoder's StreamBits reads them
|
||||
inline uint32_t bits(const uint8_t* b, unsigned at, unsigned n) {
|
||||
uint32_t v = 0;
|
||||
@@ -88,17 +104,32 @@ inline bool block(const uint8_t b[16], unsigned bw, unsigned bh) {
|
||||
range = range - 2 + (layout != 9 && (mode & 0x200) ? 6 : 0);
|
||||
const unsigned wbits = weight_bits(weights, range);
|
||||
if (wbits < 24 || wbits > 96) return false;
|
||||
unsigned read, cem;
|
||||
unsigned read, cem = 0, extra = 0;
|
||||
uint32_t base = 0;
|
||||
if (parts == 1) {
|
||||
cem = bits(b, 13, 4), read = 17;
|
||||
} else {
|
||||
const uint32_t base = bits(b, 23, 6);
|
||||
if (base & 3) return false; // (a mode a partition: not looked for)
|
||||
cem = base >> 2, read = 29;
|
||||
base = bits(b, 23, 6);
|
||||
read = 29;
|
||||
if (base & 3) { // (a mode a partition: its other bits are below the weights)
|
||||
if (!parts_too) return false;
|
||||
extra = parts == 2 ? 2 : parts == 3 ? 5 : 8;
|
||||
} else {
|
||||
cem = base >> 2;
|
||||
}
|
||||
}
|
||||
// (the decoder's last error, its count of the remaining bits wrapping. With one shared mode it cannot: 96 + 29 + 2)
|
||||
if (wbits + read + (dual ? 2u : 0u) > 128) return false;
|
||||
return cem == 0 || cem == 1 || cem == 6 || cem == 8 || cem == 9;
|
||||
// (the decoder's last error, its count of the remaining bits wrapping. With one shared mode it cannot: 96 + 29 + 2. With a
|
||||
// mode a partition it can: 96 + 29 + 8)
|
||||
if (wbits + read + extra + (dual ? 2u : 0u) > 128) return false;
|
||||
if (!extra) return opaque_mode(cem);
|
||||
// (the decoder's own order: (extra << 6 | base) >> 2 is a class bit a partition, then two bits a partition)
|
||||
const uint32_t all = ((bits(b, 128 - wbits - extra, extra) << 6) | base) >> 2;
|
||||
for (unsigned i = 0; i < parts; i++) {
|
||||
const unsigned cls = (base & 3) - 1 + ((all >> i) & 1);
|
||||
if (!opaque_mode((cls << 2) | ((all >> (parts + 2 * i)) & 3))) return false;
|
||||
}
|
||||
parts_passed.fetch_add(1, std::memory_order_relaxed);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Whether every one of `n` blocks (16 bytes each, in any order) is one the decoder writes opaque
|
||||
|
||||
133
gpu/vk/astc_reuse.h
Normal file
133
gpu/vk/astc_reuse.h
Normal file
@@ -0,0 +1,133 @@
|
||||
// (Triforce, 11-21, the performance audit's TEX-14, after Source A's) Decoded ASTC used again.
|
||||
//
|
||||
// The game's textures are ASTC, decoded on the GPU and for the most part encoded again as BC3 or BC1: tens of ms of GPU
|
||||
// time and the decoder's scratch for each. When the game leaves an area it finalizes its textures, and when it comes back
|
||||
// it makes them again, mostly at other addresses: the same blocks are decoded and encoded again while the image made of
|
||||
// them the first time may still be in VRAM, kept until the cache needs its room. Source A hashes an ASTC texture's blocks
|
||||
// at its first whole upload, lists a texture the game finalizes under that hash, and gives a new texture with the same
|
||||
// blocks and the same shape that image by a GPU copy, the old one going after it. Its tours of two areas four times at an
|
||||
// 8 GB budget: peak decoded ASTC 3,603 to 2,562 MB, evictions 23,378 to 16,109, the slowest frame 637 to 487 ms.
|
||||
//
|
||||
// What keeps a hit right: 128 bits of hash over the blocks as the decoder is given them; every field of the image that
|
||||
// the decode made and of the storage it was decoded from (Shape); and a source that was uploaded from memory and never
|
||||
// drawn into, still listed (used again at its own address or gone, it is unlisted).
|
||||
//
|
||||
// RX_VK_ASTC_REUSE: 0 or not set, off (the audit: off until measured in play); 2: the blocks are hashed, the finalized
|
||||
// textures listed and each hit counted, and every texture decoded as before: what a reuse would save, for no risk;
|
||||
// 1: the hit's image is copied.
|
||||
//
|
||||
// No Vulkan here: tests/test_astc_reuse.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "upload_same.h" // (content_hash: the first of the two lanes)
|
||||
|
||||
namespace astc_reuse {
|
||||
|
||||
enum Mode { kOff = 0, kCopy = 1, kCount = 2 };
|
||||
/// RX_VK_ASTC_REUSE's mode: 1 and 2 as they are, anything else off
|
||||
inline Mode mode(const char* asked) {
|
||||
if (!asked || !*asked) return kOff;
|
||||
const int v = atoi(asked);
|
||||
return v == 1 ? kCopy : v == 2 ? kCount : kOff;
|
||||
}
|
||||
|
||||
/// A second hash of n bytes, of other constants and another mix than upload_same::content_hash's: eight chains over 64
|
||||
/// bytes a step, as there, so that both read at memory's speed
|
||||
inline uint64_t lane2(const void* p, uint64_t n) {
|
||||
const uint8_t* b = static_cast<const uint8_t*>(p);
|
||||
const auto rotl = [](uint64_t x, int r) { return (x << r) | (x >> (64 - r)); };
|
||||
uint64_t h = 0xD6E8FEB86659FD93ull + n * 0x9FB21C651E98DF25ull, i = 0;
|
||||
if (n >= 512) {
|
||||
uint64_t c[8];
|
||||
for (int k = 0; k < 8; k++) c[k] = h + 0xA0761D6478BD642Full * (uint64_t)(2 * k + 1);
|
||||
for (; i + 64 <= n; i += 64) {
|
||||
uint64_t w[8];
|
||||
std::memcpy(w, b + i, 64);
|
||||
for (int k = 0; k < 8; k++) c[k] = rotl(c[k] + w[k] * 0xE7037ED1A0B428DBull, 29) * 0x8EBC6AF09C88C6E3ull;
|
||||
}
|
||||
for (int k = 0; k < 8; k++) h = rotl(h ^ c[k], 23) * 0x589965CC75374CC3ull + 0x1D8E4E27C47D124Full;
|
||||
}
|
||||
for (; i + 8 <= n; i += 8) {
|
||||
uint64_t w;
|
||||
std::memcpy(&w, b + i, 8);
|
||||
h = rotl(h + w * 0xE7037ED1A0B428DBull, 29) * 0x8EBC6AF09C88C6E3ull;
|
||||
}
|
||||
for (; i < n; i++) h = rotl(h ^ b[i], 11) * 0x589965CC75374CC3ull;
|
||||
h ^= h >> 32, h *= 0xD6E8FEB86659FD93ull, h ^= h >> 32;
|
||||
return h;
|
||||
}
|
||||
|
||||
/// The blocks' hash, 128 bits, made of pieces given in order (a big texture's storage is read 64 MB at a time): each piece
|
||||
/// goes through both lanes kStep bytes at a time, while they are in the cache, and is folded in with its length
|
||||
struct Hash {
|
||||
uint64_t a = 0, b = 0;
|
||||
bool operator==(const Hash& o) const { return a == o.a && b == o.b; }
|
||||
explicit operator bool() const { return (a | b) != 0; }
|
||||
};
|
||||
inline constexpr uint64_t kStep = 256u << 10;
|
||||
struct Hasher {
|
||||
Hash h;
|
||||
uint64_t bytes = 0;
|
||||
void add(const uint8_t* p, uint64_t n) {
|
||||
for (uint64_t at = 0; at < n; at += kStep) {
|
||||
const uint64_t m = n - at < kStep ? n - at : kStep;
|
||||
h.a = (h.a ^ upload_same::content_hash(p + at, m)) * 0x9E3779B97F4A7C15ull + m;
|
||||
h.b = (h.b + lane2(p + at, m)) * 0xC2B2AE3D27D4EB4Full ^ (m << 1 | 1);
|
||||
}
|
||||
bytes += n;
|
||||
}
|
||||
/// The hash, never all zero (zero is "not hashed")
|
||||
Hash done() const { return (h.a | h.b) ? h : Hash{1, 0}; }
|
||||
};
|
||||
|
||||
/// What has to be the same besides the blocks for one image to stand for another: the image the decode made (its format,
|
||||
/// size, levels and layers, the top levels left out and whether by the card's texture detail, kept as BC3 or BC1 or
|
||||
/// RGBA8, BC1 given up for BC3, resized for AMD) and the storage it was decoded from (the NVN format, the key's layout)
|
||||
struct Shape {
|
||||
uint32_t format, width, height, depth, levels, layers, skip, nvn_format, layout;
|
||||
bool bc, bc_resized, bc1, bc1_no, detail_skip;
|
||||
bool operator==(const Shape& o) const {
|
||||
return format == o.format && width == o.width && height == o.height && depth == o.depth && levels == o.levels &&
|
||||
layers == o.layers && skip == o.skip && nvn_format == o.nvn_format && layout == o.layout && bc == o.bc &&
|
||||
bc_resized == o.bc_resized && bc1 == o.bc1 && bc1_no == o.bc1_no && detail_skip == o.detail_skip;
|
||||
}
|
||||
};
|
||||
|
||||
/// The finalized textures whose image may serve again, by their blocks' hash. T is the cache's texture; one is in the list
|
||||
/// at most once: listed and unlisted by its pointer
|
||||
template <class T>
|
||||
class Freed {
|
||||
public:
|
||||
void list(const Hash& hash, T* t) { by_hash.emplace(hash.a, Entry{hash.b, t}); }
|
||||
void unlist(const Hash& hash, T* t) {
|
||||
auto [a, b] = by_hash.equal_range(hash.a);
|
||||
for (auto it = a; it != b; ++it)
|
||||
if (it->second.t == t) {
|
||||
by_hash.erase(it);
|
||||
return;
|
||||
}
|
||||
}
|
||||
/// One listed with this hash, both lanes, whose shape is `shape` (shape_of(T*) gives each one's) and that is not
|
||||
/// `self`; null: none
|
||||
template <class ShapeOf>
|
||||
T* find(const Hash& hash, const Shape& shape, const T* self, ShapeOf shape_of) const {
|
||||
auto [a, b] = by_hash.equal_range(hash.a);
|
||||
for (auto it = a; it != b; ++it)
|
||||
if (it->second.b == hash.b && it->second.t != self && shape_of(it->second.t) == shape) return it->second.t;
|
||||
return nullptr;
|
||||
}
|
||||
size_t size() const { return by_hash.size(); }
|
||||
|
||||
private:
|
||||
struct Entry {
|
||||
uint64_t b;
|
||||
T* t;
|
||||
};
|
||||
std::unordered_multimap<uint64_t, Entry> by_hash;
|
||||
};
|
||||
|
||||
} // namespace astc_reuse
|
||||
@@ -71,6 +71,28 @@ inline bool split_in_pass(const char* env, uint32_t vendor, bool every_draw) {
|
||||
return false; // helper threads record most frames there, and the rule stands aside for them) and no gain (90.6 / 89.2 fps
|
||||
// against 92.8 / 90.3 with the old barriers)
|
||||
}
|
||||
/// ((Triforce, 11-22, the performance audit's REPLAY-14, after Source A's) Two workarounds for Intel's drivers at a submission's
|
||||
/// splits (vk_replay.cpp maybe_split and the dry pass's points). No other vendor has either unless its switch asks.
|
||||
/// RX_VK_SPLIT_BEFORE_DRAW: a part ends only right before a plain draw. On an Arc B580 a command buffer that begins with mesh
|
||||
/// draws drew its later plain draws as exploded triangles (Source A: 40 shard frames of 1,121 before, 0 of 1,101 after). Unset:
|
||||
/// on for Intel's GPUs (vendor 0x8086); 0 off; anything else on, on any GPU
|
||||
inline bool split_before_draw(const char* env, uint32_t vendor) {
|
||||
if (env && *env) return std::strcmp(env, "0") != 0;
|
||||
return vendor == 0x8086;
|
||||
}
|
||||
/// RX_VK_SPLIT_UBFLUSH: whether a part's end writes the submission's uniform updates so far into the game's memory. Without
|
||||
/// mirrors they land in the memory later parts' other reads and the game's CPU see, mid-submission, where an unsplit submission
|
||||
/// writes them once at its end: white blocks and flashes on Intel's integrated GPUs (Source A: 1 run of 3 before, 0 of 3
|
||||
/// after). Unset: on, as before, but off on Intel's GPUs without mirrors; 0 off; anything else on
|
||||
inline bool split_ub_flush(const char* env, uint32_t vendor, bool mirrors) {
|
||||
if (env && *env) return std::strcmp(env, "0") != 0;
|
||||
return !(vendor == 0x8086 && !mirrors);
|
||||
}
|
||||
/// A part may end right before this op: a plain draw (NOP_DRAW, _IDX, _INDIRECT: not a mesh draw, not a dispatch) that will
|
||||
/// draw (rendering on, no rasterizer discard)
|
||||
constexpr bool cut_ok(bool plain_draw, bool render_enabled, bool raster_discard) {
|
||||
return plain_draw && render_enabled && !raster_discard;
|
||||
}
|
||||
/// Whether a part's split goes on in the same pass: the rule on, a pass open at the split, nothing recorded at the part's end (quiet:
|
||||
/// the work count as it was after the pass's end) and no event's write held then (written behind a full barrier before the part's
|
||||
/// write-backs, which copy its word to the game's memory: the GPU drained there all the same). ((Triforce, P4) no parallel replay's
|
||||
@@ -85,6 +107,25 @@ inline bool split_start_full(bool carry, bool recorded_between) {
|
||||
return !carry || recorded_between;
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-22, the performance audit's SYNC-66, after Source B's) RX_VK_CHUNK: how many draws and dispatches a part of a
|
||||
/// long submission has (vk_replay.cpp maybe_split). 120 on every card since the parts were made: the GPU starts early, and each
|
||||
/// split costs a queue submit, two full barriers and the part's bookkeeping, about 27 times a frame in a serial replay. Source B
|
||||
/// cuts at 128 where the GPU idles while the CPU records and at 4096 on desktop drivers; what pays here is not measured on any
|
||||
/// driver, so the default by driver is 120 for each until the lab picks one (S1 and S2 with RX_VK_PARALLEL=0 and the lab knob
|
||||
/// chunk:<n>). vendor: the card's PCI vendor; radv: Mesa's RADV
|
||||
constexpr int64_t kPartDraws = 120;
|
||||
inline int64_t part_draws_default(uint32_t vendor, bool radv) {
|
||||
(void)vendor, (void)radv;
|
||||
return kPartDraws;
|
||||
}
|
||||
/// The size in force: RX_VK_CHUNK's number when the variable is set (0 or less: whole submissions, never split); else
|
||||
/// knob_or_default, the lab knob chunk:<n>'s number in its phases and the driver's default outside them
|
||||
inline int64_t part_draws(bool env_set, int64_t env_value, int64_t knob_or_default) {
|
||||
return env_set ? env_value : knob_or_default;
|
||||
}
|
||||
/// As the size was read before: the variable's number, 120 unset
|
||||
inline int64_t part_draws_before(const char* env) { return env ? std::atoll(env) : 120; }
|
||||
|
||||
/// RX_VK_EVENT_BARRIER: "late" the rule on any card; "full" (or any other value) as before; unset or empty: with the pass barrier's
|
||||
/// game mode (pass_barrier::game: NVIDIA's driver unless RX_VK_PASS_BARRIER says otherwise)
|
||||
inline bool event_late(const char* env, bool pass_game) {
|
||||
@@ -139,5 +180,18 @@ inline bool batch_flush(Batch m, bool added_held, size_t held, size_t cap) {
|
||||
if (m == Batch::Parts && !added_held) return true;
|
||||
return held >= cap;
|
||||
}
|
||||
/// ((Triforce, 11-22, the performance audit's SYNC-09, after Source A's) RX_VK_BATCH_IDLE: a submit that joins the held batch
|
||||
/// sends it at once when the GPU has finished every batch sent. A held batch otherwise waits for a wait, a present, the 2 ms
|
||||
/// poll or the cap of 32 while the GPU sits idle beside it, until the game waits mid-frame at a capture's late pages; while the
|
||||
/// GPU is busy, submits still group. Source A has it on wherever batching is (RADV), off where guest memory stayed host-pointer
|
||||
/// imports (each submit there pays amdgpu's page check, 15-45 ms). HERE: on only with a number other than 0; unset it is OFF,
|
||||
/// on every driver, until a Linux RADV machine has run it (the audit's lab proof; with NVIDIA's driver and AMD's and Intel's on
|
||||
/// Windows nothing is held). No guard for host-pointer imports is built: with the switch off unless asked there is nothing for it to guard
|
||||
inline bool batch_idle_on(const char* env) { return env && std::atoi(env) != 0; }
|
||||
/// Whether a held batch is looked at for an idle GPU after a submit joined it: the switch on, Hold mode only (Parts sends by
|
||||
/// its own rule, when the game's submit returns), and something held (the cap's flush may just have sent it all)
|
||||
inline bool batch_idle(Batch m, bool held, bool on) {
|
||||
return on && m == Batch::Hold && held;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::barrier_rules
|
||||
|
||||
66
gpu/vk/bc_scratch.h
Normal file
66
gpu/vk/bc_scratch.h
Normal file
@@ -0,0 +1,66 @@
|
||||
// (Triforce, 11-21, the performance audit's TEX-10, after Source A's) How many layers the BC3 scratch has.
|
||||
//
|
||||
// Big ASTC textures are decoded into a 2048 x 2048 RGBA8 scratch and encoded again as BC3 on the GPU, into an output
|
||||
// buffer. Both were made with 8 layers, 128 + 32 = 160 MB (and the second scratch of the textures encoded at another
|
||||
// size, 128 MB more), whatever a run used; and a run decodes one layer (RX_VK_BC_LAYERS, the dark-line fix), so 140 MB
|
||||
// were held and never written, for good once made (RX_VK_BC_IDLE_S is 0). Sized by the layers one run uses they are
|
||||
// 16 + 4 = 20 MB. Source A measured 139 MiB less GPU memory on a Steam Deck, the median of 7 pairs, at the same fps.
|
||||
// RX_VK_BC_SCRATCH_FULL=1: all 8 layers, as before.
|
||||
//
|
||||
// The rule alone, no Vulkan: tests/test_bc_scratch_layers.py compiles it.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace bc_scratch {
|
||||
|
||||
inline constexpr uint32_t kDim = 2048, kMost = 8; // (the largest level a BC3 texture may have; the most layers a run may take)
|
||||
|
||||
/// The layers one run decodes and encodes: RX_VK_BC_LAYERS's value, 1 to kMost; not set, or not a number, 1
|
||||
inline uint32_t run_layers(const char* asked) {
|
||||
return asked ? std::clamp<uint32_t>((uint32_t)std::max(atoi(asked), 0), 1u, kMost) : 1u;
|
||||
}
|
||||
/// The scratch's layers: a run's, or all of them when full (RX_VK_BC_SCRATCH_FULL)
|
||||
inline uint32_t layers(uint32_t run, bool full) { return full ? kMost : std::clamp(run, 1u, kMost); }
|
||||
/// The output buffer's bytes: BC3's 16 bytes a block of 4 x 4, every layer
|
||||
inline uint64_t out_bytes(uint32_t layers) { return (uint64_t)(kDim / 4) * (kDim / 4) * layers * 16; }
|
||||
/// What the scratch holds, in MB: the image and its output buffer, and the second image when there is one
|
||||
inline uint64_t mb(uint32_t layers, bool second) {
|
||||
const uint64_t image = (uint64_t)kDim * kDim * 4 * layers;
|
||||
return (image + out_bytes(layers) + (second ? image : 0)) >> 20;
|
||||
}
|
||||
|
||||
// ---- (Triforce, 11-21, the performance audit's TEX-09, after Source A's) the small scratch ----
|
||||
// A run is a decode, an encode, a copy and three full barriers, and one layer a run made a 1365 x 1365 x 90 terrain array
|
||||
// of 11 levels about 990 of them: the GPU spent most of that time between the runs (Source A, Hyrule Field's entry on an
|
||||
// RX 9070 XT: 50 ms of GPU an array at one layer a run, 16-17 ms at 8; the lab's own array took 114.8 ms of an RTX 5090 in
|
||||
// one upload command buffer). Levels of kSmallDim or less each way, the arrays' second level down, go small_layers() layers
|
||||
// a run through a scratch of their own, kSmallDim x kSmallDim x 8: 32 MB, and the output buffer 8 MB for 4. One layer a
|
||||
// run was for dark lines between Hyrule Field's ground tiles on an RX 6600, layers after a run's first wrong at the small
|
||||
// levels, cause never found: so on AMD's cards the default stays 1 until the picture is checked there, and each layer of a
|
||||
// run is copied by a region of its own, at the encoder's own layer stride. A level of a texture encoded at another size
|
||||
// keeps the first scratch. RX_VK_BC_SMALL_LAYERS=<n>: the layers a run, 1 to 8; 1: no small scratch, as before.
|
||||
inline constexpr uint32_t kSmallDim = 1024;
|
||||
|
||||
/// The small scratch's layers, a run's of a small level: RX_VK_BC_SMALL_LAYERS, 1 to kMost; not set, kMost, and 1 on AMD's
|
||||
/// cards (vendor 0x1002)
|
||||
inline uint32_t small_layers(const char* asked, uint32_t vendor) {
|
||||
if (asked && *asked) return std::clamp<uint32_t>((uint32_t)std::max(atoi(asked), 0), 1u, kMost);
|
||||
return vendor == 0x1002 ? 1u : kMost;
|
||||
}
|
||||
/// Whether a level decoded at w x h goes through the small scratch: there is one, the level fits it, and the texture is not
|
||||
/// one encoded at another size (which has the second scratch)
|
||||
inline bool small_level(uint32_t w, uint32_t h, bool have_small, bool resized) {
|
||||
return have_small && !resized && w <= kSmallDim && h <= kSmallDim;
|
||||
}
|
||||
/// The small scratch's part of the output buffer, and its image, in bytes
|
||||
inline uint64_t small_out_bytes(uint32_t layers) { return (uint64_t)(kSmallDim / 4) * (kSmallDim / 4) * layers * 16; }
|
||||
inline uint64_t small_image_bytes(uint32_t layers) { return (uint64_t)kSmallDim * kSmallDim * 4 * layers; }
|
||||
/// Where layer k of a run begins in the encoder's output: host_shaders/bc_encoder.comp writes block
|
||||
/// (z * blocks_y + y) * blocks_x + x, 16 bytes a block, 8 where only the colour half is written (an image kept as BC1)
|
||||
inline uint64_t layer_offset(uint32_t k, uint32_t w, uint32_t h, bool bc1) {
|
||||
return (uint64_t)k * ((w + 3) / 4) * ((h + 3) / 4) * (bc1 ? 8 : 16);
|
||||
}
|
||||
|
||||
} // namespace bc_scratch
|
||||
164
gpu/vk/built_check.h
Normal file
164
gpu/vk/built_check.h
Normal file
@@ -0,0 +1,164 @@
|
||||
// Whether the caches still have what the light precompile's record says it built (no game, no GPU: the rules; vk_draw.cpp
|
||||
// built_check_cold, precompile_run, built_play_build). After lib_check.h, which asks the same of the stage libraries, and
|
||||
// predict_check.h, which asks it of the predicted pipelines.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's.) A light precompile (little RAM:
|
||||
// precompile_keeps_all) builds the noted pipelines for the caches, lets them go, and records their keys in <notes>.built,
|
||||
// the compute programs' as one key. A later start trusts that record on its stamp alone: what it lists is left to its first
|
||||
// draw or dispatch, to be built from the caches there. When the caches lost it (the pipeline cache file deleted or refused,
|
||||
// the play file lost, a driver's own cache cleared or past its cap), each of those first draws compiles in play while the
|
||||
// record still says built: a draw left out or waiting for a whole build, a dispatch waiting 50 ms and then left out. Source A
|
||||
// measured a cleared NVIDIA cache: 702,312 draws left out and 21.3 s of draw waits in 140 s of play, against 456 and none
|
||||
// once everything was built again before the title.
|
||||
//
|
||||
// The check. Before the record is trusted, up to kGraphics of the recorded notes (spread over them from another place each
|
||||
// start) are asked of Triforce's own caches as a draw asks, and up to kComputes of the listed compute programs as a dispatch
|
||||
// asks, where the device can ask without compiling (FAIL_ON_PIPELINE_COMPILE_REQUIRED): the answer is the caches' own.
|
||||
// Not a scratch cache of the check's own: AMD's Windows driver keeps no cache of its own (pcache_bound.h), so a question
|
||||
// put to the driver alone would always be cold there. Without the flag, a pipeline is built twice and timed, as lib_check
|
||||
// does: the second comes from the cache by construction, and a first over kRatio times it and kSlackMs more was compiled.
|
||||
// Under kAnswers answers the sample says nothing (the record is trusted, as before); at half of them or more cold it is cold.
|
||||
//
|
||||
// Cold: the record is not trusted this start, everything noted and every listed compute program is built again before the
|
||||
// title (as for a fresh record), and the record is written anew after it. <notes>.built.check keeps, for the stamp, that the
|
||||
// check built them again (kStrikes 0): cold again right after that, the caches do not keep them on this PC (often a driver's
|
||||
// shader cache that is too small), and they are left to their first draws, as before this check, with a hint in the log. A
|
||||
// warm sample forgets that.
|
||||
//
|
||||
// In play, a counter of what the check is for: recorded pipelines whose first build took kSlowUs or more, and recorded
|
||||
// compute programs the dispatch's ask did not find, logged at most every kSayMs while it grows.
|
||||
//
|
||||
// RX_VK_BUILT_CHECK: 1 by default; 0 no check and no counter, as before; 2 always cold (the lab's and the tests').
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace built_check {
|
||||
|
||||
inline constexpr int kGraphics = 24; // (recorded notes asked: Source A asks 48, Source B six computes)
|
||||
inline constexpr int kLook = 2 * kGraphics; // (notes looked at for them: one may have lost its program)
|
||||
inline constexpr int kComputes = 6; // (listed compute programs asked)
|
||||
inline constexpr int kAnswers = 16; // (fewer answers say nothing)
|
||||
inline constexpr double kRatio = 4.0; // (without the flag: a first build against the second, compiled above)
|
||||
inline constexpr double kSlackMs = 3.0; // (and this much more)
|
||||
inline constexpr int kStrikes = 0; // (built again once, and cold again right after: left to the draws)
|
||||
inline constexpr int64_t kSlowUs = 40000; // (a play build of a recorded pipeline over this was compiled)
|
||||
inline constexpr int64_t kSayMs = 10000; // (the counter's line: at most this often)
|
||||
|
||||
/// RX_VK_BUILT_CHECK: 0 off, 1 on (unset too), 2 always cold
|
||||
inline int mode(const char* env) {
|
||||
if (!env || !*env) return 1;
|
||||
const int v = std::atoi(env);
|
||||
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
|
||||
}
|
||||
|
||||
/// One pipeline asked. asked: asked of the caches only; answered_cold: they said a compile is required; made: it was made
|
||||
/// (both times, not asked). One the driver refuses says nothing of the caches
|
||||
struct Probe {
|
||||
bool asked = false, answered_cold = false, made = false;
|
||||
double first_ms = 0, again_ms = 0;
|
||||
};
|
||||
inline bool counts(const Probe& p) { return p.answered_cold || p.made; }
|
||||
inline bool cold(const Probe& p) {
|
||||
return p.answered_cold || (!p.asked && p.made && p.first_ms > kRatio * p.again_ms + kSlackMs);
|
||||
}
|
||||
|
||||
struct Tally {
|
||||
int got = 0, cold = 0, answered = 0;
|
||||
};
|
||||
inline Tally tally(const std::vector<Probe>& probes) {
|
||||
Tally t;
|
||||
for (const Probe& p : probes)
|
||||
if (counts(p)) t.got++, t.cold += cold(p), t.answered += p.answered_cold;
|
||||
return t;
|
||||
}
|
||||
|
||||
enum class Verdict { Unknown, Warm, Cold };
|
||||
/// Under kAnswers answers nothing; half of them or more cold: cold
|
||||
inline Verdict verdict(const Tally& t) {
|
||||
if (t.got < kAnswers) return Verdict::Unknown;
|
||||
return t.cold * 2 >= t.got ? Verdict::Cold : Verdict::Warm;
|
||||
}
|
||||
|
||||
/// k places spread over n, the first at seed's place in the first step: each start asks other pipelines
|
||||
inline std::vector<size_t> spots(size_t n, size_t k, uint64_t seed) {
|
||||
std::vector<size_t> at;
|
||||
if (!n || !k) return at;
|
||||
const size_t step = std::max<size_t>(1, n / k);
|
||||
for (size_t i = (size_t)(seed % step); i < n && at.size() < k; i += step) at.push_back(i);
|
||||
return at;
|
||||
}
|
||||
|
||||
// ---- <notes>.built.check: "built-check <stamp> <strikes>"
|
||||
|
||||
struct State {
|
||||
bool found = false;
|
||||
uint64_t stamp = 0;
|
||||
int strikes = 0;
|
||||
};
|
||||
inline State state_read(const std::string& path) {
|
||||
State s;
|
||||
FILE* f = fopen(path.c_str(), "rb");
|
||||
if (!f) return s;
|
||||
char line[96]{};
|
||||
const size_t n = fread(line, 1, sizeof line - 1, f);
|
||||
fclose(f);
|
||||
line[n] = 0;
|
||||
unsigned long long stamp = 0;
|
||||
int strikes = 0;
|
||||
if (sscanf(line, "built-check %16llx %d", &stamp, &strikes) == 2 && strikes >= 0)
|
||||
s.found = true, s.stamp = stamp, s.strikes = strikes;
|
||||
return s;
|
||||
}
|
||||
/// Written whole beside the old one, then put in place (a start that ends in the middle leaves the old file)
|
||||
inline bool state_write(const std::string& path, uint64_t stamp, int strikes) {
|
||||
char line[96];
|
||||
const int n = snprintf(line, sizeof line, "built-check %016llx %d\n", (unsigned long long)stamp, strikes);
|
||||
const std::string tmp = path + ".tmp";
|
||||
FILE* f = fopen(tmp.c_str(), "wb");
|
||||
if (!f) return false;
|
||||
bool ok = fwrite(line, 1, (size_t)n, f) == (size_t)n;
|
||||
ok = fclose(f) == 0 && ok;
|
||||
if (ok) {
|
||||
remove(path.c_str()); // (rename does not replace on Windows)
|
||||
ok = rename(tmp.c_str(), path.c_str()) == 0;
|
||||
}
|
||||
if (!ok) remove(tmp.c_str());
|
||||
return ok;
|
||||
}
|
||||
|
||||
enum class Do { Nothing, Again, Stop };
|
||||
/// What a start does with the verdict, the record of the last starts and the stamp now. always: mode 2, built again at every
|
||||
/// start whatever was recorded. *next: the strikes to record once they are built again (Again only)
|
||||
inline Do decide(Verdict v, const State& had, uint64_t stamp, bool always, int* next) {
|
||||
if (always) return *next = 0, Do::Again;
|
||||
if (v != Verdict::Cold) return Do::Nothing;
|
||||
const bool same = had.found && had.stamp == stamp; // (built again for this stamp, and cold again since)
|
||||
if (same && had.strikes >= kStrikes) return Do::Stop;
|
||||
*next = same ? had.strikes + 1 : 0;
|
||||
return Do::Again;
|
||||
}
|
||||
|
||||
// ---- the counter: play builds of recorded pipelines the caches did not have
|
||||
|
||||
/// Whether a build of that many microseconds was compiled rather than found
|
||||
inline bool slow(int64_t us) { return us >= kSlowUs; }
|
||||
/// Whether the counter's line is due: said_ms, when it was last written (0: never)
|
||||
inline bool say(int64_t now_ms, int64_t said_ms) { return !said_ms || now_ms - said_ms >= kSayMs; }
|
||||
/// keys sorted without repeats, for has()
|
||||
inline std::vector<uint64_t> sorted(std::vector<uint64_t> keys) {
|
||||
std::sort(keys.begin(), keys.end());
|
||||
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
||||
return keys;
|
||||
}
|
||||
inline bool has(const std::vector<uint64_t>& sorted_keys, uint64_t key) {
|
||||
return std::binary_search(sorted_keys.begin(), sorted_keys.end(), key);
|
||||
}
|
||||
|
||||
} // namespace built_check
|
||||
64
gpu/vk/cap_spare.h
Normal file
64
gpu/vk/cap_spare.h
Normal file
@@ -0,0 +1,64 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-33, after Source A's) Which spare capture buffer a capture takes, how
|
||||
// much the spares and the ones being made ahead may hold together, and a capture's size in steps.
|
||||
//
|
||||
// A capture's buffer comes from the spares (retired buffers, and the 32 to 192 MB ones made ahead on a worker), or is
|
||||
// made new on the game's submitting thread. Three things were so before:
|
||||
// - the new buffer was made with cap_free_mu held (AMD's driver: about 0.8 ms a MB), and the replay's take, holding
|
||||
// pools_mu, waits on that mutex to retire a buffer;
|
||||
// - any spare that fit was taken, so a 2 MB capture could sit in a load's 192 MB spare for the 24 submissions a capture
|
||||
// is kept, and the load then made its own on the game's thread;
|
||||
// - a spare made ahead was made once: the one a load took was not made again.
|
||||
// The rules here are the decisions alone (no Vulkan, no lock): tests/test_capture_spare_reuse.py compiles them, and
|
||||
// vk_memory.cpp's cap_buf_get, cap_prewarm_class and cap_buf_free ask them. RX_VK_CAP_SPARE_FIT=0: as before.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace cap_spare {
|
||||
|
||||
constexpr uint64_t kMB = 1ull << 20;
|
||||
constexpr uint64_t kBig = 32 * kMB; // (a load's spare: the classes made ahead start here)
|
||||
constexpr uint64_t kHeld = 768 * kMB; // (the most the spares hold, whatever the card)
|
||||
constexpr size_t kMost = 12; // (and the most there are)
|
||||
|
||||
/// A capture of size class `cls` leaves the spare of `size` bytes alone: a big one (32 MB or more) over twice its class.
|
||||
/// any_size: no spare is left alone (spares capped at 32 MB, the switch off, or the fallback after a failed allocation)
|
||||
constexpr bool skips(uint64_t size, uint64_t cls, bool any_size) { return !any_size && size >= kBig && size > 2 * cls; }
|
||||
|
||||
/// The spare a capture of `need` bytes (class `cls`) takes of n spares, size_of(i) the i-th one's bytes: the smallest that
|
||||
/// fits and is not left alone, the first of equals. n: none
|
||||
template <class SizeOf>
|
||||
constexpr size_t pick(size_t n, const SizeOf& size_of, uint64_t need, uint64_t cls, bool any_size) {
|
||||
size_t best = n;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
const uint64_t s = size_of(i);
|
||||
if (s >= need && !skips(s, cls, any_size) && (best == n || s < size_of(best))) best = i;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/// Room for one more spare of `size` bytes to be made ahead: with what the spares hold and what is being made (reserved)
|
||||
/// it stays within what spares may hold (spare_max: the card's, cap_spare_max) and 768 MB
|
||||
constexpr bool room(uint64_t held, uint64_t reserved, uint64_t size, uint64_t spare_max) {
|
||||
const uint64_t limit = spare_max < kHeld ? spare_max : kHeld;
|
||||
return held + reserved <= limit && size <= limit - (held + reserved);
|
||||
}
|
||||
|
||||
/// A buffer of `size` bytes given back is kept as a spare: fewer than 12 with the ones being made (workers), none over 256
|
||||
/// MB (keep_old, RX_VK_CAPTURE_KEEP_OLD: 64), and with the bytes being made (reserved) within the limit. With no workers
|
||||
/// and nothing reserved this is the rule as it was
|
||||
constexpr bool keeps(size_t spares, size_t workers, uint64_t held, uint64_t reserved, uint64_t size, uint64_t spare_max, bool keep_old) {
|
||||
const uint64_t limit = keep_old ? spare_max : (spare_max < kHeld ? spare_max : kHeld);
|
||||
return spares + workers < kMost && size <= (keep_old ? 64 * kMB : 256 * kMB) && held + reserved <= limit &&
|
||||
size <= limit - (held + reserved);
|
||||
}
|
||||
|
||||
/// (RX_VK_CAP_STEP_MB=<n>, 0: off) a capture over 16 and up to 32 MB in steps of step_mb MB, not the whole 32 MB class,
|
||||
/// where spares are capped at 32 MB (a card of 6.5 GB or less). cls: its class as it is (cap_buf_size)
|
||||
constexpr uint64_t stepped(uint64_t need, uint64_t cls, uint64_t step_mb, bool capped) {
|
||||
if (!step_mb || !capped || cls != kBig || need <= 16 * kMB || need > kBig) return cls;
|
||||
const uint64_t step = step_mb * kMB, v = (need + step - 1) / step * step;
|
||||
return v < cls ? v : cls;
|
||||
}
|
||||
|
||||
} // namespace cap_spare
|
||||
110
gpu/vk/capture_tables.h
Normal file
110
gpu/vk/capture_tables.h
Normal file
@@ -0,0 +1,110 @@
|
||||
// (Triforce, 11-21, the upscaler audit's UPS-02) The stored draws' two tables, kept from one frame to the next.
|
||||
//
|
||||
// While NVIDIA DLSS, AMD FSR 4 or AMD FSR 3.1 upscales, every plain draw into the scene stores its positions and is drawn
|
||||
// again into the motion vectors (vk_temporal.cpp capture_motion). Two tables go with that, both filled again every frame:
|
||||
// the draws of a model so far this frame (which of them a draw is: the n-th this frame is the n-th the frame before), and
|
||||
// the frame before's draws by their key (where each one's positions of that frame are). Both were std::unordered_map,
|
||||
// emptied and filled each frame: a node from the heap for every entry and back to it a frame later, about 1,400 of each
|
||||
// a frame in the owner's log of 11-21 ("captured motion ... of 1444 stored a frame ... 0.650 ms CPU a frame"), on the
|
||||
// thread that records the frame. Here they keep their memory: the count in one open-addressed array (flat_map.h) that is
|
||||
// emptied in place, the frame before's draws in two lists that take turns, found through such an array. A frame of about
|
||||
// the size of the one before takes nothing from the heap.
|
||||
//
|
||||
// RX_VK_CAPTURE_TABLES=0: std::unordered_map again, a node an entry, as before (kept()).
|
||||
//
|
||||
// No Vulkan: tests/test_capture_tables.py compiles this and counts the heap's calls.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "flat_map.h"
|
||||
|
||||
namespace capture_tables {
|
||||
|
||||
/// RX_VK_CAPTURE_TABLES (value, null when unset): kept unless it is 0
|
||||
inline bool kept(const char* value) { return !value || !*value || std::atoi(value) != 0; }
|
||||
|
||||
/// An array emptied for the next frame: in place, so the frame fills it without the heap; given back when it is far
|
||||
/// larger than what the frame put in it (a frame of many draws once, then few), so emptying it does not walk all of that
|
||||
template <class K, class V>
|
||||
inline void empty_for_next(FlatMap<K, V>& m) {
|
||||
if (m.capacity() > 8 * (m.size() + 512)) m.clear();
|
||||
else m.reset();
|
||||
}
|
||||
|
||||
/// The room a frame's list is given at its start: the frame before's count (kept), so it grows once and not a dozen times
|
||||
/// from nothing; 0, as before, with the switch off
|
||||
inline size_t room(size_t taken, bool keep) { return keep ? taken : 0; }
|
||||
|
||||
/// How many of a model's draws came so far this frame
|
||||
class Nth {
|
||||
public:
|
||||
explicit Nth(bool keep = true) : keep_(keep) {}
|
||||
/// The draws of `model` before this one, which is counted
|
||||
int next(uint64_t model) { return keep_ ? flat_[model]++ : node_[model]++; }
|
||||
void clear() {
|
||||
if (keep_) empty_for_next(flat_);
|
||||
else node_.clear();
|
||||
}
|
||||
size_t size() const { return keep_ ? flat_.size() : node_.size(); }
|
||||
|
||||
private:
|
||||
bool keep_;
|
||||
FlatMap<uint64_t, int> flat_;
|
||||
std::unordered_map<uint64_t, int> node_;
|
||||
};
|
||||
|
||||
/// A frame's entries by key, and the frame before's: turn() makes what was put since the last turn the frame before's,
|
||||
/// put() notes one of this frame's (the last with a key stands), before() finds the frame before's
|
||||
template <class V>
|
||||
class Before {
|
||||
public:
|
||||
explicit Before(bool keep = true) : keep_(keep) {}
|
||||
void turn() {
|
||||
if (!keep_) {
|
||||
node_old_ = std::move(node_now_);
|
||||
node_now_.clear();
|
||||
return;
|
||||
}
|
||||
std::swap(now_, old_), std::swap(now_at_, old_at_);
|
||||
now_.clear(); // (its memory kept)
|
||||
empty_for_next(now_at_);
|
||||
}
|
||||
void put(uint64_t key, const V& v) {
|
||||
if (!keep_) {
|
||||
node_now_[key] = v;
|
||||
return;
|
||||
}
|
||||
uint32_t& at = now_at_[key]; // (the entry's place in now_, plus one: 0 is none yet)
|
||||
if (at) now_[at - 1] = v;
|
||||
else now_.push_back(v), at = (uint32_t)now_.size();
|
||||
}
|
||||
/// The frame before's entry of `key`, null when it had none: good until the next turn() or clear()
|
||||
const V* before(uint64_t key) {
|
||||
if (!keep_) {
|
||||
const auto it = node_old_.find(key);
|
||||
return it == node_old_.end() ? nullptr : &it->second;
|
||||
}
|
||||
const auto* s = old_at_.find(key);
|
||||
return s ? &old_[s->second - 1] : nullptr;
|
||||
}
|
||||
/// Nothing of either frame (the chain let go: its memory too)
|
||||
void clear() {
|
||||
std::vector<V>().swap(now_), std::vector<V>().swap(old_);
|
||||
now_at_.clear(), old_at_.clear();
|
||||
node_now_.clear(), node_old_.clear();
|
||||
}
|
||||
size_t size() const { return keep_ ? now_.size() : node_now_.size(); }
|
||||
|
||||
private:
|
||||
bool keep_;
|
||||
std::vector<V> now_, old_;
|
||||
FlatMap<uint64_t, uint32_t> now_at_, old_at_;
|
||||
std::unordered_map<uint64_t, V> node_now_, node_old_;
|
||||
};
|
||||
|
||||
} // namespace capture_tables
|
||||
@@ -23,6 +23,7 @@ inline const char* name(char c) {
|
||||
case 'u': return "upload of texture";
|
||||
case 'g': return "GPU unswizzle of texture";
|
||||
case 'w': return "write-back diff pass, chunks";
|
||||
case 's': return "mirror syncs' pass, chunks"; // ((Triforce, 11-21, MEM-48))
|
||||
case 'r': return "recording begun, value";
|
||||
case 'v': return "uploads of the recording, value";
|
||||
case 'z': return "recording ended, value";
|
||||
@@ -45,6 +46,9 @@ inline const char* gap(char c) {
|
||||
return "that pool's write-back copies (its mirror -> guest memory, and the shadow's fill), then the write-back diff pass if "
|
||||
"one follows (w): with no w after it, the end of the recording (z)";
|
||||
case 'u': case 'g': return "that texture's upload or unswizzle itself";
|
||||
case 's':
|
||||
return "the mirror syncs' pass (one compute dispatch in the recording's uploads, reading a capture's buffer and writing "
|
||||
"the pool's VRAM mirror by device address; RX_VK_MSYNC_PASS=0: copy regions), then the sync's copies from guest memory";
|
||||
case 'w':
|
||||
return "the write-back diff pass (one compute dispatch over every pool's chunks, reading mirrors and shadows and writing guest "
|
||||
"memory by device address), then its barrier and the end of the recording (z)";
|
||||
|
||||
120
gpu/vk/commit_room.h
Normal file
120
gpu/vk/commit_room.h
Normal file
@@ -0,0 +1,120 @@
|
||||
// (Triforce, 11-22, the performance audit's OS-05, after Source A's) The textures give commit back near Windows' commit limit,
|
||||
// and the page files' room to grow counts as commit still to come.
|
||||
//
|
||||
// Windows charges every block of a discrete card's memory to the process's commit, as it does the game's own memory. Where the
|
||||
// system's commit runs out (RAM and the page files full) an allocation is refused, and the game ends unless the refusal passes
|
||||
// (OS-04's wait, HOS_COMMIT_WAIT_MS, helps only if commit comes back). Source A takes the shortfall under a reserve of free
|
||||
// commit (2,560 MB) off the texture budget on any card, so that eviction and compaction give blocks back. Its first rule fired
|
||||
// while a page file Windows manages could still grow (292,526 textures retired in 90 minutes, black textures); the fix counts
|
||||
// each page file's room to grow, capped by its drive's free space, as commit still free.
|
||||
//
|
||||
// The rules here, apart from Windows so that tests/test_commit_room.py can run them:
|
||||
// - reserve_of: RX_VK_TEX_COMMIT_RESERVE_MB, the reserve. Off (0) unless set: the entry's 2560 is not measured on this tree
|
||||
// - growth_on: RX_VK_TEX_COMMIT_GROWTH, whether the growth is counted: 0 never, another number always, unset where a reserve is
|
||||
// - low_line, read_growth: the growth is read only while less is free than the higher of the reserve and the line of
|
||||
// tex_pressure::commit_low (whose fall-back to the fixed size then gets the growth too: vk_texture.cpp commit_now)
|
||||
// - cap: the textures at most what they hold less the shortfall under the reserve, never under 1 GB (Source A has no floor,
|
||||
// and its false cut went to 0)
|
||||
// - growth: the page files' room to grow, from the records of NtQuerySystemInformation's class 0x90 (undocumented: each
|
||||
// record is checked against the buffer, and one that is not inside it ends the walk), a drive's page files capped together
|
||||
// by its free space. vk_texture.cpp page_file_growth asks Windows, at most once a second, and answers 0 on any failure.
|
||||
// Written from the audit's description: Source A's file is not on this PC.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace commit_room {
|
||||
|
||||
inline constexpr uint64_t kFloor = 1ull << 30; // (the textures are never capped under 1 GB)
|
||||
inline constexpr int64_t kMostMb = 65536; // (the reserve at most)
|
||||
|
||||
/// RX_VK_TEX_COMMIT_RESERVE_MB in bytes: 0 (unset, empty, no number, 0 or less): no cut, as before; at most kMostMb MB
|
||||
inline uint64_t reserve_of(const char* env) {
|
||||
if (!env || !*env) return 0;
|
||||
const long long mb = std::atoll(env);
|
||||
return mb <= 0 ? 0 : (uint64_t)std::min<long long>(mb, kMostMb) << 20;
|
||||
}
|
||||
|
||||
/// RX_VK_TEX_COMMIT_GROWTH: unset or empty, counted where a reserve is set; 0 (or no number), never; another number, always
|
||||
inline bool growth_on(const char* env, uint64_t reserve) {
|
||||
if (!env || !*env) return reserve != 0;
|
||||
return std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
/// tex_pressure::commit_low's line in bytes: want_mb MB; -1 (less than 0) the larger of 4 GB and a tenth of the limit; 0, or a
|
||||
/// limit not known (0), none
|
||||
inline uint64_t low_line(uint64_t limit, int64_t want_mb) {
|
||||
if (!limit || !want_mb) return 0;
|
||||
return want_mb > 0 ? (uint64_t)want_mb << 20 : std::max<uint64_t>(4096ull << 20, limit / 10);
|
||||
}
|
||||
|
||||
/// Whether the page files' growth is read at this look: counted (on), and less free (avail) than the higher of the reserve and
|
||||
/// commit_low's line (low)
|
||||
inline bool read_growth(bool on, uint64_t avail, uint64_t reserve, uint64_t low) { return on && avail < std::max(reserve, low); }
|
||||
|
||||
/// The textures' cap: what they hold (total) less the shortfall of what is free (avail, the growth in it) under the reserve,
|
||||
/// never under kFloor. UINT64_MAX (no cap) with no reserve, or with the reserve free
|
||||
inline uint64_t cap(uint64_t total, uint64_t avail, uint64_t reserve) {
|
||||
if (!reserve || avail >= reserve) return UINT64_MAX;
|
||||
const uint64_t shortfall = reserve - avail;
|
||||
return std::max(kFloor, total > shortfall ? total - shortfall : 0);
|
||||
}
|
||||
|
||||
/// NtQuerySystemInformation's class for the page files' records (SystemPageFileInformationEx)
|
||||
inline constexpr unsigned kPageFileClass = 0x90;
|
||||
|
||||
/// One record of that class as 64-bit Windows lays it out; sizes in pages
|
||||
struct PageFileRecord {
|
||||
uint32_t next; // (bytes from this record to the next; 0: the last)
|
||||
uint32_t total_pages; // (its size now)
|
||||
uint32_t in_use_pages, peak_pages;
|
||||
uint16_t name_bytes, name_room; // (the name: UTF-16, such as \??\C:\pagefile.sys, in the same buffer)
|
||||
const uint16_t* name;
|
||||
uint32_t min_pages, max_pages; // (the size it may grow to)
|
||||
};
|
||||
static_assert(sizeof(void*) != 8 || sizeof(PageFileRecord) == 40, "the record as 64-bit Windows lays it out");
|
||||
|
||||
/// The drive letter of a page file's name ('C' for \??\C:\pagefile.sys), 0 where it has none
|
||||
inline char drive_of(const uint16_t* name, size_t chars) {
|
||||
for (size_t i = 0; i + 1 < chars; i++) {
|
||||
const uint16_t c = name[i];
|
||||
if (name[i + 1] == ':' && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) return (char)(c >= 'a' ? c - 32 : c);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// The page files' room to grow in bytes, from the records in buf[0, len): a record's (max - size) pages of `page` bytes, the
|
||||
/// records of one drive summed and capped by that drive's free space (free_of('C'), bytes). A record that is not wholly inside
|
||||
/// the buffer, whose name is not, or whose next one would overlap it, ends the walk: what came before it counts
|
||||
template <class FreeOf>
|
||||
inline uint64_t growth(const uint8_t* buf, size_t len, uint64_t page, FreeOf free_of) {
|
||||
uint64_t by_drive[26] = {};
|
||||
const uintptr_t base = (uintptr_t)buf;
|
||||
for (size_t at = 0; len >= sizeof(PageFileRecord) && at <= len - sizeof(PageFileRecord);) {
|
||||
PageFileRecord r;
|
||||
std::memcpy(&r, buf + at, sizeof r);
|
||||
char drive = 0;
|
||||
if (r.name_bytes) {
|
||||
const uintptr_t n = (uintptr_t)r.name;
|
||||
if (n < base || n - base > len || r.name_bytes > len - (n - base)) break;
|
||||
uint16_t chars[64] = {};
|
||||
const size_t count = std::min<size_t>(r.name_bytes / 2, 64);
|
||||
std::memcpy(chars, buf + (n - base), count * 2);
|
||||
drive = drive_of(chars, count);
|
||||
}
|
||||
if (drive && r.max_pages > r.total_pages) by_drive[drive - 'A'] += (uint64_t)(r.max_pages - r.total_pages) * page;
|
||||
if (!r.next) break;
|
||||
if (r.next < sizeof r || r.next > len - at) break;
|
||||
at += r.next;
|
||||
}
|
||||
uint64_t sum = 0;
|
||||
for (int d = 0; d < 26; d++)
|
||||
if (by_drive[d]) sum += std::min<uint64_t>(by_drive[d], free_of((char)('A' + d)));
|
||||
return sum;
|
||||
}
|
||||
|
||||
} // namespace commit_room
|
||||
125
gpu/vk/compute_pin.h
Normal file
125
gpu/vk/compute_pin.h
Normal file
@@ -0,0 +1,125 @@
|
||||
// Which compute programs are held to 32 lanes (requiredSubgroupSize), by what the program can tell (no game, no GPU: a rule).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-52, after Source A's and Source B's.) The console's programs are written
|
||||
// for warps of 32 lanes, so every compute pipeline asked for subgroups of 32 (vk_draw.cpp pin_subgroup). Most programs
|
||||
// cannot tell how many lanes run beside them: they never vote, never read a lane mask or their lane's index, never
|
||||
// shuffle between lanes. Mesa's Intel driver compiles a pinned program for 32 lanes only (SIMD32), which on its large
|
||||
// programs is minutes and gigabytes: Source A measured a 730 KB program at 0.8-1 GB over 23-25 s pinned and at most 300 MB
|
||||
// unpinned (an Iris Xe, Mesa 26.2.4), with 271 of 1046 variants still pinned by this rule.
|
||||
//
|
||||
// A program is pinned when it can see its lanes:
|
||||
// - a vote, a lane mask, the lane's index, a shuffle, FSWZADD or a matrix multiply (Shader::Info's flags);
|
||||
// - a barrier of Subgroup scope in its module: this translator's own case, the barrier its lowering puts between a
|
||||
// warp's shared-memory store and load, which takes the subgroup to be the warp. No flag records it, so the module is
|
||||
// looked at (subgroup_barrier below), and a module nobody looked at counts as having one;
|
||||
// - shared memory in a workgroup of 32 invocations or fewer, where the workgroup is one warp and its lanes run in step.
|
||||
// Everything else gets the driver's own choice: no required size, and no full-subgroups flag either.
|
||||
//
|
||||
// RX_VK_COMPUTE_PIN, read once:
|
||||
// all every compute program pinned, as it was (the default on every driver but Mesa's Intel one)
|
||||
// lanes the rule above (the default on Mesa's Intel driver)
|
||||
// shared the rule above, and any shared memory pins, whatever the workgroup: for a program whose warps keep
|
||||
// shared memory in step in a workgroup over 32, which nothing else here sees
|
||||
// A doubt pins: an unknown value of the switch is the driver's default, a program without a stage is pinned.
|
||||
// The graphics stages and the mesh fallback's compute stages are not asked here: they stay with pin_subgroup.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
namespace compute_pin {
|
||||
enum class Mode { All, Lanes, Shared };
|
||||
|
||||
/// What one compute program is, as the rule needs it
|
||||
struct Facts {
|
||||
bool has_stage = false; // (false: nothing known, pinned)
|
||||
bool vote = false; // Shader::Info::uses_subgroup_vote
|
||||
bool mask = false; // uses_subgroup_mask
|
||||
bool lane_index = false; // uses_subgroup_invocation_id
|
||||
bool shuffles = false; // uses_subgroup_shuffles
|
||||
bool fswzadd = false; // uses_fswzadd
|
||||
bool matrices = false; // uses_cooperative_matrix
|
||||
bool subgroup_barrier = true; // its module has an OpControlBarrier of Subgroup scope (true until a look says no)
|
||||
uint32_t shared_memory = 0; // bytes of shared memory
|
||||
uint64_t invocations = 0; // the workgroup's x * y * z
|
||||
};
|
||||
|
||||
/// The program can tell how many lanes run in its subgroup, or which one it is
|
||||
constexpr bool sees_lanes(const Facts& f) {
|
||||
return f.vote || f.mask || f.lane_index || f.shuffles || f.fswzadd || f.matrices || f.subgroup_barrier;
|
||||
}
|
||||
|
||||
/// RX_VK_COMPUTE_PIN's value (nullptr: unset) on this driver
|
||||
inline Mode mode_for(const char* env, bool mesa_intel) {
|
||||
if (env && !std::strcmp(env, "all")) return Mode::All;
|
||||
if (env && !std::strcmp(env, "lanes")) return Mode::Lanes;
|
||||
if (env && !std::strcmp(env, "shared")) return Mode::Shared;
|
||||
return mesa_intel ? Mode::Lanes : Mode::All;
|
||||
}
|
||||
|
||||
inline const char* name(Mode m) {
|
||||
return m == Mode::All ? "all" : m == Mode::Lanes ? "lanes" : "shared";
|
||||
}
|
||||
|
||||
/// The program's pipeline asks for subgroups of 32
|
||||
constexpr bool needed(Mode m, const Facts& f) {
|
||||
if (m == Mode::All || !f.has_stage) return true;
|
||||
if (sees_lanes(f)) return true;
|
||||
if (f.shared_memory) return m == Mode::Shared || f.invocations <= 32;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Whether a SPIR-V module's entry point is a compute shader's (OpEntryPoint with the GLCompute model): the only modules
|
||||
/// the rule is asked about, so the only ones worth a note. One that does not read as SPIR-V: no
|
||||
inline bool compute_entry(const uint32_t* code, size_t words) {
|
||||
constexpr uint32_t kMagic = 0x07230203, kOpEntryPoint = 15, kOpFunction = 54, kGLCompute = 5;
|
||||
if (!code || words < 5 || code[0] != kMagic) return false;
|
||||
for (size_t at = 5; at < words;) {
|
||||
const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu;
|
||||
if (!count || at + count > words || op == kOpFunction) return false;
|
||||
if (op == kOpEntryPoint && count >= 2) return code[at + 1] == kGLCompute;
|
||||
at += count;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Whether a SPIR-V module has an OpControlBarrier whose execution scope is the constant Subgroup (3). A module that does
|
||||
/// not read as SPIR-V counts as having one. words: the module's 32-bit words
|
||||
inline bool subgroup_barrier(const uint32_t* code, size_t words) {
|
||||
constexpr uint32_t kMagic = 0x07230203, kOpConstant = 43, kOpControlBarrier = 224, kScopeSubgroup = 3;
|
||||
if (!code || words < 5 || code[0] != kMagic) return true;
|
||||
// (the scopes are constants, and constants come before the functions: one pass for the barriers' scope ids, kept small,
|
||||
// and their values looked up at the end)
|
||||
constexpr size_t kMost = 64;
|
||||
uint32_t scopes[kMost];
|
||||
size_t n_scopes = 0;
|
||||
for (size_t at = 5; at < words;) {
|
||||
const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu;
|
||||
if (!count || at + count > words) return true;
|
||||
if (op == kOpControlBarrier && count >= 4) {
|
||||
const uint32_t id = code[at + 1];
|
||||
bool seen = false;
|
||||
for (size_t i = 0; i < n_scopes; i++) seen = seen || scopes[i] == id;
|
||||
if (!seen) {
|
||||
if (n_scopes == kMost) return true; // (more scope ids than a module has: a doubt)
|
||||
scopes[n_scopes++] = id;
|
||||
}
|
||||
}
|
||||
at += count;
|
||||
}
|
||||
if (!n_scopes) return false;
|
||||
size_t resolved = 0;
|
||||
for (size_t at = 5; at < words;) {
|
||||
const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu;
|
||||
if (op == kOpConstant && count >= 4) {
|
||||
for (size_t i = 0; i < n_scopes; i++)
|
||||
if (scopes[i] == code[at + 2]) {
|
||||
if (code[at + 3] == kScopeSubgroup) return true;
|
||||
resolved++;
|
||||
}
|
||||
}
|
||||
at += count;
|
||||
}
|
||||
return resolved != n_scopes; // (a scope that is no plain constant: a doubt)
|
||||
}
|
||||
} // namespace compute_pin
|
||||
41
gpu/vk/cpu_count.h
Normal file
41
gpu/vk/cpu_count.h
Normal file
@@ -0,0 +1,41 @@
|
||||
// (Triforce, 11-22, the performance audit's OS-49, after Source B's) The logical processors the renderer sizes its pools and
|
||||
// its threads' counts by: std::thread::hardware_concurrency, or, while the lab's HOS_SIM_CPU=<cores>/<threads> holds the
|
||||
// process to fewer of the PC's cores (host/hos_sim_cpu.h), its <threads>: an affinity mask alone does not change what
|
||||
// hardware_concurrency answers, and the pools would be a 24-thread PC's on 6 cores. The host clears the variable when it did
|
||||
// not apply it, so the two agree. Windows only, as the host's part; unset, hardware_concurrency as before. Every such count in
|
||||
// gpu/ goes through cpu_count() but vk_shader.cpp's (a file the shader cache's stamp is made from):
|
||||
// tests/test_sim_cores.py checks that, and the rule.
|
||||
#pragma once
|
||||
#include <cstdlib>
|
||||
#include <thread>
|
||||
|
||||
namespace cpu_count_rule {
|
||||
/// The threads HOS_SIM_CPU asks for ("6/12": 12; "6": 6), when fewer than the PC's hw (0: not known); else 0: not set, not
|
||||
/// "<cores>[/<threads>]" with each from 1 to 256 and threads not under cores, or not fewer
|
||||
inline unsigned sim_threads(const char* e, unsigned hw) {
|
||||
if (!e || !*e) return 0;
|
||||
char* end = nullptr;
|
||||
const long c = std::strtol(e, &end, 10);
|
||||
if (end == e) return 0;
|
||||
long t = c;
|
||||
if (*end == '/') {
|
||||
const char* s = end + 1;
|
||||
t = std::strtol(s, &end, 10);
|
||||
if (end == s) return 0;
|
||||
}
|
||||
if (*end || c < 1 || c > 256 || t < c || t > 256) return 0;
|
||||
return hw && (unsigned)t >= hw ? 0 : (unsigned)t;
|
||||
}
|
||||
} // namespace cpu_count_rule
|
||||
|
||||
/// The logical processors to size by (read once)
|
||||
inline unsigned cpu_count() {
|
||||
static const unsigned n = [] {
|
||||
const unsigned hw = std::thread::hardware_concurrency();
|
||||
#ifdef _WIN32
|
||||
if (const unsigned sim = cpu_count_rule::sim_threads(std::getenv("HOS_SIM_CPU"), hw)) return sim;
|
||||
#endif
|
||||
return hw;
|
||||
}();
|
||||
return n;
|
||||
}
|
||||
39
gpu/vk/dead_count.h
Normal file
39
gpu/vk/dead_count.h
Normal file
@@ -0,0 +1,39 @@
|
||||
// (Triforce, 11-21, the performance audit's THREAD-21, its first step) What destroying the dead images costs a present.
|
||||
//
|
||||
// A texture let go waits in a list until the GPU is done with it, and texture_frame then destroys every one that is, in
|
||||
// the middle of the present being recorded: a render target's image goes to the pool (cheap), every other image is
|
||||
// destroyed with its views. The audit's port bounds that to a share a present, after the present's submit, as Source B
|
||||
// does on its GPU thread's idle moments. Its own proof asks for the cost first: "if no present spends 3 ms in the loop,
|
||||
// stop". This is that count, and nothing else changes: a line for a present whose loop took kSayMs or more, and every
|
||||
// 600 presents the totals and the worst present.
|
||||
//
|
||||
// No Vulkan here: tests/test_dead_count.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
namespace dead_count {
|
||||
|
||||
inline constexpr double kSayMs = 2.0; // (a present's loop that gets a line of its own)
|
||||
inline constexpr double kWorthMs = 3.0; // (the audit's line: under it in every present, the bounded step is not built)
|
||||
inline constexpr uint64_t kReport = 600; // (presents between the totals' lines)
|
||||
|
||||
struct Counts {
|
||||
uint64_t presents = 0, destroyed = 0, pooled = 0, over_say = 0, over_worth = 0;
|
||||
double ms = 0, worst_ms = 0;
|
||||
uint64_t worst_destroyed = 0, worst_left = 0;
|
||||
/// A present's loop: n_destroyed images destroyed and n_pooled put in the pool in loop_ms, left still waiting. True
|
||||
/// when the present gets a line of its own
|
||||
bool note(uint64_t n_destroyed, uint64_t n_pooled, double loop_ms, uint64_t left) {
|
||||
presents++, destroyed += n_destroyed, pooled += n_pooled, ms += loop_ms;
|
||||
if (loop_ms > worst_ms) worst_ms = loop_ms, worst_destroyed = n_destroyed, worst_left = left;
|
||||
if (loop_ms >= kWorthMs) over_worth++;
|
||||
if (loop_ms < kSayMs) return false;
|
||||
over_say++;
|
||||
return true;
|
||||
}
|
||||
/// Whether the totals' line is due, at present `present`
|
||||
bool due(uint64_t present) const { return presents != 0 && present % kReport == 0; }
|
||||
void reset() { *this = Counts{}; }
|
||||
};
|
||||
|
||||
} // namespace dead_count
|
||||
67
gpu/vk/desc_counts.h
Normal file
67
gpu/vk/desc_counts.h
Normal file
@@ -0,0 +1,67 @@
|
||||
// A program's descriptor counts kept with the program (no game, no GPU: a rule; vk_shader.h Program::desc_n, vk_draw.cpp
|
||||
// build_descriptors).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's DESC-22, after Source A's.) build_descriptors sums the program's buffers,
|
||||
// images and texel views at every build, six lists a stage, only to size three reserve() calls that keep the pointers
|
||||
// it hands the writes stable. The sums are fixed once the program is translated: its stages are filled before it is
|
||||
// ready and never after. So the three are packed into one word kept on the program, stored at its first build once
|
||||
// ready and read at every later one; a word that is not a count walks as before.
|
||||
//
|
||||
// The word is 32 bits because that is what the program has room for: the padding after Program::ready. The struct must
|
||||
// not grow: sizeof(Program) is in the program cache's stamp, and a change of it throws every player's caches away. So
|
||||
// three counts of 10 bits; a program with more than 1023 of a kind is not kept (kUnset is what its pack gives) and walks
|
||||
// at every build, as before.
|
||||
//
|
||||
// A short count would let a vector move in the middle of a build and leave the writes with stale pointers. It cannot
|
||||
// come from the walk that stores it, but the build checks all the same that none of the three moved, and a program
|
||||
// whose did is marked kNever, walked from then on, and built again at once.
|
||||
//
|
||||
// RX_VK_DESC_COUNTS=0: the walk at every build, as before. The lab's knob "descwalk" does the same in its phases.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mkwvk::desc_counts {
|
||||
|
||||
inline constexpr uint32_t kUnset = ~0u; // not counted yet, or too many of a kind to pack
|
||||
inline constexpr uint32_t kNever = 0xFFFFFFFEu; // its kept counts were short once: walked at every build from then on
|
||||
inline constexpr uint32_t kMost = 1023; // the most of a kind a word holds (10 bits)
|
||||
|
||||
/// RX_VK_DESC_COUNTS: on unless set to 0
|
||||
inline bool on(const char* env) {
|
||||
return !env || std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
/// The three counts as one word, or kUnset where one does not fit
|
||||
constexpr uint32_t pack(size_t bufs, size_t imgs, size_t views) {
|
||||
if (bufs > kMost || imgs > kMost || views > kMost) return kUnset;
|
||||
return (uint32_t)bufs | (uint32_t)imgs << 10 | (uint32_t)views << 20;
|
||||
}
|
||||
|
||||
/// Whether the word holds counts (its two top bits are clear: kUnset and kNever have them set)
|
||||
constexpr bool holds(uint32_t word) {
|
||||
return (word >> 30) == 0;
|
||||
}
|
||||
|
||||
/// The counts of a word that holds them; false, and nothing written, for one that does not (walk)
|
||||
constexpr bool unpack(uint32_t word, size_t* bufs, size_t* imgs, size_t* views) {
|
||||
if (!holds(word)) return false;
|
||||
*bufs = word & kMost, *imgs = (word >> 10) & kMost, *views = (word >> 20) & kMost;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Whether a build that walked stores its counts: the rule on, the program translated (its stages do not change after),
|
||||
/// and the word still unset (never over kNever)
|
||||
constexpr bool stores(bool on, bool ready, uint32_t word) {
|
||||
return on && ready && word == kUnset;
|
||||
}
|
||||
|
||||
/// The guard at a build's end: whether one of the three vectors' storage moved since the reserve (before/after: their
|
||||
/// data() pointers), in a build that went by kept counts. A build that walked reserved exactly, as it always did
|
||||
constexpr bool moved(bool walked, const void* bufs0, const void* bufs1, const void* imgs0, const void* imgs1, const void* views0,
|
||||
const void* views1) {
|
||||
return !walked && (bufs0 != bufs1 || imgs0 != imgs1 || views0 != views1);
|
||||
}
|
||||
|
||||
} // namespace mkwvk::desc_counts
|
||||
97
gpu/vk/desc_pool_grow.h
Normal file
97
gpu/vk/desc_pool_grow.h
Normal file
@@ -0,0 +1,97 @@
|
||||
// A recording slot's descriptor pool, made at half size and made whole once a recording runs out of it (no game, no GPU:
|
||||
// a rule; vk_draw.cpp make_pool, slot_reset_locked, draw_begin_recording).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's DESC-33, after Source A's.) Each of the recording slots (48 at most) keeps a
|
||||
// descriptor pool of 8192 sets and up to 32768 descriptors a type, and the driver may keep host memory for all of that,
|
||||
// 48 times. Few recordings come near it: Source A saw one take at most 289 sets on a desktop card, and no pool at half
|
||||
// size run out in three scenes on a handheld. So a slot's pool starts at half (4096 sets, 16384 uniform buffers and
|
||||
// combined image samplers, and so on down the types).
|
||||
//
|
||||
// A recording that runs out is given a whole extra pool, as it always was (bind_descriptors), and that is the mark: at
|
||||
// the slot's next reset, where the extra pools are destroyed anyway, a slot that has any and whose own pool is not whole
|
||||
// has it destroyed and made again whole. From then on the slot is as it was before this rule. A busy slot so pays one
|
||||
// extra pool, once.
|
||||
//
|
||||
// RX_VK_DESC_POOL_GROW=0: whole pools from the start, as before. RX_VK_DESC_POOL_DIV=<n> still divides both sizes.
|
||||
// What the recordings took (sets and descriptors a type, the most one recording took of each) and how many pools were
|
||||
// added and slots made whole go into the log's "recording streams" line, so a size can be judged from a run's log.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mkwvk::desc_pool_grow {
|
||||
|
||||
/// RX_VK_DESC_POOL_GROW: on unless set to 0
|
||||
inline bool on(const char* env) {
|
||||
return !env || std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
/// A whole pool's sets, and its descriptors by type in the order of Took::n
|
||||
inline constexpr uint32_t kSets = 8192;
|
||||
enum Kind { kUniform, kStorage, kSampled, kStorageImage, kUniformTexel, kStorageTexel, kKinds };
|
||||
inline constexpr uint32_t kWhole[kKinds] = {32768, 16384, 32768, 4096, 2048, 2048};
|
||||
|
||||
/// The shift a new slot's pool is made with: half with the rule on, whole without it
|
||||
constexpr uint32_t first_shift(bool on) {
|
||||
return on ? 1u : 0u;
|
||||
}
|
||||
|
||||
/// A count of a pool made at `shift` (0 whole, 1 half) and divided by RX_VK_DESC_POOL_DIV's n (1..32): never 0
|
||||
constexpr uint32_t count(uint32_t whole, uint32_t shift, uint32_t div) {
|
||||
const uint32_t n = (whole >> (shift > 8 ? 8 : shift)) / (div ? div : 1);
|
||||
return n ? n : 1;
|
||||
}
|
||||
|
||||
/// Whether a slot's own pool is made again whole at its reset: its last recording ran out (it has extra pools) and the
|
||||
/// pool is not whole yet
|
||||
constexpr bool remake(uint32_t shift, size_t extra_pools) {
|
||||
return shift != 0 && extra_pools != 0;
|
||||
}
|
||||
|
||||
/// Which of the six a Vulkan descriptor type is (VkDescriptorType's numbers, so that this header needs no Vulkan):
|
||||
/// kKinds for any other
|
||||
constexpr int kind(uint32_t vk_type) {
|
||||
switch (vk_type) {
|
||||
case 6: return kUniform; // VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER
|
||||
case 7: return kStorage; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
|
||||
case 1: return kSampled; // VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER
|
||||
case 3: return kStorageImage; // VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
|
||||
case 4: return kUniformTexel; // VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER
|
||||
case 5: return kStorageTexel; // VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
|
||||
default: return kKinds;
|
||||
}
|
||||
}
|
||||
|
||||
/// What one recording took of its slot's pools: sets, and descriptors by type
|
||||
struct Took {
|
||||
uint32_t sets = 0;
|
||||
uint32_t n[kKinds] = {};
|
||||
};
|
||||
|
||||
/// One more set allocated, with these writes (W: VkWriteDescriptorSet, or a test's own with the same two fields)
|
||||
template <class W>
|
||||
inline void took(Took& t, const W* writes, size_t count_of) {
|
||||
t.sets++;
|
||||
for (size_t i = 0; i < count_of; i++) {
|
||||
const int k = kind((uint32_t)writes[i].descriptorType);
|
||||
if (k < kKinds) t.n[k] += (uint32_t)writes[i].descriptorCount;
|
||||
}
|
||||
}
|
||||
|
||||
/// The most any recording took of each, raised by one recording's
|
||||
inline void peak(Took& most, const Took& t) {
|
||||
if (t.sets > most.sets) most.sets = t.sets;
|
||||
for (int k = 0; k < kKinds; k++)
|
||||
if (t.n[k] > most.n[k]) most.n[k] = t.n[k];
|
||||
}
|
||||
|
||||
/// Whether what a recording took fits a pool made at `shift` and divided by div (the tests', and the log's reader's)
|
||||
constexpr bool fits(const Took& t, uint32_t shift, uint32_t div) {
|
||||
if (t.sets > count(kSets, shift, div)) return false;
|
||||
for (int k = 0; k < kKinds; k++)
|
||||
if (t.n[k] > count(kWhole[k], shift, div)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::desc_pool_grow
|
||||
@@ -4,6 +4,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
@@ -16,6 +17,50 @@ inline bool scene_size_fits(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) {
|
||||
return w && h && ow && oh && w <= ow && h <= oh && (uint64_t)w * 3 >= ow && (uint64_t)h * 3 >= oh;
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-22, the performance audit's PRESENT-38, after Source A's) One quality mode's range of render sizes for
|
||||
/// an output, as NGX answers it (ok: it answered one).
|
||||
struct ModeRange {
|
||||
uint32_t min_w = 0, min_h = 0, max_w = 0, max_h = 0;
|
||||
bool ok = false;
|
||||
};
|
||||
/// How far outside a range a render size may be and still be that mode's: a 64th of the range's largest size on that
|
||||
/// axis, 2 pixels at least. The Render resolution row's shares are twelfths of the output rounded up, NGX's own sizes
|
||||
/// are rounded down: 854x480 against NGX's 853x480 for Ultra Performance at 2560x1440.
|
||||
constexpr uint32_t mode_slack(uint32_t largest) { return largest / 64 > 2 ? largest / 64 : 2; }
|
||||
/// The mode (an index into ranges) that takes a render size of w x h for an output of ow x oh, or -1. The first range
|
||||
/// that holds the size; else, with slack on, the first it is outside of by no more than mode_slack on each axis, and
|
||||
/// only for a size DLSS scales at all (scene_size_fits: the output 1 to 3 times it). *widened: taken by the slack.
|
||||
inline int mode_for(const ModeRange* ranges, size_t n, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh, bool slack, bool* widened) {
|
||||
*widened = false;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
const ModeRange& r = ranges[i];
|
||||
if (r.ok && w >= r.min_w && h >= r.min_h && w <= r.max_w && h <= r.max_h) return (int)i;
|
||||
}
|
||||
if (!slack || !scene_size_fits(w, h, ow, oh)) return -1;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
const ModeRange& r = ranges[i];
|
||||
if (!r.ok) continue;
|
||||
const uint32_t sw = mode_slack(r.max_w), sh = mode_slack(r.max_h);
|
||||
if (w + sw >= r.min_w && h + sh >= r.min_h && w <= r.max_w + sw && h <= r.max_h + sh) return *widened = true, (int)i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-22, the performance audit's PRESENT-32, after Source A's) Whether a feature is made in a one-size mode
|
||||
/// (NGX's DLAA mode: vk_dlss.cpp prepare's `same` list): its render size is the output's. Before, whenever the two sizes
|
||||
/// were equal. NVIDIA DLSS with the Render resolution row at Auto starts at the output's size, so it got a one-size
|
||||
/// feature, and the game's dynamic resolution left that size some frames later: a second feature made in play (Source
|
||||
/// A's save load-in: 119 ms, then 29.5 ms 17 frames on). With the rule (rule: RX_VK_DLSS_PREWARM=1), one-size only where
|
||||
/// NVIDIA DLAA is the Upscaler's choice (dlaa); DLSS at the output's size takes the scaled mode whose range holds it
|
||||
/// (mode_for: Quality's reaches the output's size), and Auto's smaller sizes then draw with that same feature. Off by
|
||||
/// default: DLSS at full size then draws with the scaled mode's network, not DLAA's, a picture nobody has compared yet
|
||||
inline bool dlaa_feature(bool rule, bool dlaa, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh) {
|
||||
const bool full = rw == ow && rh == oh;
|
||||
return rule ? full && dlaa : full;
|
||||
}
|
||||
/// RX_VK_DLSS_PREWARM (value, null when unset): the rule above is on only when it is 1
|
||||
inline bool prewarm_rule(const char* text) { return text && *text && std::atoi(text) == 1; }
|
||||
|
||||
/// Whether s is all ASCII: NGX's Linux library takes only such paths (vk_dlss.cpp wide())
|
||||
inline bool ascii(std::string_view s) {
|
||||
for (const char c : s)
|
||||
@@ -84,4 +129,28 @@ inline std::wstring utf8_to_wide(std::string_view s, bool* bad = nullptr) {
|
||||
return out;
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-21, the performance audit's PRESENT-37, after Source A's) The preset DLSS draws with while the DLSS model
|
||||
/// row is stepped through (vk_dlss.cpp prepare). The row goes K, M, L, NVIDIA's default and applies live, so K to L passes
|
||||
/// M: a feature was made for each preset passed, tens of ms of CPU each, and NGX keeps a released feature's video memory for
|
||||
/// some seconds: Source A's 16 GB card at 3440x1440 held 1419 MB of DLSS's for K, M and L at once against 813 MB for L
|
||||
/// alone, and where memory was short NGX refused the next feature. A new preset is taken once the row has held it for
|
||||
/// `frames`; until then the feature already made draws on with its own. With no feature (the first, or one let go) the
|
||||
/// row's preset is taken at once
|
||||
struct PresetSettle {
|
||||
unsigned wanted = ~0u; // (the row's preset, as last seen)
|
||||
uint64_t since = 0; // (the frame it was first seen in)
|
||||
/// The preset for frame `now`: want, the row's; have, the feature's; has: there is a feature
|
||||
unsigned pick(unsigned want, unsigned have, bool has, uint64_t now, uint64_t frames) {
|
||||
if (want != wanted) wanted = want, since = now;
|
||||
if (!has || want == have) return want;
|
||||
return now - since >= frames ? want : have;
|
||||
}
|
||||
};
|
||||
/// RX_VK_DLSS_MODEL_SETTLE's frames: 30 when not set; 0 (and what is no number, or less): at once, as before
|
||||
inline uint64_t settle_frames(const char* text) {
|
||||
if (!text || !*text) return 30;
|
||||
const long v = std::atol(text);
|
||||
return v > 0 ? (uint64_t)v : 0;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::dlss_checks
|
||||
|
||||
58
gpu/vk/dlss_feature.h
Normal file
58
gpu/vk/dlss_feature.h
Normal file
@@ -0,0 +1,58 @@
|
||||
// (Triforce, 11-21, the upscaler audit's UPS-01) The render size NVIDIA DLSS's feature is made for.
|
||||
//
|
||||
// vk_dlss.cpp made every feature for the largest render size its quality mode allows, so that the game's dynamic
|
||||
// resolution could change the render size under it without the feature being made again. NGX answers that largest size as
|
||||
// the output's own (the owner's log of 11-21: "feature 1 made ...: 2944x1243 (2208x932 to 4416x1864) -> 4416x1864, mode
|
||||
// 2, preset M"), so with the Render resolution row at a fixed share, where the render size never changes, the feature was
|
||||
// still made for 2.25 times the pixels it was ever given at Quality, 4 times at Performance and 9 at Ultra Performance,
|
||||
// and the chain's three input images with it. NVIDIA's guide (DLSS Programming Guide 310.6.0, 3.2 and 5.2.8) has the
|
||||
// optimal render size passed to the creation "exactly as given", and says of the second generation transformer's presets
|
||||
// L and M that their cost follows the size the feature is initialized for, which is why it is to be the dynamic range's
|
||||
// largest only where there is a dynamic range. The same log has DLSS's own video memory at 1069 MB for that feature, where
|
||||
// NVIDIA's table has 456 MB for preset M at 4K.
|
||||
//
|
||||
// So: a fixed share gets a feature of its own render size (exact), as AMD FSR 4 gets a context of its own
|
||||
// (temporal_rules.h plan_context). Where the render size is the game's to change (the row's Auto), or once it has changed
|
||||
// under one output in this run, the feature is made for the mode's whole range, as before. DLAA's is the output's either
|
||||
// way.
|
||||
//
|
||||
// RX_VK_DLSS_FEATURE_MAX=1: every feature for its mode's largest render size, as before (whole_range()).
|
||||
//
|
||||
// No NGX, no Vulkan: tests/test_dlss_feature.py compiles this.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace dlss_feature {
|
||||
|
||||
/// RX_VK_DLSS_FEATURE_MAX (value, null when unset): not 0, every feature for its mode's largest render size, as before
|
||||
inline bool whole_range(const char* value) { return value && *value && std::atoi(value) != 0; }
|
||||
|
||||
/// What a feature is made for: w x h, and whether that is the one render size it takes (exact) or the largest of a range
|
||||
struct Size {
|
||||
uint32_t w, h;
|
||||
bool exact;
|
||||
};
|
||||
|
||||
/// The size to make a feature for, for a frame of rw x rh in a mode whose render sizes go up to max_w x max_h. aa: DLAA,
|
||||
/// the render size the output's (its range is that one size). range: the render size may change under this output (the
|
||||
/// row's Auto, the size seen changing, or the switch)
|
||||
inline Size size(bool aa, bool range, uint32_t rw, uint32_t rh, uint32_t max_w, uint32_t max_h) {
|
||||
if (aa || range || rw > max_w || rh > max_h) return {max_w, max_h, false};
|
||||
return {rw, rh, true};
|
||||
}
|
||||
|
||||
/// Whether a feature made for fw x fh takes a frame of rw x rh: its one size (exact), or any from min_w x min_h up to its own
|
||||
inline bool takes(bool exact, uint32_t fw, uint32_t fh, uint32_t min_w, uint32_t min_h, uint32_t rw, uint32_t rh) {
|
||||
if (exact) return rw == fw && rh == fh;
|
||||
return rw <= fw && rh <= fh && rw >= min_w && rh >= min_h;
|
||||
}
|
||||
|
||||
/// Whether this frame shows that the render size changes under one output: a feature of one size (exact) for the same
|
||||
/// output, preset and colour (same), and a frame of another size. The caller keeps it for the run: the next feature, and
|
||||
/// every one after it, is made for the mode's whole range
|
||||
inline bool found_range(bool same, bool exact, uint32_t fw, uint32_t fh, uint32_t rw, uint32_t rh) {
|
||||
return same && exact && (rw != fw || rh != fh);
|
||||
}
|
||||
|
||||
} // namespace dlss_feature
|
||||
@@ -93,6 +93,36 @@ inline void unjitter(float m[16], const float jitter[2]) {
|
||||
}
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-20) the owner's "DLSS flicker": the whole picture and the shadows, while the camera moves, with NVIDIA
|
||||
/// DLSS frame generation on, and not without it.) Streamline's camera matrices are in the clip space of the depth buffer it
|
||||
/// is given: its depth value is that clip space's z over w, 1 at the near plane with depthInverted. The game's projection
|
||||
/// is NVN's, clip z from -1 at the near plane to 1 at the far one, and the depth frame generation gets is FSR's copy
|
||||
/// (vk_fsr3_inputs.cpp): d = (1 - ndc z) / 2 where the game's buffer has 0 at the near plane (flipped: Camera::flip_depth),
|
||||
/// (ndc z + 1) / 2 where it is reversed already. Given the game's own matrices with that depth, a depth read as clip z put
|
||||
/// what is far a fifth of a metre from the camera and what is near at the far plane, so whatever Streamline works out from
|
||||
/// the depth and the camera's move was wrong as soon as the camera moved, and right while it stood. Only DLSS frame
|
||||
/// generation takes matrices (FSR takes the near and far planes and the field of view), so nothing else showed it.
|
||||
/// The depth's clip space is the game's with z' = (w - z) / 2 (flipped) or (w + z) / 2: Z * clip, Z the identity but for
|
||||
/// its z row, (0 0 -1/2 1/2) or (0 0 1/2 1/2).
|
||||
/// A projection (or a view-projection) into the depth's clip space: Z * m, row major (clip = M * position)
|
||||
inline void depth_clip_projection(const float m[16], bool flipped, float out[16]) {
|
||||
const float s = flipped ? -0.5f : 0.5f;
|
||||
for (int i = 0; i < 16; i++) out[i] = m[i];
|
||||
for (int c = 0; c < 4; c++) out[2 * 4 + c] = s * m[2 * 4 + c] + 0.5f * m[3 * 4 + c];
|
||||
}
|
||||
/// A clip-to-clip matrix of the game's (this frame's clip space to the frame before's: reprojection) between the depth's
|
||||
/// clip spaces: Z * m * inverse(Z), inverse(Z)'s z row (0 0 -2 1) or (0 0 2 -1). The identity stays the identity
|
||||
inline void depth_clip_reprojection(const float m[16], bool flipped, float out[16]) {
|
||||
const float k = flipped ? -2.0f : 2.0f, t = flipped ? 1.0f : -1.0f;
|
||||
float r[16];
|
||||
for (int row = 0; row < 4; row++) { // (m * inverse(Z): the z column scaled, and added to the w column)
|
||||
r[row * 4 + 0] = m[row * 4 + 0], r[row * 4 + 1] = m[row * 4 + 1];
|
||||
r[row * 4 + 2] = k * m[row * 4 + 2];
|
||||
r[row * 4 + 3] = m[row * 4 + 3] + t * m[row * 4 + 2];
|
||||
}
|
||||
depth_clip_projection(r, flipped, out); // (Z * that)
|
||||
}
|
||||
|
||||
/// A jitter in clip space as DLSS takes it: in render pixels, x right and y down (clip y is up)
|
||||
inline void jitter_to_pixels(const float clip[2], unsigned width, unsigned height, float out[2]) {
|
||||
out[0] = clip[0] * 0.5f * (float)width;
|
||||
|
||||
38
gpu/vk/dmabuf_gate.h
Normal file
38
gpu/vk/dmabuf_gate.h
Normal file
@@ -0,0 +1,38 @@
|
||||
// (Triforce, 11-22, the performance audit's OS-08, after Source A's) Whether guest memory is offered to the driver as
|
||||
// udmabuf dma-bufs at all, and what the start's line says (vk_memory.cpp import_dma_buf asks at its first call; plain code:
|
||||
// tests/test_dmabuf_driver_gate.py runs it anywhere).
|
||||
//
|
||||
// On Linux the guest window is a memfd and each pool is first offered to the driver as a dma-buf made by udmabuf. NVIDIA's
|
||||
// proprietary driver refuses every one (VK_ERROR_OUT_OF_DEVICE_MEMORY) and the pool ends as a host-pointer import anyway:
|
||||
// 49-50 refused imports a run and a kernel message at each start on Source A's RTX 4090, the first refusal 2.5-3.7 ms, one
|
||||
// of a 64 MB piece 198 ms on the presentation thread during a save load. So with that driver /dev/udmabuf is not opened
|
||||
// and the pools are host-pointer imports from the start. NVK and RADV have other driver ids and keep their dma-bufs.
|
||||
// RX_VK_DMABUF_NVIDIA=1: tried on NVIDIA's driver too, as before (for a later driver that takes them).
|
||||
#pragma once
|
||||
|
||||
namespace dmabuf_gate {
|
||||
|
||||
/// What the start found. fn: the host gave its memfd function (false also with RX_VK_NO_DMABUF). memfd: the window is a
|
||||
/// memfd. ext: the device imports dma-bufs (VK_EXT_external_memory_dma_buf and its fd properties call). nvidia: NVIDIA's
|
||||
/// proprietary driver, and RX_VK_DMABUF_NVIDIA not set
|
||||
struct Facts {
|
||||
bool fn = false, memfd = false, ext = false, nvidia = false;
|
||||
};
|
||||
|
||||
/// Whether /dev/udmabuf is opened: only then can a pool be offered as a dma-buf
|
||||
inline bool try_udmabuf(const Facts& f) { return f.fn && f.memfd && f.ext && !f.nvidia; }
|
||||
|
||||
/// As it was: NVIDIA's driver was offered them too
|
||||
inline bool try_udmabuf_before(const Facts& f) { return f.fn && f.memfd && f.ext; }
|
||||
|
||||
/// The start's "vk: guest memory as dma-bufs: ..." answer. opened: /dev/udmabuf opened (asked only after try_udmabuf)
|
||||
inline const char* answer(const Facts& f, bool opened) {
|
||||
if (try_udmabuf(f) && opened) return "yes";
|
||||
if (!f.fn) return "no (no memfd window, or RX_VK_NO_DMABUF)";
|
||||
if (!f.memfd) return "no (the window is anonymous memory)";
|
||||
if (!f.ext) return "no (VK_EXT_external_memory_dma_buf)";
|
||||
if (f.nvidia) return "no (NVIDIA's driver refuses them: host pointers)";
|
||||
return "no (/dev/udmabuf cannot be opened)";
|
||||
}
|
||||
|
||||
} // namespace dmabuf_gate
|
||||
41
gpu/vk/drawn_page_map.h
Normal file
41
gpu/vk/drawn_page_map.h
Normal file
@@ -0,0 +1,41 @@
|
||||
// (Triforce, 11-21, the performance audit's TEX-21, after Source A's) A texture the GPU drew gets its page map at its first
|
||||
// draw.
|
||||
//
|
||||
// A texture in a virtual pool (its storage mapped 64 KB a page) is checked through its mapping: texture_dirty keeps each
|
||||
// page's guest address (page_map) and says which pages were mapped anew or written since. The map is recorded at the
|
||||
// texture's first check, which for a texture uploaded from memory is its first upload. One the GPU drew first is marked
|
||||
// uploaded by the draw and never checked, so it has no map; once the game finalizes it, its first check finds none, takes
|
||||
// that as "everything was mapped since", and the whole storage is uploaded from memory over what the GPU drew: a HUD
|
||||
// label or a render target gone blank after an area change, and an upload of up to 15 MB each. Source A's sessions from
|
||||
// Hyrule Field to Castle Town: such whole uploads of HUD labels 6, other HUD 4 to 6, render targets 4 to 9, all to 0.
|
||||
// Here: the map is recorded at the first draw that says who drew it, by one walk of the mapping (the storage's size / 64 KB
|
||||
// lookups, once a texture). The later check then gives only the pages mapped anew or written by the CPU.
|
||||
// Not for a write that gives no reason ("?": a clear, the movie upscaler): those keep the path that brings a font cache's
|
||||
// letters back from memory. RX_VK_DRAWN_PAGE_MAP=0: no map at the draw, as before.
|
||||
//
|
||||
// The rule alone: tests/test_drawn_page_map.py compiles it, and texture_dirty cut from the source.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace drawn_page_map {
|
||||
|
||||
inline constexpr uint64_t kVirtual = 1ull << 40; // (a virtual pool's addresses start here)
|
||||
inline constexpr uint64_t kPage = 1ull << 16; // (and its pages are 64 KB)
|
||||
|
||||
/// RX_VK_DRAWN_PAGE_MAP: off only with 0
|
||||
inline bool on(const char* asked) { return !asked || !*asked || atoi(asked) != 0; }
|
||||
|
||||
/// The pages of a storage of guest_size bytes
|
||||
constexpr size_t pages(uint64_t guest_size) { return (size_t)((guest_size + kPage - 1) / kPage); }
|
||||
|
||||
/// Whether a write by `why` into a texture at gpu, of guest_size bytes of storage, whose map holds map_pages pages now,
|
||||
/// records its map first
|
||||
inline bool seeds(bool switch_on, const char* why, uint64_t gpu, uint64_t guest_size, size_t map_pages) {
|
||||
if (!switch_on || !why || !std::strcmp(why, "?")) return false;
|
||||
return gpu >= kVirtual && guest_size != 0 && map_pages != pages(guest_size);
|
||||
}
|
||||
|
||||
} // namespace drawn_page_map
|
||||
140
gpu/vk/driver_faults.h
Normal file
140
gpu/vk/driver_faults.h
Normal file
@@ -0,0 +1,140 @@
|
||||
// Programs the driver faulted on, kept out of it, and one build of a program at a time (no game, no GPU: the rules and the
|
||||
// gate; vk_draw.cpp fault_quarantine, build_graphics, guarded_graphics). Apart from Vulkan so that
|
||||
// tests/test_driver_faults.py runs it alone, on many made-up PCs.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-57, after Source A's.) The guards catch a fault inside the driver's
|
||||
// compiler (vk_draw.cpp guarded_graphics), and a fault during the start's own shader work starts the game again
|
||||
// (driver_fault_restart). Nothing remembered the program: the next start asked the driver for it again, faulted again and
|
||||
// restarted again, and in play a second thread could ask for the same program while the driver was still inside the
|
||||
// first. Source A saw that on AMD's Windows driver (an RX 6800 XT): in a freeze, three threads sat in the driver on two
|
||||
// programs' builds while the presenting thread waited for one of them; a preparation stuck at "6625 of 6627, nothing
|
||||
// finished for 30 s" took 32.5 s, and 4.2 s with three faulted programs kept out on the first run, 2.3 s after. Triforce
|
||||
// has the same class of report: "two of 4926 on an RX 5700 XT, stuck at 99%".
|
||||
//
|
||||
// What. On AMD's own Windows driver (or everywhere with RX_VK_DRIVER_FAULTS=2):
|
||||
// - a program whose pipeline build faulted in the driver is not asked of the driver again this run, and its key goes to
|
||||
// triforce_driver_faults.txt beside the executable, one line under the vendor and the driver's version, written by the
|
||||
// fault's handler before the restart can end the process (so a start that faults does not fault again); later starts
|
||||
// with that driver skip those programs, and say so once (delete the file to try again; another driver version ignores
|
||||
// the lines);
|
||||
// - one build of a program at a time: a second waits at the gate, outside the driver, and a probe that asks the caches
|
||||
// only is told a compile is required.
|
||||
// Elsewhere (NVIDIA's, Intel's, Mesa's drivers, AMD's on Linux) nothing of it runs, as before: the NVIDIA fault of 11-13
|
||||
// was a race in the driver, not a bad program. RX_VK_DRIVER_FAULTS=0: nowhere.
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace driver_faults {
|
||||
|
||||
/// RX_VK_DRIVER_FAULTS: 0 off, 1 (unset) AMD's Windows driver, 2 every driver
|
||||
inline int mode(const char* env) {
|
||||
if (!env || !*env) return 1;
|
||||
const int v = std::atoi(env);
|
||||
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
|
||||
}
|
||||
/// Whether the quarantine, its file and the gate are on: mode 2 anywhere, mode 1 on Windows with AMD's own driver
|
||||
/// (VK_DRIVER_ID_AMD_PROPRIETARY)
|
||||
inline bool on(int mode, bool windows, bool amd_proprietary) {
|
||||
return mode >= 2 || (mode == 1 && windows && amd_proprietary);
|
||||
}
|
||||
|
||||
inline constexpr int kLine = 35; // ("vvvvvvvv dddddddd kkkkkkkkkkkkkkkk\n")
|
||||
|
||||
/// One line of the file into out (kLine bytes and a 0): made without the C library, as the fault's handler makes it
|
||||
inline int line(char* out, uint32_t vendor, uint32_t driver, uint64_t key) {
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
int n = 0;
|
||||
for (int i = 7; i >= 0; i--) out[n++] = hex[(vendor >> (4 * i)) & 0xF];
|
||||
out[n++] = ' ';
|
||||
for (int i = 7; i >= 0; i--) out[n++] = hex[(driver >> (4 * i)) & 0xF];
|
||||
out[n++] = ' ';
|
||||
for (int i = 15; i >= 0; i--) out[n++] = hex[(key >> (4 * i)) & 0xF];
|
||||
out[n++] = '\n';
|
||||
out[n] = 0;
|
||||
return n;
|
||||
}
|
||||
/// The keys of the file's lines for this vendor and driver version; another's, and a broken line, are passed over
|
||||
inline std::vector<uint64_t> parse(const std::string& text, uint32_t vendor, uint32_t driver) {
|
||||
std::vector<uint64_t> keys;
|
||||
const auto hexval = [](const std::string& s, size_t at, int digits, uint64_t* v) {
|
||||
if (at + (size_t)digits > s.size()) return false;
|
||||
uint64_t x = 0;
|
||||
for (int i = 0; i < digits; i++) {
|
||||
const char c = s[at + i];
|
||||
const int d = c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1;
|
||||
if (d < 0) return false;
|
||||
x = x << 4 | (uint64_t)d;
|
||||
}
|
||||
*v = x;
|
||||
return true;
|
||||
};
|
||||
size_t at = 0;
|
||||
while (at < text.size()) {
|
||||
size_t end = text.find('\n', at);
|
||||
if (end == std::string::npos) end = text.size();
|
||||
std::string l = text.substr(at, end - at);
|
||||
if (!l.empty() && l.back() == '\r') l.pop_back();
|
||||
uint64_t v = 0, d = 0, k = 0;
|
||||
if (l.size() == (size_t)kLine - 1 && l[8] == ' ' && l[17] == ' ' && hexval(l, 0, 8, &v) && hexval(l, 9, 8, &d) &&
|
||||
hexval(l, 18, 16, &k) && v == vendor && d == driver)
|
||||
keys.push_back(k);
|
||||
at = end + 1;
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
/// The run's quarantine and the gate: begin before a program's build, end after it (with whether it faulted)
|
||||
class Quarantine {
|
||||
public:
|
||||
enum class Begin { Go, Busy, Faulted };
|
||||
void load(const std::vector<uint64_t>& keys) {
|
||||
std::lock_guard lk(mu_);
|
||||
bad_.insert(keys.begin(), keys.end());
|
||||
}
|
||||
/// Faulted: not asked of the driver. Busy: another build of it is inside the driver and the caller may not wait (a
|
||||
/// probe of the caches). Go: this build may go in; the caller ends it with end(). may_wait: wait at the gate instead
|
||||
Begin begin(uint64_t key, bool may_wait) {
|
||||
std::unique_lock lk(mu_);
|
||||
for (;;) {
|
||||
if (bad_.count(key)) return Begin::Faulted;
|
||||
if (!busy_.count(key)) break;
|
||||
if (!may_wait) return Begin::Busy;
|
||||
cv_.wait(lk);
|
||||
}
|
||||
busy_.insert(key);
|
||||
return Begin::Go;
|
||||
}
|
||||
/// True when it faulted now for the first time
|
||||
bool end(uint64_t key, bool faulted) {
|
||||
bool fresh = false;
|
||||
{
|
||||
std::lock_guard lk(mu_);
|
||||
busy_.erase(key);
|
||||
if (faulted) fresh = bad_.insert(key).second;
|
||||
}
|
||||
cv_.notify_all();
|
||||
return fresh;
|
||||
}
|
||||
bool faulted(uint64_t key) const {
|
||||
std::lock_guard lk(mu_);
|
||||
return bad_.count(key) != 0;
|
||||
}
|
||||
size_t count() const {
|
||||
std::lock_guard lk(mu_);
|
||||
return bad_.size();
|
||||
}
|
||||
|
||||
private:
|
||||
mutable std::mutex mu_;
|
||||
std::condition_variable cv_;
|
||||
std::unordered_set<uint64_t> bad_, busy_;
|
||||
};
|
||||
|
||||
} // namespace driver_faults
|
||||
65
gpu/vk/evict_reason.h
Normal file
65
gpu/vk/evict_reason.h
Normal file
@@ -0,0 +1,65 @@
|
||||
// Whether the write-back of a texture the eviction chose is stale: another image was drawn over its storage since its own
|
||||
// last drawing (no game, no GPU: a rule). tests/test_evict_reason.py
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's TEX-R404, after Source A's.) Over the budget, the eviction writes each chosen
|
||||
// texture the GPU drew back into the game's memory before it goes, so that one made again over that storage is uploaded
|
||||
// with what the GPU drew. The game draws its HUD labels (110 x 110, 46 x 46, 140 x 140) inside the storage of a
|
||||
// 1920 x 1080 target it no longer uses. The eviction's own tests do not see that: drawn_over looks only at images of the
|
||||
// same start, and target_overlap.h leaves out an image inside the target. So when the target went, its old texels landed
|
||||
// over the labels, the pages counted as written, and the labels, finalized, were uploaded from them as noise (Source A:
|
||||
// in 3 of 3 runs of two minutes in Hyrule Field at a 6 GB allowance, none with the rule).
|
||||
//
|
||||
// The rule: a write-back is left out when another image the GPU drew overlaps the texture's storage and was drawn in a
|
||||
// later frame than the texture's own last drawing. The texture still goes. One drawn in the same frame keeps the
|
||||
// write-back (a frame's targets drawn side by side), and so does one that only touches it. What is given up: the parts
|
||||
// of the texture the newer image does not cover are not written back either; made again there and sampled undrawn, it
|
||||
// starts from what the memory holds, as before the write-back existed. RX_VK_EVICT_STALE_FLUSH=1: written back, as before.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <vector>
|
||||
|
||||
namespace evict_reason {
|
||||
|
||||
/// RX_VK_EVICT_STALE_FLUSH: unset, empty or 0 leaves a stale write-back out; anything else writes it back, as before
|
||||
inline bool flush_stale(const char* env) { return env && *env && std::atoi(env) != 0; }
|
||||
|
||||
/// Whether the storage [lo, hi) and [olo, ohi) share a byte (ranges that touch do not)
|
||||
constexpr bool overlaps(uint64_t lo, uint64_t hi, uint64_t olo, uint64_t ohi) {
|
||||
return lo < hi && olo < ohi && lo < ohi && olo < hi;
|
||||
}
|
||||
|
||||
/// Whether a texture last drawn in frame `last` holds texels older than what is in its storage: `other` is the frame of
|
||||
/// the newest drawing of another image over that storage (0: none)
|
||||
constexpr bool write_back_stale(uint64_t last, uint64_t other) { return other > last; }
|
||||
|
||||
/// A texture whose write-back the eviction is about to make
|
||||
struct Candidate {
|
||||
uint64_t lo = 0, hi = 0; // its storage
|
||||
size_t index = 0; // the caller's own: its place in the eviction's list
|
||||
const void* self = nullptr; // itself, so that it is not taken for another image over its own storage
|
||||
uint64_t newest = 0; // the frame of the newest drawing of another image over its storage (0: none)
|
||||
};
|
||||
|
||||
/// Puts the candidates in the order of their starts, for note(); returns the longest one's size
|
||||
inline uint64_t by_start(std::vector<Candidate>& c) {
|
||||
std::sort(c.begin(), c.end(), [](const Candidate& a, const Candidate& b) { return a.lo < b.lo; });
|
||||
uint64_t longest = 0;
|
||||
for (const Candidate& k : c) longest = std::max(longest, k.hi > k.lo ? k.hi - k.lo : 0);
|
||||
return longest;
|
||||
}
|
||||
|
||||
/// One image the GPU drew, [olo, ohi), last drawn in frame `o_written`: every candidate it overlaps (but itself) takes
|
||||
/// that frame when it is newer than what the candidate has. `c` is in by_start's order, `longest` what by_start gave
|
||||
inline void note(std::vector<Candidate>& c, uint64_t longest, uint64_t olo, uint64_t ohi, uint64_t o_written, const void* other) {
|
||||
if (ohi <= olo) return;
|
||||
// (a candidate over [olo, ohi) starts before ohi and after olo less its size: the search begins at olo less the longest)
|
||||
auto it = std::lower_bound(c.begin(), c.end(), olo > longest ? olo - longest : 0,
|
||||
[](const Candidate& k, uint64_t g) { return k.lo < g; });
|
||||
for (; it != c.end() && it->lo < ohi; ++it)
|
||||
if (it->self != other && overlaps(it->lo, it->hi, olo, ohi) && o_written > it->newest) it->newest = o_written;
|
||||
}
|
||||
|
||||
} // namespace evict_reason
|
||||
73
gpu/vk/fallback_size_policy.h
Normal file
73
gpu/vk/fallback_size_policy.h
Normal file
@@ -0,0 +1,73 @@
|
||||
// Mesh-fallback compute stages too big for two Mesa drivers are not handed to them (no game, no GPU: a rule).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-56, after Source A's.) A mesh program on a GPU without mesh shaders runs
|
||||
// as compute stages (vk_draw.cpp fallback_compute_pipelines). Every stage went to the driver, and a refusal was only
|
||||
// remembered after the build. Two drivers do not get that far on the largest stages:
|
||||
// - Mesa's RADV on GCN 1 to 4 (GFX6-8: vk_vendor.h radv_gcn). Source A's numbers under drm-shim with a 6 GB cap: the
|
||||
// 79,947-phi stage not built after 45-90 s and killed (25 GB at 150 s), where GFX9 and Navi built it in 13-14 s at
|
||||
// 2.9 GB. Limit: 50,000 OpPhi in one stage (2 of about 800 modules are over it);
|
||||
// - Mesa's Intel driver. An Iris Xe: a 251-call stage hit a 14 GB cap unfinished after 27 minutes. Limits: 20,000 OpPhi
|
||||
// or 200 OpFunctionCall in one stage (10 of 376 stages left out in Source A's full preparation).
|
||||
// A program with a stage over its driver's limit goes into the failed set before anything is built, as a program the
|
||||
// driver refused does: its draws are left out. Which objects those are is not known. No other driver has a limit, and the
|
||||
// switches set none there: NVIDIA, Intel's Windows driver, AMD's own drivers and RADV from Vega on build every stage.
|
||||
//
|
||||
// RX_VK_FALLBACK_PHI_LIMIT=<n>, RX_VK_FALLBACK_CALL_LIMIT=<n>: the two limits on those two drivers; 0: no limit.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace fallback_size {
|
||||
constexpr uint32_t kOpFunctionCall = 57, kOpPhi = 245;
|
||||
constexpr uint32_t kGcnPhis = 50000, kIntelPhis = 20000, kIntelCalls = 200;
|
||||
|
||||
struct Size {
|
||||
uint32_t phis = 0, calls = 0;
|
||||
};
|
||||
struct Limits {
|
||||
uint32_t phis = 0, calls = 0; // (0: none)
|
||||
};
|
||||
enum class Driver { Other, RadvGcn, MesaIntel };
|
||||
|
||||
/// A module's OpPhi and OpFunctionCall instructions. One that does not read as SPIR-V counts 0 of both
|
||||
inline Size count(const uint32_t* code, size_t words) {
|
||||
Size s;
|
||||
if (!code || words < 5 || code[0] != 0x07230203u) return s;
|
||||
for (size_t at = 5; at < words;) {
|
||||
const uint32_t n = code[at] >> 16, op = code[at] & 0xffffu;
|
||||
if (!n || at + n > words) return Size{};
|
||||
s.phis += op == kOpPhi, s.calls += op == kOpFunctionCall;
|
||||
at += n;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
inline uint32_t spirv_phis(const uint32_t* code, size_t words) { return count(code, words).phis; }
|
||||
inline uint32_t spirv_calls(const uint32_t* code, size_t words) { return count(code, words).calls; }
|
||||
|
||||
/// A limit's switch: unset, empty or not a number, the driver's own; 0, none; else the number
|
||||
inline uint32_t parse_limit(const char* env, uint32_t own) {
|
||||
if (!env || !*env) return own;
|
||||
char* end = nullptr;
|
||||
const unsigned long long v = std::strtoull(env, &end, 10);
|
||||
if (end == env || *end) return own;
|
||||
return v > 0xffffffffull ? 0xffffffffu : (uint32_t)v;
|
||||
}
|
||||
|
||||
/// The driver: RADV on a GCN 1 to 4 GPU first (it is never Mesa's Intel driver), then Mesa's Intel driver
|
||||
constexpr Driver driver_of(bool radv_gcn, bool mesa_intel) {
|
||||
return radv_gcn ? Driver::RadvGcn : mesa_intel ? Driver::MesaIntel : Driver::Other;
|
||||
}
|
||||
|
||||
/// The limits on that driver, with the two switches' values (nullptr: unset). None on any other driver, whatever is set
|
||||
inline Limits limits(Driver d, const char* phi_env, const char* call_env) {
|
||||
if (d == Driver::Other) return Limits{};
|
||||
const Limits own = d == Driver::RadvGcn ? Limits{kGcnPhis, 0} : Limits{kIntelPhis, kIntelCalls};
|
||||
return Limits{parse_limit(phi_env, own.phis), parse_limit(call_env, own.calls)};
|
||||
}
|
||||
|
||||
/// The stage is over a limit: its program is not built
|
||||
constexpr bool refused(const Limits& l, const Size& s) {
|
||||
return (l.phis && s.phis > l.phis) || (l.calls && s.calls > l.calls);
|
||||
}
|
||||
} // namespace fallback_size
|
||||
38
gpu/vk/fg_motion.h
Normal file
38
gpu/vk/fg_motion.h
Normal file
@@ -0,0 +1,38 @@
|
||||
// (Triforce, 11-22, the owner's DLSS frame generation flicker, again) Whose motion vectors frame generation is given.
|
||||
//
|
||||
// Since FG-4 (vk_fsr3_inputs.cpp chain_motion), while NVIDIA DLSS or AMD FSR 4 upscales, frame generation's motion is the
|
||||
// upscaler chain's: the camera's from the depth, with every draw of the scene that stored its positions drawn again over
|
||||
// it (vk_temporal.cpp capture_motion). With NVIDIA DLSS frame generation that showed as the owner's flicker of 11-20 again
|
||||
// (the whole picture while the camera moved), on a build with the clip-space fix of that day (dlss_math.h
|
||||
// depth_clip_projection) in it. The owner's runs of 11-22 (DLSS at 50%, 2x frame generation, HDR, before the bloom):
|
||||
// - RX_VK_FG_OWN_MOTION=0, the camera's motion alone: no flicker;
|
||||
// - the chain's motion with the upscaler audit's commits all set back (RX_VK_CAPTURE_TABLES=0, RX_VK_REDRAW_STATE=0,
|
||||
// RX_VK_DLSS_FEATURE_MAX=1): the flicker, so it is not theirs;
|
||||
// - RX_VK_DLSS_PROBE=1 in that run: the redraw's motion of the scene's largest draw within 0.03 px of the camera's.
|
||||
// What in the chain's motion Streamline takes for a flicker is not known: its made frames are not in any log or shot. DLSS
|
||||
// frame generation finds what moves by itself with its own optical flow, and FSR's inputs give it the depth and the
|
||||
// camera, so it is given the camera's motion alone: what the first run showed. AMD's frame generation and FSR 3.1's
|
||||
// interpolation keep the chain's, as since FG-4 (no flicker of theirs is reported).
|
||||
//
|
||||
// RX_VK_DLSSG_OWN_MOTION=1: DLSS frame generation is given the chain's motion again, as before. RX_VK_FG_OWN_MOTION=0 still
|
||||
// gives every generator the camera's alone.
|
||||
//
|
||||
// No Vulkan: tests/test_fg_motion.py compiles this.
|
||||
#pragma once
|
||||
#include <cstdlib>
|
||||
|
||||
namespace fg_motion {
|
||||
|
||||
/// A switch read as the renderer reads its others: set and not 0 (value null when unset)
|
||||
inline bool set_on(const char* value) { return value && *value && std::atoi(value) != 0; }
|
||||
/// RX_VK_FG_OWN_MOTION=0 (value null when unset): the camera's motion alone, for every generator
|
||||
inline bool camera_alone(const char* value) { return value && *value && std::atoi(value) == 0; }
|
||||
|
||||
/// Whether frame generation is given the upscaler chain's motion this frame. dlssg: NVIDIA DLSS frame generation runs in
|
||||
/// this session (vk_dlssg.h active). fg_own: RX_VK_FG_OWN_MOTION; dlssg_own: RX_VK_DLSSG_OWN_MOTION (null when unset)
|
||||
inline bool chain(bool dlssg, const char* fg_own, const char* dlssg_own) {
|
||||
if (camera_alone(fg_own)) return false;
|
||||
return !dlssg || set_on(dlssg_own);
|
||||
}
|
||||
|
||||
} // namespace fg_motion
|
||||
42
gpu/vk/finalized_obsolete.h
Normal file
42
gpu/vk/finalized_obsolete.h
Normal file
@@ -0,0 +1,42 @@
|
||||
// (Triforce, 11-21, the performance audit's TEX-43, after Source A's) A finalized texture the GPU drew whose storage the
|
||||
// game has since written over.
|
||||
//
|
||||
// A texture the GPU drew in one frame is kept after the game finalizes it (texture_sweep keep_finalized, up to
|
||||
// RX_VK_KEEP_FINALIZED_MB): the game may make a texture over that storage again and sample it without drawing it, as the
|
||||
// terrain does with its 260 x 260 tiles. Within the budget only a draw over one let it go, and the game's own CPU writes
|
||||
// to that memory never did: in players' logs about 200 render target images an hour stayed for good, most of them such
|
||||
// tiles, with the blocks they pin. Source A's rule: once the CPU has written three quarters or more of the storage since
|
||||
// the release, the game has put other data there, the image holds nothing that memory holds on the console, and the
|
||||
// sweep lets it go whatever the budget says. Only one of 32 KB or more (a small one shares its pages with what the game
|
||||
// keeps beside it, and a write there says nothing of it) and not one in a virtual pool. Its laps of Hyrule Field at a
|
||||
// 6 GB allowance: render target images 129, 166, 221 against 128, 131, 129; the kept ones 131 against 41.
|
||||
// The answer is kept with the texture: once rewritten it stays so until it is released again, and one not yet rewritten
|
||||
// is looked at again at most every kRecheck presents (the look walks its pages' write generations).
|
||||
// RX_VK_KEEP_OBSOLETE=1: kept, as before.
|
||||
//
|
||||
// The rule alone: tests/test_finalized_obsolete.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace finalized_obsolete {
|
||||
|
||||
inline constexpr uint64_t kMinBytes = 32u << 10;
|
||||
inline constexpr uint64_t kVirtual = 1ull << 40;
|
||||
inline constexpr uint64_t kRecheck = 60;
|
||||
|
||||
/// RX_VK_KEEP_OBSOLETE: not 0 keeps them, as before
|
||||
inline bool keep(const char* asked) { return asked && atoi(asked) != 0; }
|
||||
|
||||
/// Whether a texture can be one: finalized after the GPU wrote it, in no virtual pool, of 32 KB of storage or more
|
||||
constexpr bool candidate(bool released, bool gpu_written, uint64_t gpu, uint64_t guest_size) {
|
||||
return released && gpu_written && gpu < kVirtual && guest_size >= kMinBytes;
|
||||
}
|
||||
|
||||
/// Whether `written` bytes of a storage of guest_size, written by the CPU since the release, make it rewritten
|
||||
constexpr bool rewritten(uint64_t written, uint64_t guest_size) { return guest_size != 0 && written * 4 >= guest_size * 3; }
|
||||
|
||||
/// Whether one not rewritten when last looked at, in present `checked` (0: never), is looked at again in present `frame`
|
||||
constexpr bool due(uint64_t frame, uint64_t checked) { return !checked || frame - checked >= kRecheck; }
|
||||
|
||||
} // namespace finalized_obsolete
|
||||
74
gpu/vk/first_start.h
Normal file
74
gpu/vk/first_start.h
Normal file
@@ -0,0 +1,74 @@
|
||||
// (Triforce, 11-22, the performance audit's PRESENT-46, after Source A's) A first start sized to the card: the Render
|
||||
// resolution row set once on a weak GPU.
|
||||
//
|
||||
// With no options file every card started the same: the Render resolution at Auto, the game's dynamic resolution. Source A
|
||||
// reads the card and puts it in a tier; its tier 1 is an integrated GPU (a handheld's is one), a card without mesh shaders,
|
||||
// or a card with 4.5 GB of video memory or less, and it draws the 3D at 50% there, or at 33% on an integrated GPU beside
|
||||
// 4 processor threads or fewer (its note: an i3-1115G4's own GPU took 60 to 220 ms a frame at 50%).
|
||||
//
|
||||
// This is that test, for a first start only (vk_menu.cpp menu_first_start): no options file, RX_VK_LOW_SPEC not set,
|
||||
// nothing saved by it, so a start after the player has saved anything is the player's own. Left out, as the audit's
|
||||
// entry says: Source A's preset row, its fingerprint that chooses again on new hardware (it would overwrite the player's
|
||||
// rows), its tuner in play, any change of the upscaler or of the output's size, and its test of the SteamDeck variable,
|
||||
// which took desktop PCs started from that launcher for handhelds: a handheld is found by its integrated GPU alone.
|
||||
//
|
||||
// RX_VK_FIRST_START_AUTO: not set or 0, off (as before; the default until the owner has decided what a first start on a
|
||||
// weak card gets). 1: on. A larger number: on, the card taken as having that many MB of video memory, for the lab.
|
||||
//
|
||||
// No Vulkan, no menu: tests/test_first_start.py compiles this.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace first_start {
|
||||
|
||||
/// What is read of the card
|
||||
struct Card {
|
||||
bool integrated = false; // (VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU)
|
||||
bool mesh_shaders = true; // (VK_EXT_mesh_shader, as the device was made)
|
||||
uint64_t vram_mb = 0; // (its video memory; 0: not known)
|
||||
unsigned threads = 0; // (the processor's threads; 0: not known)
|
||||
};
|
||||
|
||||
constexpr uint64_t kSmallVramMb = 4608; // 4.5 GB: tier 1 up to here
|
||||
constexpr unsigned kFewThreads = 4; // an integrated GPU beside this many threads or fewer: 33%
|
||||
constexpr int kRowHalf = 4, kRowThird = 6; // opts.resolution's 50% and 33% (upscaler_rows.h)
|
||||
|
||||
enum Why { kKeep = 0, kIntegrated, kNoMeshShaders, kSmallVram };
|
||||
/// The Render resolution row for the card's first start: row 0, the row left as it is (not tier 1)
|
||||
struct Choice {
|
||||
int row = 0;
|
||||
Why why = kKeep;
|
||||
};
|
||||
|
||||
inline Choice choose(const Card& c) {
|
||||
if (c.integrated) return {c.threads && c.threads <= kFewThreads ? kRowThird : kRowHalf, kIntegrated};
|
||||
if (!c.mesh_shaders) return {kRowHalf, kNoMeshShaders};
|
||||
if (c.vram_mb && c.vram_mb <= kSmallVramMb) return {kRowHalf, kSmallVram};
|
||||
return {};
|
||||
}
|
||||
|
||||
inline const char* why_text(Why w) {
|
||||
switch (w) {
|
||||
case kIntegrated: return "an integrated GPU";
|
||||
case kNoMeshShaders: return "no mesh shaders";
|
||||
case kSmallVram: return "4.5 GB of video memory or less";
|
||||
default: return "not a weak card";
|
||||
}
|
||||
}
|
||||
|
||||
/// RX_VK_FIRST_START_AUTO (value, null when unset): -1 off; 0 on, the card's own video memory; otherwise on, the card
|
||||
/// taken as having that many MB
|
||||
inline int64_t switch_mb(const char* text) {
|
||||
if (!text || !*text) return -1;
|
||||
const long long v = std::atoll(text);
|
||||
return v <= 0 ? -1 : v == 1 ? 0 : (int64_t)v;
|
||||
}
|
||||
|
||||
/// Whether the first start's choice is made at all: the switch on, no options file, RX_VK_LOW_SPEC not asked (it sets the
|
||||
/// row itself), and no switch that sets the Render resolution for this run (HOS_DYNRES)
|
||||
inline bool applies(bool has_options, bool low_spec, bool resolution_env, int64_t switch_value) {
|
||||
return switch_value >= 0 && !has_options && !low_spec && !resolution_env;
|
||||
}
|
||||
|
||||
} // namespace first_start
|
||||
@@ -28,6 +28,7 @@ struct Split {
|
||||
double wb_wait = 0; // (the waits for write-backs still on the GPU, wherever they were: memory sync's capture, a read)
|
||||
double wb_held = 0; // (of them, the part after the writer's recording was done: the timeline held, not the GPU)
|
||||
double qs_bg = 0; // (the queue submits made while a background import ran)
|
||||
double gpu_waits = 0; // ((Triforce, 11-22, SYNC-27) every thread's time in calls that block on the GPU: gpu_wait_kinds.h)
|
||||
};
|
||||
|
||||
/// Outside the replay and the present: the game's own update, its waits for the replay and the GPU among it. As the stall line's
|
||||
@@ -35,12 +36,13 @@ inline double outside(const Split& s) { return s.interval - s.replay - s.present
|
||||
|
||||
/// The columns, after the trace's first ones (their names; the line's values in this order: line)
|
||||
constexpr const char* kColumns =
|
||||
"replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms,game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms";
|
||||
"replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms,game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms,"
|
||||
"gpu_waits_ms";
|
||||
|
||||
/// The values for kColumns, comma-separated, no line end: snprintf's count
|
||||
inline int line(const Split& s, char* out, size_t cap) {
|
||||
return std::snprintf(out, cap, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f", s.replay, s.present, s.dispatch,
|
||||
s.qsubmit, s.par_wait, outside(s), s.game_wait, s.captures, s.wb_wait, s.wb_held, s.qs_bg);
|
||||
return std::snprintf(out, cap, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f", s.replay, s.present, s.dispatch,
|
||||
s.qsubmit, s.par_wait, outside(s), s.game_wait, s.captures, s.wb_wait, s.wb_held, s.qs_bg, s.gpu_waits);
|
||||
}
|
||||
|
||||
} // namespace mkwvk::frame_trace
|
||||
|
||||
49
gpu/vk/framegen_idle.h
Normal file
49
gpu/vk/framegen_idle.h
Normal file
@@ -0,0 +1,49 @@
|
||||
// (Triforce, 11-21, the performance audit's PRESENT-18, after Source A's) When frame generation's images are let go.
|
||||
//
|
||||
// The built-in interpolation's frames, lumas, flows and made-frame sets (and, for AMD FSR 4's frame generation, the four
|
||||
// images of the picture without the HUD) are made at the first frame that needs them and were kept until a resize: turned
|
||||
// off, or left for NVIDIA's own frame generation, they stayed in VRAM for the run. Source A measured 150 to 160 MiB given
|
||||
// back at 2560 x 1440 with 2x after its switch away. They are let go after kPresents presents in a row with frame
|
||||
// generation off, about two seconds at 60: a row stepped through on the way to another choice keeps them. A present that
|
||||
// recorded made frames counts as on whatever the row says by then: its set is busy until the display thread has shown
|
||||
// it, and a drain then would wait for a job that is only queued afterwards. So the count is taken after the present's
|
||||
// queue (vk_replay.cpp, after framegen_queue). They are made again at the next frame that needs them, as after a resize.
|
||||
// RX_VK_FG_KEEP=1: kept, as before.
|
||||
//
|
||||
// The rule alone, no Vulkan: tests/test_framegen_idle.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
namespace framegen_idle {
|
||||
|
||||
inline constexpr uint32_t kPresents = 120;
|
||||
|
||||
/// The presents in a row off, after this one: on now, or used at this present, starts again
|
||||
inline uint32_t next_off(uint32_t off, bool on_now, bool used) { return on_now || used ? 0 : off < UINT32_MAX ? off + 1 : off; }
|
||||
|
||||
/// Whether this present lets them go: there are some, and they have been off long enough
|
||||
inline bool due(uint32_t off, bool have) { return have && off >= kPresents; }
|
||||
|
||||
/// One present, as framegen_idle sees it after the present's queue
|
||||
struct Present {
|
||||
bool on; // frame generation on by the row, read again (framegen_on)
|
||||
bool recorded; // this present recorded made frames (framegen_record's true)
|
||||
bool hudless_wanted; // AMD FSR 4's frame generation is the row's (fsr4fg::frames_wanted)
|
||||
bool have_images; // the shared images are there (fg.w)
|
||||
bool have_hudless; // the images of the picture without the HUD are there
|
||||
};
|
||||
struct Counts {
|
||||
uint32_t off = 0, hudless_off = 0;
|
||||
};
|
||||
struct Release {
|
||||
bool shared, hudless;
|
||||
};
|
||||
/// The two counts after this present, and what it lets go. keep: RX_VK_FG_KEEP, nothing ever
|
||||
inline Release step(Counts& c, const Present& p, bool keep) {
|
||||
c.off = next_off(c.off, p.on, p.recorded);
|
||||
c.hudless_off = next_off(c.hudless_off, p.on && p.hudless_wanted, false);
|
||||
if (keep) return {false, false};
|
||||
return {due(c.off, p.have_images), due(c.hudless_off, p.have_hudless)};
|
||||
}
|
||||
|
||||
} // namespace framegen_idle
|
||||
40
gpu/vk/framegen_refresh.h
Normal file
40
gpu/vk/framegen_refresh.h
Normal file
@@ -0,0 +1,40 @@
|
||||
// (Triforce, 11-21, the upscaler audit's UPS-05) A fixed multiplier's frames, no more than the display can show.
|
||||
//
|
||||
// The Frame generation row's 2x to 8x make that many frames for each of the game's whatever the display's refresh rate
|
||||
// is; only the row's Auto goes by it. The owner's log of 11-20 has 5x on a 165 Hz display with the game near 66 frames a
|
||||
// second: 1,655 frames shown in every 5 s (331 a second, two for each refresh), each present call 2.2 to 3.0 ms where it
|
||||
// takes 0.03 ms without frame generation, and the display thread's line "0 frames shown on time, 1655 late". Half of the
|
||||
// made frames were never on the screen (the swapchain is mailbox: the newest at each refresh), and each was a pass over
|
||||
// the whole picture and a present the display thread could not keep up with.
|
||||
//
|
||||
// With the switch, a fixed multiplier makes no more frames than refreshes fall in one frame of the game: the game's
|
||||
// recent frame time over the display's period, rounded up, a tenth of a refresh forgiven so a game at a whole fraction
|
||||
// of the refresh rate is not given one more for its frames' jitter. Never more than the row asks, never fewer than the
|
||||
// game's own frame. The row's Auto and AMD FSR's own are not touched.
|
||||
//
|
||||
// RX_VK_FRAMEGEN_REFRESH=1: on. Off unless set (the row's number is the frames made, as before): it changes what a row
|
||||
// the player set shows, so it is the owner's to turn on.
|
||||
//
|
||||
// No Vulkan: tests/test_framegen_refresh.py compiles this.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace framegen_refresh {
|
||||
|
||||
/// RX_VK_FRAMEGEN_REFRESH (value, null when unset): on when it is not 0; unset, off
|
||||
inline bool on(const char* value) { return value && *value && std::atoi(value) != 0; }
|
||||
|
||||
/// The frames shown for each of the game's (the game's own among them) with a fixed multiplier m, the game's frames game_ns
|
||||
/// apart, the display refreshing every period_ns. An unknown frame time or period: m
|
||||
inline int frames(int m, int64_t game_ns, int64_t period_ns) {
|
||||
if (m <= 1 || game_ns <= 0 || period_ns <= 0) return m;
|
||||
// (the refreshes in a frame of the game, rounded up, a tenth of one forgiven: ceil(game / period - 0.1))
|
||||
const int64_t most = (game_ns * 10 + period_ns * 9 - 1) / (period_ns * 10);
|
||||
return most < 1 ? 1 : most < m ? (int)most : m;
|
||||
}
|
||||
|
||||
/// The display's period for a refresh rate in mHz (vk_main.cpp display_refresh_mhz); 0 when it is not known
|
||||
inline int64_t period_ns(int refresh_mhz) { return refresh_mhz > 0 ? (int64_t)(1000000000000LL / refresh_mhz) : 0; }
|
||||
|
||||
} // namespace framegen_refresh
|
||||
@@ -232,4 +232,24 @@ template <class Ops> void teardown(Ops& ops, bool exiting) {
|
||||
ops.destroy_images();
|
||||
}
|
||||
|
||||
// ---- (Triforce, 11-20) What a present writes into a set of frames: never past the frames the set was made with ----
|
||||
// The owner's "upscaling crash": 0xc0000005 reading 0x590 in NVIDIA's driver, inside a vkCmdCopyImage the recorder thread
|
||||
// ran, in three runs of 11-20, each with frame interpolation on and the Frame generation row stepped up in the options
|
||||
// (Auto in the first, 3x at the next start, 5x at the one after). The sets' frames are made for the row's multiplier when
|
||||
// a present begins (vk_framegen.cpp ensure_size); the present then waits for the display thread to let a set go, and
|
||||
// while frames are made the display thread is the one that runs the menu (present_image, menu_tick), so a step of the row
|
||||
// lands in that wait. The multiplier read again after it was more than the frames made, and the game's frame was copied
|
||||
// into an image that was never made (VK_NULL_HANDLE). A step down only left frames unused. The row's new multiplier takes
|
||||
// effect at the next present, which makes the sets anew.
|
||||
/// The multiplier a present records with: the row's, and no more than the frames its sets were made with
|
||||
constexpr int set_multiplier(int row_multiplier, int made_with) {
|
||||
const int most = made_with < 1 ? 1 : made_with;
|
||||
return row_multiplier < 1 ? 1 : row_multiplier > most ? most : row_multiplier;
|
||||
}
|
||||
/// The game's own frame's place in a set, after `made` made ones: the set's last frame at most
|
||||
constexpr int game_slot(int made, int made_with) {
|
||||
const int last = (made_with < 1 ? 1 : made_with) - 1;
|
||||
return made < 0 ? 0 : made > last ? last : made;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::framegen_rows
|
||||
|
||||
@@ -46,4 +46,24 @@ inline int pick(const std::string& want, const std::vector<std::string>& names,
|
||||
return menu_device(menu, (int)names.size());
|
||||
}
|
||||
|
||||
/// ((Triforce) 11-22, tickets "igpu targeted for shader cache", "Launcher \"GPU device\" setting is non-functional" (RX 6600
|
||||
/// with a Ryzen 5600G's graphics) and on 0.6.0 "still picking IGPU for building shaders") The choice that pick takes by its
|
||||
/// number alone (no saved name, or a name no card has here) when that number is the processor's graphics while a card of its
|
||||
/// own is listed: not taken, Auto instead. Such a number comes from an options file written before 11-20, when the launcher
|
||||
/// numbered the cards in Windows' order (the display class's driver keys: the processor graphics' driver first on a 5600G)
|
||||
/// and saved no name with it; the update to 0.6.0 still prepared its shaders there and the game started there, as no name
|
||||
/// came with the number until the row was chosen again. A choice saved with its name (the launcher and the menu since 11-20)
|
||||
/// is taken as it is, the processor graphics too. types: the usable devices' VkPhysicalDeviceType (1 integrated, 2 discrete),
|
||||
/// in the game's order, as names
|
||||
inline bool unnamed_integrated(const std::string& want, const std::vector<std::string>& names, const std::vector<int>& types,
|
||||
int menu) {
|
||||
const int k = pick(want, names, menu);
|
||||
if (k < 0 || (size_t)k >= types.size() || types[(size_t)k] != 1) return false;
|
||||
const std::string key = name_key(want);
|
||||
if (!key.empty() && name_key(names[(size_t)k]) == key) return false; // (chosen by its name)
|
||||
for (const int t : types)
|
||||
if (t == 2) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::gpu_choice
|
||||
|
||||
57
gpu/vk/gpu_clock.h
Normal file
57
gpu/vk/gpu_clock.h
Normal file
@@ -0,0 +1,57 @@
|
||||
// (Triforce) Whether the renderer holds an AMD GPU at amdgpu's stable power state while the game runs, and which one (Linux:
|
||||
// vk_gpu_clock.cpp; RX_VK_GPU_CLOCK). Plain code with no system calls: tests/test_gpu_clock.py runs it on made-up devices.
|
||||
//
|
||||
// The Steam Deck's automatic GPU clock drops while the frame waits on the CPU or a readback (200 MHz for 27% of a Kokiri run), so
|
||||
// the GPU work after the wait runs slow. amdgpu lets any process that opens the GPU's render node ask for a stable power state
|
||||
// through a context of its own (DRM_IOCTL_AMDGPU_CTX, AMDGPU_CTX_OP_SET_STABLE_PSTATE, Linux 5.18): no root and no sysfs write.
|
||||
// The kernel gives the level back when that context goes, at the process's end too, a crash or kill -9 included. Measured on a
|
||||
// Deck OLED (SteamOS 3.9.5, plugged in): Kokiri handheld 11.1 -> 16.2 fps, 1% low 6.8 -> 12.1, GPU time 58 -> 32 ms at the same
|
||||
// 15 W; Hyrule Field +19% for 4 W more. It costs battery where the APU was under its power cap.
|
||||
//
|
||||
// - Unset: the peak state, on the Deck's APU only (its Vulkan name, or SteamDeck=1 from Steam on an integrated AMD GPU), and
|
||||
// only while the level is amdgpu's own (power_dpm_force_performance_level "auto"): a level the player set is left alone.
|
||||
// - RX_VK_GPU_CLOCK=0: nothing opened. =peak (or 1), =standard: that state on any AMD GPU, whatever the level.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
namespace mkwvk::gpu_clock {
|
||||
|
||||
enum Want { kNone, kPeak, kStandard };
|
||||
|
||||
// (uapi/drm/amdgpu_drm.h, written out: no libdrm and no kernel headers needed to build)
|
||||
constexpr uint32_t kCtxOpAlloc = 1, kCtxOpSetStablePstate = 6;
|
||||
constexpr uint32_t kPstateStandard = 1, kPstatePeak = 4;
|
||||
constexpr uint32_t kCtxArgBytes = 16; // (union drm_amdgpu_ctx)
|
||||
/// DRM_IOCTL_AMDGPU_CTX: DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_CTX, union drm_amdgpu_ctx) = _IOWR('d', 0x40 + 0x02, 16 bytes)
|
||||
constexpr unsigned long kCtxIoctl = (3ul << 30) | ((unsigned long)kCtxArgBytes << 16) | ((unsigned long)'d' << 8) | 0x42ul;
|
||||
|
||||
/// The Deck's APU: "AMD Custom GPU 0405" (LCD) or "0932" (OLED) in its Vulkan name; or SteamDeck=1 (Steam sets it there) on an
|
||||
/// integrated GPU
|
||||
inline bool deck_apu(const char* device_name, bool integrated, const char* steam_deck_env) {
|
||||
if (device_name && (std::strstr(device_name, "Custom GPU 0405") || std::strstr(device_name, "Custom GPU 0932"))) return true;
|
||||
return integrated && steam_deck_env && !std::strcmp(steam_deck_env, "1");
|
||||
}
|
||||
|
||||
struct Choice {
|
||||
Want want;
|
||||
const char* why;
|
||||
};
|
||||
|
||||
/// switch_value: RX_VK_GPU_CLOCK (nullptr unset); level: power_dpm_force_performance_level's text, first word (nullptr: unread)
|
||||
inline Choice choose(const char* switch_value, uint32_t vendor_id, bool integrated, const char* device_name,
|
||||
const char* steam_deck_env, const char* level) {
|
||||
if (vendor_id != 0x1002) return {kNone, "not an AMD GPU"};
|
||||
if (switch_value && *switch_value) {
|
||||
if (!std::strcmp(switch_value, "0")) return {kNone, "RX_VK_GPU_CLOCK=0"};
|
||||
if (!std::strcmp(switch_value, "peak") || !std::strcmp(switch_value, "1")) return {kPeak, "RX_VK_GPU_CLOCK=peak"};
|
||||
if (!std::strcmp(switch_value, "standard")) return {kStandard, "RX_VK_GPU_CLOCK=standard"};
|
||||
}
|
||||
if (!deck_apu(device_name, integrated, steam_deck_env)) return {kNone, "not a Steam Deck (RX_VK_GPU_CLOCK=peak: any AMD GPU)"};
|
||||
if (!level) return {kNone, "the GPU's performance level could not be read"};
|
||||
if (std::strcmp(level, "auto")) return {kNone, "the GPU's performance level is set by hand"};
|
||||
return {kPeak, "the Steam Deck's GPU, its level amdgpu's own"};
|
||||
}
|
||||
|
||||
} // namespace mkwvk::gpu_clock
|
||||
86
gpu/vk/gpu_wait_kinds.h
Normal file
86
gpu/vk/gpu_wait_kinds.h
Normal file
@@ -0,0 +1,86 @@
|
||||
// (Triforce, 11-22, the performance audit's SYNC-27, after Source A's) The host's time in calls that block on the GPU,
|
||||
// counted by kind where the Vulkan calls are wrapped (vk_work.h).
|
||||
//
|
||||
// The log's "GPU-bound wait" counts only the waits for a recording slot: it reads about 0 while threads wait on the GPU
|
||||
// elsewhere (a fence of the game's, a write-back, the present's frame fence, a device idle, a query read that waits).
|
||||
// Those were timed apart or not at all. Here every blocking wait is timed in one place and summed by kind:
|
||||
// fences vkWaitForFences with a timeout above 0
|
||||
// timelines vkWaitSemaphores with a timeout above 0
|
||||
// idle vkQueueWaitIdle, vkDeviceWaitIdle
|
||||
// queries vkGetQueryPoolResults with VK_QUERY_RESULT_WAIT_BIT
|
||||
// A poll (timeout 0, no wait bit) counts nothing. A wait made inside another counts once, as the outer one (a depth of
|
||||
// the thread's). A wait inside an Exclude scope counts nothing: pacing (the driver's low-latency sleep), which is not a
|
||||
// wait for the GPU's work. The totals only grow; each reader keeps its own last values (Reader), so the log's line and
|
||||
// the frame trace's column each see every nanosecond.
|
||||
//
|
||||
// The cost is two clock reads a blocking wait. RX_VK_GPU_WAITS=0, or the lab knob nogpuwaits in its phases: the calls
|
||||
// straight through, nothing counted. No Vulkan here: tests/test_gpu_waits.py compiles it.
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "lab_ab.h"
|
||||
|
||||
namespace gpu_wait_kinds {
|
||||
|
||||
enum Kind : int { FENCE = 0, TIMELINE = 1, IDLE = 2, QUERY = 3, KINDS = 4 };
|
||||
|
||||
struct Totals {
|
||||
std::atomic<int64_t> ns[KINDS]{}, n[KINDS]{};
|
||||
};
|
||||
inline Totals totals; // (every thread's, since the start: they only grow)
|
||||
inline thread_local int depth = 0; // (this thread is inside a counted wait)
|
||||
inline thread_local int excluded = 0; // (this thread is inside an Exclude scope)
|
||||
|
||||
/// Counted at all (RX_VK_GPU_WAITS, read once; on unless 0), and not in the lab knob's phases
|
||||
inline bool on() {
|
||||
static const bool env = [] {
|
||||
const char* e = getenv("RX_VK_GPU_WAITS");
|
||||
return !(e && *e && atoi(e) == 0);
|
||||
}();
|
||||
return env && !mkwvk::lab_ab::on("nogpuwaits");
|
||||
}
|
||||
|
||||
/// The waits made on this thread while one of these lives count nothing
|
||||
struct Exclude {
|
||||
Exclude() { excluded++; }
|
||||
~Exclude() { excluded--; }
|
||||
Exclude(const Exclude&) = delete;
|
||||
Exclude& operator=(const Exclude&) = delete;
|
||||
};
|
||||
|
||||
/// call(), and its time under `kind` when it blocks (blocks: a timeout above 0, the wait bit), unless this thread is
|
||||
/// inside a counted wait or an Exclude scope, or the count is off. What call() returns
|
||||
template <class Call>
|
||||
inline auto timed(Kind kind, bool blocks, Call&& call) -> decltype(call()) {
|
||||
if (!blocks || depth || excluded || !on()) return call();
|
||||
depth++;
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
const auto result = call();
|
||||
const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
|
||||
depth--;
|
||||
totals.ns[kind].fetch_add(ns, std::memory_order_relaxed);
|
||||
totals.n[kind].fetch_add(1, std::memory_order_relaxed);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// One reader's last values: take() gives what was counted since its last take (ns and waits by kind, either may be
|
||||
/// null) and returns the nanoseconds of all kinds together
|
||||
struct Reader {
|
||||
int64_t last_ns[KINDS]{}, last_n[KINDS]{};
|
||||
int64_t take(int64_t* ns_by_kind, int64_t* n_by_kind) {
|
||||
int64_t all = 0;
|
||||
for (int k = 0; k < KINDS; k++) {
|
||||
const int64_t ns = totals.ns[k].load(std::memory_order_relaxed), n = totals.n[k].load(std::memory_order_relaxed);
|
||||
if (ns_by_kind) ns_by_kind[k] = ns - last_ns[k];
|
||||
if (n_by_kind) n_by_kind[k] = n - last_n[k];
|
||||
all += ns - last_ns[k];
|
||||
last_ns[k] = ns, last_n[k] = n;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace gpu_wait_kinds
|
||||
112
gpu/vk/guest_holes.h
Normal file
112
gpu/vk/guest_holes.h
Normal file
@@ -0,0 +1,112 @@
|
||||
// (Triforce) The all-zero upload check's reads (vk_texture.cpp upload_all_zero): which bytes of a storage it reads, split between its
|
||||
// threads. tests/test_guest_holes.py.
|
||||
//
|
||||
// On Linux, where /dev/udmabuf is usable, the guest window is a memfd's (host/hos_main.c). A page the game never wrote is a hole
|
||||
// there: it reads as zero, but a read through the shared mapping makes the kernel allocate and zero it. The check read every byte
|
||||
// of a 32 MB+ storage, so the never-written 288 MB shadow map (12288 x 12288 D16) became RAM at each Hyrule Field arrival, and
|
||||
// took 200-300 ms of the replay thread on a Steam Deck. fd_data_extents asks the memfd where its data is (lseek SEEK_DATA and
|
||||
// SEEK_HOLE): a hole is zero by definition, and SEEK_DATA may call a hole data, never the other way round (a page in swap is
|
||||
// data), so reading only the data says the same. Anywhere else, or on any failure, one extent of the whole storage.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#if defined(__linux__) || defined(GUEST_HOLES_TEST) // (the test: its own lseek)
|
||||
#include <cerrno>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace mkwvk::guest_holes {
|
||||
|
||||
struct Extent {
|
||||
uint64_t lo, hi; // (bytes from the storage's start)
|
||||
};
|
||||
|
||||
inline uint64_t bytes(const std::vector<Extent>& ex) {
|
||||
uint64_t n = 0;
|
||||
for (const Extent& e : ex) n += e.hi - e.lo;
|
||||
return n;
|
||||
}
|
||||
|
||||
/// Thread t's part of n: the extents' bytes, not their range, split into n runs of whole 4 KB (the last takes the rest); every
|
||||
/// byte of the extents in exactly one part
|
||||
inline std::vector<Extent> share(const std::vector<Extent>& ex, uint32_t t, uint32_t n) {
|
||||
std::vector<Extent> out;
|
||||
const uint64_t total = bytes(ex);
|
||||
if (!n || t >= n || !total) return out;
|
||||
const uint64_t per = ((total / n) + 4095) & ~4095ull;
|
||||
const uint64_t a = std::min<uint64_t>(total, (uint64_t)t * per), b = t + 1 == n ? total : std::min<uint64_t>(total, a + per);
|
||||
uint64_t at = 0; // (bytes of the extents before e)
|
||||
for (const Extent& e : ex) {
|
||||
const uint64_t len = e.hi - e.lo, lo = std::max(a, at), hi = std::min(b, at + len);
|
||||
if (lo < hi) out.push_back({e.lo + (lo - at), e.lo + (hi - at)});
|
||||
at += len;
|
||||
if (at >= b) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Whether any byte of p's extents is not 0, read on nthreads threads (this one among them), each stopping once one found it
|
||||
inline bool any_set(const uint8_t* p, const std::vector<Extent>& ex, uint32_t nthreads) {
|
||||
std::atomic<bool> found{false};
|
||||
if (ex.empty()) return false;
|
||||
nthreads = std::max(nthreads, 1u);
|
||||
auto part = [&](uint32_t t) {
|
||||
for (const Extent& e : share(ex, t, nthreads))
|
||||
for (uint64_t at = e.lo; at < e.hi && !found.load(std::memory_order_relaxed); at += 65536) {
|
||||
const uint64_t end = std::min<uint64_t>(e.hi, at + 65536);
|
||||
uint64_t acc = 0;
|
||||
uint64_t k = at;
|
||||
for (; k + 8 <= end; k += 8) {
|
||||
uint64_t w;
|
||||
std::memcpy(&w, p + k, 8);
|
||||
acc |= w;
|
||||
}
|
||||
for (; k < end; k++) acc |= p[k];
|
||||
if (acc) found.store(true, std::memory_order_relaxed);
|
||||
}
|
||||
};
|
||||
if (nthreads > 1) {
|
||||
std::vector<std::thread> th;
|
||||
for (uint32_t t = 1; t < nthreads; t++) th.emplace_back(part, t);
|
||||
part(0);
|
||||
for (auto& x : th) x.join();
|
||||
} else {
|
||||
part(0);
|
||||
}
|
||||
return found.load();
|
||||
}
|
||||
|
||||
#if defined(__linux__) || defined(GUEST_HOLES_TEST) // (the test: its own lseek)
|
||||
/// The data of [off, off + len) in fd, as extents from off (none: all holes). At most max extents: past them, the rest as one.
|
||||
/// False when the file cannot say (a seek failed): the caller reads it all
|
||||
inline bool fd_data_extents(int fd, uint64_t off, uint64_t len, std::vector<Extent>* out, size_t max = 1024) {
|
||||
out->clear();
|
||||
const uint64_t end = off + len;
|
||||
uint64_t pos = off;
|
||||
while (pos < end) {
|
||||
if (out->size() + 1 >= max) { // (the last one: the rest, whole)
|
||||
out->push_back({pos - off, len});
|
||||
return true;
|
||||
}
|
||||
const off_t d = lseek(fd, (off_t)pos, SEEK_DATA);
|
||||
if (d < 0) {
|
||||
if (errno == ENXIO) return true; // (no data from pos to the file's end)
|
||||
return out->clear(), false;
|
||||
}
|
||||
if ((uint64_t)d >= end) return true;
|
||||
const off_t h = lseek(fd, d, SEEK_HOLE);
|
||||
if (h < 0 || (uint64_t)h <= (uint64_t)d) return out->clear(), false;
|
||||
const uint64_t hi = std::min<uint64_t>(end, (uint64_t)h);
|
||||
out->push_back({(uint64_t)d - off, hi - off});
|
||||
pos = hi;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace mkwvk::guest_holes
|
||||
@@ -7,7 +7,7 @@
|
||||
// what it waits for comes within it, it never decides anything. The audit's trace: the helpers began a segment 0.262 ms after
|
||||
// they were given it, and the replay thread waited 1.74 ms a frame for them (r22-trace). tests/test_helper_spin.py
|
||||
//
|
||||
// RX_VK_HELPER_SPIN_US=<us>: the spin's bound (100, NX2navi's; at most 1000000); 0: no spin, straight to the wait as before
|
||||
// RX_VK_HELPER_SPIN_US=<us>: the spin's bound (20 since 11-21; 100 before, NX2navi's; at most 1000000); 0: no spin, straight to the wait as before
|
||||
#pragma once
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
@@ -20,7 +20,8 @@ namespace helper_spin {
|
||||
|
||||
/// The bound in nanoseconds that RX_VK_HELPER_SPIN_US's text gives (null: unset)
|
||||
inline int64_t bound_ns(const char* text) {
|
||||
if (!text || !*text) return 100000;
|
||||
if (!text || !*text) return 20000; // ((Triforce, 11-21) 20 us, was 100 (NX2navi's): with the game's threads the frame's limit, a
|
||||
// helper spinning 100 us took its SMT sibling from them. Lab A/B 79.3 -> 80.0, 1% low +2.0)
|
||||
const long long us = std::atoll(text);
|
||||
return us <= 0 ? 0 : us >= 1000000 ? 1000000000LL : us * 1000;
|
||||
}
|
||||
|
||||
53
gpu/vk/import_budget.h
Normal file
53
gpu/vk/import_budget.h
Normal file
@@ -0,0 +1,53 @@
|
||||
// What the idle imports' valve measures the system heap against (no game, no GPU: a rule).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's MEM-22, after Source A's.) An import is a pool of the game's memory the driver
|
||||
// pins and counts in the system heap. vk_memory.cpp imports_give_back gives idle ones back past RX_VK_IMPORT_HIGH percent
|
||||
// (85) of that heap's budget, down to RX_VK_IMPORT_LOW percent (75), but by default only where the budget is under 12 GB:
|
||||
// on a PC with more, live imports pile up for the whole session (2.0 GB at a fresh Market, 6.5 GB after ten areas), and
|
||||
// on Linux's amdgpu every submission pays for each one. Source A's numbers (Arch, RX 9070 XT, RADV, 32 GB, a 15,434 MB
|
||||
// budget): uncapped after about two hours 221 live imports (7,392 MB), queue submit 1.11-1.14 ms, read-back waits
|
||||
// 6.29-7.03 ms, 1,511 MB of swap, 44-48 fps; capped, 50-64 live (2,256-2,720 MB), 0.32-0.73 ms, 0.64-1.14 ms, no swap,
|
||||
// 63-68 fps (with other settings changed in the same runs: the cap's own share is not known).
|
||||
//
|
||||
// So with Mesa's RADV and a budget of 12 GB or more the valve is on and measures the heap against 4,500 MB, not the whole
|
||||
// budget, and a pass of it gives back 1 GB at most. Everything else is as it was:
|
||||
// - a budget under 12 GB: the whole budget, on (any driver);
|
||||
// - 12 GB or more on any other driver: off unless RX_VK_IMPORT_HIGH is set, then the whole budget;
|
||||
// - RX_VK_IMPORT_HIGH or RX_VK_IMPORT_LOW set by the player: the whole budget, as before, RADV too.
|
||||
// RX_VK_IMPORT_CAP_MB=<n>: the cap on any driver and any budget (it turns the valve on); =0: no cap, which is RADV's
|
||||
// behaviour before. A given-back import costs 5-45 ms when its pool is bound again, which is why only pools idle for 600
|
||||
// submissions go.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
namespace import_budget {
|
||||
constexpr uint64_t kMB = 1ull << 20;
|
||||
constexpr uint64_t kBigBudget = 12ull << 30; // from here the valve was off by default
|
||||
constexpr uint64_t kCapMb = 4500; // RADV's cap at a big budget
|
||||
constexpr uint64_t kPassMost = 1ull << 30; // a capped pass gives back this much at most
|
||||
|
||||
struct Rule {
|
||||
bool on = false; // the heap's trigger runs
|
||||
uint64_t cap = 0; // bytes the heap is measured against at most (0: the whole budget)
|
||||
};
|
||||
|
||||
/// budget: the system heap's budget when the rule is first asked. high_set, low_set: the player set RX_VK_IMPORT_HIGH,
|
||||
/// RX_VK_IMPORT_LOW. cap_mb: RX_VK_IMPORT_CAP_MB (-1: unset). radv: the driver is Mesa's RADV
|
||||
constexpr Rule rule(uint64_t budget, bool high_set, bool low_set, int64_t cap_mb, bool radv) {
|
||||
const bool before = high_set || budget < kBigBudget; // (as it was: on where the budget is small, or asked for)
|
||||
if (cap_mb > 0) return Rule{true, (uint64_t)cap_mb * kMB};
|
||||
if (cap_mb == 0 || high_set || low_set || !radv || budget < kBigBudget) return Rule{before, 0};
|
||||
return Rule{true, kCapMb * kMB};
|
||||
}
|
||||
|
||||
/// What the heap's use is measured against now (the budget moves; the cap does not)
|
||||
constexpr uint64_t measured(const Rule& r, uint64_t budget) {
|
||||
return r.cap && r.cap < budget ? r.cap : budget;
|
||||
}
|
||||
|
||||
/// How much a pass by the heap's trigger wants back: down to the target, and 1 GB at most where the cap is what measures
|
||||
constexpr uint64_t pass_want(const Rule& r, uint64_t budget, uint64_t used, uint64_t target) {
|
||||
const uint64_t want = used > target ? used - target : 0;
|
||||
return r.cap && r.cap < budget && want > kPassMost ? kPassMost : want;
|
||||
}
|
||||
} // namespace import_budget
|
||||
30
gpu/vk/index_rebase.h
Normal file
30
gpu/vk/index_rebase.h
Normal file
@@ -0,0 +1,30 @@
|
||||
// (Triforce, 11-21, the performance audit's DESC-12, after Source A's) An indexed draw's index buffer bound at offset 0,
|
||||
// with the draw's offset given as its first index.
|
||||
//
|
||||
// The game's draws share one buffer at different offsets, so bind_index's skip (the same buffer, offset and type as the
|
||||
// one on the command buffer) rarely held: nearly every indexed draw called vkCmdBindIndexBuffer, about 2,900 calls a frame
|
||||
// in the Market. Bound at 0 with the offset as the draw's firstIndex, the skip holds until the buffer or the index type
|
||||
// changes. Only where the offset is a whole number of indices that fits a first index; otherwise the draw binds at its
|
||||
// offset as before. Indirect indexed draws are not touched: their first index is in the GPU's records.
|
||||
// RX_VK_INDEX_REBASE=0: every draw binds at its own offset, as before.
|
||||
//
|
||||
// The rule alone, no Vulkan: tests/test_index_rebase.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace index_rebase {
|
||||
|
||||
/// The switch's value: on unless it is set to 0
|
||||
inline bool on(const char* asked) { return !(asked && *asked && atoi(asked) == 0); }
|
||||
|
||||
/// Whether a draw whose indices start offset bytes into its buffer, unit bytes an index, can be drawn from the buffer bound
|
||||
/// at 0: *first is then its first index. Not when the offset falls inside an index, nor past what a first index the draw's
|
||||
/// own range is added to can hold
|
||||
inline bool rebase(uint64_t offset, uint32_t unit, uint32_t* first) {
|
||||
if (!unit || offset % unit || offset / unit > 0x7fffffffull) return false;
|
||||
*first = (uint32_t)(offset / unit);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace index_rebase
|
||||
@@ -10,12 +10,21 @@
|
||||
A knob named d_<site> is a delay: a busy wait of <value> microseconds at that site, at most once a frame (delay(), below):
|
||||
what the frame's time answers to a part of it taking longer, which is whether that part is on the frame's critical path
|
||||
(1 ms in for 1 ms of delay) or has slack (nothing). Any other name is a switch the code reads with on("<name>").
|
||||
(Triforce, 11-22, the performance audit's REPLAY-51, after Source B's) A knob named e_<event> is a delay at every
|
||||
occurrence of an event, not once a frame: a busy wait of <value> NANOseconds at each draw (e_draw) or dispatch
|
||||
(e_dispatch), first in vk_draw.cpp's draw() and dispatch() so that the profiles leave it out, on whichever thread
|
||||
records, counted (each(), below). The frame's change against the time spun says how much of that path is on the frame's
|
||||
critical path: 200 ns at ~3,050 draws a frame is 0.6 ms of work. One line a phase gives each knob's spins and ms a second
|
||||
(each_report). d_present_after is d_present's after: the same wait just after the present's drain(0), where d_present is
|
||||
just before it. Pin RX_VK_PARALLEL for e_ knobs: Automatic moves draws between threads (the replay warns).
|
||||
Not for players: nothing reads these without the variable. */
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
@@ -32,6 +41,17 @@ inline std::atomic<uint64_t> frame_no{0}; // (presents counted here)
|
||||
inline std::atomic<uint64_t> fired[16]; // (the frame each delay last waited in)
|
||||
inline std::atomic<uint32_t> pres_thread{0}; // (the thread that presents: the game's Presentation thread, in the plain mode)
|
||||
inline int64_t phase_frames = 300, skip_frames = 30;
|
||||
// ((Triforce, 11-22, REPLAY-51) the e_ knobs: each event's wait in this phase (ns, 0: none), and what was spun)
|
||||
enum Each { E_DRAW = 0, E_DISPATCH = 1, E_COUNT = 2 };
|
||||
inline const char* const each_names[E_COUNT] = {"e_draw", "e_dispatch"};
|
||||
inline std::atomic<int64_t> each_ns[E_COUNT];
|
||||
inline std::atomic<uint64_t> each_spins[E_COUNT], each_spun_ns[E_COUNT];
|
||||
struct EachPhase { // (the presenting thread's: the phase being counted)
|
||||
int knob = 0;
|
||||
std::chrono::steady_clock::time_point since{};
|
||||
};
|
||||
inline EachPhase each_phase;
|
||||
inline char each_line[256]; // (the line of a phase just over, for the replay to log; empty: none)
|
||||
|
||||
inline void init() {
|
||||
if (n_knobs >= 0) return;
|
||||
@@ -53,6 +73,53 @@ inline void init() {
|
||||
if (const char* s = getenv("RX_VK_LAB_AB_SKIP")) skip_frames = atoll(s);
|
||||
n_knobs = n;
|
||||
}
|
||||
/// ((Triforce, 11-22, REPLAY-51) Whether any knob is an e_ one (the replay warns once unless RX_VK_PARALLEL is pinned)
|
||||
inline bool any_each() {
|
||||
for (int i = 0; i < n_knobs; i++)
|
||||
for (const char* name : each_names)
|
||||
if (!strcmp(knobs[i].name, name)) return true;
|
||||
return false;
|
||||
}
|
||||
/// (phase_tick, the presenting thread) The phase is knob k (0: none): at a change, the line of the phase that ended (each_line,
|
||||
/// when it had an e_ knob: its spins and ms a second), and the events' waits for this one
|
||||
inline void each_tick(int k) {
|
||||
if (k == each_phase.knob && each_phase.since.time_since_epoch().count()) return;
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
each_line[0] = 0;
|
||||
const int was = each_phase.knob;
|
||||
if (was > 0 && each_phase.since.time_since_epoch().count()) {
|
||||
const double s = std::chrono::duration<double>(now - each_phase.since).count();
|
||||
for (int e = 0; e < E_COUNT; e++) {
|
||||
if (strcmp(knobs[was - 1].name, each_names[e])) continue;
|
||||
const uint64_t spins = each_spins[e].exchange(0, std::memory_order_relaxed);
|
||||
const uint64_t ns = each_spun_ns[e].exchange(0, std::memory_order_relaxed);
|
||||
snprintf(each_line, sizeof each_line, "vk: lab A/B: phase %s:%lld over: %.0f waits a second, %.2f ms a second spun "
|
||||
"(%llu waits in %.1f s)", each_names[e], (long long)knobs[was - 1].value, s > 0 ? spins / s : 0.0,
|
||||
s > 0 ? ns / 1e6 / s : 0.0, (unsigned long long)spins, s);
|
||||
}
|
||||
}
|
||||
for (int e = 0; e < E_COUNT; e++)
|
||||
each_ns[e].store(k > 0 && !strcmp(knobs[k - 1].name, each_names[e]) ? std::max<int64_t>(0, knobs[k - 1].value) : 0,
|
||||
std::memory_order_relaxed);
|
||||
each_phase.knob = k, each_phase.since = now;
|
||||
}
|
||||
/// At every occurrence of an event (first in draw() and dispatch()): one relaxed load without its knob; with it, a busy wait
|
||||
/// of the knob's nanoseconds, counted
|
||||
inline void each(int event) {
|
||||
const int64_t ns = each_ns[event].load(std::memory_order_relaxed);
|
||||
if (!ns) return;
|
||||
const auto t0 = std::chrono::steady_clock::now(), until = t0 + std::chrono::nanoseconds(ns);
|
||||
auto now = t0;
|
||||
while (now < until) {
|
||||
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
|
||||
__builtin_ia32_pause();
|
||||
#endif
|
||||
now = std::chrono::steady_clock::now();
|
||||
}
|
||||
each_spins[event].fetch_add(1, std::memory_order_relaxed);
|
||||
each_spun_ns[event].fetch_add((uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(now - t0).count(),
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
/// Once a present, on the presenting thread: the phase this frame belongs to (the trace's column)
|
||||
inline int phase_tick() {
|
||||
init();
|
||||
@@ -61,6 +128,7 @@ inline int phase_tick() {
|
||||
const uint64_t phase = f / (uint64_t)phase_frames; // (0 none, 1 knob 1, 2 none, 3 knob 2, ...)
|
||||
const int k = (phase & 1) ? (int)((phase / 2) % (uint64_t)n_knobs) + 1 : 0;
|
||||
active.store(k, std::memory_order_relaxed);
|
||||
each_tick(k); // ((Triforce, 11-22, REPLAY-51) the e_ knobs' waits for this phase, and the line of the one before)
|
||||
const bool settling = (int64_t)(f % (uint64_t)phase_frames) < skip_frames;
|
||||
return settling ? -(k + 1) : k; // (-1: none, settling; -(k + 1): knob k, settling)
|
||||
}
|
||||
|
||||
49
gpu/vk/late_inline.h
Normal file
49
gpu/vk/late_inline.h
Normal file
@@ -0,0 +1,49 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-80, after Source B's) Which of take_capture's no-wait rules for late pages run.
|
||||
//
|
||||
// A late page is one the CPU wrote while a submission still on the GPU writes it back (vk_memory.cpp capture_make). By default
|
||||
// every late page is waited for (RX_VK_LATE_QUEUE: at the queue's next submit) and taken again. RX_VK_LATE_NEW=1 (the lab
|
||||
// knob latenew) turns four rules on that take the wait away from some of them: pages of pools with no mirror skipped, pages
|
||||
// with the bytes they were last synced with left as they are, pages written back in part synced in their other parts (the
|
||||
// partial rule, late_partial), pages written back whole put off. Each brought brightness steps back at the Castle Town save.
|
||||
//
|
||||
// RX_VK_LATE_INLINE=1 (the lab knob lateinline), OFF unless set: the partial rule ALONE, Source B's analogue (the CPU's
|
||||
// bytes go in, the GPU's ranges stay the GPU's, nothing waits); the other three stay off, and the pages they would take
|
||||
// (no mirror, texture storage, whole write-backs, unknown ranges) are waited for as before.
|
||||
//
|
||||
// And the partial rule asks which ranges are still on the GPU against the GPU's progress capture_make read before its
|
||||
// copies, not a second read at the take: a write-back that landed between the copy and the take is then still counted in
|
||||
// flight, so its range is not synced from the older copy (the bug Source B fixed with the same change). This is so under
|
||||
// RX_VK_LATE_NEW too.
|
||||
//
|
||||
// The rule alone: no Vulkan, no game. tests/test_late_inline.py.
|
||||
#pragma once
|
||||
#include <cstdlib>
|
||||
|
||||
namespace late_inline {
|
||||
|
||||
/// RX_VK_LATE_INLINE: a number other than 0 on, 0 off; unset: the lab knob lateinline (on only in its phases)
|
||||
inline bool on(const char* env, bool knob) {
|
||||
if (env) return std::atoi(env) != 0;
|
||||
return knob;
|
||||
}
|
||||
|
||||
struct Rules {
|
||||
bool any = false; // (any rule: the late pages looked at one by one; else every one waited for, as by default)
|
||||
bool skip = false; // (a page of a pool with no mirror: not waited for)
|
||||
bool same = false; // (a page with the bytes it was last synced with: left as it is)
|
||||
bool part = false; // (a page written back in part: its other parts synced, no wait)
|
||||
bool defer = false; // (a page written back whole: put off)
|
||||
};
|
||||
/// late_new: RX_VK_LATE_NEW or its knob; inline_on: on() above; same_on, part_on, defer_on: RX_VK_LATE_SAME, RX_VK_LATE_PART,
|
||||
/// RX_VK_LATE_DEFER as read today (on, on, off unless set)
|
||||
inline Rules rules(bool late_new, bool inline_on, bool same_on, bool part_on, bool defer_on) {
|
||||
Rules r;
|
||||
r.any = late_new || inline_on;
|
||||
r.skip = late_new;
|
||||
r.same = late_new && same_on;
|
||||
r.part = r.any && part_on;
|
||||
r.defer = late_new && defer_on;
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace late_inline
|
||||
153
gpu/vk/linux_pressure.h
Normal file
153
gpu/vk/linux_pressure.h
Normal file
@@ -0,0 +1,153 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-141, after Source A's) A memory pressure hold for Linux and the Steam Deck that
|
||||
// counts the swap too (vk_memory.cpp steps it from memory_ram_watch, off Windows; this is the rule, plain code:
|
||||
// tests/test_linux_pressure.py runs it anywhere).
|
||||
//
|
||||
// The RAM rules looked at MemAvailable alone. A Linux PC with swap keeps MemAvailable up by pushing the game's pages out:
|
||||
// on a 16 GB Steam Deck Source A saw the swap fill (4,036 MB at the peak, 0 MB free at the lowest) while MemAvailable said
|
||||
// little, and the stalls were the pages coming back. So the swap left counts: the PC is short when MemAvailable is under
|
||||
// 1536 MB (the imports' own line), or when MemAvailable and SwapFree together are under the reserve (RX_VK_SWAP_RESERVE_MB).
|
||||
// While it is short a hold is on and the renderer gives memory back sooner:
|
||||
// - a capture buffer is kept as a spare only up to 32 MB of spares (not 768);
|
||||
// - the staging arenas the GPU is done with are freed when the hold begins, then after 5 s unused (not 30);
|
||||
// - malloc_trim is asked every 600 presents from 64 MB free (not every 1800 from 192);
|
||||
// - idle imports are given back, in all of one episode at most as many bytes as the PC was short at its worst (not up to
|
||||
// 1 GB a pass: the thrash of imports given back and made again was seen here before), and still only after 3,600 idle
|
||||
// submissions;
|
||||
// - on an integrated GPU, whose "VRAM" is the same RAM, the texture budget is less by what the PC is short (never under
|
||||
// 512 MB), and comes back 256 MB a sample.
|
||||
// The hold ends once MemAvailable and the sum are both 256 MB clear of their lines. A PC without swap, or one whose swap
|
||||
// cannot be read, is never short by the sum: not known is not short. The program modules are let go by the program pack
|
||||
// already (vk_shader.cpp), so the hold has no valve for them.
|
||||
//
|
||||
// RX_VK_SWAP_RESERVE_MB=<MB>: the reserve; 0 or not set: no hold at all, as before. Source A's is 2560; it is off here until
|
||||
// a Steam Deck run has shown it. RX_VK_TEST_SWAP_MB=<n> plays that much swap free, as RX_VK_TEST_AVAIL_MB plays MemAvailable.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace linux_pressure {
|
||||
|
||||
constexpr uint64_t kMB = 1ull << 20;
|
||||
constexpr uint64_t kAvailLine = 1536 * kMB; // (short under this much MemAvailable, whatever the swap)
|
||||
constexpr uint64_t kClear = 256 * kMB; // (the hold ends once both are this far clear of their lines)
|
||||
constexpr uint64_t kSourceReserve = 2560 * kMB; // (Source A's reserve: MemAvailable + SwapFree)
|
||||
constexpr uint64_t kSampleEvery = 30; // (presents between two looks)
|
||||
constexpr uint64_t kSpareCap = 32 * kMB; // (capture spares while held)
|
||||
constexpr int64_t kArenaIdleS = 5; // (a staging arena unused this long goes, while held)
|
||||
constexpr uint64_t kTrimEvery = 600; // (presents between two malloc_trim looks, while held)
|
||||
constexpr uint64_t kTrimMB = 64; // (free MB in the arenas from which malloc_trim is asked, while held)
|
||||
constexpr uint64_t kBudgetFloor = 512 * kMB; // (an integrated GPU's textures are never cut under this)
|
||||
constexpr uint64_t kBudgetBack = 256 * kMB; // (the cut comes back this much a sample)
|
||||
|
||||
/// RX_VK_SWAP_RESERVE_MB's text as bytes; 0 (not set, empty, 0, not a number): no hold
|
||||
inline uint64_t reserve_of(const char* e) {
|
||||
const long long mb = e && *e ? atoll(e) : 0;
|
||||
return mb > 0 ? (uint64_t)mb * kMB : 0;
|
||||
}
|
||||
|
||||
/// What /proc/meminfo said. A field that is not there is not known (has_*), never 0
|
||||
struct Meminfo {
|
||||
uint64_t total = 0, avail = 0, swap_total = 0, swap_free = 0;
|
||||
bool has_total = false, has_avail = false, has_swap_total = false, has_swap_free = false;
|
||||
};
|
||||
|
||||
/// /proc/meminfo's text ("Name: <n> kB" a line)
|
||||
inline Meminfo parse(const char* text) {
|
||||
Meminfo m;
|
||||
auto field = [text](const char* name, uint64_t* out) {
|
||||
const size_t len = strlen(name);
|
||||
for (const char* p = text; p && *p; p = strchr(p, '\n'), p = p ? p + 1 : p) {
|
||||
if (strncmp(p, name, len) || p[len] != ':') continue;
|
||||
const char* v = p + len + 1;
|
||||
while (*v == ' ' || *v == '\t') v++;
|
||||
if (*v < '0' || *v > '9') return false;
|
||||
*out = strtoull(v, nullptr, 10) << 10; // (kB)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
m.has_total = field("MemTotal", &m.total);
|
||||
m.has_avail = field("MemAvailable", &m.avail);
|
||||
m.has_swap_total = field("SwapTotal", &m.swap_total);
|
||||
m.has_swap_free = field("SwapFree", &m.swap_free);
|
||||
return m;
|
||||
}
|
||||
|
||||
/// One look at the PC's memory. swap_known: there is swap and what is free of it was read
|
||||
struct Sample {
|
||||
bool avail_known = false, swap_known = false;
|
||||
uint64_t avail = 0, swap_free = 0;
|
||||
};
|
||||
|
||||
/// The sample of a meminfo, with the lab's played values over it (negative: not played). Swapless (SwapTotal 0), or SwapFree
|
||||
/// or SwapTotal unreadable: the swap is not known. A played swap counts as known
|
||||
inline Sample sample_of(const Meminfo& m, long long test_avail_mb = -1, long long test_swap_mb = -1) {
|
||||
Sample s;
|
||||
s.avail_known = m.has_avail, s.avail = m.avail;
|
||||
s.swap_known = m.has_swap_total && m.has_swap_free && m.swap_total > 0, s.swap_free = s.swap_known ? m.swap_free : 0;
|
||||
if (test_avail_mb >= 0) s.avail_known = true, s.avail = (uint64_t)test_avail_mb * kMB;
|
||||
if (test_swap_mb >= 0) s.swap_known = true, s.swap_free = (uint64_t)test_swap_mb * kMB;
|
||||
return s;
|
||||
}
|
||||
|
||||
/// How many bytes the PC is short: the larger of what MemAvailable lacks to its line and what the sum lacks to the reserve.
|
||||
/// 0: not short, or not known
|
||||
inline uint64_t shortfall(const Sample& s, uint64_t reserve) {
|
||||
if (!reserve || !s.avail_known) return 0;
|
||||
const uint64_t by_avail = s.avail < kAvailLine ? kAvailLine - s.avail : 0;
|
||||
const uint64_t sum = s.avail + s.swap_free;
|
||||
const uint64_t by_sum = s.swap_known && sum < reserve ? reserve - sum : 0;
|
||||
return std::max(by_avail, by_sum);
|
||||
}
|
||||
|
||||
/// Both lines cleared by kClear (the hold may end). Not known: clear
|
||||
inline bool clear(const Sample& s, uint64_t reserve) {
|
||||
if (!reserve || !s.avail_known) return true;
|
||||
if (s.avail < kAvailLine + kClear) return false;
|
||||
return !s.swap_known || s.avail + s.swap_free >= reserve + kClear;
|
||||
}
|
||||
|
||||
/// The hold: on from a short sample until a clear one. One episode is one stretch of it
|
||||
struct Hold {
|
||||
bool held = false;
|
||||
uint64_t deepest = 0; // (the episode's largest shortfall)
|
||||
uint64_t given = 0; // (the bytes of imports given back in this episode)
|
||||
uint64_t now = 0; // (the last sample's shortfall)
|
||||
uint64_t cut = 0; // (an integrated GPU's texture budget is less by this)
|
||||
uint64_t episodes = 0;
|
||||
|
||||
enum Step { Same, Entered, Left };
|
||||
/// One sample. reserve 0: never held
|
||||
Step step(const Sample& s, uint64_t reserve) {
|
||||
const uint64_t lacking = shortfall(s, reserve);
|
||||
Step r = Same;
|
||||
if (!held && lacking) held = true, deepest = 0, given = 0, episodes++, r = Entered;
|
||||
else if (held && clear(s, reserve)) held = false, r = Left;
|
||||
now = held ? lacking : 0;
|
||||
if (held) deepest = std::max(deepest, lacking);
|
||||
// (the budget's cut: down at once to what is lacking, back up kBudgetBack a sample)
|
||||
cut = now > cut ? now : std::max(now, cut > kBudgetBack ? cut - kBudgetBack : 0);
|
||||
return r;
|
||||
}
|
||||
/// The bytes of idle imports this episode may still give back: its largest shortfall, less what it gave
|
||||
uint64_t episode_want() const { return held && deepest > given ? deepest - given : 0; }
|
||||
void gave(uint64_t bytes) { given += bytes; }
|
||||
};
|
||||
|
||||
/// The valves' sizes, held or not
|
||||
inline uint64_t spare_cap(bool held, uint64_t usual) { return held ? std::min(usual, kSpareCap) : usual; }
|
||||
/// (idle_s 0: the player keeps the arenas: kept while held too)
|
||||
inline int64_t arena_idle_s(bool held, int64_t usual) { return held && usual > kArenaIdleS ? kArenaIdleS : usual; }
|
||||
inline uint64_t trim_every(bool held, uint64_t usual) { return held ? std::min(usual, kTrimEvery) : usual; }
|
||||
/// (trim_mb 0: never, held or not)
|
||||
inline uint64_t trim_mb(bool held, uint64_t usual) { return held && usual ? std::min(usual, kTrimMB) : usual; }
|
||||
/// An integrated GPU's texture budget less the cut: never under the floor, never more than it was
|
||||
inline uint64_t cut_budget(uint64_t budget, uint64_t cut) {
|
||||
if (!cut || budget <= kBudgetFloor) return budget;
|
||||
return std::max(kBudgetFloor, budget > cut ? budget - cut : 0);
|
||||
}
|
||||
|
||||
} // namespace linux_pressure
|
||||
213
gpu/vk/lock_census.h
Normal file
213
gpu/vk/lock_census.h
Normal file
@@ -0,0 +1,213 @@
|
||||
// (Triforce, 11-22, the performance audit's THREAD-68, after Source B's) The lock census: who holds the lock a thread waits for.
|
||||
//
|
||||
// The log has totals of the waits for the renderer's locks (the memory sync's lock wait, the virtual cores' line, the order lock's
|
||||
// slow takes), none of which names the holder. With RX_VK_LOCK_CENSUS=1 a lock that is busy when a thread comes for it notes the
|
||||
// waiter, the thread that held it, and how long the wait was; every 10 s a thread of the census's own logs the 12 pairs that
|
||||
// waited longest:
|
||||
// vk: lock census: Presentation Thread <- MainThread (replay_mu) 17.2 ms/s (2900/s, 6 us each)
|
||||
// Counted: replay_mu (DrainingMutex), sync_order_mu (OrderMutex), held_mu, and in vk_memory.cpp pools_mu, sync_mu and cap_mu.
|
||||
// Only the counting is ported: Source B's also samples the holders' stacks, which disturbs a millisecond or two a second.
|
||||
//
|
||||
// Off (unset, or 0): every lock is taken exactly as before, behind one branch on a value read once. On: a free lock is one
|
||||
// try_lock and a store of its holder; only a busy lock reads the clock, twice. The holder is read without the lock, so a holder
|
||||
// that lets go at that moment can be misnamed, never more. The census's own mutex is taken last, with the counted lock held, and
|
||||
// nothing that holds it ever takes a counted lock. With RX_VK_LAB_AB naming a knob lockcensus it counts only in that knob's phases.
|
||||
//
|
||||
// No windows.h here (vk_recorder.h, which includes this, has none): a thread's name is read through g_thread_name, which
|
||||
// vk_replay.cpp sets. tests/test_lock_census.py cuts DrainingMutex and OrderMutex out and compiles them with this.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
namespace mkwvk {
|
||||
void logf(const char* fmt, ...); // (vk_main.cpp)
|
||||
|
||||
namespace lock_census {
|
||||
|
||||
/// RX_VK_LOCK_CENSUS, read once: 1 (any number but 0) counts
|
||||
inline bool enabled() {
|
||||
static const bool v = [] {
|
||||
const char* e = getenv("RX_VK_LOCK_CENSUS");
|
||||
return e && atoi(e) != 0;
|
||||
}();
|
||||
return v;
|
||||
}
|
||||
inline std::atomic<bool> phase_on{true}; // (lab: false outside the knob's phases)
|
||||
inline bool counting() { return enabled() && phase_on.load(std::memory_order_relaxed); }
|
||||
|
||||
/// The clock a busy lock reads (a test counts the reads through it)
|
||||
inline int64_t steady_ns() {
|
||||
return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
inline int64_t (*now_ns)() = steady_ns;
|
||||
inline std::atomic<int> period_ms{10000}; // (between two reports)
|
||||
constexpr int kTop = 12;
|
||||
|
||||
struct Pair {
|
||||
int64_t ns = 0, n = 0;
|
||||
};
|
||||
/// The census's own state. Names are kept as pointers to strings that are never freed, one per distinct name
|
||||
struct Book {
|
||||
std::mutex mu;
|
||||
std::map<std::string, const char*> names;
|
||||
std::map<std::tuple<const char*, const char*, const char*>, Pair> pairs; // (waiter, holder, lock)
|
||||
bool reporter = false;
|
||||
int64_t since_ns = 0;
|
||||
};
|
||||
inline Book& book() {
|
||||
static Book* b = new Book; // (never destroyed: a thread may count while the process ends)
|
||||
return *b;
|
||||
}
|
||||
/// the one pointer for a name (book().mu held)
|
||||
inline const char* intern(Book& b, const char* name) {
|
||||
auto it = b.names.find(name);
|
||||
if (it == b.names.end()) {
|
||||
char* kept = new char[strlen(name) + 1];
|
||||
strcpy(kept, name);
|
||||
it = b.names.emplace(name, kept).first;
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
/// A thread as the census names it: its description (the game's threads and the renderer's own are named), else its number. The
|
||||
/// slot is never freed, so a lock's note of its holder can be read after that thread has ended
|
||||
struct Slot {
|
||||
std::atomic<const char*> name{"?"};
|
||||
int64_t named_ns = 0; // (its own thread's: when the name was last read)
|
||||
};
|
||||
/// (set by vk_replay.cpp, which has windows.h: the calling thread's description into out, left empty when it has none; and what
|
||||
/// the reporting thread does first, which names it)
|
||||
inline void (*g_thread_name)(char* out, size_t cap) = nullptr;
|
||||
inline void (*g_report_start)() = nullptr;
|
||||
inline void name_slot(Slot* s) {
|
||||
char text[96] = "";
|
||||
if (g_thread_name) g_thread_name(text, sizeof text);
|
||||
if (!text[0]) snprintf(text, sizeof text, "thread %zu", std::hash<std::thread::id>()(std::this_thread::get_id()) % 100000);
|
||||
Book& b = book();
|
||||
std::lock_guard lk(b.mu);
|
||||
s->name.store(intern(b, text), std::memory_order_relaxed);
|
||||
}
|
||||
/// the calling thread's slot; named at its first use, and again by `again_at` (a waiter's clock read: a thread named after its
|
||||
/// first lock is not "thread 1234" for the whole session) when a second has passed
|
||||
inline const Slot* me(int64_t again_at = 0) {
|
||||
thread_local Slot* mine = nullptr;
|
||||
if (!mine) {
|
||||
mine = new Slot;
|
||||
mine->named_ns = again_at; // (0 on a free lock, which reads no clock: named again at this thread's first wait)
|
||||
name_slot(mine);
|
||||
} else if (again_at && again_at - mine->named_ns > 1000000000) {
|
||||
mine->named_ns = again_at;
|
||||
name_slot(mine);
|
||||
}
|
||||
return mine;
|
||||
}
|
||||
|
||||
/// The pairs counted since the last call, the longest waited first, kTop at most, as the log's lines; all counts start again
|
||||
inline std::vector<std::string> report(int64_t now) {
|
||||
Book& b = book();
|
||||
std::vector<std::pair<std::tuple<const char*, const char*, const char*>, Pair>> rows;
|
||||
double seconds = 0;
|
||||
{
|
||||
std::lock_guard lk(b.mu);
|
||||
rows.assign(b.pairs.begin(), b.pairs.end());
|
||||
b.pairs.clear();
|
||||
seconds = (double)(now - b.since_ns) / 1e9;
|
||||
b.since_ns = now;
|
||||
}
|
||||
if (seconds <= 0) seconds = 1e-9;
|
||||
std::sort(rows.begin(), rows.end(), [](const auto& x, const auto& y) { return x.second.ns > y.second.ns; });
|
||||
std::vector<std::string> out;
|
||||
for (size_t i = 0; i < rows.size() && i < (size_t)kTop; i++) {
|
||||
const auto& [key, p] = rows[i];
|
||||
char line[320];
|
||||
snprintf(line, sizeof line, "vk: lock census: %s <- %s (%s) %.1f ms/s (%.0f/s, %.0f us each)", std::get<0>(key), std::get<1>(key),
|
||||
std::get<2>(key), (double)p.ns / 1e6 / seconds, (double)p.n / seconds, p.n ? (double)p.ns / 1e3 / (double)p.n : 0.0);
|
||||
out.emplace_back(line);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
inline void report_main() {
|
||||
if (g_report_start) g_report_start();
|
||||
for (;;) {
|
||||
// (slept in steps, the period read at each: a test shortens it while this thread sleeps)
|
||||
for (int slept = 0; slept < period_ms.load(std::memory_order_relaxed); slept += 50)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
||||
for (const std::string& line : report(now_ns())) logf("%s", line.c_str());
|
||||
}
|
||||
}
|
||||
/// A wait is over: self waited ns for the lock `lock`, which `holder` held when it came (null: not known). The first one starts
|
||||
/// the reporting thread
|
||||
inline void waited(const Slot* self, const Slot* holder, const char* lock, int64_t ns, int64_t now) {
|
||||
Book& b = book();
|
||||
bool start = false;
|
||||
{
|
||||
std::lock_guard lk(b.mu);
|
||||
if (!b.since_ns) b.since_ns = now - ns;
|
||||
Pair& p = b.pairs[{self->name.load(std::memory_order_relaxed), holder ? holder->name.load(std::memory_order_relaxed) : intern(b, "?"),
|
||||
intern(b, lock)}];
|
||||
p.ns += ns, p.n++;
|
||||
if (!b.reporter) b.reporter = start = true;
|
||||
}
|
||||
if (start) std::thread(report_main).detach();
|
||||
}
|
||||
|
||||
/// m.lock() for a counted lock. owner: where that lock keeps its holder (set here, cleared by unlocking())
|
||||
template <class M>
|
||||
inline void lock(M& m, std::atomic<const Slot*>& owner, const char* name) {
|
||||
if (!counting()) { // (off: the lock as it was)
|
||||
m.lock();
|
||||
return;
|
||||
}
|
||||
if (m.try_lock()) { // (free: no clock read)
|
||||
owner.store(me(), std::memory_order_relaxed);
|
||||
return;
|
||||
}
|
||||
const Slot* holder = owner.load(std::memory_order_relaxed);
|
||||
const int64_t t0 = now_ns();
|
||||
m.lock();
|
||||
const int64_t t1 = now_ns();
|
||||
const Slot* self = me(t1);
|
||||
owner.store(self, std::memory_order_relaxed);
|
||||
waited(self, holder, name, t1 - t0, t1);
|
||||
}
|
||||
/// m.try_lock() for a counted lock
|
||||
template <class M>
|
||||
inline bool try_lock(M& m, std::atomic<const Slot*>& owner) {
|
||||
if (!m.try_lock()) return false;
|
||||
if (counting()) owner.store(me(), std::memory_order_relaxed);
|
||||
return true;
|
||||
}
|
||||
/// just before a counted lock's unlock: it has no holder
|
||||
inline void unlocking(std::atomic<const Slot*>& owner) {
|
||||
if (owner.load(std::memory_order_relaxed)) owner.store(nullptr, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
/// A std::mutex the census counts, for std::lock_guard and std::unique_lock
|
||||
struct CensusMutex {
|
||||
const char* name;
|
||||
std::mutex m;
|
||||
std::atomic<const Slot*> owner{nullptr};
|
||||
explicit CensusMutex(const char* n) : name(n) {}
|
||||
void lock() { lock_census::lock(m, owner, name); }
|
||||
bool try_lock() { return lock_census::try_lock(m, owner); }
|
||||
void unlock() {
|
||||
unlocking(owner);
|
||||
m.unlock();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace lock_census
|
||||
} // namespace mkwvk
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -210,5 +211,30 @@ inline uint64_t own_for_keep(uint64_t ws, uint64_t cache_read, bool amd) {
|
||||
if (amd) return ws;
|
||||
return ws > cache_read ? ws - cache_read : 0;
|
||||
}
|
||||
/// (11-22, after Casual Gamer's fix) Whether the game is told its mapped pages are used (host/hos_mem.c query_used), from the
|
||||
/// host's switches as it reads them: HOS_MEM_QUERY_USED not "0", and the census on (hos_mem_census_on: HOS_MEM_CENSUS not 0 or
|
||||
/// less, or HOS_MEM_GIVE_BACK above 0)
|
||||
inline bool query_used_on(const char* query_used, const char* census, const char* give_back) {
|
||||
const bool census_on = !(census && atoi(census) <= 0) || (give_back && atoi(give_back) > 0);
|
||||
return !(query_used && query_used[0] == '0' && !query_used[1]) && census_on;
|
||||
}
|
||||
/// (11-22) Whether the precompile keeps every pipeline for a PC of under 20 GB (vk_draw.cpp precompile_keeps_all): on a discrete
|
||||
/// GPU with the game told its mapped pages are used (query_on), which lets old areas go; total 0: not known
|
||||
inline bool keep_all_small_pc(bool discrete, uint64_t total, bool query_on) {
|
||||
return discrete && query_on && total && total < (20ull << 30);
|
||||
}
|
||||
|
||||
/// (11-22, after Casual Gamer's fix) Whether the retired imports kept for good (vk_memory.cpp retired_imports_give_back: Windows,
|
||||
/// AMD's and Intel's drivers) are given back all the same: the system heap past high percent of its budget (used of budget;
|
||||
/// a budget of 0: not known), or an import refused for want of room within the last kRefusedHoldMs (refused_ms: when, as now_ms
|
||||
/// counts; 0: never)
|
||||
constexpr int64_t kRefusedHoldMs = 60000;
|
||||
inline bool retired_pressed(uint64_t used, uint64_t budget, int high, int64_t now_ms, int64_t refused_ms) {
|
||||
return (budget && high > 0 && used > budget / 100 * (uint64_t)high) || (refused_ms && now_ms - refused_ms < kRefusedHoldMs);
|
||||
}
|
||||
/// (11-22, after Casual Gamer's fix) Whether the capture's spare buffers are held to 32 MB (vk_memory.cpp cap_spare_max): on a
|
||||
/// card the driver allows 6.5 GB or less (small_card), or on a PC of under 20 GB of RAM (ram_total; 0: not known)
|
||||
constexpr uint64_t kSpareRamBytes = 20ull << 30;
|
||||
inline bool spare_capped(bool small_card, uint64_t ram_total) { return small_card || (ram_total && ram_total < kSpareRamBytes); }
|
||||
|
||||
} // namespace mem_parts
|
||||
|
||||
73
gpu/vk/memory_priority.h
Normal file
73
gpu/vk/memory_priority.h
Normal file
@@ -0,0 +1,73 @@
|
||||
// VK_EXT_memory_priority: which device memory the system should keep in video memory when the card is short (no game, no
|
||||
// GPU: the switch's reading and the tiers). tests/test_memory_priority.py
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's TEX-47, after Source A's.) When a card is short of video memory, Windows moves
|
||||
// some of a program's device memory to system memory, which the GPU then reads over the bus, and it picks what goes. With
|
||||
// this extension a program ranks its memory from 0 to 1: the lower goes first. Source A gives the render targets 1.0,
|
||||
// the sampled textures 0.25 and the rest 0.5, and measured it on an RTX 4090 with the video memory taken by other
|
||||
// programs and the textures held to an 8 GB card's blocks: with 1.3 GB moved out, the GPU's median frame was 109 to
|
||||
// 179 ms unset and 24 to 29 ms on. The textures' low rank is the lever: with them at 0.5 (`1,0.5`) it ended at 68 ms.
|
||||
//
|
||||
// Off unless RX_VK_MEMORY_PRIORITY is set: no small card has run it, drivers may ignore it, and in Source A's runs the
|
||||
// wrong tiers were worse than none. Unset, the extension is not asked for and nothing here is used.
|
||||
// RX_VK_MEMORY_PRIORITY unset, empty, 0, off, no: off
|
||||
// 1: render targets 1.0, textures 0.25, the rest 0.5
|
||||
// <targets>,<textures>: those two, each from 0 to 1; the rest 0.5
|
||||
// anything else (one number that is not 0 or 1, three numbers, a number outside 0 to 1): off
|
||||
// With the extension on, the allocator (VMA) gives every pool and every allocation of its own the priority its create
|
||||
// info names, and one that names none gets 0, below the textures: so every one in vk_vma.cpp names its tier.
|
||||
#pragma once
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace memory_priority {
|
||||
|
||||
inline constexpr float kDefault = 0.5f; // (what a driver gives memory that names no priority)
|
||||
|
||||
struct Tiers {
|
||||
bool on = false;
|
||||
float targets = kDefault, textures = kDefault, other = kDefault;
|
||||
};
|
||||
|
||||
/// RX_VK_MEMORY_PRIORITY as written (see the top of the file); what cannot be read is off
|
||||
inline Tiers parse(const char* env) {
|
||||
Tiers t;
|
||||
if (!env) return t;
|
||||
while (*env == ' ' || *env == '\t') env++;
|
||||
char word[32] = {};
|
||||
size_t n = std::strlen(env);
|
||||
while (n && (env[n - 1] == ' ' || env[n - 1] == '\t')) n--;
|
||||
if (!n || n >= sizeof word) return t;
|
||||
for (size_t i = 0; i < n; i++) word[i] = (char)std::tolower((unsigned char)env[i]);
|
||||
if (!std::strcmp(word, "0") || !std::strcmp(word, "off") || !std::strcmp(word, "no")) return t;
|
||||
if (!std::strcmp(word, "1")) {
|
||||
t.on = true, t.targets = 1.0f, t.textures = 0.25f;
|
||||
return t;
|
||||
}
|
||||
// (two numbers and a comma between them, nothing else)
|
||||
char* end = nullptr;
|
||||
const double a = std::strtod(word, &end);
|
||||
if (end == word || *end != ',') return t;
|
||||
const char* second = end + 1;
|
||||
const double b = std::strtod(second, &end);
|
||||
if (end == second || *end) return t;
|
||||
if (!std::isfinite(a) || !std::isfinite(b) || a < 0 || a > 1 || b < 0 || b > 1) return t;
|
||||
t.on = true, t.targets = (float)a, t.textures = (float)b;
|
||||
return t;
|
||||
}
|
||||
|
||||
/// The tiers a run uses: the switch's where the device has the extension and its feature, else none (all the default)
|
||||
inline Tiers active(const char* env, bool device_has_it) { return device_has_it ? parse(env) : Tiers{}; }
|
||||
|
||||
/// What a piece of device memory is for. Held: the lab's ballast (RX_VK_TEST_CARD_MB), which must stay on the card
|
||||
enum class Kind { Target, Texture, Other, Held };
|
||||
|
||||
/// Its priority under the tiers; the default for everything while they are off
|
||||
inline float of(const Tiers& t, Kind k) {
|
||||
if (!t.on) return kDefault;
|
||||
return k == Kind::Target ? t.targets : k == Kind::Texture ? t.textures : k == Kind::Held ? 1.0f : t.other;
|
||||
}
|
||||
|
||||
} // namespace memory_priority
|
||||
38
gpu/vk/mesa_spilling.h
Normal file
38
gpu/vk/mesa_spilling.h
Normal file
@@ -0,0 +1,38 @@
|
||||
// Mesa's Intel shader compiler told to spill registers at rate 2, before the driver is loaded (no game, no GPU: a rule).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-55, after Source A's.) Mesa's drivers read their options (driconf) from
|
||||
// the environment by name. shader_spilling_rate is read by the Intel compiler alone: after a register allocation that
|
||||
// failed, it spills that many times more registers at once, so a program that does not fit is built in fewer, larger
|
||||
// steps. A program that does not spill builds the same. Source A's numbers on an Iris Xe (i9-13900H, 32 GB, Mesa 26.2.4):
|
||||
// one compute program not built after 20 minutes and 13 GB at the driver's own rate, 32 s and 1.7 GB at rate 2; another
|
||||
// 531 s and 7.6 GB against 87 s and 4 GB; rate 3 took 165 s there and rate 4 did not finish in 8 minutes. The driver's
|
||||
// own estimate for a spilling program's speed at rate 2 is from 12% better to 53% worse.
|
||||
//
|
||||
// The Linux build sets it in create_instance (vk_main.cpp) without overwriting, so a value the player set wins.
|
||||
// RX_VK_NO_SPILLING_RATE=1 leaves the driver as it is. The Windows build sets nothing: under Proton the driver reads the
|
||||
// Linux environment, which the Windows build cannot write, and those players put shader_spilling_rate=2 in the launch
|
||||
// options.
|
||||
// Nothing here is read by NVIDIA's or AMD's drivers, RADV included.
|
||||
#pragma once
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mesa_spilling {
|
||||
constexpr const char* kName = "shader_spilling_rate";
|
||||
constexpr const char* kRate = "2";
|
||||
|
||||
/// The build sets the rate: off Windows, unless RX_VK_NO_SPILLING_RATE is set to something other than 0. off: its value
|
||||
inline bool wanted(bool windows_build, const char* off) {
|
||||
return !windows_build && !(off && std::atoi(off) != 0);
|
||||
}
|
||||
|
||||
/// What the driver will read once the build has done its part: the player's own value where there is one (set without
|
||||
/// overwriting), else the rate where it is wanted, else nullptr (the driver's own rate)
|
||||
inline const char* effective(const char* players, bool is_wanted) {
|
||||
return players ? players : is_wanted ? kRate : nullptr;
|
||||
}
|
||||
|
||||
/// The log names the rate on Mesa's Intel driver alone
|
||||
constexpr bool told(bool mesa_intel) {
|
||||
return mesa_intel;
|
||||
}
|
||||
} // namespace mesa_spilling
|
||||
27
gpu/vk/mirror_budget.h
Normal file
27
gpu/vk/mirror_budget.h
Normal file
@@ -0,0 +1,27 @@
|
||||
// (Triforce) The VRAM mirrors' budget by the card (vk_memory.cpp mirror_budget; RX_VK_MIRROR_MB=<MB> goes over it). Plain code with
|
||||
// no Vulkan calls: tests/test_mirror_budget.py runs it on made-up cards.
|
||||
//
|
||||
// - A discrete card over 4.5 GB: a quarter of its video memory, at most 3 GB. On a 6 GB card (RTX 3060 Laptop, Hyrule Field,
|
||||
// whole mirrors) 512 MB of mirrors gave 17.6 fps, 1024 MB 30.7 and a quarter (1498 MB) 39.2: the pools left in system memory
|
||||
// were read over PCIe each frame. The eighth those cards had (768 MB) sat between two sizes measured as losses. Textures on
|
||||
// cards of 8.5 GB or less are 45% of the allowance and give memory back at 90% and 97% of it (vk_texture.cpp), the room a
|
||||
// quarter needs; the eighth was set for a 6 GB laptop before that rule, when a quarter and the textures went past the card.
|
||||
// - 4.5 GB or less, and an integrated GPU up to 6.5 GB: an eighth (no smaller size was measured on a real small card, and an
|
||||
// integrated GPU's "video memory" is the system's).
|
||||
// - Video memory not known: 3 GB.
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
namespace mkwvk::mirror_size {
|
||||
|
||||
constexpr uint64_t kMB = 1ull << 20;
|
||||
|
||||
constexpr uint64_t bytes(uint64_t vram, bool discrete) {
|
||||
if (!vram) return 3072 * kMB;
|
||||
if (vram <= 4608 * kMB || (!discrete && vram <= 6656 * kMB)) return vram / 8;
|
||||
return std::min<uint64_t>(vram / 4, 3072 * kMB);
|
||||
}
|
||||
|
||||
} // namespace mkwvk::mirror_size
|
||||
42
gpu/vk/mirror_spare.h
Normal file
42
gpu/vk/mirror_spare.h
Normal file
@@ -0,0 +1,42 @@
|
||||
// (Triforce, 11-21, the performance audit's MEM-27, after Source A's: its first step) What the mirrors' sparse buffers cost.
|
||||
//
|
||||
// A mirror in pieces is a sparse buffer: address space, not memory, yet the driver charges by its size. Source A measured
|
||||
// about 0.06 ms a MB to create one and 0.07 to destroy it (NVIDIA 617.42), 52 to 64 ms batches of destructions at an area
|
||||
// change, and the game frees its pools there and makes them again at the same sizes. Its answer is to keep a freed
|
||||
// mirror's buffer, unbound, for the next mirror of exactly that size. Before that is built here, this counts: the creates
|
||||
// and destroys, their time, the longest, and how many creates a buffer freed earlier, of exactly their size, could have
|
||||
// served. The line "vk: mirrors' sparse buffers" every 1200 submissions has them (vk_memory.cpp).
|
||||
//
|
||||
// The count alone, no Vulkan: tests/test_mirror_spare.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
|
||||
namespace mirror_spare {
|
||||
|
||||
struct Counts {
|
||||
uint64_t creates = 0, destroys = 0, servable = 0; // (servable: creates a freed buffer of their size was there for)
|
||||
double create_ms = 0, destroy_ms = 0, longest_create_ms = 0, longest_destroy_ms = 0;
|
||||
uint64_t create_bytes = 0, destroy_bytes = 0, over = 0; // (over: calls of more than kLongMs)
|
||||
std::map<uint64_t, uint32_t> freed; // (sizes freed and not yet matched by a create)
|
||||
};
|
||||
inline constexpr double kLongMs = 4.0;
|
||||
|
||||
/// A sparse buffer of size bytes was destroyed in ms
|
||||
inline void destroyed(Counts& c, uint64_t size, double ms) {
|
||||
c.destroys++, c.destroy_ms += ms, c.destroy_bytes += size, c.over += ms > kLongMs;
|
||||
if (ms > c.longest_destroy_ms) c.longest_destroy_ms = ms;
|
||||
c.freed[size]++;
|
||||
}
|
||||
/// A sparse buffer of size bytes was created in ms; true when one freed before, of exactly that size, could have served
|
||||
inline bool created(Counts& c, uint64_t size, double ms) {
|
||||
c.creates++, c.create_ms += ms, c.create_bytes += size, c.over += ms > kLongMs;
|
||||
if (ms > c.longest_create_ms) c.longest_create_ms = ms;
|
||||
auto it = c.freed.find(size);
|
||||
if (it == c.freed.end()) return false;
|
||||
if (!--it->second) c.freed.erase(it);
|
||||
c.servable++;
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace mirror_spare
|
||||
@@ -16,6 +16,7 @@
|
||||
// No ImGui and no Vulkan: tests/test_mods_entry_anim.py compiles it alone.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
|
||||
namespace mods_entry {
|
||||
@@ -69,6 +70,16 @@ inline int64_t in_at(float amount) {
|
||||
return (int64_t)((double)hi * (double)kFrameNs);
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-20) the game's own alpha of its buttons as the amount the entry is drawn with. The game blends its
|
||||
/// buttons in linear light; the overlay blends in the window's code values unless the window is HDR. A white word at alpha
|
||||
/// a over a dark picture is a^(1/2.2) of the way to white in code values when blended in linear light: drawn at a itself
|
||||
/// in code values, the entry was dimmer than the game's buttons all through a fade (a quarter of the way where they were
|
||||
/// half way), though it began and ended with them. linear: the overlay blends in linear light too (HDR): a itself)
|
||||
inline float list_amount(float game_alpha, bool linear) {
|
||||
const float a = std::clamp(game_alpha, 0.0f, 1.0f);
|
||||
return linear ? a : std::pow(a, 1.0f / 2.2f);
|
||||
}
|
||||
|
||||
/// The entry's state from one drawing to the next: how chosen it looks (focus) and how much of it shows with the list
|
||||
struct Anim {
|
||||
bool sel = false; // (it had the cursor at the last step)
|
||||
@@ -78,8 +89,12 @@ struct Anim {
|
||||
int64_t list_ns = 0; // (when the list came)
|
||||
float gone_from = 0; // (shown when the list went)
|
||||
float focus = 0, shown = 0;
|
||||
/// One drawing at now: whether the entry has the cursor, how long since the host last told of the list
|
||||
void step(int64_t now, bool selected, int64_t list_age_ns) {
|
||||
/// One drawing at now: whether the entry has the cursor, how long since the host last told of the list.
|
||||
/// ((Triforce, 11-20) list_alpha: how much of its own buttons the game shows now (0 to 1, the host's word each frame:
|
||||
/// HOS_OPT_MODS_ALPHA); the entry shows as much while the list is there, so it comes and goes with them, frame for
|
||||
/// frame, whatever animation the game plays (the title's opening, a second long, after a second and a half of
|
||||
/// nothing; the 15 frames as the list comes back). Under 0: not told, the list's In by this clock, as before)
|
||||
void step(int64_t now, bool selected, int64_t list_age_ns, float list_alpha = -1.0f) {
|
||||
if (!since_ns || selected != sel) from = since_ns ? focus : (selected ? 1.0f : 0.0f), sel = selected, since_ns = now;
|
||||
const float f = frames(now - since_ns);
|
||||
focus = sel ? std::max(from, curve(kSelect, f)) : from * curve(kUnSelect, f);
|
||||
@@ -87,7 +102,8 @@ struct Anim {
|
||||
if (seen && !there) list_ns = now - in_at(shown); // ((Triforce, 11-19) from where it is: no blink to nothing)
|
||||
if (!seen && there) gone_from = shown;
|
||||
there = seen;
|
||||
shown = seen ? curve(kListIn, frames(now - list_ns)) : gone_from * curve(kUnSelect, frames(list_age_ns - kListGoneNs));
|
||||
shown = !seen ? gone_from * curve(kUnSelect, frames(list_age_ns - kListGoneNs))
|
||||
: list_alpha >= 0 ? list_alpha : curve(kListIn, frames(now - list_ns));
|
||||
shown = std::clamp(shown, 0.0f, 1.0f), focus = std::clamp(focus, 0.0f, 1.0f);
|
||||
}
|
||||
/// LoopRandom's value now (it runs from the cursor's coming)
|
||||
|
||||
48
gpu/vk/mods_page_anim.h
Normal file
48
gpu/vk/mods_page_anim.h
Normal file
@@ -0,0 +1,48 @@
|
||||
// (Triforce, 11-20) The Mods screen as it opens. The owner: "Mod menu fade in". The entry that opens it is animated as the
|
||||
// game's own entries are (mods_entry_anim.h on the title, focus_anim.h under the pause menu's System list), and Quit's
|
||||
// dialog opens as the game's SystemWindow_00; the screen itself came at once, whole, on the frame after the press. It
|
||||
// fades in now as the game's lists and windows do: the alpha of Title_00's list and of SystemWindow_00's N_In_00 (the
|
||||
// same keys in both layouts: 0 to 1 over 15 frames, slow to start, slopes 0.02 and 0.14), with no rise, as a whole screen
|
||||
// has none. It takes input once that has played, as the game's own screens and Quit's dialog do. Back is a cut, as the
|
||||
// game's windows go in 2 frames.
|
||||
// No ImGui and no Vulkan: tests/test_mods_page_anim.py compiles it alone.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include "mods_entry_anim.h" // (the layouts' curves: Key, curve, frames)
|
||||
|
||||
namespace mods_page {
|
||||
|
||||
using mods_entry::curve;
|
||||
using mods_entry::frames;
|
||||
using mods_entry::Key;
|
||||
using mods_entry::kFrameNs;
|
||||
|
||||
constexpr Key kInAlpha[] = {{0, 0, 0.02f}, {15, 1, 0.14f}};
|
||||
constexpr int64_t kInNs = 15 * kFrameNs;
|
||||
|
||||
/// How much of the screen shows t ns after it opened (0 to 1)
|
||||
inline float in_alpha(int64_t t) {
|
||||
if (t <= 0) return 0;
|
||||
return std::clamp(curve(kInAlpha, frames(t)), 0.0f, 1.0f);
|
||||
}
|
||||
|
||||
/// The screen from one tick to the next: whether it is shown, and since when
|
||||
struct Page {
|
||||
bool shown = false; // (it was the page shown at the last step)
|
||||
int64_t since_ns = 0; // (when it came; 0 while it is not shown)
|
||||
/// One tick at now: whether the Mods screen is the page shown. A screen that comes starts its In; one that goes has none
|
||||
void step(int64_t now, bool on) {
|
||||
if (on && !shown) since_ns = now;
|
||||
if (!on) since_ns = 0;
|
||||
shown = on;
|
||||
}
|
||||
/// How much of it shows at now. settle: as it stands once its In is done (a still, a picture that compares). A screen
|
||||
/// asked for and not stepped yet (the page was set this tick, the step comes at the tick's end) shows nothing: drawn in
|
||||
/// between at full, it flashed whole for a frame before its In began
|
||||
float alpha(int64_t now, bool settle = false) const { return settle ? 1.0f : !shown ? 0.0f : in_alpha(now - since_ns); }
|
||||
/// (it takes input once its In has played, as the game's)
|
||||
bool ready(int64_t now) const { return shown && now - since_ns >= kInNs; }
|
||||
};
|
||||
|
||||
} // namespace mods_page
|
||||
95
gpu/vk/msync_pass.h
Normal file
95
gpu/vk/msync_pass.h
Normal file
@@ -0,0 +1,95 @@
|
||||
// (Triforce, 11-21, the performance audit's MEM-48, after Source A's) A mirror sync's pages from the captures by one compute
|
||||
// dispatch: the rules.
|
||||
//
|
||||
// A pool the GPU reads has a mirror in VRAM, and a sync brings the pages the CPU wrote into it. Those that come from a
|
||||
// capture's buffer (host memory) went as copy regions, a page or a few each, 700 to 1,300 a frame: the GPU reads each
|
||||
// region over PCIe after the one before. The write-back diff's pass (shaders/wb_diff.comp) reads a source, compares it
|
||||
// with a shadow and writes a destination where they differ; with the shadow and the destination both the mirror it is a
|
||||
// copy, a workgroup a chunk, every chunk of a sync in one dispatch. Source A's RTX 5080 in Hyrule Field: the syncs' GPU
|
||||
// time 1.41 to 0.69 ms a frame, 79.7 to 83.1 fps where the GPU bounds the frame.
|
||||
//
|
||||
// What keeps it right:
|
||||
// - a run goes as chunks only in whole words at word-aligned addresses (words); any other run of a sync is copied;
|
||||
// - the chunk tables are host memory in a ring of blocks, each table from an atom of its own (the flush covers whole
|
||||
// atoms), a block taken again only once the GPU is done with the recording that last read it (Ring); a sync whose
|
||||
// table finds no room is copied, as before;
|
||||
// - copies into one buffer are in order among themselves, a dispatch is not: where a mirror is written twice in one
|
||||
// upload command buffer and one of the two is the pass, a full barrier goes between (barrier_first). The pass's own
|
||||
// chunks and a sync's other copies never share a page.
|
||||
// RX_VK_MSYNC_PASS: on for NVIDIA when not set (measured there; no gain on an RX 5600 XT), 1: on every GPU, 0: copies.
|
||||
//
|
||||
// No Vulkan here: tests/test_msync_pass.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace msync_pass {
|
||||
|
||||
inline constexpr int kBlocks = 16;
|
||||
inline constexpr uint64_t kBlockBytes = 256u << 10; // (8192 chunks of 32 bytes: a frame's syncs took about 1,300)
|
||||
inline constexpr uint64_t kChunkBytes = 64u << 10; // (the pass's chunk at most)
|
||||
|
||||
/// Whether the pass is used: asked, RX_VK_MSYNC_PASS's value; not set: on NVIDIA's GPUs
|
||||
inline bool on(const char* asked, bool nvidia) {
|
||||
if (!asked || !*asked) return nvidia;
|
||||
return atoi(asked) != 0;
|
||||
}
|
||||
|
||||
/// Whether a run from src to dst (addresses) of size bytes can go as chunks
|
||||
constexpr bool words(uint64_t src, uint64_t dst, uint64_t size) { return size != 0 && !((src | dst | size) & 3); }
|
||||
|
||||
/// A run's chunks: emit(source, destination, count, vectors), the count in 16-byte vectors or in words. split(at, size,
|
||||
/// e(at, bytes, vectors)) is the write-back diff's (wbd_split: the words before the first 16-byte line, the lines 64 KB
|
||||
/// at a time, the words after). Addresses not alike within 16 bytes go in words alone
|
||||
template <class Split, class Emit>
|
||||
void run_chunks(uint64_t src, uint64_t dst, uint64_t size, Split&& split, Emit&& emit) {
|
||||
if ((src ^ dst) & 15) {
|
||||
for (uint64_t at = 0; at < size; at += kChunkBytes) {
|
||||
const uint64_t len = size - at < kChunkBytes ? size - at : kChunkBytes;
|
||||
emit(src + at, dst + at, (uint32_t)(len / 4), false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
split(dst, size, [&](uint64_t a, uint64_t len, bool vecs) { emit(src + (a - dst), a, (uint32_t)(len / (vecs ? 16 : 4)), vecs); });
|
||||
}
|
||||
|
||||
/// What wrote a mirror so far in an upload command buffer
|
||||
enum : uint8_t { kCopies = 1, kPass = 2 };
|
||||
/// Whether a full barrier goes before a write of kind `now` into a mirror that `before` wrote earlier in the same
|
||||
/// upload command buffer: the pass after anything, and anything after the pass
|
||||
constexpr bool barrier_first(uint8_t before, uint8_t now) { return now == kPass ? before != 0 : (before & kPass) != 0; }
|
||||
|
||||
/// The tables' blocks: which are made, the newest recording that reads each, and how far each is filled
|
||||
struct Ring {
|
||||
struct Block {
|
||||
bool made = false;
|
||||
uint64_t value = 0, fill = 0;
|
||||
};
|
||||
Block b[kBlocks];
|
||||
int cur = -1;
|
||||
/// Room for a table of `bytes` read by the recording rec_value, the GPU done up to `completed`: the block's index
|
||||
/// and *at, the table's offset in it, a multiple of align; -1: none (too big, or the next block is still read).
|
||||
/// A block returned with made false is the caller's to make before it writes
|
||||
int get(uint64_t bytes, uint64_t align, uint64_t rec_value, uint64_t completed, uint64_t* at) {
|
||||
if (!bytes || bytes > kBlockBytes || !align) return -1;
|
||||
if (cur >= 0) {
|
||||
Block& c = b[cur];
|
||||
const uint64_t start = (c.fill + align - 1) / align * align;
|
||||
if (start + bytes <= kBlockBytes) {
|
||||
c.fill = start + bytes;
|
||||
if (rec_value > c.value) c.value = rec_value;
|
||||
*at = start;
|
||||
return cur;
|
||||
}
|
||||
}
|
||||
const int next = (cur + 1) % kBlocks;
|
||||
Block& n = b[next];
|
||||
if (n.made && n.value > completed) return -1;
|
||||
cur = next;
|
||||
n.fill = bytes, n.value = rec_value;
|
||||
*at = 0;
|
||||
return next;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace msync_pass
|
||||
49
gpu/vk/new_wait.h
Normal file
49
gpu/vk/new_wait.h
Normal file
@@ -0,0 +1,49 @@
|
||||
// (Triforce, 11-22, the performance audit's OS-04, after Source A's and Source B's) A `new` of the renderer's that fails at
|
||||
// Windows' commit limit waits before it throws.
|
||||
//
|
||||
// A refused allocation in the renderer was an uncaught std::bad_alloc: the game ended. At the commit limit (RAM and the page
|
||||
// file full) a refusal often ends in moments, as the page file grows or commit is given back, so the renderer's new handler
|
||||
// (vk_main.cpp new_refused, set in mkwvk_init) sleeps 50 ms and lets the allocation be tried again, for up to 5 s a thread
|
||||
// (HOS_COMMIT_WAIT_MS, the host's switch for the same wait: 0, none), and throws after that. Only while the system has less
|
||||
// than 4 GiB of commit left: a `new` refused with more than that free was not refused for the limit (a size no memory could
|
||||
// hold), and throws at once, as before. Source A's is the wait; the 4 GiB rule is Source B's, written anew here.
|
||||
//
|
||||
// A `new (std::nothrow)` whose caller has its own answer to a refusal (a staging block, a copy that is only a saving, the
|
||||
// recorder's ring) is not held up: NoWait around it makes the handler throw at once, and the nothrow new answers null.
|
||||
//
|
||||
// The rule alone, no system calls: tests/test_commit_limit.py compiles it.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mkwvk::new_wait {
|
||||
|
||||
constexpr uint64_t kNear = 4ull << 30; // (the commit left under which a refusal is taken for the limit's)
|
||||
constexpr int kStepMs = 50, kWaitMs = 5000, kMostMs = 60000;
|
||||
|
||||
/// HOS_COMMIT_WAIT_MS in ms: not set (or empty) 5000; a number that many, 0 and less none, 60000 at most
|
||||
inline int wait_ms(const char* env) {
|
||||
if (!env || !*env) return kWaitMs;
|
||||
const long v = std::atol(env);
|
||||
return v <= 0 ? 0 : v > kMostMs ? kMostMs : (int)v;
|
||||
}
|
||||
|
||||
inline thread_local int t_waited_ms = 0; // (what this thread has waited in all: its 5 s are not given twice)
|
||||
inline thread_local int t_no_wait = 0; // (NoWait scopes open on this thread)
|
||||
|
||||
/// Around a new (std::nothrow) that must answer at once
|
||||
struct NoWait {
|
||||
NoWait() { t_no_wait++; }
|
||||
~NoWait() { t_no_wait--; }
|
||||
NoWait(const NoWait&) = delete;
|
||||
NoWait& operator=(const NoWait&) = delete;
|
||||
};
|
||||
|
||||
/// A `new` was refused. true: sleep a step and let it be tried again; false: throw. known and avail: the system's commit left,
|
||||
/// when it could be read; waited_ms: this thread's waits so far; max_ms: wait_ms's; no_wait: NoWait scopes open
|
||||
inline bool waits(bool known, uint64_t avail, int waited_ms, int max_ms, int no_wait) {
|
||||
return !no_wait && known && avail < kNear && waited_ms < max_ms;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::new_wait
|
||||
@@ -15,6 +15,8 @@
|
||||
// The game also predicates draws on these words (render conditions, NOP_RENDER_COND): as NX2navi decides them
|
||||
// (runtime/vk_exec_xfer.c cond_enabled), a word whose newest report is a samples-passed one with its count still to come is
|
||||
// unknown, and the condition renders; a word whose count was written is read from memory, where a real 0 can skip the draw.
|
||||
// (11-22) A count over a draw that such a skip left out is marked (Counter::gate, Writers::kGatedSrc, the line counts them);
|
||||
// RX_VK_OCC_GATE=1: a condition on it renders (cond_decide), so a skip's 0 cannot hide the same object again and again.
|
||||
//
|
||||
// Counter one counting, from a reset to its report, over the queries its draws began: the replay's (one per NVN
|
||||
// queue's state, under replay_mu) or a parallel replay's segment's (the helper thread's: a segment starts with
|
||||
@@ -95,6 +97,7 @@ struct Rep {
|
||||
uint32_t first = 0, n = 0;
|
||||
uint64_t pos = 0;
|
||||
int64_t present = 0;
|
||||
bool gated = false; // (a draw of its count left out by a condition that read a written count: Counter::gate)
|
||||
};
|
||||
|
||||
/// One samples-passed counting. The replay calls reset() at a SAMPLES_PASSED reset, draw_enter() as a draw goes into draw(),
|
||||
@@ -113,8 +116,10 @@ struct Counter {
|
||||
int open = -1; // (the query begun and not ended, its index in the block)
|
||||
bool lost = false; // (a draw in another recording, or the block full)
|
||||
uint32_t ops = 0, issued = 0; // (draws that went into draw() while counting, and those it recorded)
|
||||
enum Why { kNone, kSkipped, kOtherCmd, kFull };
|
||||
enum Why { kNone, kSkipped, kOtherCmd, kFull, kColour };
|
||||
Why why = kNone;
|
||||
bool gated = false; // (since the reset a draw was left out by a render condition that read a written count)
|
||||
bool colour = false; // ((Triforce, 11-22, GPUWORK-01) a draw that writes colour came in since the reset)
|
||||
|
||||
/// A parallel replay's segment starts (or ends), or the replay's counting is given up: nothing is counting
|
||||
void segment() { *this = Counter{}; }
|
||||
@@ -126,7 +131,35 @@ struct Counter {
|
||||
state = kCounting, slot = none ? -1 : slot_, serial = none ? 0 : serial_, first = next, open = -1, lost = none;
|
||||
ops = issued = 0;
|
||||
why = none ? kOtherCmd : kNone;
|
||||
gated = false;
|
||||
colour = false;
|
||||
}
|
||||
/// (11-22) A draw the replay passes with rendering off because a condition read a written count (a 0 of an earlier
|
||||
/// report): while counting, this count is gated, NVN counts nothing for that draw and neither do the queries. Such a count
|
||||
/// is a skip's result, not the object's visibility (Writers::kGatedSrc)
|
||||
void gate(bool keep) {
|
||||
if (counting(keep)) gated = true;
|
||||
}
|
||||
/// ((Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's) A draw that writes colour comes in, its pass
|
||||
/// open, before begin(). The game's occlusion tests draw depth only (a bounding box against the depth buffer); a query
|
||||
/// that stays open over colour draws makes the GPU wait at its end for the work it encloses. What the caller does with
|
||||
/// the answer: first, this is the window's first such draw, and open, a query is open over it (both counted, whether or
|
||||
/// not the rule is on); lose, the rule is on (depth_only) and the count is given up: the caller ends the open query and
|
||||
/// calls lose_colour(). Nothing for a count not counting or lost already
|
||||
struct Colour {
|
||||
bool first = false, open = false, lose = false;
|
||||
};
|
||||
Colour colour_draw(bool keep, bool depth_only) {
|
||||
Colour r;
|
||||
if (!counting(keep) || lost) return r;
|
||||
r.first = !colour, colour = true;
|
||||
r.open = open >= 0;
|
||||
r.lose = depth_only;
|
||||
return r;
|
||||
}
|
||||
/// The count given up for a colour draw (its open query ended by the caller): no query begins until the next reset,
|
||||
/// its report is incomplete, so the record keeps the placeholder and its render condition renders
|
||||
void lose_colour() { lost = true, why = kColour; }
|
||||
void draw_enter(bool keep) {
|
||||
if (counting(keep)) ops++;
|
||||
}
|
||||
@@ -162,7 +195,7 @@ struct Counter {
|
||||
if (!complete && !lost) why = kSkipped;
|
||||
state = kReported;
|
||||
if (!complete) return kIncomplete;
|
||||
rep->first = first, rep->n = next - first;
|
||||
rep->first = first, rep->n = next - first, rep->gated = gated;
|
||||
return kMeasured;
|
||||
}
|
||||
};
|
||||
@@ -170,57 +203,55 @@ struct Counter {
|
||||
/// A map from 64-bit keys, open addressing with linear probes and backward-shift deletion (no tombstones): no allocation per
|
||||
/// entry. The replay placed one entry for every samples-passed report (~2,600 a frame at the Market) in a std::unordered_map,
|
||||
/// a node allocated and freed each time: 0.76 ms a frame of the Presentation Thread (all-thread sample, 2026-11-20)
|
||||
// ((Triforce, 11-21) The keys, the used marks and the values in three arrays: a probe reads 9 bytes a slot, not a 48-byte slot.
|
||||
// The game's report addresses rotate through ~49,000 a 600-present period, ~27,000 live at once (kKeepAge): the table of slots was
|
||||
// ~3 MB, past the core's L2, and its probes and the prune's walk over it were 5% of the Presentation Thread (exact profile of a
|
||||
// line-table build, 2026-11-21). Same entries, same order of probes and shifts as before)
|
||||
template <class V>
|
||||
class FlatU64 {
|
||||
public:
|
||||
struct Slot {
|
||||
uint64_t key = 0;
|
||||
V val{};
|
||||
bool used = false;
|
||||
};
|
||||
V* find(uint64_t k) {
|
||||
if (!n_) return nullptr;
|
||||
for (size_t i = idx(k);; i = (i + 1) & mask_) {
|
||||
Slot& s = s_[i];
|
||||
if (!s.used) return nullptr;
|
||||
if (s.key == k) return &s.val;
|
||||
if (!used_[i]) return nullptr;
|
||||
if (keys_[i] == k) return &vals_[i];
|
||||
}
|
||||
}
|
||||
const V* find(uint64_t k) const { return const_cast<FlatU64*>(this)->find(k); }
|
||||
V& operator[](uint64_t k) {
|
||||
if (V* v = find(k)) return *v;
|
||||
if ((n_ + 1) * 10 > s_.size() * 7) grow();
|
||||
if ((n_ + 1) * 10 > keys_.size() * 7) grow();
|
||||
size_t i = idx(k);
|
||||
while (s_[i].used) i = (i + 1) & mask_;
|
||||
s_[i].key = k, s_[i].val = V{}, s_[i].used = true;
|
||||
while (used_[i]) i = (i + 1) & mask_;
|
||||
keys_[i] = k, vals_[i] = V{}, used_[i] = 1;
|
||||
n_++;
|
||||
return s_[i].val;
|
||||
return vals_[i];
|
||||
}
|
||||
size_t count(uint64_t k) const { return find(k) ? 1 : 0; }
|
||||
size_t erase(uint64_t k) {
|
||||
if (!n_) return 0;
|
||||
size_t i = idx(k);
|
||||
for (;; i = (i + 1) & mask_) {
|
||||
if (!s_[i].used) return 0;
|
||||
if (s_[i].key == k) break;
|
||||
if (!used_[i]) return 0;
|
||||
if (keys_[i] == k) break;
|
||||
}
|
||||
// backward shift: the entries after the hole that would be found from a slot at or before it move into it
|
||||
for (size_t j = (i + 1) & mask_;; j = (j + 1) & mask_) {
|
||||
if (!s_[j].used) break;
|
||||
const size_t home = idx(s_[j].key);
|
||||
if (!used_[j]) break;
|
||||
const size_t home = idx(keys_[j]);
|
||||
const bool moves = i <= j ? (home <= i || home > j) : (home <= i && home > j);
|
||||
if (moves) s_[i] = s_[j], i = j;
|
||||
if (moves) keys_[i] = keys_[j], vals_[i] = vals_[j], i = j;
|
||||
}
|
||||
s_[i] = Slot{};
|
||||
keys_[i] = 0, vals_[i] = V{}, used_[i] = 0;
|
||||
n_--;
|
||||
return 1;
|
||||
}
|
||||
template <class F>
|
||||
void erase_if(F pred) { // (the keys to go first, then each erased: a shift during the walk would skip or revisit entries)
|
||||
std::vector<uint64_t> gone;
|
||||
for (const Slot& s : s_)
|
||||
if (s.used && pred(s.key, s.val)) gone.push_back(s.key);
|
||||
for (uint64_t k : gone) erase(k);
|
||||
gone_.clear();
|
||||
for (size_t i = 0; i < keys_.size(); i++)
|
||||
if (used_[i] && pred(keys_[i], vals_[i])) gone_.push_back(keys_[i]);
|
||||
for (uint64_t k : gone_) erase(k);
|
||||
}
|
||||
bool empty() const { return n_ == 0; }
|
||||
size_t size() const { return n_; }
|
||||
@@ -228,18 +259,23 @@ public:
|
||||
private:
|
||||
size_t idx(uint64_t k) const { return (size_t)((k * 0x9E3779B97F4A7C15ull) >> shift_) & mask_; }
|
||||
void grow() {
|
||||
std::vector<Slot> old;
|
||||
old.swap(s_);
|
||||
const size_t cap = old.empty() ? 1024 : old.size() * 2;
|
||||
s_.assign(cap, Slot{});
|
||||
std::vector<uint64_t> ok;
|
||||
std::vector<uint8_t> ou;
|
||||
std::vector<V> ov;
|
||||
ok.swap(keys_), ou.swap(used_), ov.swap(vals_);
|
||||
const size_t cap = ok.empty() ? 1024 : ok.size() * 2;
|
||||
keys_.assign(cap, 0), used_.assign(cap, 0), vals_.assign(cap, V{});
|
||||
mask_ = cap - 1;
|
||||
shift_ = 64;
|
||||
for (size_t c = cap; c > 1; c >>= 1) shift_--;
|
||||
n_ = 0;
|
||||
for (const Slot& s : old)
|
||||
if (s.used) (*this)[s.key] = s.val;
|
||||
for (size_t i = 0; i < ok.size(); i++)
|
||||
if (ou[i]) (*this)[ok[i]] = ov[i];
|
||||
}
|
||||
std::vector<Slot> s_;
|
||||
std::vector<uint64_t> keys_;
|
||||
std::vector<uint8_t> used_;
|
||||
std::vector<V> vals_;
|
||||
std::vector<uint64_t> gone_; // (erase_if's list, kept: no allocation a prune)
|
||||
size_t n_ = 0, mask_ = 0;
|
||||
int shift_ = 64;
|
||||
};
|
||||
@@ -259,12 +295,15 @@ public:
|
||||
kWritten, // (its count is in the record: completion wrote it)
|
||||
kFinal // (no count comes: skip list, knob off, a clear flag, not counted whole, lost, late, the CPU wrote it)
|
||||
};
|
||||
enum Src { kNone, kPendingSrc, kWrittenSrc, kFinalSrc }; // (what a render condition reading the word finds)
|
||||
// (what a render condition reading the word finds. kGatedSrc: written, a count over a draw a written count's condition
|
||||
// left out, see Counter::gate)
|
||||
enum Src { kNone, kPendingSrc, kWrittenSrc, kFinalSrc, kGatedSrc };
|
||||
struct Placed {
|
||||
uint64_t pos = 0;
|
||||
int64_t present = 0;
|
||||
uint64_t value = 0; // (what the replay, or a count written since, put in the word: a count is written only over it)
|
||||
State state = kFinal;
|
||||
bool gated = false; // (its count, written, was over a draw a written count's condition left out)
|
||||
};
|
||||
/// A report's placeholder `value` written at the replay, at pos in the stream (the newest there from now on). write:
|
||||
/// its count is to be written once read back (not on RX_VK_OCC_QUERY_SKIP, the lab knob on, not a clear flag)
|
||||
@@ -287,10 +326,11 @@ public:
|
||||
const auto* it = m_.find(addr);
|
||||
return !it ? 0 : it->value;
|
||||
}
|
||||
/// The count of the report at (addr, pos) is in its record now
|
||||
void wrote(uint64_t addr, uint64_t pos, uint64_t count) {
|
||||
/// The count of the report at (addr, pos) is in its record now (gated: counted over a draw a written count's condition
|
||||
/// left out)
|
||||
void wrote(uint64_t addr, uint64_t pos, uint64_t count, bool gated = false) {
|
||||
auto* it = m_.find(addr);
|
||||
if (it && it->pos == pos) it->state = kWritten, it->value = count;
|
||||
if (it && it->pos == pos) it->state = kWritten, it->value = count, it->gated = gated;
|
||||
}
|
||||
/// No count comes for the report at (addr, pos) (not counted whole, lost, late, never sent, the CPU wrote the record):
|
||||
/// a condition reads its word from memory from now on, as before
|
||||
@@ -317,7 +357,7 @@ public:
|
||||
if (!it) return kNone;
|
||||
switch (it->state) {
|
||||
case kPending: return now - it->present > kLateAge ? kFinalSrc : kPendingSrc;
|
||||
case kWritten: return kWrittenSrc;
|
||||
case kWritten: return it->gated ? kGatedSrc : kWrittenSrc;
|
||||
case kFinal: break;
|
||||
}
|
||||
return kFinalSrc;
|
||||
@@ -339,23 +379,33 @@ private:
|
||||
/// they differ) over the words' sources (Writers::word), after NX2navi's cond_enabled: a samples-passed count not done yet
|
||||
/// renders; a written one, as any other word, is read from memory (base_enabled: the replay's answer from memory). active:
|
||||
/// RX_VK_OCC_QUERY=1 or the lab knob's phase (else the answer is the base one, whatever the table says); cond_on: false
|
||||
/// with RX_VK_OCC_COND=0, every condition on a samples-passed word renders
|
||||
/// with RX_VK_OCC_COND=0, every condition on a samples-passed word renders.
|
||||
/// (11-22) gate_on (RX_VK_OCC_GATE=1): a word whose count was written over a draw that a written count's condition left
|
||||
/// out (Writers::kGatedSrc) is unknown, and the condition renders. The count is a skip's result, not the object's: on the
|
||||
/// console the game's CPU reads a 0 before it records the next frame and draws the object's proxy instead; here the count
|
||||
/// often lands after the CPU has read the placeholder (4096: visible), the game goes on drawing the object under the
|
||||
/// condition, the condition reads the skip's 0, the next count is 0 again, and the object stays hidden for good. Off: as a
|
||||
/// written count (read from memory)
|
||||
struct Cond {
|
||||
enum Kind { kNotSamples, kBase, kUnknown, kForced, kWrittenOn, kWrittenOff, kMemory };
|
||||
enum Kind { kNotSamples, kBase, kUnknown, kForced, kWrittenOn, kWrittenOff, kMemory, kGated };
|
||||
bool enabled;
|
||||
Kind kind;
|
||||
};
|
||||
inline Cond cond_decide(bool base_enabled, Writers::Src sx, Writers::Src sy, bool active, bool cond_on) {
|
||||
inline Cond cond_decide(bool base_enabled, Writers::Src sx, Writers::Src sy, bool active, bool cond_on, bool gate_on = false) {
|
||||
if (sx == Writers::kNone && sy == Writers::kNone) return {base_enabled, Cond::kNotSamples};
|
||||
if (!active) return {base_enabled, Cond::kBase};
|
||||
if (!cond_on) return {true, Cond::kForced};
|
||||
if (sx == Writers::kPendingSrc || sy == Writers::kPendingSrc) return {true, Cond::kUnknown};
|
||||
if (sx == Writers::kWrittenSrc || sy == Writers::kWrittenSrc)
|
||||
const bool gated = sx == Writers::kGatedSrc || sy == Writers::kGatedSrc;
|
||||
if (gated && gate_on) return {true, Cond::kGated};
|
||||
if (sx == Writers::kWrittenSrc || sy == Writers::kWrittenSrc || gated)
|
||||
return {base_enabled, base_enabled ? Cond::kWrittenOn : Cond::kWrittenOff};
|
||||
return {base_enabled, Cond::kMemory};
|
||||
}
|
||||
inline const char* src_name(Writers::Src s) {
|
||||
return s == Writers::kPendingSrc ? "pending" : s == Writers::kWrittenSrc ? "written" : s == Writers::kFinalSrc ? "memory, no count to come" : "memory";
|
||||
return s == Writers::kPendingSrc ? "pending" : s == Writers::kWrittenSrc ? "written"
|
||||
: s == Writers::kGatedSrc ? "written, over a draw a written count skipped"
|
||||
: s == Writers::kFinalSrc ? "memory, no count to come" : "memory";
|
||||
}
|
||||
|
||||
/// What happened since the last line (under the caller's lock)
|
||||
@@ -370,9 +420,16 @@ struct Stats {
|
||||
int64_t rewritten = 0; // records rewritten in the stream after their report (a small clear, another counter type)
|
||||
int64_t measured = 0, no_reset = 0, incomplete = 0, doubles = 0; // counted whole by queries, and those not
|
||||
int64_t inc_skipped = 0, inc_other_cmd = 0, inc_full = 0; // why not whole
|
||||
// ((Triforce, 11-22, GPUWORK-01) and of the incomplete ones, those given up for a colour draw in their window
|
||||
// (RX_VK_OCC_DEPTH_ONLY); the windows that met a colour draw, the colour draws under an open query, the counts given up
|
||||
// (colour_line, a line of its own: line() is at the log's length already)
|
||||
int64_t inc_colour = 0, colour_windows = 0, colour_open = 0, colour_lost = 0;
|
||||
// render conditions that read a samples-passed report's word (Cond::Kind), and the work they skipped
|
||||
int64_t conds = 0, cond_base = 0, cond_unknown = 0, cond_forced = 0, cond_written_on = 0, cond_written_off = 0, cond_memory = 0;
|
||||
int64_t draws_skipped = 0, clears_skipped = 0;
|
||||
// (11-22) counts over a draw a written count's condition left out (Counter::gate): counted whole, written (of them 0),
|
||||
// the conditions that read one, and those RX_VK_OCC_GATE=1 rendered
|
||||
int64_t gated = 0, gated_written = 0, gated_zeros = 0, cond_gated_read = 0, cond_gated = 0;
|
||||
// at completion
|
||||
int64_t written = 0, zeros = 0; // counts written into their records (of them 0: the object hidden)
|
||||
int64_t lost = 0; // a query not available once its recording was done: 4096 stays
|
||||
@@ -389,7 +446,17 @@ struct Stats {
|
||||
if (k == Cond::kNotSamples) return;
|
||||
conds++;
|
||||
(k == Cond::kBase ? cond_base : k == Cond::kUnknown ? cond_unknown : k == Cond::kForced ? cond_forced
|
||||
: k == Cond::kWrittenOn ? cond_written_on : k == Cond::kWrittenOff ? cond_written_off : cond_memory)++;
|
||||
: k == Cond::kWrittenOn ? cond_written_on : k == Cond::kWrittenOff ? cond_written_off : k == Cond::kGated ? cond_gated
|
||||
: cond_memory)++;
|
||||
}
|
||||
/// ((Triforce, 11-22, the performance audit's GPUWORK-01) The colour draws' line, into out. on: the rule is on
|
||||
int colour_line(char* out, size_t size, int64_t presents, bool on) const {
|
||||
return snprintf(out, size,
|
||||
"vk: occlusion queries (RX_VK_OCC_QUERY), %lld presents (3): %lld counting windows met a draw that "
|
||||
"writes colour, %lld such draws were recorded under an open query; depth-only counting "
|
||||
"(RX_VK_OCC_DEPTH_ONLY) is %s: %lld counts given up at a colour draw, %lld reports kept 4096 for it",
|
||||
(long long)presents, (long long)colour_windows, (long long)colour_open, on ? "on" : "off",
|
||||
(long long)colour_lost, (long long)inc_colour);
|
||||
}
|
||||
/// The line, into out
|
||||
int line(char* out, size_t size, int64_t presents, size_t tracked) const {
|
||||
@@ -407,7 +474,8 @@ struct Stats {
|
||||
"%lld (%lld as before: knob occq off; %lld rendered: a count pending (unknown); %lld rendered: "
|
||||
"RX_VK_OCC_COND=0; %lld rendered from a written count; %lld skipped from a written count; %lld from "
|
||||
"memory, no count to come); skipped by them: %lld draws and dispatches, %lld clears; %lld addresses "
|
||||
"this period, %zu tracked",
|
||||
"this period, %zu tracked; counts over a draw such a skip left out: %lld counted whole, %lld written "
|
||||
"(%lld of them 0), %lld conditions read one (%lld rendered: RX_VK_OCC_GATE=1)",
|
||||
(long long)presents, (long long)reports, (long long)in_segments, (long long)skipped, (long long)fixed,
|
||||
(long long)flagged, (long long)rewritten, (long long)measured, (long long)no_reset,
|
||||
(long long)incomplete, (long long)inc_skipped, (long long)inc_other_cmd, (long long)inc_full,
|
||||
@@ -417,7 +485,8 @@ struct Stats {
|
||||
(long long)at[kAtWait], (long long)at[kAtBegin], (long long)at[kAtPresent], (long long)at[kAtReuse],
|
||||
(long long)conds, (long long)cond_base, (long long)cond_unknown, (long long)cond_forced,
|
||||
(long long)cond_written_on, (long long)cond_written_off, (long long)cond_memory,
|
||||
(long long)draws_skipped, (long long)clears_skipped, (long long)addrs, tracked);
|
||||
(long long)draws_skipped, (long long)clears_skipped, (long long)addrs, tracked, (long long)gated,
|
||||
(long long)gated_written, (long long)gated_zeros, (long long)cond_gated_read, (long long)cond_gated);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -458,9 +527,10 @@ void complete(const std::vector<Rep>& reps, const uint64_t* pairs, uint32_t used
|
||||
w.settle(p.addr, p.pos);
|
||||
continue;
|
||||
}
|
||||
w.wrote(p.addr, p.pos, sum);
|
||||
w.wrote(p.addr, p.pos, sum, p.gated);
|
||||
const int64_t lat = std::max<int64_t>(0, now - p.present);
|
||||
st.written++, st.zeros += sum ? 0 : 1, st.lat_sum += lat, st.lat_ge1 += lat >= 1, st.lat_ge2 += lat >= 2;
|
||||
if (p.gated) st.gated_written++, st.gated_zeros += sum ? 0 : 1;
|
||||
}
|
||||
}
|
||||
/// A recording's reports that never went to the GPU: no count comes for them
|
||||
|
||||
82
gpu/vk/occ_lean.h
Normal file
82
gpu/vk/occ_lean.h
Normal file
@@ -0,0 +1,82 @@
|
||||
// (Triforce, 11-22, the performance audit's DESC-82, after Source B's: its first step, the census) The game's occlusion tests
|
||||
// draw proxies that write no colour, and each such draw runs its whole fragment shader and fills its fragment descriptors.
|
||||
// Source B makes a sibling pipeline without the fragment stage for a counted draw that writes no colour and whose fragment
|
||||
// shader has no effect but colour (no depth or sample-mask output, no demote, no stores, image writes, atomics or global
|
||||
// memory) and whose state has neither alpha to coverage nor an alpha test. It was never on by default nor measured; its
|
||||
// census at Kakariko counted about 2,570 colour-off draws of about 3,420 counted. The entry's own order is the census first,
|
||||
// and the lean pipelines only if the census finds more than one fragment descriptor a proxy to save.
|
||||
//
|
||||
// This is the census alone. RX_VK_OCC_LEAN=3: each draw under an open occlusion query that writes no colour is classed
|
||||
// (classify: lean, or the first refusal), and its fragment stage's descriptors added to its class; every 600 presents one
|
||||
// line gives each class's draws and descriptors a frame (vk_replay.cpp status_tick). No pipeline, key or descriptor changes.
|
||||
// 1 (the lean pipelines) and 2 (their check) are not built: they read as off. Unset: nothing of it runs.
|
||||
//
|
||||
// The rules alone: tests/test_occ_lean.py compiles them.
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace occ_lean {
|
||||
|
||||
/// RX_VK_OCC_LEAN: 3 for the census; anything else is off (1 and 2: the steps after the census, not built)
|
||||
inline int mode_of(const char* e) { return e && *e && std::atoi(e) == 3 ? 3 : 0; }
|
||||
|
||||
enum Class { kLean, kNoFragment, kDepth, kSampleMask, kDemote, kStores, kImageWrites, kAtomics, kGlobal, kA2c, kAlphaTest, kClasses };
|
||||
inline const char* const kNames[kClasses] = {"lean", "no fragment stage", "depth out", "sample mask", "demote", "stores",
|
||||
"image writes", "atomics", "global memory", "alpha to coverage", "alpha test"};
|
||||
|
||||
/// What the class reads of a draw: its fragment stage's (Shader::Info) and its state's
|
||||
struct Facts {
|
||||
bool has_fragment = true;
|
||||
bool depth_out = false, sample_mask = false, demote = false, stores = false, image_writes = false, atomics = false,
|
||||
global = false;
|
||||
bool a2c = false, alpha_test = false;
|
||||
};
|
||||
|
||||
/// The draw's class: no fragment stage (nothing to leave out), the first refusal of the shader's then of the state's, or lean
|
||||
inline Class classify(const Facts& f) {
|
||||
if (!f.has_fragment) return kNoFragment;
|
||||
if (f.depth_out) return kDepth;
|
||||
if (f.sample_mask) return kSampleMask;
|
||||
if (f.demote) return kDemote;
|
||||
if (f.stores) return kStores;
|
||||
if (f.image_writes) return kImageWrites;
|
||||
if (f.atomics) return kAtomics;
|
||||
if (f.global) return kGlobal;
|
||||
if (f.a2c) return kA2c;
|
||||
if (f.alpha_test) return kAlphaTest;
|
||||
return kLean;
|
||||
}
|
||||
|
||||
/// The counts, any recording thread (relaxed)
|
||||
struct Census {
|
||||
std::atomic<uint64_t> draws[kClasses]{};
|
||||
std::atomic<uint64_t> descriptors[kClasses]{};
|
||||
void note(Class c, uint32_t frag_descriptors) {
|
||||
draws[c].fetch_add(1, std::memory_order_relaxed);
|
||||
descriptors[c].fetch_add(frag_descriptors, std::memory_order_relaxed);
|
||||
}
|
||||
/// The line for `presents` presents (draws and fragment descriptors a frame, the lean ones' descriptors a draw, then each
|
||||
/// refusal that met any), and the counts set back. Its length
|
||||
int line(char* out, size_t n, uint64_t presents) {
|
||||
uint64_t d[kClasses], s[kClasses], all = 0;
|
||||
for (int i = 0; i < kClasses; i++)
|
||||
d[i] = draws[i].exchange(0, std::memory_order_relaxed), s[i] = descriptors[i].exchange(0, std::memory_order_relaxed),
|
||||
all += d[i];
|
||||
const double p = presents ? (double)presents : 1.0;
|
||||
int at = std::snprintf(out, n, "vk: occlusion proxies (RX_VK_OCC_LEAN=3, %llu presents): %.0f colour-off counted draws a frame; "
|
||||
"lean %.0f a frame with %.2f fragment descriptors each", (unsigned long long)presents, all / p,
|
||||
d[kLean] / p, d[kLean] ? (double)s[kLean] / (double)d[kLean] : 0.0);
|
||||
for (int i = 1; i < kClasses && at > 0 && (size_t)at < n; i++)
|
||||
if (d[i])
|
||||
at += std::snprintf(out + at, n - (size_t)at, "; %s %.0f (%.2f descriptors each)", kNames[i], d[i] / p,
|
||||
(double)s[i] / (double)d[i]);
|
||||
return at;
|
||||
}
|
||||
};
|
||||
inline Census census;
|
||||
|
||||
} // namespace occ_lean
|
||||
100
gpu/vk/page_marks.h
Normal file
100
gpu/vk/page_marks.h
Normal file
@@ -0,0 +1,100 @@
|
||||
// (Triforce, 11-22, the performance audit's THREAD-25, after Source A's) Pages that something ever happened to, asked without a lock;
|
||||
// and the question an invalidate asks of them first.
|
||||
//
|
||||
// The game invalidates each shape's 64-byte occlusion slot, per view: tens of thousands of nvnBufferInvalidateMappedRange calls in a
|
||||
// few seconds. The host gives the calling thread's virtual core up before every one and queues for it again after, as the call can
|
||||
// wait for the replay or for a write-back still on the GPU; nearly always it has nothing to wait for. What it would wait for is
|
||||
// known from three things: a write-back ever made into the range's pages, a texture the GPU wrote ever listed there, and a live
|
||||
// texture's storage there. The first two are tables under their own locks; PageMarks keeps, beside each, one bit a 4 KB page that is
|
||||
// only ever set, set before the table changes and under the table's lock, and read with no lock. A range with no mark in either and
|
||||
// no texture storage is one invalidate() returns from without waiting, so the host may keep the core and not make the call
|
||||
// (invalidate_quick in vk_replay.cpp, asked through mkwvk_invalidate_quick by host/rx_backend.c).
|
||||
//
|
||||
// A mark is never taken back: a page written back once declines for good, and waits as before. Addresses at or past 512 GB are
|
||||
// not kept and answer as marked. The header is written from the audit's description of Source A's, whose file is not here.
|
||||
//
|
||||
// RX_VK_INVALIDATE_QUICK=1 turns the question on; off unless set (see docs/development.md for why). tests/test_invalidate_quick.py
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mkwvk {
|
||||
|
||||
struct PageMarks {
|
||||
static constexpr uint64_t kSpan = 1ull << 39; // (512 GB: the guest window, and the GPU's addresses the pools lie in)
|
||||
static constexpr uint64_t kPageShift = 12, kPages = kSpan >> kPageShift;
|
||||
static constexpr uint64_t kChunkPages = 1ull << 18; // (a chunk: 1 GB of addresses, 32 KB of bits, made at its first mark)
|
||||
static constexpr uint64_t kChunks = kPages / kChunkPages, kChunkWords = kChunkPages / 64;
|
||||
std::atomic<std::atomic<uint64_t>*> chunks[kChunks] = {};
|
||||
|
||||
/// every page with a byte in [lo, hi) marked (what lies at or past 512 GB is not kept: any() answers true there)
|
||||
void mark(uint64_t lo, uint64_t hi) {
|
||||
if (hi > kSpan) hi = kSpan;
|
||||
if (hi <= lo) return;
|
||||
const uint64_t last = (hi - 1) >> kPageShift;
|
||||
for (uint64_t p = lo >> kPageShift; p <= last;) {
|
||||
std::atomic<uint64_t>* c = chunks[p / kChunkPages].load(std::memory_order_acquire);
|
||||
if (!c) {
|
||||
std::atomic<uint64_t>* made = new std::atomic<uint64_t>[kChunkWords];
|
||||
for (uint64_t i = 0; i < kChunkWords; i++) made[i].store(0, std::memory_order_relaxed);
|
||||
if (chunks[p / kChunkPages].compare_exchange_strong(c, made, std::memory_order_acq_rel)) c = made;
|
||||
else delete[] made; // (another thread made it first: c is its chunk)
|
||||
}
|
||||
// (the pages of this word from p to the word's end, or to `last`)
|
||||
const uint64_t in_chunk = p % kChunkPages, bit = in_chunk % 64;
|
||||
const uint64_t word_last = p - bit + 63, to = word_last < last ? word_last : last;
|
||||
const uint64_t n = to - p + 1;
|
||||
const uint64_t mask = (n == 64 ? ~0ull : ((1ull << n) - 1)) << bit;
|
||||
c[in_chunk / 64].fetch_or(mask, std::memory_order_seq_cst);
|
||||
p = to + 1;
|
||||
}
|
||||
}
|
||||
/// whether a page with a byte in [lo, lo + size) was ever marked; true for a range that reaches 512 GB or wraps
|
||||
bool any(uint64_t lo, uint64_t size) const {
|
||||
if (!size) return false;
|
||||
if (lo >= kSpan || size > kSpan - lo) return true;
|
||||
const uint64_t last = (lo + size - 1) >> kPageShift;
|
||||
for (uint64_t p = lo >> kPageShift; p <= last;) {
|
||||
const std::atomic<uint64_t>* c = chunks[p / kChunkPages].load(std::memory_order_seq_cst);
|
||||
if (!c) { // (a chunk never marked: skipped whole)
|
||||
p = (p / kChunkPages + 1) * kChunkPages;
|
||||
continue;
|
||||
}
|
||||
const uint64_t in_chunk = p % kChunkPages, bit = in_chunk % 64;
|
||||
const uint64_t word_last = p - bit + 63, to = word_last < last ? word_last : last;
|
||||
const uint64_t n = to - p + 1;
|
||||
const uint64_t mask = (n == 64 ? ~0ull : ((1ull << n) - 1)) << bit;
|
||||
if (c[in_chunk / 64].load(std::memory_order_seq_cst) & mask) return true;
|
||||
p = to + 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
namespace inv_quick { // (the rule of vk_replay.cpp's invalidate_quick)
|
||||
|
||||
/// Whether the question is asked at all. quick: RX_VK_INVALIDATE_QUICK (a number other than 0 turns it on; off unless set). The
|
||||
/// three others are switches that make invalidate() log every call or wait for every range: with one set, every call is made
|
||||
inline bool on(const char* quick, const char* log_invalidate, const char* log_readback, const char* wait_all) {
|
||||
return quick && std::atoi(quick) != 0 && !log_invalidate && !log_readback && !wait_all;
|
||||
}
|
||||
|
||||
/// What is known of a range when the question is asked
|
||||
struct Look {
|
||||
uint64_t gpu = 0, size = 0;
|
||||
bool ticket_pending = false; // (a fast fence poll's order ticket not yet left: invalidate() waits for that section first)
|
||||
bool tex_storage = false; // (a live texture's storage in the range's pages: tex_pages_any)
|
||||
bool vw_marked = false; // (a texture the GPU wrote ever listed there)
|
||||
bool wb_marked = false; // (a write-back ever made into its pages in the game's memory)
|
||||
};
|
||||
/// true: invalidate(gpu, size) would return without waiting for anything and without work: the call may be left out
|
||||
inline bool nothing_to_wait_for(const Look& l) {
|
||||
if (!l.size) return true;
|
||||
if (l.gpu >= PageMarks::kSpan || l.size > PageMarks::kSpan - l.gpu) return false;
|
||||
return !l.ticket_pending && !l.tex_storage && !l.vw_marked && !l.wb_marked;
|
||||
}
|
||||
|
||||
} // namespace inv_quick
|
||||
} // namespace mkwvk
|
||||
@@ -25,6 +25,7 @@
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <new>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
@@ -87,6 +88,7 @@ enum class Read {
|
||||
OverBound, // (longer than the bound: not a byte of it read)
|
||||
NoMemory, // (no block of its size)
|
||||
Cut, // (the read ended early: nothing of it is used)
|
||||
Foreign, // ((Triforce) its header is another GPU's or driver's: not read past it)
|
||||
};
|
||||
|
||||
inline const char* why(Read r) {
|
||||
@@ -96,13 +98,19 @@ inline const char* why(Read r) {
|
||||
case Read::OverBound: return "it is over the bound";
|
||||
case Read::NoMemory: return "there is no memory for a block of its size";
|
||||
case Read::Cut: return "reading it ended early";
|
||||
case Read::Foreign: return "it is another GPU's or driver's";
|
||||
default: return "";
|
||||
}
|
||||
}
|
||||
|
||||
/// The whole file into out, or nothing: out is empty unless the answer is Ok. *len: the file's length whenever it could be
|
||||
/// asked, read or not. least: a file of at most this many bytes is refused; bound: one longer than this too (0: none)
|
||||
inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<uint8_t>& out, uint64_t* len) {
|
||||
/// asked, read or not. least: a file of at most this many bytes is refused; bound: one longer than this too (0: none).
|
||||
/// ((Triforce) head_ok: when given, the file's first kHead bytes (a pipeline cache's header) are read alone and handed to it
|
||||
/// with the file's length; refused, the rest is not read (Foreign). The file's stamp names only the translated shaders, so
|
||||
/// after a driver update a file of 0.65-1.6 GB was read whole only for its header to be refused)
|
||||
constexpr size_t kHead = 32;
|
||||
inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<uint8_t>& out, uint64_t* len,
|
||||
bool (*head_ok)(const uint8_t* head, uint64_t size) = nullptr) {
|
||||
out.clear();
|
||||
*len = 0;
|
||||
FILE* f = fopen(path, "rb");
|
||||
@@ -120,7 +128,14 @@ inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<u
|
||||
else if ((uint64_t)n <= least) r = Read::Short;
|
||||
else if (!within((uint64_t)n, bound)) r = Read::OverBound;
|
||||
else if ((uint64_t)n > SIZE_MAX) r = Read::NoMemory;
|
||||
else {
|
||||
uint8_t head[kHead];
|
||||
size_t got = 0;
|
||||
if (r == Read::Ok && head_ok && (uint64_t)n >= kHead) {
|
||||
got = fread(head, 1, kHead, f);
|
||||
if (got != kHead) r = Read::Cut;
|
||||
else if (!head_ok(head, (uint64_t)n)) r = Read::Foreign;
|
||||
}
|
||||
if (r == Read::Ok) {
|
||||
try {
|
||||
out.resize((size_t)n);
|
||||
} catch (const std::bad_alloc&) {
|
||||
@@ -128,7 +143,7 @@ inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<u
|
||||
} catch (const std::length_error&) {
|
||||
r = Read::NoMemory;
|
||||
}
|
||||
size_t got = 0;
|
||||
if (r == Read::Ok && got) std::memcpy(out.data(), head, got);
|
||||
while (r == Read::Ok && got < out.size()) {
|
||||
const size_t piece = std::min(out.size() - got, kPiece);
|
||||
const size_t had = fread(out.data() + got, 1, piece, f);
|
||||
|
||||
107
gpu/vk/pcache_libs.h
Normal file
107
gpu/vk/pcache_libs.h
Normal file
@@ -0,0 +1,107 @@
|
||||
// A pipeline cache file of stage libraries and compute pipelines alone, on NVIDIA PCs of little RAM (no game, no GPU: the
|
||||
// rule, the driver's cache folder and its size, the record; vk_draw.cpp pcache_libs_decide, pcache_init, build_graphics,
|
||||
// pcache_play_open). Apart from Vulkan so that tests/test_pcache_libs.py runs it on many made-up PCs.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-78, after Source A's and Source B's.) mkw_vk_pipelines.bin is read whole at
|
||||
// every start and held by the driver all session: about 650 MB fresh, 1.1 to 1.6 GB with the play's pipelines on a 16 GB PC
|
||||
// (pcache_bound.h), where paging is what hitches. Source A leaves its file unread on NVIDIA's own driver with 20 GB of RAM or
|
||||
// less and 512 MB or more in the driver's own disk cache (GLCache), whole pipelines then coming from that cache: on a 16 GB
|
||||
// RTX 3070 Ti, peak memory 12,481 -> 12,029 and 12,638 -> 11,822 MB over a tour, fps 54.3 -> 54.6, a pipeline's load 1.9 ->
|
||||
// 5.0 ms. Triforce cannot drop the file: a stage library asked under the compile-required flag must be in the cache it hands
|
||||
// the driver (21 of 21 cold otherwise). So the file keeps the libraries and the compute pipelines, and only whole graphics
|
||||
// pipelines are left to the driver's cache, with no play file.
|
||||
//
|
||||
// Decided only when a fresh file begins (none read for these shaders), and kept in <file>.lean ("lean <stamp>") for that
|
||||
// file's life: a full file stays full and a lean one lean until the next fresh one. Not in a start that only prepares.
|
||||
//
|
||||
// RX_VK_PCACHE_LIBS_ONLY: unset or 0, off (every pipeline in the file, as before); 1, by the rule; 2, at every fresh file
|
||||
// whatever the PC (the lab's and the tests'). Off by default: the driver's cache is shared and capped, and on a 32 GB PC
|
||||
// Source A measured p99 17.23 -> 18.58 ms.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
|
||||
namespace pcache_libs {
|
||||
|
||||
inline constexpr uint64_t kRamMost = 20ull << 30; // (20 GB of RAM or less)
|
||||
inline constexpr uint64_t kGlCacheLeast = 512ull << 20; // (512 MB or more in the driver's own cache)
|
||||
|
||||
/// RX_VK_PCACHE_LIBS_ONLY: 0 off (unset too), 1 by the rule, 2 forced
|
||||
inline int mode(const char* env) {
|
||||
if (!env || !*env) return 0;
|
||||
const int v = std::atoi(env);
|
||||
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
|
||||
}
|
||||
|
||||
struct Pc {
|
||||
bool nvidia_proprietary = false; // (VK_DRIVER_ID_NVIDIA_PROPRIETARY)
|
||||
uint64_t ram = 0; // (the PC's RAM, bytes; 0 unknown)
|
||||
uint64_t glcache = 0; // (bytes in the driver's own cache folder)
|
||||
bool gpl = false; // (the stage libraries are wanted and kept in our cache: gpl_wanted, gpl_libs_cached)
|
||||
bool prepare_only = false; // (a start that only prepares)
|
||||
};
|
||||
/// Whether the fresh file is a lean one
|
||||
inline bool choose(int mode, const Pc& pc) {
|
||||
if (mode <= 0 || pc.prepare_only) return false;
|
||||
if (mode >= 2) return true;
|
||||
return pc.nvidia_proprietary && pc.gpl && pc.ram && pc.ram <= kRamMost && pc.glcache >= kGlCacheLeast;
|
||||
}
|
||||
|
||||
/// The folder NVIDIA's driver keeps its Vulkan and OpenGL cache in: on Windows <LOCALAPPDATA>\NVIDIA\GLCache; elsewhere
|
||||
/// __GL_SHADER_DISK_CACHE_PATH, else <XDG_CACHE_HOME>/nvidia/GLCache, else <HOME>/.cache/nvidia/GLCache. Empty: unknown
|
||||
inline std::string nvidia_cache_dir(bool windows, const char* localappdata, const char* gl_path, const char* xdg, const char* home) {
|
||||
if (windows) return localappdata && *localappdata ? std::string(localappdata) + "\\NVIDIA\\GLCache" : std::string();
|
||||
if (gl_path && *gl_path) return gl_path;
|
||||
if (xdg && *xdg) return std::string(xdg) + "/nvidia/GLCache";
|
||||
if (home && *home) return std::string(home) + "/.cache/nvidia/GLCache";
|
||||
return {};
|
||||
}
|
||||
/// The bytes of the files under a folder, at most most_entries looked at (a folder that is not there: 0)
|
||||
inline uint64_t folder_bytes(const std::string& dir, size_t most_entries = 200000) {
|
||||
uint64_t sum = 0;
|
||||
if (dir.empty()) return 0;
|
||||
std::error_code ec;
|
||||
const std::filesystem::path root(std::u8string(reinterpret_cast<const char8_t*>(dir.data()), dir.size())); // (UTF-8)
|
||||
std::filesystem::recursive_directory_iterator it(root, std::filesystem::directory_options::skip_permission_denied, ec);
|
||||
for (size_t n = 0; !ec && it != std::filesystem::recursive_directory_iterator() && n < most_entries; it.increment(ec), n++) {
|
||||
std::error_code e2;
|
||||
if (it->is_regular_file(e2)) {
|
||||
const uint64_t s = it->file_size(e2);
|
||||
if (!e2) sum += s;
|
||||
}
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
// ---- <file>.lean: "lean <stamp>"
|
||||
|
||||
inline bool lean_read(const std::string& path, const char* stamp16) {
|
||||
FILE* f = fopen(path.c_str(), "rb");
|
||||
if (!f) return false;
|
||||
char line[40]{};
|
||||
const size_t n = fread(line, 1, sizeof line - 1, f);
|
||||
fclose(f);
|
||||
line[n] = 0;
|
||||
return n >= 21 && !memcmp(line, "lean ", 5) && !memcmp(line + 5, stamp16, 16);
|
||||
}
|
||||
inline bool lean_write(const std::string& path, const char* stamp16) {
|
||||
FILE* f = fopen(path.c_str(), "wb");
|
||||
if (!f) return false;
|
||||
const bool ok = fprintf(f, "lean %.16s\n", stamp16) > 0;
|
||||
return fclose(f) == 0 && ok;
|
||||
}
|
||||
|
||||
/// What a start does: fresh (no file read for these shaders), whether the rule says lean now, and the record of the file
|
||||
/// read. Lean for the session: a fresh file the rule makes lean, or a file read whose record says lean (and the switch on)
|
||||
inline bool lean_now(int mode, bool fresh, bool chose, bool recorded) {
|
||||
if (mode <= 0) return false;
|
||||
return fresh ? chose : recorded;
|
||||
}
|
||||
|
||||
} // namespace pcache_libs
|
||||
@@ -22,6 +22,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace pcache_play {
|
||||
|
||||
@@ -33,6 +34,13 @@ inline bool routes(bool main_cache, bool play_cache, bool preparing) {
|
||||
return main_cache && play_cache && !preparing;
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-22, the performance audit's PIPE-95, after Source A's) RX_VK_PCACHE_OWN_PASSES: whether the back end's own
|
||||
/// compute passes (FSR 3.1's, the anti-aliasing's, the texture converters': vk_draw.cpp shared_cache_compute) are made
|
||||
/// through the play cache as the game's are, on unless set to 0 (0: bare into the main cache, as before)
|
||||
inline bool own_passes(const char* env) {
|
||||
return !env || std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
/// At the start, the play file read: whether it is folded into the main cache now (fold_bytes 0: never)
|
||||
inline bool fold_due(uint64_t play_bytes, uint64_t fold_bytes) {
|
||||
return fold_bytes && play_bytes >= fold_bytes;
|
||||
|
||||
62
gpu/vk/piece_idle.h
Normal file
62
gpu/vk/piece_idle.h
Normal file
@@ -0,0 +1,62 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-26, its first step, after Source A's) What a mirror in pieces holds that
|
||||
// no game pool covers any more, and how much of that no binding has reached for a while: counted, nothing released.
|
||||
//
|
||||
// A mirror in pieces gives a piece its memory when a binding first reaches it, and keeps it until the whole mirror goes.
|
||||
// The mirror is an import's (a chunk of guest memory around what was bound), and the game's pools inside it come and go:
|
||||
// a long-lived import keeps the memory of every piece any area reached, also where no game pool is left. Source A gives
|
||||
// such memory back after two waits of 600 submissions. Before that is ported (the unbind and the free must wait for the
|
||||
// GPU and for the bind timeline: memory freed under a compute pass that reads it lost devices before) this counts what
|
||||
// it would find, every 1200 submissions, in the "mirrors in pieces" line: under about 100 MB at the end of a tour on a
|
||||
// 6 GB card, the release is not worth having.
|
||||
//
|
||||
// Counted by piece. A release frees whole runs of pieces (one allocation a run), so this is the most it could give back.
|
||||
// The rule alone, no Vulkan: tests/test_mirror_piece_idle.py compiles it against a page-by-page model.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace piece_idle {
|
||||
|
||||
using Ranges = std::vector<std::pair<uint64_t, uint64_t>>; // ([first, second))
|
||||
|
||||
/// The ranges sorted and merged: disjoint, ascending, none empty (touching ones joined)
|
||||
inline void merge(Ranges& r) {
|
||||
std::sort(r.begin(), r.end());
|
||||
size_t m = 0;
|
||||
for (const auto& e : r) {
|
||||
if (e.second <= e.first) continue;
|
||||
if (m && e.first <= r[m - 1].second) r[m - 1].second = std::max(r[m - 1].second, e.second);
|
||||
else r[m++] = e;
|
||||
}
|
||||
r.resize(m);
|
||||
}
|
||||
|
||||
struct Count {
|
||||
uint64_t held = 0; // (bytes of the pieces that have memory)
|
||||
uint64_t uncovered = 0; // (of them, the pieces no game pool touches)
|
||||
uint64_t idle = 0; // (of those, the ones no binding reached for idle_for submissions; 0 when that is not tracked)
|
||||
};
|
||||
|
||||
/// A mirror at guest address lo of `size` bytes, in pieces of 1 << shift bytes: resident(i) says piece i has memory,
|
||||
/// last(i) the submission a binding last reached it at. pools: the game's pools, merged (merge above). now: this
|
||||
/// submission; idle_for: 0, or how many submissions without a binding make a piece idle. Adds to out
|
||||
template <class Resident, class Last>
|
||||
inline void count(uint64_t lo, uint64_t size, uint32_t shift, const Resident& resident, const Last& last, const Ranges& pools, uint64_t now,
|
||||
uint64_t idle_for, Count& out) {
|
||||
const uint64_t piece = 1ull << shift, n = (size + piece - 1) >> shift;
|
||||
size_t at = 0; // (the first pool that ends after the piece's start: the pieces come ascending)
|
||||
for (uint64_t i = 0; i < n; i++) {
|
||||
if (!resident(i)) continue;
|
||||
const uint64_t a = lo + (i << shift), b = lo + std::min(size, (i + 1) << shift);
|
||||
out.held += b - a;
|
||||
while (at < pools.size() && pools[at].second <= a) at++;
|
||||
if (at < pools.size() && pools[at].first < b) continue; // (a game pool touches it)
|
||||
out.uncovered += b - a;
|
||||
if (idle_for && now - last(i) >= idle_for) out.idle += b - a;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace piece_idle
|
||||
48
gpu/vk/pipe_ahead.h
Normal file
48
gpu/vk/pipe_ahead.h
Normal file
@@ -0,0 +1,48 @@
|
||||
// Where the pipelines the draws first need stand in the last sessions' order, against where the draws are (no game, no GPU:
|
||||
// the rules; vk_draw.cpp order_met, graphics_pipeline, draw_left_out_take). Apart from Vulkan so that tests/test_pipe_ahead.py
|
||||
// runs them.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-81, after Source B's: its first step.) Where the precompile keeps no
|
||||
// pipelines (light mode: precompile_keeps_all), every pipeline of a load is new at its first draw: linked there from its
|
||||
// stage libraries, or by the warm a moment earlier (pipe_warm.h, pipe_order.h). The entry proposes building whole pipelines
|
||||
// just ahead of the draws on light frames, from the order's next places, and keeping them; and it says to count first:
|
||||
// each pipeline first drawn, its place ahead of where the draws are (pipe_order.h Front) or none, and whether the warm had
|
||||
// linked it. Under a third within the window: stop, the builder is not worth its RAM and its compiles.
|
||||
//
|
||||
// This is that count, and only it: nothing is built or kept. A line every 600 presents while it grows (with the order used):
|
||||
// within the window ahead, further ahead, behind, not in the order, and how many of those within the window the warm had
|
||||
// linked; and whether the builder would be worth it by the entry's rule.
|
||||
//
|
||||
// RX_VK_PIPE_AHEAD=<places>: the window, 1024 unset (1 to 4096); 0: not counted.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace pipe_ahead {
|
||||
|
||||
inline constexpr int64_t kWindow = 1024;
|
||||
inline constexpr int64_t kMost = 4096;
|
||||
|
||||
/// RX_VK_PIPE_AHEAD: the window in places, kWindow unset; 0 off; clamped to 1..kMost
|
||||
inline int64_t window(const char* env) {
|
||||
if (!env || !*env) return kWindow;
|
||||
const long long v = std::atoll(env);
|
||||
return v <= 0 ? 0 : v > kMost ? kMost : (int64_t)v;
|
||||
}
|
||||
|
||||
enum class Where { None, Near, Far, Behind };
|
||||
/// A pipeline first drawn at place in the order (-1: not in it), the draws at front (Front::furthest; -1: not yet placed,
|
||||
/// counted from the order's start)
|
||||
inline Where where(int64_t place, int64_t front, int64_t window_places) {
|
||||
if (place < 0) return Where::None;
|
||||
const int64_t from = front < 0 ? 0 : front;
|
||||
if (place < from) return Where::Behind;
|
||||
return place - from <= window_places ? Where::Near : Where::Far;
|
||||
}
|
||||
/// The entry's rule for the builder: worth it when a third or more of the pipelines first drawn were within the window
|
||||
inline bool worth(int64_t within, int64_t all) {
|
||||
return all > 0 && within * 3 >= all;
|
||||
}
|
||||
|
||||
} // namespace pipe_ahead
|
||||
@@ -32,6 +32,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <iterator>
|
||||
@@ -45,7 +46,11 @@
|
||||
|
||||
namespace mkwvk::pipe_order {
|
||||
|
||||
constexpr size_t kKeysMax = program_order::kKeysMax; // (pipelines a session's order keeps, and the file)
|
||||
// ((Triforce, 11-21) 16384, was the programs' 4096: one ten-area tour needs 3918 pipelines, so an adult tour pushed half of the
|
||||
// child tour's out and its next tour linked 828 of 2274 ahead where it had linked 1455 of 2012 (pipeline stalls 1.3 -> 2.6 s,
|
||||
// the worst 174 -> 464 ms). The file only: what the warm holds stays bounded by kLinksMax and kLibsMax below, and the programs'
|
||||
// read-ahead (its RAM) by program_order's own 4096. 8 bytes a pipeline: 128 KB at most)
|
||||
constexpr size_t kKeysMax = 16384; // (pipelines a session's order keeps, and the file)
|
||||
constexpr size_t kPerVariant = 32; // (the order's pipelines made for a program variant, at most: as the notes')
|
||||
constexpr size_t kLinksMax = 1024; // (links held for draws that have not come: a load's, beside the title's taken)
|
||||
constexpr size_t kLibsMax = 2048; // (stage libraries the warm made that no draw has had: two a program variant of 1024)
|
||||
@@ -58,9 +63,13 @@ inline bool on(const char* env) { return !(env && env[0] == '0' && !env[1]); }
|
||||
|
||||
/// The file: program_order.h's form, its own magic
|
||||
inline std::vector<uint8_t> write(const std::vector<uint64_t>& keys) { return program_order::write(keys, kMagic); }
|
||||
inline std::vector<uint64_t> read(const uint8_t* data, size_t n) { return program_order::read(data, n, kMagic); }
|
||||
using program_order::merge;
|
||||
using Record = program_order::Record;
|
||||
inline std::vector<uint64_t> read(const uint8_t* data, size_t n) { return program_order::read(data, n, kMagic, kKeysMax); }
|
||||
inline std::vector<uint64_t> merge(const std::vector<uint64_t>& now, const std::vector<uint64_t>& before, size_t max = kKeysMax) {
|
||||
return program_order::merge(now, before, max);
|
||||
}
|
||||
struct Record : program_order::Record {
|
||||
Record() { max = kKeysMax; }
|
||||
};
|
||||
|
||||
/// One pipeline of the order: its place, its note's record in the notes file
|
||||
struct Entry {
|
||||
@@ -154,12 +163,19 @@ public:
|
||||
by_place_.emplace(std::make_pair(place, key), o);
|
||||
place_of_.emplace(key, place);
|
||||
while (by_place_.size() > max_) {
|
||||
const auto last = std::prev(by_place_.end());
|
||||
gone.push_back(last->second);
|
||||
place_of_.erase(last->first.second);
|
||||
by_place_.erase(last);
|
||||
// ((Triforce, 11-21, soft) one behind the draws first (the farthest behind), then the farthest ahead as before
|
||||
const bool back = soft_ && by_place_.begin()->first.first < behind_at_;
|
||||
const auto out = back ? by_place_.begin() : std::prev(by_place_.end());
|
||||
gone.push_back(out->second);
|
||||
place_of_.erase(out->first.second);
|
||||
by_place_.erase(out);
|
||||
}
|
||||
}
|
||||
/// ((Triforce, 11-21) RX_VK_ORDER_SOFT_BEHIND) Soft: behind() lets nothing go, it marks where behind begins; those held there
|
||||
/// stay for a draw that comes back to them (a tour's shared pipelines sit in another session's region of the merged order,
|
||||
/// and the draws went back and forth: 1265 of 3048 links let go as behind in a child tour after an adult one) and go first
|
||||
/// when a new link needs the room. The bound (max) is the same
|
||||
void soft(bool on) { soft_ = on; }
|
||||
/// The link held for key, taken out (a draw's from now on): false when none is
|
||||
bool take(uint64_t key, One& out) {
|
||||
const auto it = place_of_.find(key);
|
||||
@@ -172,6 +188,10 @@ public:
|
||||
}
|
||||
/// Those placed before at, taken out into gone
|
||||
void behind(int64_t at, std::vector<One>& gone) {
|
||||
if (soft_) {
|
||||
behind_at_ = std::max(behind_at_, at);
|
||||
return;
|
||||
}
|
||||
const auto end = by_place_.lower_bound({at, 0});
|
||||
for (auto it = by_place_.begin(); it != end;) {
|
||||
gone.push_back(it->second);
|
||||
@@ -193,7 +213,8 @@ public:
|
||||
}
|
||||
/// Whether a link at place would stay held: room left, or nearer than the farthest held (made, not let go at once)
|
||||
bool room(int64_t place) const {
|
||||
return by_place_.size() < max_ || (!by_place_.empty() && place < std::prev(by_place_.end())->first.first);
|
||||
return by_place_.size() < max_ || (!by_place_.empty() && place < std::prev(by_place_.end())->first.first) ||
|
||||
(soft_ && !by_place_.empty() && by_place_.begin()->first.first < behind_at_);
|
||||
}
|
||||
/// Whether a link held was linked from library lib
|
||||
bool uses(uint64_t lib) const {
|
||||
@@ -216,6 +237,8 @@ private:
|
||||
return false;
|
||||
}
|
||||
size_t max_;
|
||||
bool soft_ = false;
|
||||
int64_t behind_at_ = INT64_MIN;
|
||||
std::map<std::pair<int64_t, uint64_t>, One> by_place_; // ((place, key): the nearest first)
|
||||
std::unordered_map<uint64_t, int64_t> place_of_;
|
||||
};
|
||||
|
||||
@@ -39,6 +39,13 @@
|
||||
// libraries after it, so the draws of such a pipeline waited for the whole build. The same objects are made either way.
|
||||
// A draw that waits runs the first job itself (vk_jobs.cpp jobs_wait): the libraries and the link, not the whole build.
|
||||
// RX_VK_PIPE_FAST_WORKERS=0: the whole build first, as before.
|
||||
// - choose(), depth_new ((Triforce, 11-22, the performance audit's PIPE-29, after Source B's): a depth-tested draw into a
|
||||
// pass drawn once whose shaders write no memory, outside a samples-passed count and the item icon capture's kept targets):
|
||||
// from the frame's wait pool, then left out, and a pipeline that ran out of time is left out at once until it is built
|
||||
// (vk_draw.cpp). Source B froze 11-13 s twice on an Arc B580 compiling some fifty new pipelines for such passes, and made
|
||||
// them wait at most 50 ms a frame. RX_VK_PIPE_DEPTH_BUDGET=1; off by default on every driver, Intel's too, where the
|
||||
// entry would have it on: a render into a new depth target that is left out can stay missing until it is drawn again,
|
||||
// and that trade is the owner's to make (depth_budget).
|
||||
// A wait from a budget is vk_jobs.cpp jobs_wait_for: the pipeline's job goes to the front of the queue, and is not run on the
|
||||
// draw's thread (a compile there would outlast the bound).
|
||||
#pragma once
|
||||
@@ -46,6 +53,7 @@
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace pipe_wait {
|
||||
|
||||
@@ -139,7 +147,12 @@ private:
|
||||
// ---- the draw's choice
|
||||
|
||||
enum class Act { Skip, Wait, WaitAll };
|
||||
enum Kind { K_NONE, K_ONESHOT, K_FULL, K_LAST };
|
||||
enum Kind { K_NONE, K_ONESHOT, K_FULL, K_LAST, K_DEPTH };
|
||||
|
||||
/// ((Triforce, 11-22, PIPE-29) RX_VK_PIPE_DEPTH_BUDGET: on when set to anything but 0; unset, off on every driver)
|
||||
inline bool depth_budget(const char* env) {
|
||||
return env && *env && std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
struct Rules {
|
||||
bool always_wait = false; // (RX_VK_PIPE_SKIP=0: every pipeline waited for)
|
||||
@@ -155,9 +168,16 @@ struct Choice {
|
||||
|
||||
/// A draw whose pipeline is not ready. kept: its pass (kept()); writes: its shaders write memory; full: full_screen(); held:
|
||||
/// the hold of the last picture would take this frame if it lacks a draw (present_hold.h DEFERS, the pipelines counted);
|
||||
/// pool_ns, last_ns: what the frame's wait pool and its last wait have left
|
||||
inline Choice choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns) {
|
||||
/// pool_ns, last_ns: what the frame's wait pool and its last wait have left. depth_new ((Triforce, 11-22, PIPE-29)
|
||||
/// RX_VK_PIPE_DEPTH_BUDGET, the caller's: a depth-tested draw into a pass drawn once whose shaders write no memory): from the
|
||||
/// pool, then left out
|
||||
inline Choice choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns,
|
||||
bool depth_new = false) {
|
||||
if (r.always_wait) return {Act::WaitAll, 0, K_NONE};
|
||||
if (!kept && !writes && depth_new) {
|
||||
if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_DEPTH};
|
||||
return {Act::Skip, 0, K_NONE};
|
||||
}
|
||||
if (!kept || writes) {
|
||||
if (!r.oneshot_budget) return {Act::WaitAll, 0, K_NONE};
|
||||
if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_ONESHOT};
|
||||
|
||||
92
gpu/vk/precompile_save.h
Normal file
92
gpu/vk/precompile_save.h
Normal file
@@ -0,0 +1,92 @@
|
||||
// The pipeline cache saved before the title after a long precompile, and after a preparation without stage libraries
|
||||
// (no game, no GPU: a rule; vk_draw.cpp precompile_run, prepare_run, prepare_compute_pipeline, pipeline_cache_save).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-23, after Source A's.) A precompile that keeps every pipeline (enough
|
||||
// RAM: precompile_keeps_all) ended without a save: what it compiled was first written in play, at present 300 and only
|
||||
// after 32 more builds, and on Intel only at a clean close. A crash, a kill or a power cut before that cost the whole
|
||||
// build again at the next start, minutes of it when the driver's own cache did not have them either. Source A wrote its
|
||||
// cache before the title once the precompile had built for 30 s or more (after a 1,454 s compile on a handheld: 439 MB
|
||||
// in 821 ms, and the next start's precompile took 13 s).
|
||||
//
|
||||
// Here the same, with two differences that are this project's:
|
||||
// - the save is the play file's alone while there is a play cache (pcache_play.h): the precompile's compiles went into
|
||||
// it. The main cache's hits count as builds too (cached_create), and on NVIDIA the main cache grows by the driver's
|
||||
// copies, so an ordinary forced save would write the whole main file, 2 to 3 GB, before the title. With no play
|
||||
// cache (a start that folded the play file, or no pipelineCreationCacheControl) the main file is what there is, and
|
||||
// it is saved;
|
||||
// - not where something else saves or the workers still build: a precompile the game went on from (its workers build
|
||||
// behind the game: no fetch while they do, Intel's driver crashed there), a stopped one (the stop's own save), a
|
||||
// start that only prepares (its end saves), and the light precompile (it saves before its record, as before).
|
||||
//
|
||||
// The second gap: a preparation without stage libraries compiled its compute pipelines into the main cache, counted
|
||||
// them nowhere the save looks (pcache_main_built) and ended without a save, where the one with libraries saves after
|
||||
// them (warm_libraries). Now they are counted, and that preparation saves before its record says it is made.
|
||||
//
|
||||
// RX_VK_PRECOMPILE_SAVE_S=<seconds>: the build time from which the precompile's end saves, 30 by default, at most a day;
|
||||
// 0: never, and the preparation's count and save are off too, all as before.
|
||||
#pragma once
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mkwvk::precompile_save {
|
||||
|
||||
inline constexpr int kAfterS = 30; // Source A's
|
||||
inline constexpr int kMostS = 86400;
|
||||
|
||||
/// RX_VK_PRECOMPILE_SAVE_S: the seconds of building from which the precompile's end saves (0: never)
|
||||
inline int after_s(const char* env) {
|
||||
if (!env || !*env) return kAfterS;
|
||||
const long v = std::atol(env);
|
||||
return v <= 0 ? 0 : v > kMostS ? kMostS : (int)v;
|
||||
}
|
||||
|
||||
/// Whether the rule is on at all (the preparation's count and its save go by this too)
|
||||
constexpr bool on(int save_s) {
|
||||
return save_s > 0;
|
||||
}
|
||||
|
||||
/// Whether a precompile that built for build_s seconds is a long one: only those say anything
|
||||
constexpr bool long_build(double build_s, int save_s) {
|
||||
return save_s > 0 && build_s >= (double)save_s;
|
||||
}
|
||||
|
||||
/// Whether the keep-all precompile's end saves now. gone_on: the game went on while workers still build; stopped: a stop
|
||||
/// was asked; prepare_only: a start that only prepares
|
||||
constexpr bool due(double build_s, int save_s, bool gone_on, bool stopped, bool prepare_only) {
|
||||
return long_build(build_s, save_s) && !gone_on && !stopped && !prepare_only;
|
||||
}
|
||||
|
||||
enum class Outcome {
|
||||
Saved, // (written: the line gives its size and time)
|
||||
Nothing, // (the cache holds nothing its files lack)
|
||||
GoneOn,
|
||||
Stopped,
|
||||
PrepareOnly,
|
||||
NoCache, // (no pipeline cache this session)
|
||||
};
|
||||
|
||||
/// Why a long precompile's end does not save (only asked when due() said no, or with no cache)
|
||||
constexpr Outcome why_not(bool gone_on, bool stopped, bool prepare_only, bool cache) {
|
||||
return !cache ? Outcome::NoCache : stopped ? Outcome::Stopped : gone_on ? Outcome::GoneOn :
|
||||
prepare_only ? Outcome::PrepareOnly : Outcome::Nothing;
|
||||
}
|
||||
|
||||
/// The log line's words after "vk: precompile: N s of building: "
|
||||
constexpr const char* said(Outcome o) {
|
||||
switch (o) {
|
||||
case Outcome::Saved: return "the pipeline cache saved before the title";
|
||||
case Outcome::Nothing: return "the pipeline cache holds nothing new to save before the title";
|
||||
case Outcome::GoneOn: return "the pipeline cache is not saved before the title: the game went on and the workers still build";
|
||||
case Outcome::Stopped: return "the pipeline cache is not saved here: the stop's own save writes it";
|
||||
case Outcome::PrepareOnly: return "the pipeline cache is not saved here: this start only prepares and saves at its end";
|
||||
case Outcome::NoCache: return "there is no pipeline cache to save";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/// Whether a preparation without stage libraries saves the cache before its record is written: the rule on, not behind
|
||||
/// the game, and compute pipelines built into the main cache
|
||||
constexpr bool prepare_saves(int save_s, bool libraries_warmed, bool background, unsigned long long computes_built) {
|
||||
return on(save_s) && !libraries_warmed && !background && computes_built > 0;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::precompile_save
|
||||
211
gpu/vk/predict_check.h
Normal file
211
gpu/vk/predict_check.h
Normal file
@@ -0,0 +1,211 @@
|
||||
// Whether the driver still has the predicted pipelines, at a start whose record says they were all built (no game, no
|
||||
// GPU: the rules; vk_draw.cpp predict_check_cold, predict_background, predicted_play_build). After lib_check.h, which
|
||||
// asks the same of the stage libraries.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-68, after Source B's and Source A's.) The predicted pipelines (the ones
|
||||
// the notes predict for programs no note names) are built once for the driver's own cache and let go, and a record
|
||||
// (<notes>.predicted) says which programs are done; once it says all, no start looks again. But the driver's cache is
|
||||
// the driver's: NVIDIA's has a size cap, AMD's Windows driver kept 30 to 44% of what Source B had prepared, and a cache
|
||||
// folder can be cleared. Source B found 105 dispatches waiting 115 s in all on prepared pipelines the driver had dropped,
|
||||
// while its record still said built. Then each of those programs' first draw compiles in play: the freezes on arrival
|
||||
// the prediction is there to take away.
|
||||
//
|
||||
// The check. At a start whose record says all built, on the builder's idle thread after its delay and, as the builder,
|
||||
// only while the frame shown is a light one: up to kSample predicted pipelines from the plan, spread over it from
|
||||
// another place each start, each built for the driver's cache alone and timed, then built once more at once. The second
|
||||
// comes from the driver's memory by construction, so it is this driver's warm time on this PC and no fixed time is
|
||||
// needed: a first build over kRatio times the second and kSlackMs more was compiled. Not asked with
|
||||
// FAIL_ON_PIPELINE_COMPILE_REQUIRED: under that flag a driver answers only from the cache it is handed, and these are in
|
||||
// none of ours. The sample ends once its majority is decided.
|
||||
//
|
||||
// Cold (more than half of a sample): the record is removed, so the builder builds them all again behind the game at
|
||||
// light frames, this session and the next, and <notes>.predicted.check takes a strike for this stamp. Cold again at a
|
||||
// start after they were built again: this driver does not keep them. That is said once and recorded, and no later start
|
||||
// checks for this stamp (the compiles of the check itself are not free), so the pipelines are left to their first draw
|
||||
// as before the prediction. Warm: the strike is forgotten. Another stamp (a new driver, new shaders) begins anew.
|
||||
//
|
||||
// The counter, for what the check is worth: builds in play of programs the plan predicts that took over 20 ms, which a
|
||||
// driver that had them does not need. Logged at most every 10 s while it grows.
|
||||
//
|
||||
// RX_VK_PREDICT_CHECK: 1 by default; 0: no check and no counter, as before; 2: checked at every start, a recorded
|
||||
// "does not keep them" too (the lab's).
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace predict_check {
|
||||
|
||||
inline constexpr int kSample = 12; // (pipelines sampled, as lib_check's)
|
||||
inline constexpr int kLook = 2 * kSample; // (plan entries looked at for them: one may have lost its notes or its program)
|
||||
inline constexpr double kRatio = 4.0; // (the first build against the second: compiled above)
|
||||
inline constexpr double kSlackMs = 3.0; // (and this much more: lookups of a few milliseconds say nothing)
|
||||
inline constexpr int kStrikes = 1; // (the record removed once; cold again after that: stop)
|
||||
inline constexpr int64_t kSlowUs = 20000; // (a play build of a predicted program over this was compiled)
|
||||
inline constexpr int64_t kSayMs = 10000; // (the counter's line: at most this often)
|
||||
|
||||
/// RX_VK_PREDICT_CHECK: 0 off, 1 on (unset too), 2 every start
|
||||
inline int mode(const char* env) {
|
||||
if (!env || !*env) return 1;
|
||||
const int v = std::atoi(env);
|
||||
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
|
||||
}
|
||||
|
||||
/// One pipeline sampled: made both times, and what each took (ms). One the driver refuses says nothing of its cache
|
||||
struct Probe {
|
||||
bool made = false;
|
||||
double first_ms = 0, again_ms = 0;
|
||||
};
|
||||
inline bool cold(const Probe& p) {
|
||||
return p.made && p.first_ms > kRatio * p.again_ms + kSlackMs;
|
||||
}
|
||||
|
||||
struct Tally {
|
||||
int got = 0, cold = 0;
|
||||
double first_median = 0, again_median = 0;
|
||||
};
|
||||
inline double median(std::vector<double> v) {
|
||||
if (v.empty()) return 0;
|
||||
std::sort(v.begin(), v.end());
|
||||
return v[v.size() / 2];
|
||||
}
|
||||
inline Tally tally(const std::vector<Probe>& probes) {
|
||||
Tally t;
|
||||
std::vector<double> first, again;
|
||||
for (const Probe& p : probes) {
|
||||
if (!p.made) continue;
|
||||
t.got++, t.cold += cold(p);
|
||||
first.push_back(p.first_ms), again.push_back(p.again_ms);
|
||||
}
|
||||
t.first_median = median(first), t.again_median = median(again);
|
||||
return t;
|
||||
}
|
||||
|
||||
enum class Verdict { Unknown, Warm, Cold };
|
||||
/// Cold when more than half of the sample is; under half a sample says nothing
|
||||
inline Verdict verdict(const Tally& t) {
|
||||
if (t.got * 2 < kSample) return Verdict::Unknown;
|
||||
return t.cold * 2 > t.got ? Verdict::Cold : Verdict::Warm;
|
||||
}
|
||||
/// Whether the sample needs no more probes: it is full, or what is still to come cannot change its verdict
|
||||
inline bool decided(const Tally& t) {
|
||||
return t.got >= kSample || t.cold * 2 > kSample || (t.got - t.cold) * 2 >= kSample;
|
||||
}
|
||||
|
||||
/// k places spread over n, the first at seed's place in the first step: each start samples other pipelines
|
||||
inline std::vector<size_t> spots(size_t n, size_t k, uint64_t seed) {
|
||||
std::vector<size_t> at;
|
||||
if (!n || !k) return at;
|
||||
const size_t step = std::max<size_t>(1, n / k);
|
||||
for (size_t i = (size_t)(seed % step); i < n && at.size() < k; i += step) at.push_back(i);
|
||||
return at;
|
||||
}
|
||||
|
||||
/// The records of some notes in the notes file f, read from its start: a head of {stamp, rec_bytes}, then records of
|
||||
/// rec_bytes whose first 8 bytes are the key (0: the first after the head). Only the keys are read, each record's rest
|
||||
/// stepped over, and the walk ends once every wanted key is found. A key twice keeps its first record. Empty when the
|
||||
/// head is another build's
|
||||
inline std::unordered_map<uint64_t, uint32_t> find(FILE* f, uint64_t stamp, uint64_t rec_bytes, const std::vector<uint64_t>& keys) {
|
||||
std::unordered_map<uint64_t, uint32_t> at;
|
||||
uint64_t head[2] = {0, 0};
|
||||
if (!f || rec_bytes < 8 || rec_bytes > (1u << 30) || fread(head, 8, 2, f) != 2 || head[0] != stamp || head[1] != rec_bytes) return at;
|
||||
std::unordered_map<uint64_t, bool> want;
|
||||
for (const uint64_t k : keys) want.emplace(k, false);
|
||||
size_t left = want.size();
|
||||
uint64_t key = 0;
|
||||
for (uint32_t i = 0; left && fread(&key, 8, 1, f) == 1; i++) {
|
||||
const auto w = want.find(key);
|
||||
if (w != want.end() && !w->second) w->second = true, at.emplace(key, i), left--;
|
||||
if (fseek(f, (long)(rec_bytes - 8), SEEK_CUR) != 0) break;
|
||||
}
|
||||
return at;
|
||||
}
|
||||
|
||||
// ---- <notes>.predicted.check: "predict-check <stamp> <strikes>"
|
||||
|
||||
struct State {
|
||||
bool found = false;
|
||||
uint64_t stamp = 0;
|
||||
int strikes = 0;
|
||||
};
|
||||
inline State state_read(const std::string& path) {
|
||||
State s;
|
||||
FILE* f = fopen(path.c_str(), "rb");
|
||||
if (!f) return s;
|
||||
char line[96]{};
|
||||
const size_t n = fread(line, 1, sizeof line - 1, f);
|
||||
fclose(f);
|
||||
line[n] = 0;
|
||||
unsigned long long stamp = 0;
|
||||
int strikes = 0;
|
||||
if (sscanf(line, "predict-check %16llx %d", &stamp, &strikes) == 2 && strikes >= 0)
|
||||
s.found = true, s.stamp = stamp, s.strikes = strikes;
|
||||
return s;
|
||||
}
|
||||
/// Written whole beside the old one, then put in place (a start that ends in the middle leaves the old file)
|
||||
inline bool state_write(const std::string& path, uint64_t stamp, int strikes) {
|
||||
char line[96];
|
||||
const int n = snprintf(line, sizeof line, "predict-check %016llx %d\n", (unsigned long long)stamp, strikes);
|
||||
const std::string tmp = path + ".tmp";
|
||||
FILE* f = fopen(tmp.c_str(), "wb");
|
||||
if (!f) return false;
|
||||
bool ok = fwrite(line, 1, (size_t)n, f) == (size_t)n;
|
||||
ok = fclose(f) == 0 && ok;
|
||||
if (ok) {
|
||||
remove(path.c_str()); // (rename does not replace on Windows)
|
||||
ok = rename(tmp.c_str(), path.c_str()) == 0;
|
||||
}
|
||||
if (!ok) remove(tmp.c_str());
|
||||
return ok;
|
||||
}
|
||||
|
||||
/// Whether the record says this driver does not keep them, for this stamp: cold again after they were built again
|
||||
inline bool stopped(const State& had, uint64_t stamp) {
|
||||
return had.found && had.stamp == stamp && had.strikes > kStrikes;
|
||||
}
|
||||
/// Whether a start checks at all: the switch, and (but for mode 2) not once the record says stop for this stamp
|
||||
inline bool runs(int mode, const State& had, uint64_t stamp) {
|
||||
return mode >= 2 || (mode == 1 && !stopped(had, stamp));
|
||||
}
|
||||
|
||||
enum class Do { Nothing, Again, Stop };
|
||||
/// What a start does with a sample's verdict, the record of the last starts and the stamp now. *next: the strikes to
|
||||
/// record (Again and Stop). Again: the record of what is built is removed, and the builder builds them again
|
||||
inline Do decide(Verdict v, const State& had, uint64_t stamp, int* next) {
|
||||
if (v != Verdict::Cold) return Do::Nothing;
|
||||
const bool same = had.found && had.stamp == stamp; // (found cold for this stamp before, and built again since)
|
||||
*next = same ? had.strikes + 1 : 1;
|
||||
return same && had.strikes >= kStrikes ? Do::Stop : Do::Again;
|
||||
}
|
||||
|
||||
// ---- the counter: play builds of predicted programs the driver compiled
|
||||
|
||||
/// Whether a build of that many microseconds was compiled rather than found
|
||||
inline bool slow(int64_t us) {
|
||||
return us > kSlowUs;
|
||||
}
|
||||
/// Whether the counter's line is due: said_ms, when it was last written (0: never)
|
||||
inline bool say(int64_t now_ms, int64_t said_ms) {
|
||||
return !said_ms || now_ms - said_ms >= kSayMs;
|
||||
}
|
||||
/// The plan's programs as a sorted list without repeats, for has() (Plan: entries with .prog)
|
||||
template <class Plan>
|
||||
std::vector<uint64_t> programs(const std::vector<Plan>& plan) {
|
||||
std::vector<uint64_t> keys;
|
||||
keys.reserve(plan.size());
|
||||
for (const Plan& p : plan) keys.push_back(p.prog);
|
||||
std::sort(keys.begin(), keys.end());
|
||||
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
|
||||
return keys;
|
||||
}
|
||||
inline bool has(const std::vector<uint64_t>& sorted, uint64_t prog) {
|
||||
return std::binary_search(sorted.begin(), sorted.end(), prog);
|
||||
}
|
||||
|
||||
} // namespace predict_check
|
||||
74
gpu/vk/predict_stream.h
Normal file
74
gpu/vk/predict_stream.h
Normal file
@@ -0,0 +1,74 @@
|
||||
// (Triforce, 11-21, the performance audit's PIPE-83, after Source B's) The predicted builder's plan as record numbers, its
|
||||
// notes read when their program's turn comes.
|
||||
//
|
||||
// predict_background (vk_draw.cpp) builds, behind the game, the predicted pipelines the start did not get to: on the starts
|
||||
// after a translator or driver update that is most of them, and it rests while the game is in play, so it runs for much of
|
||||
// a session. It read every note of the notes file (a PipeNote is ~27 KB), made a full note of every predicted pipeline, and
|
||||
// held both until it ended: about 1 GB with the game's 14,535 notes and 24,182 predicted pipelines. The draws, the
|
||||
// read-ahead and the warm read a note by its record already (pipe_note_at). Here: the notes file is gone through once into
|
||||
// one record's buffer for each key's record number, the plan becomes {variant, the vertex note's record, the output note's
|
||||
// record} for each program still to build, 16 bytes a pipeline, and a worker reads the two notes through its own handle
|
||||
// when it builds that pipeline. The same programs and pipelines; a note gone since drops its pipeline, as before.
|
||||
// RX_VK_PREDICT_BG_STREAM=0: every note and predicted note held, as before.
|
||||
//
|
||||
// No Vulkan and no PipeNote here: tests/test_predict_stream.py compiles it with a file of its own.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace predict_stream {
|
||||
|
||||
/// One predicted pipeline: its program's variant, and the records of the note whose vertex layout and of the note whose
|
||||
/// targets and state it takes
|
||||
struct Entry {
|
||||
uint64_t variant;
|
||||
uint32_t vertex_rec, output_rec;
|
||||
};
|
||||
/// A program still to build and its pipelines, in the plan's order
|
||||
struct Program {
|
||||
uint64_t prog;
|
||||
std::vector<Entry> entries;
|
||||
};
|
||||
|
||||
/// Each note's key to its record (0: the first after the head), of the notes file f at its start: a head of {stamp,
|
||||
/// rec_bytes}, then records of rec_bytes whose first 8 bytes are the key. rec: a buffer of rec_bytes each is read into;
|
||||
/// sane(): whether the one in it is used (it may mend it). A key twice keeps its first record, as pipe_notes_read. Empty
|
||||
/// when the head is another build's
|
||||
template <class Sane>
|
||||
std::unordered_map<uint64_t, uint32_t> index(FILE* f, uint64_t stamp, uint64_t rec_bytes, void* rec, Sane&& sane) {
|
||||
std::unordered_map<uint64_t, uint32_t> at;
|
||||
uint64_t head[2] = {0, 0};
|
||||
if (!f || rec_bytes < 8 || fread(head, 8, 2, f) != 2 || head[0] != stamp || head[1] != rec_bytes) return at;
|
||||
for (uint32_t i = 0; fread(rec, (size_t)rec_bytes, 1, f) == 1; i++) {
|
||||
if (!sane()) continue;
|
||||
uint64_t key;
|
||||
std::memcpy(&key, rec, 8);
|
||||
at.emplace(key, i);
|
||||
}
|
||||
return at;
|
||||
}
|
||||
|
||||
/// The plan (entries with prog, variant, vertex_note, output_note: PredictPlan) as the programs still to build: those not
|
||||
/// in had, in the order the plan first names them, each with its pipelines in the plan's order. A pipeline one of whose
|
||||
/// notes is not in the index is dropped, and a program left with none is not listed
|
||||
template <class Plan>
|
||||
std::vector<Program> group(const std::vector<Plan>& plan, const std::unordered_map<uint64_t, uint32_t>& at,
|
||||
const std::unordered_set<uint64_t>& had) {
|
||||
std::vector<Program> out;
|
||||
std::unordered_map<uint64_t, size_t> where;
|
||||
for (const Plan& p : plan) {
|
||||
if (had.count(p.prog)) continue;
|
||||
const auto v = at.find(p.vertex_note), o = at.find(p.output_note);
|
||||
if (v == at.end() || o == at.end()) continue;
|
||||
const auto [it, fresh] = where.try_emplace(p.prog, out.size());
|
||||
if (fresh) out.push_back(Program{p.prog, {}});
|
||||
out[it->second].entries.push_back(Entry{p.variant, v->second, o->second});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace predict_stream
|
||||
96
gpu/vk/predict_types.h
Normal file
96
gpu/vk/predict_types.h
Normal file
@@ -0,0 +1,96 @@
|
||||
// The prediction's checks (no game, no GPU: vk_predict.cpp predict_pipelines and vk_draw.cpp predict_run,
|
||||
// predict_background): whether a predicted vertex layout can feed a program's vertex inputs, and the switch.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-44, after Source A's.) The predicted pipelines take a program no note names
|
||||
// and give it the vertex layout of notes of the same vertex module, or else the layout most programs with its vertex inputs
|
||||
// had. A program's key already says which of its inputs it reads as integers (vk_shader.cpp graphics_key), so a layout
|
||||
// borrowed from another program can hand an integer input a float attribute, or the other signedness: such a pipeline is
|
||||
// invalid (VUID-VkGraphicsPipelineCreateInfo-Input-08733), built for nothing or worse. Source A reads each vertex input's
|
||||
// base type from the module's SPIR-V and drops a predicted layout whose attribute types definitely differ: 8,209 of 64,797
|
||||
// predicted pipelines on its PC, none by colour. Scaled formats count as integers here (the translator reads them so).
|
||||
//
|
||||
// And a program's predictions can give the same pipeline twice (two layouts that differ only in what is dynamic, a stride):
|
||||
// Source A found 420 built twice. Built once each now, by the pipeline's key, as play keys them: the same pipelines.
|
||||
//
|
||||
// RX_VK_PREDICT_TYPES: unset, each pipeline built once (the same pipelines as before, fewer builds); 1, that and the type
|
||||
// check, which drops pipelines (off by default: it changes which pipelines exist, and a plan is made once for the shaders,
|
||||
// so it applies to a plan made after it is set); 0, as before.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace predict_types {
|
||||
|
||||
/// RX_VK_PREDICT_TYPES: 0 as before, 1 (unset) built once, 2 built once and the type check
|
||||
inline int mode(const char* env) {
|
||||
if (!env || !*env) return 1;
|
||||
return std::atoi(env) == 0 ? 0 : std::atoi(env) == 1 ? 2 : 1;
|
||||
}
|
||||
inline bool once(int mode) { return mode >= 1; }
|
||||
inline bool typed(int mode) { return mode >= 2; }
|
||||
|
||||
/// A module's vertex inputs by location (under 32): read, read as an integer, read as a signed integer
|
||||
struct Locations {
|
||||
uint32_t mask = 0, ints = 0, sign = 0;
|
||||
};
|
||||
|
||||
/// The Input (storage class 1) variables with a Location in a SPIR-V module, and each one's base type: through its
|
||||
/// pointer type to a vector's or an array's element, an OpTypeInt (signedness) or anything else (float). Built-ins carry
|
||||
/// no Location and are not counted
|
||||
inline Locations locations(const std::vector<uint32_t>& w) {
|
||||
std::unordered_map<uint32_t, uint32_t> loc, pointee, element;
|
||||
std::unordered_map<uint32_t, int> int_sign; // (OpTypeInt id: 1 signed, 0 unsigned)
|
||||
std::vector<std::pair<uint32_t, uint32_t>> inputs; // (variable id, its pointer type)
|
||||
for (size_t i = 5; i < w.size();) {
|
||||
const uint32_t op = w[i] & 0xFFFF, n = w[i] >> 16;
|
||||
if (!n || i + n > w.size()) break;
|
||||
if (op == 71 && n >= 4 && w[i + 2] == 30) loc[w[i + 1]] = w[i + 3]; // OpDecorate Location
|
||||
else if (op == 59 && n >= 4 && w[i + 3] == 1) inputs.push_back({w[i + 2], w[i + 1]}); // OpVariable Input
|
||||
else if (op == 32 && n >= 4) pointee[w[i + 1]] = w[i + 3]; // OpTypePointer
|
||||
else if ((op == 23 || op == 28) && n >= 4) element[w[i + 1]] = w[i + 2]; // OpTypeVector, OpTypeArray
|
||||
else if (op == 21 && n >= 4) int_sign[w[i + 1]] = w[i + 3] ? 1 : 0; // OpTypeInt
|
||||
i += n;
|
||||
}
|
||||
Locations out;
|
||||
for (const auto& [var, ptr] : inputs) {
|
||||
const auto l = loc.find(var);
|
||||
if (l == loc.end() || l->second >= 32) continue;
|
||||
const auto p = pointee.find(ptr);
|
||||
if (p == pointee.end()) continue;
|
||||
uint32_t t = p->second;
|
||||
for (int depth = 0; depth < 4; depth++) { // (a vector of, an array of vectors of)
|
||||
const auto e = element.find(t);
|
||||
if (e == element.end()) break;
|
||||
t = e->second;
|
||||
}
|
||||
const uint32_t bit = 1u << l->second;
|
||||
out.mask |= bit;
|
||||
if (const auto s = int_sign.find(t); s != int_sign.end()) {
|
||||
out.ints |= bit;
|
||||
if (s->second) out.sign |= bit;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// What an attribute of the layout hands its location: nothing known (unset: it reads zeros, as play's), a float, an
|
||||
/// unsigned or a signed integer (scaled formats are read as integers)
|
||||
enum class Kind : uint8_t { Unknown, Float, Uint, Sint };
|
||||
|
||||
/// Whether a layout's attributes (kinds by location) can feed the module's inputs: false only where a read location's
|
||||
/// attribute is definitely of the other kind (integer against float, or the other signedness)
|
||||
inline bool fits(const Locations& in, const Kind kinds[16]) {
|
||||
for (int i = 0; i < 16; i++) {
|
||||
if (!(in.mask >> i & 1) || kinds[i] == Kind::Unknown) continue;
|
||||
const bool want_int = in.ints >> i & 1, want_sign = in.sign >> i & 1;
|
||||
if ((kinds[i] != Kind::Float) != want_int) return false;
|
||||
if (want_int && (kinds[i] == Kind::Sint) != want_sign) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace predict_types
|
||||
79
gpu/vk/prepare_again.h
Normal file
79
gpu/vk/prepare_again.h
Normal file
@@ -0,0 +1,79 @@
|
||||
// (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Two ways a start took its shaders for prepared when they were not, apart
|
||||
// from Vulkan so that tests/test_prepare_again.py runs them (vk_draw.cpp prepare_run).
|
||||
//
|
||||
// The program cache made again for another GPU. triforce_prepared.bin says the shaders were translated for this stamp
|
||||
// (program_cache_stamp: the translator and what of the GPU and its driver changes a translation). A start on another GPU or
|
||||
// with other settings that did not prepare (or did not rewrite the record) makes triforce_programs.bin again for its own
|
||||
// stamp: back here the record still matches, prepare_check says nothing to do, the program cache opens with none of these
|
||||
// shaders (vk_shader.cpp disk_init throws the other stamp's away), and every program is translated in play. So when the
|
||||
// record says nothing to do, the program cache's head (kDiskMagic, its stamp) and size are read: a cache of another stamp, or
|
||||
// of its head alone, holds none of them, and the shaders are prepared again. A cache carried over an update keeps the stamp in
|
||||
// its head (its second word, the build that wrote it, is another: program_carry.h) and passes.
|
||||
//
|
||||
// The mark of the driver's part without stage libraries, <prepared>.warming: "this step began and did not end", an empty file.
|
||||
// A start that found it left the part out for good, on whatever GPU and driver came next. Now it holds what the part is made
|
||||
// for (vk_draw.cpp libraries_stamp: the GPU, the driver, the programs), as the libraries' journal does (gpl_rules.h
|
||||
// journal_left): a mark of another stamp is another driver's, and the part goes on. An empty one (an older build's) is this
|
||||
// driver's, as before.
|
||||
// RX_VK_PREPARE_RECHECK=0: neither, as before.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
|
||||
namespace prepare_again {
|
||||
|
||||
inline constexpr uint32_t kProgramsMagic = 0x31505348; // (vk_shader.cpp kDiskMagic: the program cache's head)
|
||||
inline constexpr uint64_t kProgramsHead = 16; // (the magic, the build that wrote it, the stamp)
|
||||
|
||||
/// RX_VK_PREPARE_RECHECK: on unless it is 0
|
||||
inline bool on(const char* env) { return !(env && env[0] == '0' && !env[1]); }
|
||||
|
||||
/// Whether a program cache, by its first n bytes (head) and its size, holds none of the shaders of stamp: no program cache's
|
||||
/// head, another stamp's, or its head alone
|
||||
inline bool programs_none(const uint8_t* head, size_t n, uint64_t size, uint64_t stamp) {
|
||||
if (n < kProgramsHead || size <= kProgramsHead) return true;
|
||||
uint32_t magic;
|
||||
uint64_t was;
|
||||
std::memcpy(&magic, head, 4), std::memcpy(&was, head + 8, 8);
|
||||
return magic != kProgramsMagic || was != stamp;
|
||||
}
|
||||
/// The same for the file at path (none there: none of them). *size: its size, 0 when there is none
|
||||
inline bool programs_none_at(const std::string& path, uint64_t stamp, uint64_t* size = nullptr) {
|
||||
std::error_code ec;
|
||||
const uint64_t n = std::filesystem::is_regular_file(path, ec) && !ec ? (uint64_t)std::filesystem::file_size(path, ec) : 0;
|
||||
const uint64_t whole = ec ? 0 : n;
|
||||
if (size) *size = whole;
|
||||
uint8_t head[kProgramsHead]{};
|
||||
size_t got = 0;
|
||||
if (FILE* f = std::fopen(path.c_str(), "rb")) {
|
||||
got = std::fread(head, 1, sizeof head, f);
|
||||
std::fclose(f);
|
||||
}
|
||||
return programs_none(head, got, whole, stamp);
|
||||
}
|
||||
|
||||
/// What the last start left of the driver's part without stage libraries: no mark, a mark of this stamp (or an empty one, an
|
||||
/// older build's; or any, with by_stamp false: RX_VK_PREPARE_RECHECK=0), or one of another stamp
|
||||
enum class Mark { None, Ours, Other };
|
||||
inline Mark mark_left(const std::string& path, uint64_t stamp, bool by_stamp) {
|
||||
FILE* f = std::fopen(path.c_str(), "rb");
|
||||
if (!f) return Mark::None;
|
||||
uint64_t was = 0;
|
||||
const bool stamped = std::fread(&was, 8, 1, f) == 1;
|
||||
std::fclose(f);
|
||||
return !by_stamp || !stamped || was == stamp ? Mark::Ours : Mark::Other;
|
||||
}
|
||||
/// The mark written before the part: the stamp it is made for (empty with by_stamp false, as before). False: not written
|
||||
inline bool mark_write(const std::string& path, uint64_t stamp, bool by_stamp) {
|
||||
FILE* f = std::fopen(path.c_str(), "wb");
|
||||
if (!f) return false;
|
||||
const bool ok = !by_stamp || std::fwrite(&stamp, 8, 1, f) == 1;
|
||||
return std::fclose(f) == 0 && ok;
|
||||
}
|
||||
|
||||
} // namespace prepare_again
|
||||
@@ -58,4 +58,14 @@ inline unsigned translate(unsigned all, bool amd_older, bool asked) {
|
||||
return amd_older && !asked ? std::min(all, kAmdOlderMost) : all;
|
||||
}
|
||||
|
||||
/// (Triforce) Which heap the process is on (Windows), by the signature word at byte 16 of GetProcessHeap()'s header: the
|
||||
/// segment heap's (an exe built with HOOKSHOT_SEGMENT_HEAP=1, nx2/win_manifest.py) or the NT heap's; said in the log's line
|
||||
/// on the preparation's threads, so that a run says which of the two exes it was
|
||||
inline constexpr uint32_t kSegmentHeapSignature = 0xDDEEDDEEu;
|
||||
inline constexpr uint32_t kNtHeapSignature = 0xFFEEFFEEu;
|
||||
inline const char* heap_name(uint32_t signature) {
|
||||
return signature == kSegmentHeapSignature ? "segment" : signature == kNtHeapSignature ? "NT" : "not known";
|
||||
}
|
||||
inline bool segment_heap(uint32_t signature) { return signature == kSegmentHeapSignature; }
|
||||
|
||||
} // namespace prepare_threads
|
||||
|
||||
@@ -100,11 +100,11 @@ inline std::vector<uint8_t> write(const std::vector<uint64_t>& keys, uint32_t ma
|
||||
}
|
||||
|
||||
/// The keys of a file's bytes; none when they are not a whole file of this version (cut short, another file, a bad check word)
|
||||
inline std::vector<uint64_t> read(const uint8_t* data, size_t n, uint32_t magic = kMagic) {
|
||||
inline std::vector<uint64_t> read(const uint8_t* data, size_t n, uint32_t magic = kMagic, size_t max_keys = kKeysMax) {
|
||||
uint32_t head[4];
|
||||
if (!data || n < 24) return {};
|
||||
std::memcpy(head, data, 16);
|
||||
if (head[0] != magic || head[1] != kVersion || head[2] > kKeysMax || n != 16 + (size_t)head[2] * 8 + 8) return {};
|
||||
if (head[0] != magic || head[1] != kVersion || head[2] > max_keys || n != 16 + (size_t)head[2] * 8 + 8) return {};
|
||||
std::vector<uint64_t> keys(head[2]);
|
||||
if (!keys.empty()) std::memcpy(keys.data(), data + 16, keys.size() * 8);
|
||||
uint64_t c;
|
||||
@@ -129,8 +129,9 @@ struct Record {
|
||||
std::vector<uint64_t> keys;
|
||||
std::unordered_set<uint64_t> seen;
|
||||
size_t saved = 0; // (keys.size() at the last write)
|
||||
size_t max = kKeysMax; // ((Triforce, 11-21) the pipelines' order keeps more: pipe_order.h kKeysMax)
|
||||
bool add(uint64_t key) {
|
||||
if (!key || keys.size() >= kKeysMax || !seen.insert(key).second) return false;
|
||||
if (!key || keys.size() >= max || !seen.insert(key).second) return false;
|
||||
keys.push_back(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
37
gpu/vk/range_walk.h
Normal file
37
gpu/vk/range_walk.h
Normal file
@@ -0,0 +1,37 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-42, after Source A's) Is an address in a sorted list of ranges, asked for
|
||||
// the pages of a whole loop: a cursor that only moves forward.
|
||||
//
|
||||
// The capture and collection loops ask of every page whether it lies in a range list sorted by first address (the
|
||||
// captures' ranges, the texture storage's), and each asked with a binary search. A loop's pages come ascending, or
|
||||
// nearly, so one pass over the ranges serves the whole loop. An address below the last one asked searches again, so any
|
||||
// order gives the answers the search gave: tests/test_range_walk.py compares the two.
|
||||
//
|
||||
// A walk lives inside one loop, under the lock of its list: the list must not change while it is used.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace range_walk {
|
||||
|
||||
using Ranges = std::vector<std::pair<uint64_t, uint64_t>>; // ([first, second), sorted by first)
|
||||
|
||||
struct RangeWalk {
|
||||
const Ranges& r;
|
||||
bool search = false; // (always search, as before: RX_VK_RANGE_WALK=0, the lab knob norangewalk)
|
||||
size_t next = 0; // (the first range that begins after the last address asked)
|
||||
uint64_t last = 0;
|
||||
|
||||
/// a lies in one of the ranges: std::upper_bound(r, {a, ~0}) != begin && a < prev->second, as the loops asked
|
||||
bool in(uint64_t a) {
|
||||
if (search || a < last) next = (size_t)(std::upper_bound(r.begin(), r.end(), std::make_pair(a, ~uint64_t(0))) - r.begin());
|
||||
else
|
||||
while (next < r.size() && r[next].first <= a) next++;
|
||||
last = a;
|
||||
return next != 0 && a < r[next - 1].second;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace range_walk
|
||||
123
gpu/vk/readback_late.h
Normal file
123
gpu/vk/readback_late.h
Normal file
@@ -0,0 +1,123 @@
|
||||
// (Triforce, 11-22, the performance audit's SYNC-19, after Source A's) Late readbacks: ranges the CPU reads back without
|
||||
// the wait for their write-back. The replay's invalidate (vk_replay.cpp) waits for the write-back of every range the game is
|
||||
// about to read: without it the game read 0 from its exposure meter one frame and twice the sum the next (the white flashes).
|
||||
// Some ranges the game reads right after it submitted the work that fills them, so the console cannot count on that work
|
||||
// either: there the wait is the GPU's whole latency for nothing but a value one frame newer. Source A names such ranges and
|
||||
// reads them from what guest memory holds, the last write-back made:
|
||||
// a size in hex every range of that size (Source A's default is 3980: the culling camera block, 0x3980 bytes)
|
||||
// newest a range whose writer is the newest submission handed out (the game cannot have waited for it)
|
||||
// RX_VK_READBACK_LATE=<rule>[,<rule>...]. Unset: NO rule here until the lab has run it (the audit's lab proof: a handheld or an
|
||||
// AMD iGPU, where the waits are long); =0: none either. The lab knob rblate (lab_ab.h) turns the rules a later default would
|
||||
// have (default_rules: 3980) on in its phases when the variable is unset. A late range keeps its note for the CPU's
|
||||
// write-back (memory_note_readback) and skips the catch-up and the wait, nothing else.
|
||||
// What it risks: a late read hands the game the frame before's values; with mirrors, when the CPU resets a range read late,
|
||||
// the pending write-back lands over its reset (the exposure meter's 2x flashes). `newest` can catch that meter, so it is
|
||||
// never a default. RX_VK_LOG_READBACK_RANGES=1 counts the readbacks by range (RangeLog) to find such ranges on other hardware.
|
||||
// tests/test_readback_late.py
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace mkwvk::readback_late {
|
||||
|
||||
constexpr int kMaxSizes = 8;
|
||||
/// The size Source A reads late by default: the culling camera block
|
||||
constexpr uint64_t kCameraBlock = 0x3980;
|
||||
|
||||
struct Rules {
|
||||
uint64_t sizes[kMaxSizes] = {};
|
||||
int n = 0;
|
||||
bool newest = false;
|
||||
int bad = 0; // (parts of the text that were neither a hex size nor "newest", or sizes past kMaxSizes: left out)
|
||||
bool any() const { return n > 0 || newest; }
|
||||
bool has(uint64_t size) const {
|
||||
for (int i = 0; i < n; i++)
|
||||
if (sizes[i] == size) return true;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/// The rules a default would have once the lab has passed them (and the knob rblate has now): the camera block's size alone
|
||||
inline Rules default_rules() {
|
||||
Rules r;
|
||||
r.sizes[r.n++] = kCameraBlock;
|
||||
return r;
|
||||
}
|
||||
|
||||
/// RX_VK_READBACK_LATE's text. Unset or empty: no rule (until the lab passes; then default_rules()). "0": none. Otherwise parts
|
||||
/// separated by commas: "newest", or a size in hex with or without 0x. A part that is neither is left out and counted in bad
|
||||
inline Rules parse(const char* text) {
|
||||
Rules r;
|
||||
if (!text || !*text) return r;
|
||||
for (const char* p = text; *p;) {
|
||||
const char* end = std::strchr(p, ',');
|
||||
const size_t len = end ? (size_t)(end - p) : std::strlen(p);
|
||||
if (len == 6 && !std::strncmp(p, "newest", 6)) {
|
||||
r.newest = true;
|
||||
} else if (len > 0 && len < 20) {
|
||||
char part[24] = {};
|
||||
std::memcpy(part, p, len);
|
||||
char* stop = nullptr;
|
||||
const unsigned long long v = std::strtoull(part, &stop, 16);
|
||||
if (stop == part || *stop) r.bad++;
|
||||
else if (v == 0) {} // ("0": none, and no rule of its own)
|
||||
else if (r.has(v)) {}
|
||||
else if (r.n < kMaxSizes) r.sizes[r.n++] = v;
|
||||
else r.bad++;
|
||||
} else if (len) {
|
||||
r.bad++;
|
||||
}
|
||||
if (!end) break;
|
||||
p = end + 1;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
/// Whether a readback of `size` bytes is read late. writer_newest: its write-back's value is at or past the newest value
|
||||
/// handed out (the fence a game's submit got last)
|
||||
inline bool wanted(const Rules& r, uint64_t size, bool writer_newest) {
|
||||
return (r.newest && writer_newest) || r.has(size);
|
||||
}
|
||||
/// The decision invalidate makes: only a range whose write-back is still ahead of the GPU (pending) can be late; one whose
|
||||
/// write-back is done waits for nothing anyway
|
||||
inline bool late(const Rules& r, uint64_t size, uint64_t writer_value, uint64_t gpu_done, uint64_t newest_handed_out) {
|
||||
return r.any() && writer_value > gpu_done && wanted(r, size, writer_value >= newest_handed_out);
|
||||
}
|
||||
|
||||
/// RX_VK_LOG_READBACK_RANGES: the readbacks by range. Not thread safe: the caller locks
|
||||
struct RangeLog {
|
||||
struct Row {
|
||||
int64_t readbacks = 0, waits = 0, wait_ns = 0, newest = 0, late = 0;
|
||||
};
|
||||
std::map<std::pair<uint64_t, uint64_t>, Row> rows; // ((address, size))
|
||||
size_t cap = 4096; // (the game reads a ring of slots at new addresses: past this, the rest is counted together)
|
||||
Row other;
|
||||
void note(uint64_t gpu, uint64_t size, bool waited, int64_t ns, bool writer_newest, bool was_late) {
|
||||
Row* row = &other;
|
||||
auto it = rows.find({gpu, size});
|
||||
if (it != rows.end()) row = &it->second;
|
||||
else if (rows.size() < cap) row = &rows[{gpu, size}];
|
||||
row->readbacks++, row->waits += waited, row->wait_ns += waited ? ns : 0, row->newest += writer_newest, row->late += was_late;
|
||||
}
|
||||
struct Line {
|
||||
uint64_t gpu = 0, size = 0;
|
||||
Row row;
|
||||
};
|
||||
/// The n ranges that waited longest in all, longest first (ties: the lower address first)
|
||||
std::vector<Line> top(size_t n) const {
|
||||
std::vector<Line> all;
|
||||
all.reserve(rows.size());
|
||||
for (const auto& [key, row] : rows) all.push_back({key.first, key.second, row});
|
||||
std::stable_sort(all.begin(), all.end(), [](const Line& a, const Line& b) { return a.row.wait_ns > b.row.wait_ns; });
|
||||
if (all.size() > n) all.resize(n);
|
||||
return all;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace mkwvk::readback_late
|
||||
56
gpu/vk/recorder_auto.h
Normal file
56
gpu/vk/recorder_auto.h
Normal file
@@ -0,0 +1,56 @@
|
||||
// (Triforce) The recorder thread's default (vk_recorder.h g_auto_thread, RX_VK_RECORDER unset), chosen once the device is known
|
||||
// (vk_main.cpp pick_device). Plain code with no Vulkan or system calls: tests/test_recorder_auto.py runs it on made-up PCs.
|
||||
//
|
||||
// - NVIDIA: on with 12 logical processors or more, as since 11-16 (lab, RTX 4070 SUPER + i7-13700K: Market +11%; held to 8
|
||||
// logical processors -10%: the recorder needs a core of its own).
|
||||
// - AMD: on with 6 physical cores or more in the process's affinity. AMD's driver spends about twice NVIDIA's CPU time on a call,
|
||||
// so the replay thread's vkCmd* calls cost it more; taken out of a renderer of the same per-call design, the densest Market view
|
||||
// (~2,500 draws) fell 16-25% behind on RX 7900 XTX, 9070 XT and 9060 XT cards. Physical cores, not logical processors: the
|
||||
// recorder spins on a core while the replay records, and an SMT sibling of a busy core is not one (a 4-core Steam Deck: off).
|
||||
// Not on AMD's older Windows driver (vk_vendor.h amd_older_compiler, RX 5000 and RX 6000): it lost the device with commands
|
||||
// recorded beside the replay thread (replay_threads.h), so it waits for a lab of its own.
|
||||
// - Anything else (Intel, a GPU not named): off, nothing measured.
|
||||
// RX_VK_RECORDER=thread or =off goes over this either way.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace mkwvk::recorder_default {
|
||||
|
||||
enum Why : int { kNvidia, kNvidiaFew, kAmd, kAmdFew, kAmdUnknown, kAmdOlder, kOther };
|
||||
|
||||
/// Why the recorder is on or off by default. logical: cpu_count() (cpu_count.h: hardware_concurrency, or the lab's HOS_SIM_CPU); physical: physical_cores (0: unknown)
|
||||
constexpr Why why(uint32_t vendor_id, unsigned logical, unsigned physical, bool amd_older) {
|
||||
if (vendor_id == 0x10DE) return logical >= 12 ? kNvidia : kNvidiaFew;
|
||||
if (vendor_id == 0x1002) {
|
||||
if (amd_older) return kAmdOlder;
|
||||
if (!physical) return kAmdUnknown;
|
||||
return physical >= 6 ? kAmd : kAmdFew;
|
||||
}
|
||||
return kOther;
|
||||
}
|
||||
constexpr bool on(Why w) { return w == kNvidia || w == kAmd; }
|
||||
constexpr bool rule(uint32_t vendor_id, unsigned logical, unsigned physical, bool amd_older) {
|
||||
return on(why(vendor_id, logical, physical, amd_older));
|
||||
}
|
||||
constexpr const char* text(Why w) {
|
||||
switch (w) {
|
||||
case kNvidia: return "NVIDIA, 12 logical processors or more";
|
||||
case kNvidiaFew: return "NVIDIA, under 12 logical processors";
|
||||
case kAmd: return "AMD, 6 physical cores or more";
|
||||
case kAmdFew: return "AMD, under 6 physical cores";
|
||||
case kAmdUnknown: return "AMD, the physical cores not known";
|
||||
case kAmdOlder: return "AMD's driver for RX 5000 and RX 6000 (not tried there)";
|
||||
default: return "not an NVIDIA or AMD GPU (not measured)";
|
||||
}
|
||||
}
|
||||
|
||||
/// The physical cores with at least one logical processor in affinity: masks[i] is core i's logical processors (one
|
||||
/// RelationProcessorCore entry each: GetLogicalProcessorInformation, which on Linux reads sysfs's package and core ids)
|
||||
inline unsigned physical_cores(const uint64_t* masks, int n, uint64_t affinity) {
|
||||
unsigned cores = 0;
|
||||
for (int i = 0; i < n; i++) cores += (masks[i] & affinity) != 0;
|
||||
return cores;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::recorder_default
|
||||
57
gpu/vk/redraw_state.h
Normal file
57
gpu/vk/redraw_state.h
Normal file
@@ -0,0 +1,57 @@
|
||||
// (Triforce, 11-21, the upscaler audit's UPS-03) What a redraw of the stored draws has to set before it draws.
|
||||
//
|
||||
// The stored draws are drawn again into the motion vectors in one pass (vk_temporal.cpp capture_motion), about 900 of
|
||||
// them a frame in the owner's log of 11-21. Each one set the primitive topology, the primitive restart and the index
|
||||
// buffer before it drew, whatever the draw before had set: three commands a draw beside its push constants and the draw
|
||||
// itself, and nearly every draw of the scene is a list of triangles without restart. Dynamic state and the index buffer
|
||||
// stay as they were set until they are set again in the same command buffer, so a command that would set what is set
|
||||
// already says nothing new. Here is what the pass has set so far, and whether a draw's own differs from it.
|
||||
//
|
||||
// A pass starts knowing nothing (a new Pass): its first draw sets everything, as before.
|
||||
// RX_VK_REDRAW_STATE=0: every draw sets all three, as before (every()).
|
||||
//
|
||||
// No Vulkan: tests/test_redraw_state.py compiles this.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace redraw_state {
|
||||
|
||||
/// RX_VK_REDRAW_STATE (value, null when unset): 0, every draw sets its state whatever is set, as before
|
||||
inline bool every(const char* value) { return value && *value && std::atoi(value) == 0; }
|
||||
|
||||
/// What one pass has set so far, from its first draw on
|
||||
struct Pass {
|
||||
bool all = false; // (every(): each asks true)
|
||||
bool have_topology = false, have_restart = false, have_index = false;
|
||||
uint32_t topology = 0;
|
||||
bool restart = false;
|
||||
uint64_t index_buffer = 0, index_offset = 0;
|
||||
uint32_t index_type = 0;
|
||||
uint64_t set = 0, left = 0; // (commands recorded, and left out: for a count in the log)
|
||||
};
|
||||
|
||||
/// Whether the topology is to be set for a draw of `topology` (and it is then what the pass has set)
|
||||
inline bool topology(Pass& p, uint32_t topology) {
|
||||
if (!p.all && p.have_topology && p.topology == topology) return p.left++, false;
|
||||
p.have_topology = true, p.topology = topology;
|
||||
return p.set++, true;
|
||||
}
|
||||
|
||||
/// The same for primitive restart
|
||||
inline bool restart(Pass& p, bool restart) {
|
||||
if (!p.all && p.have_restart && p.restart == restart) return p.left++, false;
|
||||
p.have_restart = true, p.restart = restart;
|
||||
return p.set++, true;
|
||||
}
|
||||
|
||||
/// The same for an indexed draw's index buffer: the buffer's handle as a number, the offset into it and the index type,
|
||||
/// all three the same as the one bound
|
||||
inline bool index(Pass& p, uint64_t buffer, uint64_t offset, uint32_t type) {
|
||||
if (!p.all && p.have_index && p.index_buffer == buffer && p.index_offset == offset && p.index_type == type)
|
||||
return p.left++, false;
|
||||
p.have_index = true, p.index_buffer = buffer, p.index_offset = offset, p.index_type = type;
|
||||
return p.set++, true;
|
||||
}
|
||||
|
||||
} // namespace redraw_state
|
||||
64
gpu/vk/release_queue.h
Normal file
64
gpu/vk/release_queue.h
Normal file
@@ -0,0 +1,64 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-85, after Source B's idea) Texture and program releases queued for the
|
||||
// replaying thread, not waited for.
|
||||
//
|
||||
// The game's nvnTextureFinalize and nvnProgramFinalize reach vk_replay.cpp texture_released_of and program_released_locked.
|
||||
// Without the replay thread (RX_VK_ASYNC=0, the default) a release is held for the replaying thread only while a queue is
|
||||
// stopped at a wait (PR 430: after the stopped work, which may still use it); otherwise the game's thread takes replay_mu,
|
||||
// which a replay holds through whole submissions: the finalizing thread waited 45% of its time in Kokiri before PR 430.
|
||||
//
|
||||
// RX_VK_RELEASE_QUEUE=1 (OFF unless set; the lab knob relq in its phases while it is unset): every such release is pushed to
|
||||
// a list under held_mu, stopped queue or not, and applied by the thread that replays next, at the start of a submission's
|
||||
// replay or a present (replay_mu held there already), when no queue is stopped; with one stopped, update_blocked applies
|
||||
// them after the stopped work, as PR 430's. A texture's release carries the write generation of its call, so the CPU's
|
||||
// writes after the call still count as written after the release (vk_texture.cpp release_gen). The RX_VK_ASYNC job path is
|
||||
// not touched. Unset, a release goes the way it went: way() is the old decision then (tests/test_release_queue.py).
|
||||
//
|
||||
// And a count, on by default: the releases that went the old way, their time in sync_drain and the replay's lock, the
|
||||
// longest; a line every 600 releases when one of them waited 0.5 ms or more.
|
||||
//
|
||||
// The rule alone: no Vulkan, no game, no clock.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace release_queue {
|
||||
|
||||
/// RX_VK_RELEASE_QUEUE: a number other than 0 on, 0 off; unset: the lab knob relq (on only in its phases)
|
||||
inline bool on(const char* env, bool knob) {
|
||||
if (env) return std::atoi(env) != 0;
|
||||
return knob;
|
||||
}
|
||||
|
||||
/// Where a release without the replay thread goes. Now: sync_drain(1) and the replay's lock on the caller's thread, as
|
||||
/// before; Held: PR 430's list, while a queue is stopped (hold: RX_VK_RELEASE_DRAIN unset; sync_waits: the event ordering
|
||||
/// on); Queued: the switch's list, whatever is stopped
|
||||
enum class Way { Now, Held, Queued };
|
||||
inline Way way(bool queue_on, bool hold, bool sync_waits, bool blocked) {
|
||||
if (queue_on) return Way::Queued;
|
||||
return hold && sync_waits && blocked ? Way::Held : Way::Now;
|
||||
}
|
||||
/// The old decision, for the test: held or not
|
||||
inline bool held_before(bool hold, bool sync_waits, bool blocked) { return hold && sync_waits && blocked; }
|
||||
|
||||
/// Whether the queued releases are applied at a submission's replay start or a present: something queued, no queue stopped
|
||||
inline bool apply_now(size_t queued, int blocked) { return queued != 0 && blocked == 0; }
|
||||
|
||||
/// The releases that went the old way: how many, how many waited kWaitNs or more, their time, the longest
|
||||
struct Waits {
|
||||
static constexpr int64_t kWaitNs = 500000; // (0.5 ms: a replay met, not the release's own work)
|
||||
static constexpr int64_t kEvery = 600;
|
||||
int64_t n = 0, waited = 0, ns = 0, max_ns = 0, queued = 0;
|
||||
void note(int64_t took_ns) {
|
||||
n++, ns += took_ns;
|
||||
if (took_ns >= kWaitNs) waited++;
|
||||
if (took_ns > max_ns) max_ns = took_ns;
|
||||
}
|
||||
void note_queued() { queued++; }
|
||||
/// Every kEvery releases (either way) the counts start again; a line when one of them waited, or went to the switch's list
|
||||
bool due() const { return n + queued >= kEvery; }
|
||||
bool worth_a_line() const { return waited > 0 || queued > 0; }
|
||||
void reset() { *this = Waits{}; }
|
||||
};
|
||||
|
||||
} // namespace release_queue
|
||||
77
gpu/vk/replay_pin.h
Normal file
77
gpu/vk/replay_pin.h
Normal file
@@ -0,0 +1,77 @@
|
||||
// (Triforce, 11-22, the performance audit's THREAD-09, after Source A's) Where a thread may run while it replays, on a processor with
|
||||
// cores of more than one kind.
|
||||
//
|
||||
// A thread that replays is pinned for that time to the core the host keeps for the replay (vk_replay.cpp ReplayPin,
|
||||
// hos_reserved_cpus). That core is kept from the game's threads, not from other programs': on a 6+8-core hybrid processor Source A
|
||||
// saw Windows park one of its two logical processors while another program's thread at priority 15 held the other 30 to 40 ms at a
|
||||
// time, and the pinned replay, raised only to the highest of the normal levels, waited for it ready to run: 14 to 37 frames over
|
||||
// 40 ms in a 40 s run. So on a processor of two kinds of core the pin is soft: the thread may run on the reserved core and on the
|
||||
// fastest-kind cores among its own processors, its ideal processor is the reserved core's last logical processor, and the core it
|
||||
// is on when the pin starts is left out for that pin, so that it moves at once, to the reserved core when that is free. Widening
|
||||
// alone would leave it where it is: that is the "only preferring the core" that gained nothing. After the replay it has the
|
||||
// affinity and the ideal processor it had.
|
||||
//
|
||||
// A processor of one kind of core keeps the strict pin: there the pin itself is what was measured to help, and nothing changes.
|
||||
// RX_VK_REPLAY_PIN_SOFT unset or 1: soft where the cores are of more than one kind; 0: strict everywhere, as before;
|
||||
// 2: soft everywhere (the lab).
|
||||
// Plain rules over masks, no system calls: tests/test_replay_pin_soft.py runs them over made-up processors.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace mkwvk::replay_pin {
|
||||
|
||||
constexpr int kStrict = 0, kHybrid = 1, kAlways = 2;
|
||||
|
||||
/// RX_VK_REPLAY_PIN_SOFT's value: unset (or empty, or 1) kHybrid, 0 (or anything that is no number above 0) kStrict, 2 and more kAlways
|
||||
inline int mode(const char* env) {
|
||||
if (!env || !*env) return kHybrid;
|
||||
const int v = std::atoi(env);
|
||||
return v <= 0 ? kStrict : v == 1 ? kHybrid : kAlways;
|
||||
}
|
||||
|
||||
struct Kinds {
|
||||
uint64_t fastest = 0; // (the logical processors of the cores with the highest efficiency class)
|
||||
bool hybrid = false; // (some core is of a lower class)
|
||||
};
|
||||
/// The fastest kind of core among n cores (each core's logical processors and its EfficiencyClass, higher the faster, as Windows
|
||||
/// lists them; a core with no processors is not one). hybrid only when a listed core is not of that kind
|
||||
inline Kinds kinds(const uint64_t* masks, const unsigned char* classes, int n) {
|
||||
Kinds k;
|
||||
int best = -1;
|
||||
uint64_t all = 0;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (!masks[i]) continue;
|
||||
all |= masks[i];
|
||||
if ((int)classes[i] > best) best = classes[i], k.fastest = masks[i];
|
||||
else if ((int)classes[i] == best) k.fastest |= masks[i];
|
||||
}
|
||||
k.hybrid = k.fastest != all;
|
||||
return k;
|
||||
}
|
||||
|
||||
/// Whether the pin is soft: by the switch and the processor's kinds
|
||||
inline bool soft(int mode, bool hybrid) {
|
||||
return mode >= kAlways || (mode == kHybrid && hybrid);
|
||||
}
|
||||
|
||||
/// The soft pin's processors: the reserved core, and the fastest-kind ones among the thread's own (own: its affinity before the
|
||||
/// pin, which keeps a thread held to one die, or to anything else, inside what it was held to). Never fewer than the reserved core
|
||||
inline uint64_t wide(uint64_t reserved, uint64_t fastest, uint64_t own) {
|
||||
return reserved | (fastest & own);
|
||||
}
|
||||
|
||||
/// The same without the core the thread is on now (now: that core's logical processors, 0 when not known), so that the thread
|
||||
/// moves at once. Left whole when it is on the reserved core already or when nothing else would be left
|
||||
inline uint64_t moved(uint64_t wide, uint64_t reserved, uint64_t now) {
|
||||
if (!now || (now & reserved) || !(wide & ~now)) return wide;
|
||||
return wide & ~now;
|
||||
}
|
||||
|
||||
/// The ideal processor of a replaying thread: the reserved core's last logical processor (-1: no core reserved)
|
||||
inline int ideal(uint64_t reserved) {
|
||||
return reserved ? 63 - __builtin_clzll(reserved) : -1;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::replay_pin
|
||||
@@ -25,7 +25,8 @@
|
||||
// they record only those trial frames. (Triforce, 11-15) And its rule (rule below): where the replay is plainly the frame's
|
||||
// limit they go on without a trial, and where they cannot pay no trial is made. (Triforce, 11-16) And once a way has held,
|
||||
// trials come ever rarer (up to some 14,400 presents apart) and shorter, and early only when the scene clearly changes (the
|
||||
// back-off, at Tuner).
|
||||
// back-off, at Tuner). (Triforce, 11-22) And both judge settled scenes only: the rule no window of a load or an arrival, and a
|
||||
// trial the scene changed under decides nothing (settled, at Tuner).
|
||||
//
|
||||
// The parallel replay was audited for this by reading (nothing could be run). What was found and is fixed, each with a
|
||||
// check in tests/test_parallel_fixes.py:
|
||||
@@ -236,6 +237,18 @@ inline Verdict rule(const Frame& m, int threads) {
|
||||
/// trial whose first and last parts (the same way) moved so had the scene change under it: the windows after it are the
|
||||
/// new scene's reference, so the back-off starts again there too, the next trial kEvery on.
|
||||
/// A change of way, by a trial or by the rule, starts the back-off again. tests/test_replay_threads_tuner.py "backoff"
|
||||
/// (Triforce, 11-22) Settled (settled; vk_replay.cpp par_settled_on, RX_VK_TUNER_SETTLED=0: off, as before), after
|
||||
/// Kaefergeneral's #dev-fixes post: on a 5900X with an RTX 3090, where the helpers lost in every settled scene (1.2 to 4.5 ms a
|
||||
/// frame), Automatic was 6 to 10% slower than Off. The rule turned them on in the window of an area's load (long frames, the
|
||||
/// replay 71 to 82% of them, nearly every draw cuttable), not in the settled Market (the replay 62%), and the first trial after
|
||||
/// the arrival straddled the load ("parallel 4.76 ms a frame, the other way 24.10, parallel again 17.25: as it was"): the
|
||||
/// scene's change was seen (moved) but the verdict stood, for kEvery presents or more. So:
|
||||
/// - the rule judges a window only if the one before it, on the replay thread alone, describes the same scene (scene_moved,
|
||||
/// the back-off's test): a load's window and the first after it get no verdict, neither "on" nor "no trial". After a trial
|
||||
/// or a change of way the first window has none before it, so the rule's "on" comes a window later than it did;
|
||||
/// - a trial whose first and last parts moved so decides nothing, either way (not better, no recheck), and the next one comes
|
||||
/// kWindow after a window of the way in use like the one before it (settle_look), not kEvery later, and never while the rule
|
||||
/// says no trial. kHold and the thresholds as they were. tests/test_replay_threads_tuner.py "settled"
|
||||
class Tuner {
|
||||
public:
|
||||
static constexpr int kEvery = 900, kTake = 80, kSettle = 24, kWindow = 120, kHold = 4 * kEvery;
|
||||
@@ -243,6 +256,7 @@ public:
|
||||
static constexpr double kClear = 1.10, kSceneDraws = 0.25, kSceneDrawsMin = 200, kScenePart = 0.30, kScenePartMs = 1.0;
|
||||
bool use_rule = true;
|
||||
bool backoff = true; // (the back-off above; false: every step as before it)
|
||||
bool settled = true; // (the rule and the trials only on a settled scene, above; false: as before)
|
||||
int threads = 4; // (the threads that record segments: rule()'s)
|
||||
struct Step {
|
||||
bool flip = false; // change the way in use
|
||||
@@ -262,6 +276,9 @@ public:
|
||||
bool moved = false; // (the scene changed under the trial that ended: the back-off starts again)
|
||||
bool sure = false; // (the trial kept the way, the other way longer than both its parts: the back-off grows)
|
||||
int64_t next = 0; // (the present the next trial may begin at: set at a trial's end and at a scene change)
|
||||
// (settled's)
|
||||
bool unsettled = false; // (a window the rule did not judge, or a trial that decided nothing: the scene moved)
|
||||
bool settle = false; // (the scene settled after such a trial: the next trial at next)
|
||||
};
|
||||
/// f: the frame just presented; on: the helper threads in use for it; order_only: both ways are the replay thread alone
|
||||
Step step(const Frame& f, bool on, bool order_only) {
|
||||
@@ -272,31 +289,44 @@ public:
|
||||
if (f.frame <= 60.0) win_.add(f);
|
||||
if (++win_frames_ >= kWindow) {
|
||||
if (win_.n >= kWindow * 3 / 4 && !order_only) {
|
||||
s.ruled = true, s.window = win_.mean(), s.verdict = rule(s.window, threads);
|
||||
const bool turn_on = s.verdict.kind == Verdict::kOn && frames_ >= hold_until_;
|
||||
s.rule_new = s.verdict.kind != last_ || turn_on;
|
||||
last_ = s.verdict.kind;
|
||||
if (turn_on) {
|
||||
s.flip = true;
|
||||
next_trial_ = frames_ + kEvery;
|
||||
if (backoff) held_ = 0, fresh_ = true, scene_restart();
|
||||
} else if (s.verdict.kind == Verdict::kOff) {
|
||||
next_trial_ = std::max(next_trial_, frames_ + kEvery);
|
||||
no_trial_until_ = frames_ + kEvery;
|
||||
// (settled: a window is judged only if the one before it, on the replay thread alone too, describes
|
||||
// the same scene)
|
||||
s.window = win_.mean();
|
||||
const bool same = !settled || (win_prev_set_ && !scene_moved(win_prev_, s.window));
|
||||
win_prev_ = s.window, win_prev_set_ = true;
|
||||
if (!same) {
|
||||
s.unsettled = true;
|
||||
} else {
|
||||
s.ruled = true, s.verdict = rule(s.window, threads);
|
||||
const bool turn_on = s.verdict.kind == Verdict::kOn && frames_ >= hold_until_;
|
||||
s.rule_new = s.verdict.kind != last_ || turn_on;
|
||||
last_ = s.verdict.kind;
|
||||
if (turn_on) {
|
||||
s.flip = true;
|
||||
next_trial_ = frames_ + kEvery;
|
||||
waiting_ = false, win_prev_set_ = false; // (the way changed: a trial kEvery on, as after a change)
|
||||
if (backoff) held_ = 0, fresh_ = true, scene_restart();
|
||||
} else if (s.verdict.kind == Verdict::kOff) {
|
||||
next_trial_ = std::max(next_trial_, frames_ + kEvery);
|
||||
no_trial_until_ = frames_ + kEvery;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
win_prev_set_ = false; // (a load, or DLSS / FSR kept in order: the next window has none before it)
|
||||
}
|
||||
win_ = Part{}, win_frames_ = 0;
|
||||
if (s.flip) return s;
|
||||
}
|
||||
} else {
|
||||
win_ = Part{}, win_frames_ = 0;
|
||||
win_ = Part{}, win_frames_ = 0, win_prev_set_ = false;
|
||||
}
|
||||
if (backoff) scene_look(f, s);
|
||||
if (waiting_) settle_look(f, s);
|
||||
else if (backoff) scene_look(f, s);
|
||||
if (frames_ < next_trial_) return s;
|
||||
if (order_only) return s;
|
||||
phase_ = 1, n_ = 0;
|
||||
short_ = backoff && held_ >= 2;
|
||||
win_ = Part{}, win_frames_ = 0; // (the rule's next window begins after the trial)
|
||||
win_ = Part{}, win_frames_ = 0, win_prev_set_ = false; // (the rule's next window begins after the trial)
|
||||
sum_[0] = sum_[1] = sum_[2] = 0, cnt_[0] = cnt_[1] = cnt_[2] = 0;
|
||||
part_[0] = part_[1] = part_[2] = Part{};
|
||||
return s;
|
||||
@@ -318,6 +348,11 @@ public:
|
||||
s.a = cnt_[0] ? sum_[0] / cnt_[0] : 0, s.b = cnt_[1] ? sum_[1] / cnt_[1] : 0, s.c = cnt_[2] ? sum_[2] / cnt_[2] : 0;
|
||||
s.better = s.ok && s.b < std::min(s.a, s.c) * 0.96;
|
||||
s.short_trial = short_;
|
||||
// (settled: a trial whose first and last parts, the same way, describe two scenes measured the change of scene, not
|
||||
// the way: it decides nothing, either way, and the next one waits for the scene to settle)
|
||||
const bool moved = (backoff || settled) && cnt_[0] && cnt_[2] && scene_moved(part_[0].mean(), part_[2].mean());
|
||||
s.moved = moved;
|
||||
if (settled && moved) s.better = false, s.unsettled = true;
|
||||
if (s.better && short_) s.better = false, s.recheck = true; // (a full trial decides: at once, below)
|
||||
if (s.better) {
|
||||
s.flip = true, changes_++;
|
||||
@@ -334,7 +369,6 @@ public:
|
||||
// put off ever longer; such a trial costs little, its two ways' frames being about as long. The first trial
|
||||
// after a change of scene or way that is too close to tell: kEvery, a second look soon. The trials' means
|
||||
// drift with the scene, and right after a change the first is the one most often wrong)
|
||||
s.moved = cnt_[0] && cnt_[2] && scene_moved(part_[0].mean(), part_[2].mean());
|
||||
const double top = std::max(s.a, s.c);
|
||||
s.sure = s.ok && !s.better && !s.recheck && !s.moved && s.b > top;
|
||||
const bool again = !s.sure && fresh_;
|
||||
@@ -345,6 +379,10 @@ public:
|
||||
last_end_ = frames_;
|
||||
scene_restart();
|
||||
}
|
||||
if (s.unsettled) { // (the next trial kWindow after a window like the one before it: settle_look)
|
||||
waiting_ = true, next_trial_ = INT64_MAX;
|
||||
settle_win_ = Part{}, settle_frames_ = 0, settle_prev_set_ = false;
|
||||
}
|
||||
s.next = next_trial_;
|
||||
s.trial = true, s.parts = part_;
|
||||
}
|
||||
@@ -367,6 +405,22 @@ public:
|
||||
|
||||
private:
|
||||
void scene_restart() { scene_win_ = Part{}, scene_frames_ = 0, ref_set_ = false, strikes_ = 0; }
|
||||
/// (settled: after a trial the scene moved under, the way's windows each against the one before; the first like it, and
|
||||
/// the next trial kWindow on, never while the rule says no trial. A load breaks the run)
|
||||
void settle_look(const Frame& f, Step& s) {
|
||||
if (f.frame <= 60.0) settle_win_.add(f);
|
||||
if (++settle_frames_ < kWindow) return;
|
||||
const bool counted = settle_win_.n >= kWindow * 3 / 4;
|
||||
const Frame m = settle_win_.mean();
|
||||
settle_win_ = Part{}, settle_frames_ = 0;
|
||||
const bool same = counted && settle_prev_set_ && !scene_moved(settle_prev_, m);
|
||||
settle_prev_ = m, settle_prev_set_ = counted;
|
||||
if (!same) return;
|
||||
waiting_ = false, s.settle = true;
|
||||
next_trial_ = std::max(frames_ + kWindow, no_trial_until_);
|
||||
s.next = next_trial_;
|
||||
if (backoff) scene_restart(); // (the settled scene is the back-off's reference)
|
||||
}
|
||||
/// (between trials: the scene's windows against the reference, the next trial brought forward when two in a row moved)
|
||||
void scene_look(const Frame& f, Step& s) {
|
||||
if (f.frame <= 60.0) scene_win_.add(f);
|
||||
@@ -406,6 +460,11 @@ private:
|
||||
int scene_frames_ = 0, strikes_ = 0;
|
||||
bool ref_set_ = false;
|
||||
Frame ref_;
|
||||
// (settled's: the rule's window before, and after a trial the scene moved under, the wait for it to settle and its windows)
|
||||
Frame win_prev_, settle_prev_;
|
||||
bool win_prev_set_ = false, waiting_ = false, settle_prev_set_ = false;
|
||||
Part settle_win_;
|
||||
int settle_frames_ = 0;
|
||||
};
|
||||
|
||||
} // namespace replay_threads
|
||||
|
||||
85
gpu/vk/retired_give_back.h
Normal file
85
gpu/vk/retired_give_back.h
Normal file
@@ -0,0 +1,85 @@
|
||||
// (Triforce, 11-22, the performance audit's MEM-21, after Source A's) When a retired pool's import is given back.
|
||||
//
|
||||
// A pool remade larger is retired: never bound again, and once the GPU is past it its import is pinned RAM nothing uses.
|
||||
// vk_memory.cpp retired_imports_give_back gives such imports back at once on Linux and with NVIDIA's driver on Windows, and
|
||||
// keeps them for the session with AMD's and Intel's drivers on Windows: given back at once there, an Intel Arc B390, an
|
||||
// Ally X and an RX 6600 lost the device right after. It never gives back the import of a pool whose mirror was refused
|
||||
// (RX_VK_NO_MIRROR_POOLS, a pool over 1 GB, VRAM refused), on any OS.
|
||||
//
|
||||
// Source A gives them back on Windows AMD and Intel after a wait instead: 30 s after the pool retired, or 2 s while the
|
||||
// system heap is past 85% of its budget; and the refused mirrors' imports too. Here both are OFF unless asked:
|
||||
// RX_VK_KEEP_RETIRED=wait the wait, on any driver (RX_VK_RETIRE_SECS, 30; RX_VK_RETIRE_MIN_SECS, 2)
|
||||
// RX_VK_RETIRED_REFUSED=1 the refused mirrors' imports too, by the mode in force (with `wait`: on unless =0)
|
||||
// Unset, and every value RX_VK_KEEP_RETIRED took before, decide exactly as before: tests/test_retired_give_back.py compares
|
||||
// want() with the old condition on random inputs. NVIDIA's driver on Windows keeps its behaviour.
|
||||
//
|
||||
// The rule alone, no Vulkan, no clock: the caller passes the times in milliseconds.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace retired_give_back {
|
||||
|
||||
enum class Mode { AtOnce, Wait, Keep };
|
||||
|
||||
/// RX_VK_KEEP_RETIRED: "wait" the wait; any other value as it was read before (a value beginning with 0 gives back at once,
|
||||
/// anything else keeps, the empty text too); unset: kept on Windows unless the card is NVIDIA's, given back at once elsewhere
|
||||
inline Mode mode(const char* env, bool windows, bool nvidia) {
|
||||
if (env) {
|
||||
if (std::strcmp(env, "wait") == 0) return Mode::Wait;
|
||||
return env[0] != '0' ? Mode::Keep : Mode::AtOnce;
|
||||
}
|
||||
return windows && !nvidia ? Mode::Keep : Mode::AtOnce;
|
||||
}
|
||||
/// The old reading, for the test: whether retired imports were kept
|
||||
inline bool keep_before(const char* env, bool windows, bool nvidia) {
|
||||
if (windows) return env ? env[0] != '0' : !nvidia;
|
||||
return env && env[0] != '0';
|
||||
}
|
||||
|
||||
/// RX_VK_RETIRED_REFUSED: a number other than 0 on, 0 off; unset: on with the wait (Source A's rule whole), off otherwise
|
||||
inline bool refused_too(const char* env, Mode m) {
|
||||
if (env) return std::atoi(env) != 0;
|
||||
return m == Mode::Wait;
|
||||
}
|
||||
|
||||
/// RX_VK_RETIRE_SECS, RX_VK_RETIRE_MIN_SECS: whole seconds, 0 or more; unset or not a number of 0 or more: the default
|
||||
inline int64_t secs(const char* env, int64_t fallback) {
|
||||
if (!env || !*env) return fallback;
|
||||
char* end = nullptr;
|
||||
const long long v = std::strtoll(env, &end, 10);
|
||||
return end && *end == 0 && v >= 0 ? (int64_t)v : fallback;
|
||||
}
|
||||
|
||||
/// Whether the system heap is short: used past 85% of its budget (no budget known: not short)
|
||||
inline bool pressure(uint64_t used, uint64_t budget) {
|
||||
return budget && used > budget / 100 * 85;
|
||||
}
|
||||
|
||||
/// The wait over: now_ms - retired_ms at least min_s seconds while the heap is short, wait_s seconds otherwise
|
||||
inline bool due(int64_t now_ms, int64_t retired_ms, int64_t wait_s, int64_t min_s, bool short_heap) {
|
||||
return now_ms - retired_ms >= (short_heap ? min_s : wait_s) * 1000;
|
||||
}
|
||||
|
||||
/// Whether a retired pool's import goes back now. mirror_none, mirror_refused: its mirror's state; inherit: the pool that took
|
||||
/// it in still copies from its mirror; gpu_past: the GPU is past its retirement and its last use; is_due: due() (the wait only)
|
||||
inline bool want(Mode m, bool refused, bool mirror_none, bool mirror_refused, bool inherit, bool gpu_past, bool is_due) {
|
||||
if (m == Mode::Keep) return false;
|
||||
if (!mirror_none && !(refused && mirror_refused)) return false;
|
||||
if (inherit || !gpu_past) return false;
|
||||
return m == Mode::AtOnce || is_due;
|
||||
}
|
||||
/// The old condition, for the test
|
||||
inline bool want_before(bool keep, bool mirror_none, bool inherit, bool gpu_past) {
|
||||
return !keep && mirror_none && !inherit && gpu_past;
|
||||
}
|
||||
|
||||
/// The system heap's use as the pressure give-back (imports_give_back) should read it with the wait on: less the imports given
|
||||
/// back and not freed yet (dead_imports, freed once the GPU is past them), which the driver still counts. One overshoot,
|
||||
/// one valve: without this a burst of retired imports given back made it give back live ones as well
|
||||
inline uint64_t settled(uint64_t used, uint64_t dead_bytes) {
|
||||
return used > dead_bytes ? used - dead_bytes : 0;
|
||||
}
|
||||
|
||||
} // namespace retired_give_back
|
||||
58
gpu/vk/settle_gate.h
Normal file
58
gpu/vk/settle_gate.h
Normal file
@@ -0,0 +1,58 @@
|
||||
// (Triforce, 11-22, the performance audit's SYNC-29, after Source A's) A CPU read looks at the stopped queues only after a
|
||||
// new CPU signal. A read that settles (vk_replay.cpp sync_settle: the stopped queues whose waits are met go on, none forced)
|
||||
// takes sync_order_mu whenever a queue is stopped, and that lock is held through whole replays and presents: the readback's
|
||||
// look (RX_VK_RB_SETTLE, off) and the two looks invalidate makes for a range a texture may be in.
|
||||
//
|
||||
// What can make a stopped queue's wait met:
|
||||
// a signal in the stream replayed by a thread that holds sync_order_mu, and every such holder runs schedule() and ends its
|
||||
// burst (burst_end: finish_recording) before it lets the lock go: it looked itself;
|
||||
// a signal of the CPU's event_signal_queued: queued, then counted (signals);
|
||||
// a plain store into the memory of an event nothing has set yet (the wait reads its value from guest memory,
|
||||
// event_value_in): not seen by anything, so such a wait keeps every look (wait_in_memory).
|
||||
// So a look is needed only while a queue is stopped and the CPU has signalled since the last look, or a stopped queue waits
|
||||
// on an event read from memory. A look notes the signals' count at its start (burst_begin) and publishes it once its recording
|
||||
// is finished (burst_end, after finish_recording): a reader that sees signals == settled finds the work those signals let on
|
||||
// recorded. Publishing the count as it is at the look's END would cover a signal that came after the look's schedule() and
|
||||
// was never acted on; counting a signal BEFORE it is queued would let a look publish it without having seen it. The test runs
|
||||
// both mistakes and finds them.
|
||||
//
|
||||
// RX_VK_SETTLE_GATE: unset or 0, no gate (every look as before); 1 the readback's look (the one RX_VK_RB_SETTLE turns on);
|
||||
// 2 the texture sites' looks too. Those two order the exposure meter's readback: the trial that skipped them outright made
|
||||
// the Market flicker at once, and Source A left them ungated, so 2 is new ground and needs the lab's flicker count. The lab
|
||||
// knobs settlegate (1) and texgate (2) in their phases. tests/test_settle_gate.py
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
namespace mkwvk::settle_gate {
|
||||
|
||||
/// The sites that settle
|
||||
enum Site { kReadback = 0, kTexture = 1, kSites = 2 };
|
||||
|
||||
/// RX_VK_SETTLE_GATE's text: "1" and "2"; unset, empty, "0" and anything else: 0, no gate
|
||||
inline int level(const char* text) {
|
||||
if (!text || !text[0] || text[1]) return 0;
|
||||
return text[0] == '1' ? 1 : text[0] == '2' ? 2 : 0;
|
||||
}
|
||||
/// The level in force: the variable's, raised by the lab knobs in their phases
|
||||
inline int level_now(int env, bool knob_settlegate, bool knob_texgate) {
|
||||
int l = env;
|
||||
if (knob_settlegate && l < 1) l = 1;
|
||||
if (knob_texgate && l < 2) l = 2;
|
||||
return l;
|
||||
}
|
||||
/// Whether a site's look goes through the gate at this level
|
||||
inline bool gated(int level, int site) { return site == kReadback ? level >= 1 : level >= 2; }
|
||||
/// Whether a gated site looks: a queue stopped, and a CPU signal since the last look or a stopped queue waiting on an event
|
||||
/// whose value is read from guest memory
|
||||
inline bool look(bool blocked, uint64_t signals, uint64_t settled, bool wait_in_memory) {
|
||||
return blocked && (signals != settled || wait_in_memory);
|
||||
}
|
||||
/// The whole decision of a site: 0 nothing to do (no queue stopped: as before), 1 look, 2 skipped by the gate
|
||||
enum What { kNothing = 0, kLook = 1, kSkip = 2 };
|
||||
inline int decide(int level, int site, bool blocked, uint64_t signals, uint64_t settled, bool wait_in_memory) {
|
||||
if (!blocked) return kNothing;
|
||||
if (gated(level, site) && !look(true, signals, settled, wait_in_memory)) return kSkip;
|
||||
return kLook;
|
||||
}
|
||||
|
||||
} // namespace mkwvk::settle_gate
|
||||
16
gpu/vk/shaders/sim_gpu_burn.comp
Normal file
16
gpu/vk/shaders/sim_gpu_burn.comp
Normal file
@@ -0,0 +1,16 @@
|
||||
#version 460
|
||||
// (Triforce, 11-22, the performance audit's OS-49, after Source B's) RX_VK_SIM_GPU_SLOW's burn (gpu/vk/sim_gpu.h): one
|
||||
// invocation runs a chain of dependent multiply-adds `loops` times, so that a dispatch takes a time the loop count sets; the
|
||||
// result goes to a word nothing reads, so that the chain is not left out.
|
||||
// glslangValidator -V --target-env vulkan1.3 --vn SIM_GPU_BURN_COMP_SPV -o sim_gpu_burn_comp_spv.h sim_gpu_burn.comp
|
||||
layout(local_size_x = 1) in;
|
||||
layout(push_constant) uniform P {
|
||||
uint loops;
|
||||
float seed;
|
||||
} pc;
|
||||
layout(std430, set = 0, binding = 0) buffer Sink { float v; } sink;
|
||||
void main() {
|
||||
float x = pc.seed;
|
||||
for (uint i = 0u; i < pc.loops; i++) x = x * 0.99999994 + 0.5;
|
||||
sink.v = x;
|
||||
}
|
||||
43
gpu/vk/shaders/sim_gpu_burn_comp_spv.h
Normal file
43
gpu/vk/shaders/sim_gpu_burn_comp_spv.h
Normal file
@@ -0,0 +1,43 @@
|
||||
// 1116.6.0
|
||||
#pragma once
|
||||
const uint32_t SIM_GPU_BURN_COMP_SPV[] = {
|
||||
0x07230203,0x00010600,0x0008000b,0x00000031,0x00000000,0x00020011,0x00000001,0x0006000b,
|
||||
0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001,
|
||||
0x0007000f,0x00000005,0x00000004,0x6e69616d,0x00000000,0x0000000c,0x0000002a,0x00060010,
|
||||
0x00000004,0x00000011,0x00000001,0x00000001,0x00000001,0x00030003,0x00000002,0x000001cc,
|
||||
0x00040005,0x00000004,0x6e69616d,0x00000000,0x00030005,0x00000008,0x00000078,0x00030005,
|
||||
0x0000000a,0x00000050,0x00050006,0x0000000a,0x00000000,0x706f6f6c,0x00000073,0x00050006,
|
||||
0x0000000a,0x00000001,0x64656573,0x00000000,0x00030005,0x0000000c,0x00006370,0x00030005,
|
||||
0x00000013,0x00000069,0x00040005,0x00000028,0x6b6e6953,0x00000000,0x00040006,0x00000028,
|
||||
0x00000000,0x00000076,0x00040005,0x0000002a,0x6b6e6973,0x00000000,0x00030047,0x0000000a,
|
||||
0x00000002,0x00050048,0x0000000a,0x00000000,0x00000023,0x00000000,0x00050048,0x0000000a,
|
||||
0x00000001,0x00000023,0x00000004,0x00030047,0x00000028,0x00000002,0x00050048,0x00000028,
|
||||
0x00000000,0x00000023,0x00000000,0x00040047,0x0000002a,0x00000021,0x00000000,0x00040047,
|
||||
0x0000002a,0x00000022,0x00000000,0x00020013,0x00000002,0x00030021,0x00000003,0x00000002,
|
||||
0x00030016,0x00000006,0x00000020,0x00040020,0x00000007,0x00000007,0x00000006,0x00040015,
|
||||
0x00000009,0x00000020,0x00000000,0x0004001e,0x0000000a,0x00000009,0x00000006,0x00040020,
|
||||
0x0000000b,0x00000009,0x0000000a,0x0004003b,0x0000000b,0x0000000c,0x00000009,0x00040015,
|
||||
0x0000000d,0x00000020,0x00000001,0x0004002b,0x0000000d,0x0000000e,0x00000001,0x00040020,
|
||||
0x0000000f,0x00000009,0x00000006,0x00040020,0x00000012,0x00000007,0x00000009,0x0004002b,
|
||||
0x00000009,0x00000014,0x00000000,0x0004002b,0x0000000d,0x0000001b,0x00000000,0x00040020,
|
||||
0x0000001c,0x00000009,0x00000009,0x00020014,0x0000001f,0x0004002b,0x00000006,0x00000022,
|
||||
0x3f7fffff,0x0004002b,0x00000006,0x00000024,0x3f000000,0x0003001e,0x00000028,0x00000006,
|
||||
0x00040020,0x00000029,0x0000000c,0x00000028,0x0004003b,0x00000029,0x0000002a,0x0000000c,
|
||||
0x00040020,0x0000002c,0x0000000c,0x00000006,0x00040017,0x0000002e,0x00000009,0x00000003,
|
||||
0x0004002b,0x00000009,0x0000002f,0x00000001,0x0006002c,0x0000002e,0x00000030,0x0000002f,
|
||||
0x0000002f,0x0000002f,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8,
|
||||
0x00000005,0x0004003b,0x00000007,0x00000008,0x00000007,0x0004003b,0x00000012,0x00000013,
|
||||
0x00000007,0x00050041,0x0000000f,0x00000010,0x0000000c,0x0000000e,0x0004003d,0x00000006,
|
||||
0x00000011,0x00000010,0x0003003e,0x00000008,0x00000011,0x0003003e,0x00000013,0x00000014,
|
||||
0x000200f9,0x00000015,0x000200f8,0x00000015,0x000400f6,0x00000017,0x00000018,0x00000000,
|
||||
0x000200f9,0x00000019,0x000200f8,0x00000019,0x0004003d,0x00000009,0x0000001a,0x00000013,
|
||||
0x00050041,0x0000001c,0x0000001d,0x0000000c,0x0000001b,0x0004003d,0x00000009,0x0000001e,
|
||||
0x0000001d,0x000500b0,0x0000001f,0x00000020,0x0000001a,0x0000001e,0x000400fa,0x00000020,
|
||||
0x00000016,0x00000017,0x000200f8,0x00000016,0x0004003d,0x00000006,0x00000021,0x00000008,
|
||||
0x00050085,0x00000006,0x00000023,0x00000021,0x00000022,0x00050081,0x00000006,0x00000025,
|
||||
0x00000023,0x00000024,0x0003003e,0x00000008,0x00000025,0x000200f9,0x00000018,0x000200f8,
|
||||
0x00000018,0x0004003d,0x00000009,0x00000026,0x00000013,0x00050080,0x00000009,0x00000027,
|
||||
0x00000026,0x0000000e,0x0003003e,0x00000013,0x00000027,0x000200f9,0x00000015,0x000200f8,
|
||||
0x00000017,0x0004003d,0x00000006,0x0000002b,0x00000008,0x00050041,0x0000002c,0x0000002d,
|
||||
0x0000002a,0x0000001b,0x0003003e,0x0000002d,0x0000002b,0x000100fd,0x00010038
|
||||
};
|
||||
@@ -53,6 +53,8 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "new_wait.h" // ((Triforce, 11-22, OS-04) a nothrow new that must answer at once: NoWait)
|
||||
|
||||
namespace shadow_prev {
|
||||
|
||||
inline constexpr uint64_t kPage = 4096;
|
||||
@@ -217,7 +219,10 @@ inline bool keep_old(std::unique_ptr<uint8_t[]>& b, uint64_t& hash, Vers& v, int
|
||||
v.n = (uint8_t)(maxn - 1);
|
||||
} else {
|
||||
if (alive().load(std::memory_order_relaxed) >= cap) return false;
|
||||
fresh.reset(new (std::nothrow) uint8_t[kPage]);
|
||||
{
|
||||
mkwvk::new_wait::NoWait at_once; // ((Triforce, OS-04) no page: none kept; not a wait at the commit limit)
|
||||
fresh.reset(new (std::nothrow) uint8_t[kPage]);
|
||||
}
|
||||
if (!fresh) return false;
|
||||
alive().fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
56
gpu/vk/sim_gpu.h
Normal file
56
gpu/vk/sim_gpu.h
Normal file
@@ -0,0 +1,56 @@
|
||||
// (Triforce, 11-22, the performance audit's OS-49, after Source B's) A lab switch: a slower GPU, so that a lab PC's renderer
|
||||
// meets the GPU-bound frames a weaker card's player has (the write-back waits, the capture's late pages, the recording slots,
|
||||
// the pacing), on the same 5090. RX_VK_SIM_GPU_SLOW=<factor> (over 1, at most 10; 2.5 is read): for each ms of work the
|
||||
// recordings measure, (factor - 1) ms more, a one-invocation compute loop (shaders/sim_gpu_burn.comp) at the end of the
|
||||
// replay's own recordings, between full barriers, inside the span the GPU time is measured over, so that the frame's GPU time
|
||||
// and Resolution Auto see the slower card (vk_replay.cpp sim_burn). What a burn took is measured by a third timestamp between
|
||||
// the recording's work and the burn, and the difference is carried: the debt below. It repeats the fast card's relative
|
||||
// answer only: no bandwidth, no other driver, no upscaler's cost. Unset: nothing of it runs, two timestamps a recording.
|
||||
//
|
||||
// The rules alone, no Vulkan: tests/test_sim_cores.py compiles them.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace sim_gpu {
|
||||
|
||||
inline constexpr int64_t kLeastNs = 1'000'000; // (a burn is planned for 1 ms at least: less owed waits)
|
||||
inline constexpr int64_t kMostNs = 16'000'000; // (and 16 ms at most: the rest goes to the next recordings)
|
||||
inline constexpr int64_t kDebtLeast = -50'000'000; // (the time owed is held between these: a burn that ran long is not
|
||||
inline constexpr int64_t kDebtMost = 100'000'000; // carried for seconds, nor a backlog after a hitch)
|
||||
inline constexpr double kFirstNsPerLoop = 4.0; // (a loop's time before the first measure)
|
||||
|
||||
/// RX_VK_SIM_GPU_SLOW: the factor, over 1 and at most 10; 0 (off) for anything else
|
||||
inline double factor_of(const char* e) {
|
||||
if (!e || !*e) return 0;
|
||||
char* end = nullptr;
|
||||
const double f = std::strtod(e, &end);
|
||||
if (end == e || *end || !(f > 1.0) || f > 10.0) return 0;
|
||||
return f;
|
||||
}
|
||||
/// The time owed after a recording measured `work_ns` of work and `added_ns` of burn, before the bounds
|
||||
inline double owed(int64_t debt, int64_t work_ns, int64_t added_ns, double factor) {
|
||||
return (double)debt + (double)work_ns * (factor - 1.0) - (double)added_ns;
|
||||
}
|
||||
/// And within them
|
||||
inline int64_t held(double owed_ns) { return (int64_t)std::clamp(owed_ns, (double)kDebtLeast, (double)kDebtMost); }
|
||||
/// The time the next burn is planned for: what is owed, less the burns planned and not measured yet; 0 under kLeastNs
|
||||
inline int64_t plan_ns(int64_t debt, int64_t in_flight) {
|
||||
const int64_t o = debt - in_flight;
|
||||
return o < kLeastNs ? 0 : std::min(o, kMostNs);
|
||||
}
|
||||
/// The loop count for a planned time, at the measured time a loop takes
|
||||
inline uint32_t loops_for(int64_t ns, double ns_per_loop) {
|
||||
if (ns <= 0) return 0;
|
||||
const double l = (double)ns / (ns_per_loop > 0.01 ? ns_per_loop : 0.01);
|
||||
return (uint32_t)std::clamp(l, 1.0, (double)(1u << 30));
|
||||
}
|
||||
/// The time a loop takes after a burn of `loops` took `added_ns`: 0.8 of what it was, 0.2 of what this one measured
|
||||
inline double calibrated(double ns_per_loop, int64_t added_ns, uint32_t loops) {
|
||||
if (!loops || added_ns <= 0) return ns_per_loop;
|
||||
const double m = (double)added_ns / loops;
|
||||
return ns_per_loop > 0 ? 0.8 * ns_per_loop + 0.2 * m : m;
|
||||
}
|
||||
|
||||
} // namespace sim_gpu
|
||||
49
gpu/vk/slot_retire.h
Normal file
49
gpu/vk/slot_retire.h
Normal file
@@ -0,0 +1,49 @@
|
||||
// (Triforce, 11-21, the performance audit's TEX-76, after Source B's) When a depth target that another size has replaced at its
|
||||
// start is let go.
|
||||
//
|
||||
// The game draws each character's shadow map (D32, one level) at a fixed address, sized from the camera every frame. An
|
||||
// image is kept by its storage, width and height, so each new size is a new texture, and the one before it has been drawn
|
||||
// over. The sweep took such a texture only once it was unused for 120 presents (60 over a fixed budget): a pile of dead maps
|
||||
// at every slot, in VRAM for two seconds each. Source B's rule, as it ended after its earlier ones retired targets the game
|
||||
// drew into again (after 1 frame; colour targets; depth of the same size): a depth target of one level and layer at the
|
||||
// same start and of the same format as a newer target of another width or height, the newer one written after its last
|
||||
// use, goes once it is unused for kWait presents. Its VRAM peak in the night market fell from 5,972 to 5,482 MiB, and no
|
||||
// retired image was wanted again within 16 frames (29 were with a wait of 1). Nothing is written back: the newer target's
|
||||
// texels are in that storage. The image goes to the render targets' pool by its create info, as any dead target's.
|
||||
// RX_VK_SLOT_RETIRE=<presents>: the wait, 16 when not set; 0: only after the sweep's own 120, as before.
|
||||
//
|
||||
// The rule alone, no Vulkan: tests/test_slot_retire.py compiles it.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace slot_retire {
|
||||
|
||||
inline constexpr uint64_t kWait = 16;
|
||||
|
||||
/// The wait in presents: asked, RX_VK_SLOT_RETIRE's value; not set or empty kWait; 0 or less (and what is no number): 0, off
|
||||
inline uint64_t wait(const char* asked) {
|
||||
if (!asked || !*asked) return kWait;
|
||||
const long v = atol(asked);
|
||||
return v > 0 ? (uint64_t)v : 0;
|
||||
}
|
||||
|
||||
/// What the rule looks at of a texture
|
||||
struct Target {
|
||||
bool gpu_written, depth; // (the GPU drew it; its aspect is depth)
|
||||
uint32_t levels, layers, deep; // (deep: its depth in texels, 1 for a 2D one)
|
||||
int nvn_format;
|
||||
uint32_t width, height;
|
||||
uint64_t last_frame, last_written; // (the present it was last used in; the frame the GPU last wrote it in)
|
||||
};
|
||||
|
||||
/// Whether old, unused for `unused` presents, is let go because newer took its place: both at the same start
|
||||
inline bool retired_by(const Target& old, const Target& newer, uint64_t unused, uint64_t wait_presents) {
|
||||
if (!wait_presents || unused < wait_presents) return false;
|
||||
if (!old.gpu_written || !old.depth || old.levels != 1 || old.layers != 1 || old.deep > 1) return false;
|
||||
if (newer.nvn_format != old.nvn_format) return false;
|
||||
if (newer.width == old.width && newer.height == old.height) return false; // (the same size: the game may draw into it again)
|
||||
return newer.last_written > old.last_frame; // (written after its last use: drawn over)
|
||||
}
|
||||
|
||||
} // namespace slot_retire
|
||||
41
gpu/vk/staging_keep.h
Normal file
41
gpu/vk/staging_keep.h
Normal file
@@ -0,0 +1,41 @@
|
||||
// How much of the idle staging arenas a present may keep (no game, no GPU: rules). tests/test_staging_burst.py
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's TEX-28, after Source A's.) The uploads stage their bytes in host-visible
|
||||
// arenas of 64 MB (vk_texture.cpp staging()). An arena is taken again once the GPU is done with the submission that read
|
||||
// it, which texture_frame learns a present later: frames that stage one after another need two arenas in turn. Each
|
||||
// present gives the idle ones back beyond what is kept, 256 MB, and 64 MB on a card of 6.5 GB or less, where an arena may
|
||||
// be VRAM behind a resizable BAR. With 64 MB kept, the arena that had its turn was idle at every second present of a load,
|
||||
// was freed, and was made again the present after: Source A counted 16 arenas made, 225 ms of them on the thread that
|
||||
// replays, in one arrival on a handheld (15.5 ms each), and 4 with this rule.
|
||||
//
|
||||
// The rule: an idle arena that something staged into less than RX_VK_STAGING_BURST_S seconds ago (2) is one of a burst's,
|
||||
// and while such an arena is looked at the idle ones may add up to 256 MB instead of what is kept. Not while VRAM is over
|
||||
// 90% of the allowance (the render targets' pool's own flag, pool_pressed): a full card keeps what it kept. Where 256 MB
|
||||
// are kept anyway, every card over 6.5 GB, nothing changes. The arenas over 64 MB still go at once, and the 30 s rule
|
||||
// (RX_VK_STAGING_IDLE_S) stays. RX_VK_STAGING_BURST_S=0: as before.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace staging_keep {
|
||||
|
||||
inline constexpr uint64_t kBurstKeep = 256ull << 20; // (what the idle arenas of a burst may add up to)
|
||||
inline constexpr int64_t kBurstS = 2; // (how long after its last staging an arena is one of a burst's)
|
||||
|
||||
/// RX_VK_STAGING_BURST_S: unset or empty kBurstS; a number of seconds, at most an hour; 0, less or not a number: no rule
|
||||
inline int64_t burst_s(const char* env) {
|
||||
if (!env || !*env) return kBurstS;
|
||||
const long long v = std::atoll(env);
|
||||
return v > 0 ? (int64_t)std::min<long long>(v, 3600) : 0;
|
||||
}
|
||||
|
||||
/// Whether an arena last staged into `since_used_ms` ago is one of a burst's: `pressed` is VRAM over 90% of the allowance
|
||||
constexpr bool recent(int64_t burst_s, bool pressed, int64_t since_used_ms) {
|
||||
return burst_s > 0 && !pressed && since_used_ms >= 0 && since_used_ms < burst_s * 1000;
|
||||
}
|
||||
|
||||
/// What the arenas looked at so far and this one may add up to for this one to stay: `keep` is what a present keeps anyway
|
||||
constexpr uint64_t allowance(uint64_t keep, bool recent) { return recent ? std::max(keep, kBurstKeep) : keep; }
|
||||
|
||||
} // namespace staging_keep
|
||||
@@ -80,6 +80,9 @@ struct Frame {
|
||||
double wb_unsent_ms = 0;
|
||||
int64_t wb_held = 0; // ((Triforce, wp20) how many went on after the writer's recording was done on the GPU, and
|
||||
double wb_held_ms = 0; // that part of them: the submissions' timeline held by work stopped at a wait, wb_rec_wait.h)
|
||||
// ((Triforce, 11-22, the performance audit's SYNC-122) that part by caller, ms: a capture's take, a read of a page put off,
|
||||
// a binding of one, a readback's fallback (wb_rec_wait.h Caller; the readbacks' is in none of the counts above)
|
||||
double wb_held_by[4] = {0, 0, 0, 0};
|
||||
double pipe_wait = 0; // (waits for shaders and pipelines)
|
||||
int64_t pipe_waits = 0, left_out = 0; // (how many; draws left out while theirs were built)
|
||||
int64_t pipes_first = 0, pipes_ahead = 0, pipes_order = 0; // ((Triforce, wp18) vk_draw.cpp graphics_pipeline: the pipelines the
|
||||
@@ -186,6 +189,10 @@ inline int stall_line(const Frame& f, char* out, size_t cap) {
|
||||
(long long)f.wb_held, f.wb_held_ms);
|
||||
put("%s", ")");
|
||||
}
|
||||
// ((Triforce, 11-22, SYNC-122) the held part by caller, when there is any: the reads' side of RX_VK_WB_WAIT_REC)
|
||||
if (f.wb_held_by[0] + f.wb_held_by[1] + f.wb_held_by[2] + f.wb_held_by[3] > 0)
|
||||
put("; the timeline held after a writer's recording was done, by caller: takes %.1f, reads %.1f, binds %.1f, readbacks %.1f ms",
|
||||
f.wb_held_by[0], f.wb_held_by[1], f.wb_held_by[2], f.wb_held_by[3]);
|
||||
if (f.pipe_waits || f.left_out)
|
||||
put("; %lld waits for shaders or pipelines (%.1f ms), %lld draws left out", (long long)f.pipe_waits, f.pipe_wait,
|
||||
(long long)f.left_out);
|
||||
|
||||
53
gpu/vk/stub_same_size.h
Normal file
53
gpu/vk/stub_same_size.h
Normal file
@@ -0,0 +1,53 @@
|
||||
// A gameplay upscale network that cannot run: its input drawn into its output, also where the two are one size (no
|
||||
// game, no GPU: the rule; vk_draw.cpp upscale_stub).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's GPUWORK-21, after Source B's and Source A's.) The game draws its 3D through
|
||||
// a small network every frame: fd029a317a9a4b94 from 3/4 size, 9dc98871eb7fc101 at full size. upscale_stub stands in
|
||||
// for a network that was left out or did not translate (RX_VK_NO_F16, a card or driver that cannot build it), by
|
||||
// drawing the network's input into its output. It declined a dispatch whose output is no larger than its input, as not
|
||||
// an upscale's last pass, so at Render resolution 100% a 9dc98871 that could not run was skipped whole: nothing wrote
|
||||
// the picture, a black 3D under the HUD. Source B turns its bypass on by itself where its network cannot be built, and
|
||||
// Source A lets that pass through for the gameplay networks.
|
||||
//
|
||||
// So, for the two gameplay networks only:
|
||||
// - an output as large as its input is accepted too (any other program is still declined: a pass of that shape that
|
||||
// is no gameplay network is not an upscale's last one);
|
||||
// - both take the typed upscale first, which reads the scene's shared-exponent picture as what it is and undoes the
|
||||
// network's square-root encoding (shaders/taau_upscale.comp). It was fd029a317a9a4b94's alone, so a 9dc98871 that
|
||||
// upscaled went to the blit, which copies the encoded bits as they lie;
|
||||
// - the blit, where the typed upscale cannot be made, warns of wrong colours for both.
|
||||
// The stand-in looks like the upscaler Off (GPUWORK-20): softer, none of the network's anti-aliasing. Only the path of
|
||||
// a network that cannot run changes; where it runs, nothing does.
|
||||
//
|
||||
// RX_VK_STUB_SAME_SIZE=0: the decline and the first network alone in the typed upscale, as before.
|
||||
// RX_VK_NO_UPSCALE_STUB still turns the whole stub off.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace stub_same_size {
|
||||
|
||||
/// RX_VK_STUB_SAME_SIZE: on unless set to 0
|
||||
inline bool on(const char* env) {
|
||||
return !env || std::atoi(env) != 0;
|
||||
}
|
||||
|
||||
/// Whether the stub takes a dispatch by its sizes: an output larger than its input, or (the rule on) a gameplay
|
||||
/// network's as large as it. An output smaller than its input never
|
||||
constexpr bool sized(uint64_t out_area, uint64_t in_area, bool gameplay, bool on) {
|
||||
return out_area > in_area || (on && gameplay && out_area == in_area && out_area != 0);
|
||||
}
|
||||
|
||||
/// Whether the typed upscale is tried first: any program that is not a gameplay network, the 3/4 network, and (the
|
||||
/// rule on) the other gameplay network too
|
||||
constexpr bool typed(bool gameplay, bool taau, bool on) {
|
||||
return !gameplay || taau || on;
|
||||
}
|
||||
|
||||
/// Whether the blit that follows a typed upscale that could not be made warns of wrong colours: the 3/4 network's, and
|
||||
/// (the rule on) every gameplay network's
|
||||
constexpr bool warns(bool gameplay, bool taau, bool on) {
|
||||
return taau || (on && gameplay);
|
||||
}
|
||||
|
||||
} // namespace stub_same_size
|
||||
39
gpu/vk/surface_lost.h
Normal file
39
gpu/vk/surface_lost.h
Normal file
@@ -0,0 +1,39 @@
|
||||
// (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) Presentation made again after the window's
|
||||
// surface is lost.
|
||||
//
|
||||
// An acquire, a present or a swapchain's creation can answer VK_ERROR_SURFACE_LOST_KHR: the surface itself is gone (a
|
||||
// display driver reset, a monitor change, a remote session), and no swapchain can be made for it again. Nothing handled
|
||||
// that result: every acquire after it failed without a word, so the picture stayed frozen while the game ran on, until a
|
||||
// restart.
|
||||
//
|
||||
// Now the renderer lets go of what hangs on the surface, in the order Vulkan asks (vk_main.cpp invalidate_lost_surface:
|
||||
// the device idle, the swapchain's views, its semaphores, the swapchain, then the surface), and the next swapchain's
|
||||
// creation first makes a surface for the window again with the startup's own calls (open_window_surface): only while the
|
||||
// window has a size (a minimized window waits), one try each kRetryMs at most, and with the presents' queue chosen again
|
||||
// for the new surface.
|
||||
//
|
||||
// RX_VK_SURFACE_RECOVER=0: as before, the result is not handled. Nor is it while NVIDIA DLSS frame generation runs: the
|
||||
// swapchain is Streamline's there, and that path has not been tried.
|
||||
//
|
||||
// No Vulkan: tests/test_present_surface_lost.py compiles this, and runs vk_main.cpp's and vk_replay.cpp's side against a
|
||||
// model of a driver that loses its surface.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace surface_lost {
|
||||
|
||||
/// RX_VK_SURFACE_RECOVER (value, null when unset): on unless it is 0
|
||||
inline bool on(const char* value) { return !(value && *value && std::atoi(value) == 0); }
|
||||
|
||||
/// A Vulkan result is a lost surface that is recovered from: lost (the result is VK_ERROR_SURFACE_LOST_KHR), the switch
|
||||
/// on, and the swapchain the renderer's own (not Streamline's)
|
||||
inline bool recovers(bool lost, bool on, bool dlssg_active) { return lost && on && !dlssg_active; }
|
||||
|
||||
/// The least time between two tries to make the surface again
|
||||
constexpr int64_t kRetryMs = 250;
|
||||
|
||||
/// A try is due at now_ms: none made yet (last_ms 0), or the last one kRetryMs ago or more
|
||||
inline bool retry_due(int64_t now_ms, int64_t last_ms) { return !last_ms || now_ms - last_ms >= kRetryMs; }
|
||||
|
||||
} // namespace surface_lost
|
||||
97
gpu/vk/table_stream.h
Normal file
97
gpu/vk/table_stream.h
Normal file
@@ -0,0 +1,97 @@
|
||||
// (Triforce, 11-22, the performance audit's SYNC-04, after Source A's) The guest page table's changes of one recording,
|
||||
// streamed into transfers of at most 64 KiB.
|
||||
//
|
||||
// A change to the table is a run: `count` pages from page `first`, each given `address` plus its offset (or 0 when the
|
||||
// address is 0: unmapped). The recording's flush made a (page, value) pair for every page of every run, sorted them all
|
||||
// and kept the last of each page: 16 bytes a page and a sort, on the recording thread, for ranges that are hundreds of
|
||||
// MB when a pool is imported ahead or merged (150,000 pages: about 8 ms, and 2.4 MB of scratch).
|
||||
//
|
||||
// stream() writes the same transfers in the same order without the pairs: runs that come in page order and do not
|
||||
// overlap, which is nearly always, are appended one after the other; any other set is swept by page with a heap of
|
||||
// the runs covering the page, in which the newest run (the highest index) owns it. One batch of 8,192 entries is reused
|
||||
// for every transfer. The transfers are those of the sort it replaces, entry for entry: tests/test_page_table_stream.py
|
||||
// compares the two on random runs. RX_VK_TABLE_STREAM=0 (vk_memory.cpp memory_table_flush): the sort, as before.
|
||||
//
|
||||
// No Vulkan here: emit(first_page, values, n) is the caller's vkCmdUpdateBuffer.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
|
||||
namespace table_stream {
|
||||
|
||||
constexpr size_t kBatch = 8192; // (entries a transfer: 64 KiB, the most vkCmdUpdateBuffer takes)
|
||||
|
||||
struct Scratch {
|
||||
std::vector<uint64_t> batch; // (the transfer being filled, kept from flush to flush)
|
||||
std::vector<uint32_t> order, heap; // (the sweep's: an index a run)
|
||||
};
|
||||
|
||||
/// The runs' changes (the later run of a page wins) as transfers of consecutive pages, ascending, each of kBatch entries
|
||||
/// at most: emit(first_page, values, n). Run: {first, count, address}. How many entries were written
|
||||
template <class Run, class Emit>
|
||||
uint64_t stream(const Run* runs, size_t n, Scratch& s, Emit&& emit) {
|
||||
s.batch.clear();
|
||||
if (s.batch.capacity() < kBatch) s.batch.reserve(kBatch);
|
||||
uint64_t batch_first = 0, written = 0;
|
||||
const auto send = [&] {
|
||||
if (!s.batch.empty()) emit(batch_first, (const uint64_t*)s.batch.data(), s.batch.size());
|
||||
s.batch.clear();
|
||||
};
|
||||
// (count pages from `page`, the first one's value `address`, 0: all zero. A transfer ends at a gap or when it is full)
|
||||
const auto append = [&](uint64_t page, uint64_t count, uint64_t address) {
|
||||
written += count;
|
||||
while (count) {
|
||||
if (!s.batch.empty() && (page != batch_first + s.batch.size() || s.batch.size() == kBatch)) send();
|
||||
if (s.batch.empty()) batch_first = page;
|
||||
const uint64_t room = kBatch - s.batch.size(), take = count < room ? count : room;
|
||||
const size_t at = s.batch.size();
|
||||
s.batch.resize(at + (size_t)take);
|
||||
uint64_t* const out = s.batch.data() + at;
|
||||
if (address)
|
||||
for (uint64_t i = 0; i < take; i++) out[i] = address + (i << 12);
|
||||
else
|
||||
std::fill(out, out + take, uint64_t(0));
|
||||
page += take, count -= take;
|
||||
if (address) address += take << 12;
|
||||
}
|
||||
};
|
||||
// (in page order and apart: each run ends at or before the next one's first page)
|
||||
const bool in_order = std::adjacent_find(runs, runs + n, [](const Run& a, const Run& b) { return a.first + a.count > b.first; }) == runs + n;
|
||||
if (in_order) {
|
||||
for (size_t i = 0; i < n; i++) append(runs[i].first, runs[i].count, runs[i].address);
|
||||
send();
|
||||
return written;
|
||||
}
|
||||
// (overlapping or out of order: the runs by first page; of those covering a page the newest owns it. A run stays in
|
||||
// the heap past its end until it is the top, and is dropped there)
|
||||
s.order.resize(n);
|
||||
std::iota(s.order.begin(), s.order.end(), uint32_t(0));
|
||||
std::sort(s.order.begin(), s.order.end(), [&](uint32_t a, uint32_t b) { return runs[a].first != runs[b].first ? runs[a].first < runs[b].first : a < b; });
|
||||
s.heap.clear();
|
||||
uint64_t pos = 0;
|
||||
size_t k = 0;
|
||||
while (k < n || !s.heap.empty()) {
|
||||
if (s.heap.empty()) pos = runs[s.order[k]].first;
|
||||
while (k < n && runs[s.order[k]].first <= pos) {
|
||||
s.heap.push_back(s.order[k++]);
|
||||
std::push_heap(s.heap.begin(), s.heap.end());
|
||||
}
|
||||
while (!s.heap.empty() && runs[s.heap.front()].first + runs[s.heap.front()].count <= pos) {
|
||||
std::pop_heap(s.heap.begin(), s.heap.end());
|
||||
s.heap.pop_back();
|
||||
}
|
||||
if (s.heap.empty()) continue;
|
||||
const Run& r = runs[s.heap.front()];
|
||||
uint64_t stop = r.first + r.count; // (this run owns the pages up to its end, or to where a run begins that may be newer)
|
||||
if (k < n && runs[s.order[k]].first < stop) stop = runs[s.order[k]].first;
|
||||
append(pos, stop - pos, r.address ? r.address + ((pos - r.first) << 12) : 0);
|
||||
pos = stop;
|
||||
}
|
||||
send();
|
||||
return written;
|
||||
}
|
||||
|
||||
} // namespace table_stream
|
||||
@@ -17,4 +17,32 @@ inline bool shifted_over(uint64_t lo, uint64_t hi, uint64_t last_used, uint64_t
|
||||
return b > a && (b - a) * 8 >= hi - lo;
|
||||
}
|
||||
|
||||
/// ((Triforce, 11-21, the performance audit's TEX-30, after Source A's) The same question for all the other render targets
|
||||
/// together. Judged one by one, an eighth each, a target that two others had drawn over between them, one over its head
|
||||
/// and one over its tail, each less than an eighth, was kept for good: its texels are as gone from that memory. Here the
|
||||
/// parts the others cover inside [lo, hi) are added up, each byte once: add() takes the others in the order of their starts
|
||||
/// (vk_texture.cpp's by_start), so a part never starts before the one before it and one pass merges them. A target with
|
||||
/// the same start, one inside it and one not written after its last use count nothing, as in shifted_over
|
||||
struct ShiftedOver {
|
||||
uint64_t lo, hi, last_used;
|
||||
uint64_t closed = 0, run_lo = 0, run_hi = 0; // (the parts merged and done with, and the one still growing)
|
||||
|
||||
void add(uint64_t olo, uint64_t ohi, uint64_t o_written) {
|
||||
if (hi <= lo || ohi <= olo || olo == lo || o_written <= last_used) return;
|
||||
if (olo >= lo && ohi <= hi) return;
|
||||
const uint64_t a = std::max(lo, olo), b = std::min(hi, ohi);
|
||||
if (b <= a) return;
|
||||
if (a <= run_hi) {
|
||||
run_hi = std::max(run_hi, b);
|
||||
} else {
|
||||
closed += run_hi - run_lo;
|
||||
run_lo = a, run_hi = b;
|
||||
}
|
||||
}
|
||||
/// The bytes of it the others cover
|
||||
uint64_t covered() const { return closed + (run_hi - run_lo); }
|
||||
/// Whether that is an eighth of it or more
|
||||
bool an_eighth() const { return hi > lo && covered() * 8 >= hi - lo; }
|
||||
};
|
||||
|
||||
} // namespace mkwvk::target_overlap
|
||||
|
||||
58
gpu/vk/target_pool.h
Normal file
58
gpu/vk/target_pool.h
Normal file
@@ -0,0 +1,58 @@
|
||||
// (Triforce, 11-21, the performance audit's TEX-29, after Source A's and Source B's) The render targets' pool: how much it
|
||||
// may hold, what goes first, and when.
|
||||
//
|
||||
// The pool gives a dead render target's image to the next target of its shape, so that a target the game makes again each
|
||||
// frame costs no image creation. It held a fixed 256 MB, kept every image 120 presents, was trimmed in the order of its
|
||||
// hash table, and was emptied whenever VRAM use passed 90% of the allowance. Source A found it starved at 2560 x 1440: of
|
||||
// 22,000 images it was given in 100 s it destroyed 21,879 for being over the bound, and with a larger bound, an age order
|
||||
// and tiers the images made new fell from about 20,000 to about 4,300 a run and the 1% low rose from 36 to 44 fps.
|
||||
// - The bound: a twelfth of what the driver allows, 256 MB at least and 768 at most (RX_VK_TARGET_POOL_MB sets it; 0: no
|
||||
// pool).
|
||||
// - Trimmed oldest first, through one list in the order the images came in: over the bound, the image that has waited
|
||||
// longest goes, whatever its shape.
|
||||
// - The wait: a shape never taken from the pool goes after RX_VK_POOL_IDLE presents (8): the game's per-character shadow
|
||||
// maps come in sizes of their own that nothing asks for again, and at 120 presents they filled the bound. A shape that
|
||||
// was taken before keeps 120.
|
||||
// - Tiers by the VRAM in use: above 90% of the allowance half the bound, above 97% none. With a floor of blocks kept for
|
||||
// the targets (vk_vma.cpp target_floor_kept) the tiers do nothing, as the pool was not given back there before either:
|
||||
// what it gave back went into blocks the floor keeps.
|
||||
// RX_VK_TARGET_POOL_TRIM=0: no tiers (the pool emptied above 90%), as before; with RX_VK_TARGET_POOL_MB=256 and
|
||||
// RX_VK_POOL_IDLE=120 the pool is the one it was, but for the order.
|
||||
//
|
||||
// The rules alone, no Vulkan: tests/test_target_pool_rules.py compiles them.
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace target_pool {
|
||||
|
||||
inline constexpr uint64_t kLeastMb = 256, kMostMb = 768;
|
||||
inline constexpr uint64_t kIdle = 8, kIdleTaken = 120; // (presents: a shape never taken from the pool; one taken before)
|
||||
inline constexpr size_t kTrim = 16, kTrimPressed = 64; // (images destroyed a present; while a tier holds)
|
||||
inline constexpr size_t kTakenKeys = 4096; // (the shapes remembered as taken)
|
||||
|
||||
/// The bound in MB with no switch: a twelfth of the allowance (bytes), kLeastMb to kMostMb; not known (0): kLeastMb
|
||||
inline uint64_t default_mb(uint64_t allowed) {
|
||||
return allowed ? std::clamp<uint64_t>((allowed >> 20) / 12, kLeastMb, kMostMb) : kLeastMb;
|
||||
}
|
||||
/// The tier: 0, 1 with the VRAM in use above 90% of the allowance, 2 above 97%. known: the driver gave both numbers
|
||||
inline int tier(bool known, uint64_t in_use, uint64_t allowed) {
|
||||
if (!known || !allowed) return 0;
|
||||
return in_use > allowed / 100 * 97 ? 2 : in_use > allowed / 10 * 9 ? 1 : 0;
|
||||
}
|
||||
/// What the pool may hold under a tier. tiers: RX_VK_TARGET_POOL_TRIM (off: any tier empties it, as before). floor_kept: a
|
||||
/// floor of the targets' blocks is kept, and the pool is not given back at all
|
||||
inline uint64_t cap_for(uint64_t cap, int tier, bool tiers, bool floor_kept) {
|
||||
if (tier <= 0 || floor_kept) return cap;
|
||||
return tiers && tier == 1 ? cap / 2 : 0;
|
||||
}
|
||||
/// How many images a present may destroy
|
||||
inline size_t trim_most(int tier, bool floor_kept) { return tier > 0 && !floor_kept ? kTrimPressed : kTrim; }
|
||||
/// Whether an image put in at present since has waited out its time at present now. idle: RX_VK_POOL_IDLE; taken_before:
|
||||
/// its shape was taken from the pool lately
|
||||
inline bool idle_out(uint64_t now, uint64_t since, uint64_t idle, bool taken_before) {
|
||||
return now - since > (taken_before ? std::max(idle, kIdleTaken) : idle);
|
||||
}
|
||||
|
||||
} // namespace target_pool
|
||||
@@ -13,7 +13,9 @@
|
||||
// same call with the same arguments as BC3's colour half. And BC3's alpha half for such a block is (255, 255, indices 0),
|
||||
// which decodes to 255, what the BC1 RGB format gives every texel. Which textures are all opaque: astc_opaque.h.
|
||||
//
|
||||
// A row on the Graphics tab, Off unless set, because what a test cannot show is whether the GPU decoder agrees with
|
||||
// ((Triforce, 11-21, TEX-50) On unless the row says off, since options_version 5; and with no options file too, which
|
||||
// started it at 0 until then: vk_menu.cpp tex_bc1_at_start.) It was a row on the Graphics tab, off unless set, because what
|
||||
// a test cannot show is whether the GPU decoder agrees with
|
||||
// astc_opaque.h on every block of the game's textures, and the image changes in vk_texture.cpp have not run in the game.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
40
gpu/vk/tex_count.h
Normal file
40
gpu/vk/tex_count.h
Normal file
@@ -0,0 +1,40 @@
|
||||
// What a block-compressed image the GPU samples as it is (BC, or ASTC where the driver samples it) takes: whole blocks
|
||||
// (no game, no GPU: a rule). tests/test_tex_count_blocks.py
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's TEX-86, after Source A's.) vk_texture.cpp counted such an image by its texels:
|
||||
// a byte a texel for BC, and for ASTC the 16 bytes of a block for every texel, which is 16 times its size at 4 x 4, 64
|
||||
// times at 8 x 8 and 144 times at 12 x 12; BC1 and BC4, eight bytes a block, twice. The texture budget then let textures
|
||||
// go for memory nothing held (Source A's Intel UHD 770, whose driver samples ASTC: 17,608 MB counted against a budget of
|
||||
// 2,560, 2,241 MB by blocks), and the first size estimate left out the top levels of arrays that fit. A level is
|
||||
// ceil(w / bw) x ceil(h / bh) blocks a slice, each block its bytes, times the layers. The textures the renderer encodes
|
||||
// itself (decoded ASTC kept as BC3 or BC1) were counted right already and are not asked here.
|
||||
// RX_VK_COUNT_BY_TEXELS=1: the count as it was.
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace tex_count {
|
||||
|
||||
/// RX_VK_COUNT_BY_TEXELS: unset, empty or 0 counts by blocks; anything else brings the count by texels back
|
||||
inline bool by_texels(const char* env) { return env && *env && std::atoi(env) != 0; }
|
||||
|
||||
/// The blocks of one slice of level `l`: the level's size rounded up to whole blocks each way
|
||||
constexpr uint64_t level_blocks(uint32_t width, uint32_t height, uint32_t l, uint32_t bw, uint32_t bh) {
|
||||
const uint32_t w = l < 32 && (width >> l) ? (width >> l) : 1, h = l < 32 && (height >> l) ? (height >> l) : 1;
|
||||
const uint32_t x = bw ? bw : 1, y = bh ? bh : 1;
|
||||
return (uint64_t)((w + x - 1) / x) * ((h + y - 1) / y);
|
||||
}
|
||||
|
||||
/// The bytes of the levels `first` .. `levels - 1` of an image of blocks of `bw` x `bh` texels and `block_bytes` bytes:
|
||||
/// a 3D image's level has its depth in slices of blocks (a block is flat), every layer has the whole chain
|
||||
constexpr uint64_t bytes(uint32_t width, uint32_t height, uint32_t depth, uint32_t first, uint32_t levels, uint32_t layers,
|
||||
uint32_t bw, uint32_t bh, uint32_t block_bytes) {
|
||||
uint64_t blocks = 0;
|
||||
for (uint32_t l = first; l < levels; l++) {
|
||||
const uint32_t d = l < 32 && (depth >> l) ? (depth >> l) : 1;
|
||||
blocks += level_blocks(width, height, l, bw, bh) * d;
|
||||
}
|
||||
return blocks * block_bytes * (layers ? layers : 1);
|
||||
}
|
||||
|
||||
} // namespace tex_count
|
||||
@@ -18,6 +18,9 @@
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace tex_warm {
|
||||
|
||||
@@ -49,4 +52,109 @@ inline bool ripe(uint64_t noted, uint64_t frame) { return frame >= noted + kAgeF
|
||||
/// Whether the look goes on after a texture: neither the time spent nor the frame's uploads took the budget
|
||||
inline bool goes_on(int64_t spent_ns, int64_t frame_upload_ns) { return spent_ns < kBudgetNs && frame_upload_ns < kBudgetNs; }
|
||||
|
||||
// ---- (Triforce, 11-22, the performance audit's TEX-23, after Source A's) The list of what was drawn ----
|
||||
// Every registered slot made ahead needs the 40% rule above, and on a card of 8 GB that rule stops the warming at once (40% is
|
||||
// about 2.8 GB, below the 4 GB a stand in the lab drew from). Source A keeps a list of the textures a draw used within 90 s of
|
||||
// the game making them, between runs, and makes those ahead with a room of their own: while the textures and 256 MB more fit
|
||||
// their budget. A texture is named in the list by its place: its shape and eight 64-byte samples of its bytes (the signature),
|
||||
// mixed with its storage's address, so that a texture at a new address is not taken for one the list has (it is made at its
|
||||
// draw as before, and learnt then). RX_VK_TEX_WARM_LIST=1: the listed slots first, the others after them while the 40% rule
|
||||
// allows; 2: the listed slots alone; unset or 0: every slot as above. The list is learnt only while it is on. The file is
|
||||
// triforce_textures_drawn.bin beside the game, its own magic and version; one cut short or of another version is not read.
|
||||
inline int list_mode_of(const char* env) {
|
||||
if (!env || !*env) return 0;
|
||||
const int v = std::atoi(env);
|
||||
return v == 1 || v == 2 ? v : 0;
|
||||
}
|
||||
|
||||
inline constexpr int64_t kLearnNs = 90'000'000'000; // (a texture drawn within this of the game making it is learnt)
|
||||
inline constexpr size_t kListMost = 32768; // (places in the list: past it the list starts again)
|
||||
inline constexpr uint64_t kListRoom = 256ull << 20; // (listed textures are made while the textures and this fit their budget)
|
||||
inline constexpr uint64_t kSaveEvery = 300; // (presents between saves of a list that changed)
|
||||
inline constexpr int kSamples = 8;
|
||||
inline constexpr uint64_t kSampleBytes = 64;
|
||||
inline constexpr char kListMagic[8] = {'T', 'F', 'D', 'R', 'A', 'W', 'N', 0};
|
||||
inline constexpr uint32_t kListVersion = 1;
|
||||
|
||||
/// What a texture's signature takes of its shape (its descriptor and its storage's size)
|
||||
struct Shape {
|
||||
uint32_t width, height, depth, levels, format, target, layers;
|
||||
uint64_t size;
|
||||
};
|
||||
|
||||
/// Whether a texture the game made at born_ns (0: not known) and drawn at now_ns is learnt: within kLearnNs of it
|
||||
inline bool learnable(int64_t born_ns, int64_t now_ns) { return born_ns > 0 && now_ns >= born_ns && now_ns - born_ns <= kLearnNs; }
|
||||
|
||||
/// Where sample i (0 to kSamples - 1) of a storage of `size` bytes starts: spread from its start to its end, 64-byte aligned
|
||||
inline uint64_t sample_at(uint64_t size, int i) {
|
||||
if (size <= kSampleBytes) return 0;
|
||||
return ((size - kSampleBytes) * (uint64_t)i / (kSamples - 1)) & ~(kSampleBytes - 1);
|
||||
}
|
||||
/// How many bytes each sample reads: kSampleBytes, or the whole of a smaller storage
|
||||
inline uint64_t sample_len(uint64_t size) { return size < kSampleBytes ? size : kSampleBytes; }
|
||||
|
||||
/// The signature: the shape and the samples (kSamples * kSampleBytes bytes, sample i at i * kSampleBytes), FNV-1a
|
||||
inline uint64_t signature(const Shape& s, const uint8_t* samples) {
|
||||
uint64_t h = 0xCBF29CE484222325ull;
|
||||
const auto add = [&h](uint64_t v) {
|
||||
for (int i = 0; i < 8; i++) h = (h ^ ((v >> (8 * i)) & 0xFF)) * 1099511628211ull;
|
||||
};
|
||||
add(s.width), add(s.height), add(s.depth), add(s.levels), add(s.format), add(s.target), add(s.layers), add(s.size);
|
||||
for (uint64_t i = 0; i < kSamples * kSampleBytes; i++) h = (h ^ samples[i]) * 1099511628211ull;
|
||||
return h;
|
||||
}
|
||||
/// A signature at a storage address: what the list holds (never 0)
|
||||
inline uint64_t place(uint64_t sig, uint64_t gpu) {
|
||||
uint64_t h = sig ^ (gpu * 0x9E3779B97F4A7C15ull);
|
||||
h ^= h >> 31, h *= 0xBF58476D1CE4E5B9ull, h ^= h >> 29, h *= 0x94D049BB133111EBull, h ^= h >> 32;
|
||||
return h | 1;
|
||||
}
|
||||
/// A place learnt: true when the list changed. A full list (kListMost) starts again, so that it follows what the game draws now
|
||||
inline bool remember(std::unordered_set<uint64_t>& list, uint64_t p) {
|
||||
if (!p || list.count(p)) return false;
|
||||
if (list.size() >= kListMost) list.clear();
|
||||
list.insert(p);
|
||||
return true;
|
||||
}
|
||||
/// Whether a listed texture may be made: the target pool is not being given back (VRAM near the allowance), and the textures
|
||||
/// with kListRoom more fit their budget (0 or all ones: none is set, no room is known)
|
||||
inline bool listed_room(bool pressed, uint64_t texture_bytes, uint64_t budget) {
|
||||
if (pressed || !budget || budget == ~0ull) return false;
|
||||
return texture_bytes <= budget && budget - texture_bytes >= kListRoom;
|
||||
}
|
||||
|
||||
inline uint64_t list_check(const uint8_t* b, size_t n) {
|
||||
uint64_t h = 0xCBF29CE484222325ull;
|
||||
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 1099511628211ull;
|
||||
return h;
|
||||
}
|
||||
/// The file's bytes: the magic, the version, the count, the places, and a check of the places
|
||||
inline std::vector<uint8_t> encode(const std::unordered_set<uint64_t>& list) {
|
||||
std::vector<uint8_t> o(16 + list.size() * 8 + 8);
|
||||
const uint32_t head[2] = {kListVersion, (uint32_t)list.size()};
|
||||
std::memcpy(o.data(), kListMagic, 8), std::memcpy(o.data() + 8, head, 8);
|
||||
size_t at = 16;
|
||||
for (const uint64_t p : list) std::memcpy(o.data() + at, &p, 8), at += 8;
|
||||
const uint64_t check = list_check(o.data() + 16, list.size() * 8);
|
||||
std::memcpy(o.data() + at, &check, 8);
|
||||
return o;
|
||||
}
|
||||
/// The places of a file's bytes into `out`; false, and `out` left empty, for anything that is not a whole file of this version
|
||||
inline bool decode(const uint8_t* b, size_t n, std::unordered_set<uint64_t>& out) {
|
||||
out.clear();
|
||||
if (!b || n < 24 || std::memcmp(b, kListMagic, 8)) return false;
|
||||
uint32_t head[2];
|
||||
std::memcpy(head, b + 8, 8);
|
||||
if (head[0] != kListVersion || head[1] > kListMost || n != 16 + (size_t)head[1] * 8 + 8) return false;
|
||||
uint64_t check;
|
||||
std::memcpy(&check, b + n - 8, 8);
|
||||
if (check != list_check(b + 16, (size_t)head[1] * 8)) return false;
|
||||
for (uint32_t i = 0; i < head[1]; i++) {
|
||||
uint64_t p;
|
||||
std::memcpy(&p, b + 16 + (size_t)i * 8, 8);
|
||||
if (p) out.insert(p);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace tex_warm
|
||||
|
||||
71
gpu/vk/translate_wait.h
Normal file
71
gpu/vk/translate_wait.h
Normal file
@@ -0,0 +1,71 @@
|
||||
// A bounded wait for a draw whose program is still translating (no game, no GPU: the switch, the choice and the notes of
|
||||
// which programs write memory; vk_draw.cpp draw, translate_note). Apart from Vulkan so that tests/test_translate_wait.py runs
|
||||
// it alone.
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's PIPE-30, after Source A's.) A draw whose program is still being translated
|
||||
// waits for it, its thread running the job if it is still queued (jobs_wait); RX_VK_PIPE_SKIP=2 leaves such a draw out
|
||||
// when its targets hold an earlier frame, with no bound and no regard for what its shaders write. Source A, with its switch
|
||||
// at 8 ms: a draw into targets that hold an earlier frame waits at most that much in all a frame, never running the job
|
||||
// itself, then is left out (new targets and compute still wait). On its RTX 4070 SUPER, title to Castle Town to the field
|
||||
// on a cold bank, fps 44.9/45.3 -> 50.0/47.4, the summed wait 4.94/4.85 -> 1.80/1.70 s, frames over 100 ms 39/34 -> 22/23,
|
||||
// but 4.4k -> 21-23k draws left out, mostly in black load frames.
|
||||
//
|
||||
// With the guard the audit asked for: an untranslated program cannot be asked whether it writes memory, so only a program
|
||||
// key whose variants translated before are known to write none is bounded (the variants of a key share their stage code).
|
||||
// A key never translated, or one with a variant that writes, waits as before, and so does a draw into new targets.
|
||||
//
|
||||
// RX_VK_TRANSLATE_WAIT_MS=<ms>: 1 to 100 ms a frame in all, digits only (8 to try); unset, 0 or anything else: off, every
|
||||
// such draw waits, as before. RX_VK_PIPE_SKIP=0 and =2 keep their meaning.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace translate_wait {
|
||||
|
||||
inline constexpr int64_t kMs = 1000000; // (ns)
|
||||
inline constexpr int kMostMs = 100;
|
||||
|
||||
/// RX_VK_TRANSLATE_WAIT_MS: the milliseconds a frame, strictly: digits only, 1 to kMostMs; anything else 0 (off)
|
||||
inline int parse(const char* env) {
|
||||
if (!env || !*env) return 0;
|
||||
int v = 0;
|
||||
for (const char* c = env; *c; c++) {
|
||||
if (*c < '0' || *c > '9') return 0;
|
||||
v = v * 10 + (*c - '0');
|
||||
if (v > kMostMs) return 0;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
enum class Act { AsBefore, Bounded, Skip };
|
||||
/// A draw whose program is still translating. kept: its targets hold an earlier frame (pass_kept); clean: its program key is
|
||||
/// known to write no memory; left_ns: what the frame's budget has left. Bounded: wait at most left_ns, then leave it out;
|
||||
/// Skip: the budget is spent, left out at once; AsBefore: waited for
|
||||
inline Act choose(int ms, bool kept, bool clean, int64_t left_ns) {
|
||||
if (ms <= 0 || !kept || !clean) return Act::AsBefore;
|
||||
return left_ns >= kMs ? Act::Bounded : Act::Skip;
|
||||
}
|
||||
|
||||
/// Which program keys write memory, from their variants once translated: a key is clean only when every variant noted
|
||||
/// wrote none; one that writes sticks
|
||||
class Notes {
|
||||
public:
|
||||
void note(uint64_t key, bool writes) {
|
||||
std::lock_guard lk(mu_);
|
||||
bool& w = by_[key];
|
||||
w = w || writes;
|
||||
}
|
||||
bool clean(uint64_t key) const {
|
||||
std::lock_guard lk(mu_);
|
||||
const auto it = by_.find(key);
|
||||
return it != by_.end() && !it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
mutable std::mutex mu_;
|
||||
std::unordered_map<uint64_t, bool> by_; // (key -> some variant writes)
|
||||
};
|
||||
|
||||
} // namespace translate_wait
|
||||
52
gpu/vk/ub_batch.h
Normal file
52
gpu/vk/ub_batch.h
Normal file
@@ -0,0 +1,52 @@
|
||||
// (Triforce, 11-22, the performance audit's SYNC-21, after Source A's) A recording's uniform updates as copies of many
|
||||
// regions, ordered where they overlap: when a run of them ends.
|
||||
//
|
||||
// A recording's uniform updates are written into guest memory at its end, between two full barriers: one
|
||||
// vkCmdUpdateBuffer an update (about 80 a frame), and nothing between two updates of the same bytes, whose order on the
|
||||
// GPU is then not defined (transfer writes with no dependency between them; so far a validation finding only). With
|
||||
// RX_VK_UB_BATCH=1 the updates' bytes go into the recording's stream and each run of them into one buffer is one
|
||||
// vkCmdCopyBuffer of many regions. The regions of one copy may not overlap, and two writes of the same bytes need an
|
||||
// order: a run ends where a piece would write bytes the run already wrote, or where the destination buffer changes, and
|
||||
// the next run starts behind a barrier. This header is that rule alone (no Vulkan): vk_replay.cpp flush_ub_batched asks
|
||||
// it, tests/test_ub_update_batch.py runs both and takes each part of the rule away in turn.
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace ub_batch {
|
||||
|
||||
/// What has to happen before a piece joins the run: nothing; the regions so far copied (their source buffer is another,
|
||||
/// no barrier); the run ended by an overlap; the run ended by another destination. The last two: the regions so far
|
||||
/// copied, then a barrier, then a new run
|
||||
enum Before : int { JOIN = 0, COPY = 1, RUN_OVERLAP = 2, RUN_IMPORT = 3 };
|
||||
|
||||
template <class Buffer>
|
||||
struct Run {
|
||||
Buffer src{}, dst{}; // (the stream buffer the pending regions' bytes are in; the buffer the run writes)
|
||||
bool any = false; // (a run is open: dst is set)
|
||||
size_t pending = 0; // (regions not copied yet)
|
||||
std::vector<std::pair<uint64_t, uint64_t>> written; // ([at, end) in dst written since the run began, copied or not)
|
||||
|
||||
bool overlaps(uint64_t at, uint64_t end) const {
|
||||
for (const auto& w : written)
|
||||
if (at < w.second && w.first < end) return true;
|
||||
return false;
|
||||
}
|
||||
/// Before a piece from stream buffer s into [at, end) of d joins
|
||||
Before before(Buffer s, Buffer d, uint64_t at, uint64_t end) const {
|
||||
if (!any) return JOIN;
|
||||
if (d != dst) return RUN_IMPORT;
|
||||
if (overlaps(at, end)) return RUN_OVERLAP;
|
||||
if (pending && s != src) return COPY;
|
||||
return JOIN;
|
||||
}
|
||||
void clear() { any = false, pending = 0, written.clear(); }
|
||||
void begin(Buffer d) { dst = d, any = true, pending = 0, written.clear(); } // (a new run, into d)
|
||||
void add(Buffer s, uint64_t at, uint64_t end) { src = s, pending++, written.push_back({at, end}); } // (a region joins)
|
||||
void wrote(uint64_t at, uint64_t end) { written.push_back({at, end}); } // (bytes of dst written some other way in this run)
|
||||
void copied() { pending = 0; } // (the pending regions went into a copy)
|
||||
};
|
||||
|
||||
} // namespace ub_batch
|
||||
34
gpu/vk/upscale_off.h
Normal file
34
gpu/vk/upscale_off.h
Normal file
@@ -0,0 +1,34 @@
|
||||
// Which of the game's network dispatches the plain stretch stands in for, and how it draws (no game, no GPU: a rule).
|
||||
//
|
||||
// (Triforce, 11-22, the performance audit's GPUWORK-20, after Source A's.) The game upscales its 3D with a small neural
|
||||
// network every frame: fd029a317a9a4b94 from 3/4 size, 9dc98871eb7fc101 at full size, and one network of its own for each
|
||||
// other share of the Render resolution row. vk_draw.cpp taau_stand_in already draws the first one's output with a
|
||||
// stretch of its input (shaders/taau_upscale.comp, which undoes the network's square-root encoding) where the network
|
||||
// runs on software matrix multiplies and costs most of a frame; 9dc98871 still ran every frame there (2.6 ms of GPU
|
||||
// time at 2560x1440 on a mid-range AMD card, as Source A measured its like).
|
||||
//
|
||||
// "Off" leaves every gameplay upscale network out: its input stretched linearly into its output, no FXAA of the
|
||||
// stand-in's and no history bound (the game's own SMAA has already run on the input and stays). It is opt-in
|
||||
// (RX_VK_UPSCALE_OFF=1, or the lab's "upoff" phases): the picture is softer below 100% and has none of the network's
|
||||
// sharpening. The networks are still compiled as before, so leaving Off again needs nothing made.
|
||||
#pragma once
|
||||
|
||||
namespace upscale_off {
|
||||
|
||||
struct Choice {
|
||||
bool stand_in; // the stretch draws the dispatch's output; the network is not dispatched
|
||||
bool temporal; // as the stand-in was before: its history bound and FXAA. false: the plain linear stretch
|
||||
bool by_shape; // taken for its shape alone: the caller must still see the scene's formats at both ends
|
||||
};
|
||||
|
||||
/// listed: the program is one RX_VK_TAAU_KEYS names (by default fd029a317a9a4b94 on software multiplies);
|
||||
/// off: the upscaler is Off;
|
||||
/// by_key: the program is one of the two gameplay networks known by key;
|
||||
/// shaped: it has a gameplay upscale's shape (a network sampling one texture into one image).
|
||||
constexpr Choice choose(bool listed, bool off, bool by_key, bool shaped) {
|
||||
if (off && by_key) return {true, false, false};
|
||||
if (off && shaped && !listed) return {true, false, true};
|
||||
return {listed, true, false};
|
||||
}
|
||||
|
||||
} // namespace upscale_off
|
||||
@@ -2,6 +2,8 @@
|
||||
Upscaler, and the names it shows for DLSS's (and FSR 3.1's and FSR 4's) modes. No Vulkan, no menu: tests/test_upscaler_rows.cpp checks them. */
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace mkwvk::upscaler_rows {
|
||||
|
||||
/// The Upscaler row (opts.dlss, RX_VK_DLSS): the game's own network, NVIDIA DLSS, NVIDIA DLAA, AMD FSR 3.1 (built from AMD's
|
||||
@@ -189,7 +191,18 @@ inline int window_filter(unsigned picture_w, unsigned picture_h, unsigned dest_w
|
||||
/// generation transformer, preset M (NVIDIA's own pick for Performance); 2 the same generation as tuned for Ultra
|
||||
/// Performance, preset L; 3 NVIDIA's default, preset 0 (the runtime picks per mode, and a runtime update can change it)
|
||||
enum { MODEL_K = 0, MODEL_M = 1, MODEL_L = 2, MODEL_DEFAULT = 3, kModels = 4 };
|
||||
inline unsigned model_preset(int model) {
|
||||
/// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The K row on a card with less
|
||||
/// than 7 GB of video memory asks for preset E, NVIDIA's last CNN model: less GPU time and less of NGX's own memory than
|
||||
/// the transformer there (Source A's, a 6 GB laptop RTX 3050 at 1080p DLSS Performance: 15.75 to 14.20 ms of GPU a frame,
|
||||
/// 85 to 58 MB), with more ghosting on grass and thin objects at the smallest render sizes, and NVIDIA marks it deprecated.
|
||||
/// vram: the card's video memory in bytes; 0 when the rule is off (vk_dlss.cpp dlss_cnn_vram: RX_VK_DLSS_CNN_VRAM is not
|
||||
/// 1, the default) or the size is not known: K as chosen. M, L and NVIDIA's default stay as chosen, and RX_VK_DLSS_PRESET
|
||||
/// goes over all of it
|
||||
constexpr unsigned kPresetE = 5;
|
||||
constexpr uint64_t kCnnVramBytes = 7ull << 30;
|
||||
inline bool cnn_for_vram(int model, uint64_t vram) { return model == MODEL_K && vram && vram < kCnnVramBytes; }
|
||||
inline unsigned model_preset(int model, uint64_t vram = 0) {
|
||||
if (cnn_for_vram(model, vram)) return kPresetE;
|
||||
switch (model) {
|
||||
case MODEL_M: return 13;
|
||||
case MODEL_L: return 12;
|
||||
@@ -202,8 +215,8 @@ inline unsigned model_preset(int model) {
|
||||
inline bool preset_env(const char* env) {
|
||||
return env && ((*env >= 'A' && *env <= 'Z') || (*env >= 'a' && *env <= 'z') || (env[0] == '0' && !env[1]));
|
||||
}
|
||||
inline unsigned dlss_preset(int model, const char* env) {
|
||||
if (!preset_env(env)) return model_preset(model);
|
||||
inline unsigned dlss_preset(int model, const char* env, uint64_t vram = 0) {
|
||||
if (!preset_env(env)) return model_preset(model, vram);
|
||||
if (*env == '0') return 0;
|
||||
return (unsigned)(*env >= 'a' ? *env - 'a' + 1 : *env - 'A' + 1);
|
||||
}
|
||||
|
||||
@@ -150,6 +150,27 @@ enum { HOS_MODS_PRESS_UP = 1, HOS_MODS_PRESS_DOWN = 2, HOS_MODS_PRESS_LEFT = 4,
|
||||
text. HOS_OPT_TEXT_CHAR, option_set (the window's thread, host/display.c): a code point typed, or 8 Backspace, 127
|
||||
Ctrl+Backspace (a pasted text comes as its characters) */
|
||||
enum { HOS_OPT_TEXT_CHAR = 159, HOS_OPT_TEXT_WANTED };
|
||||
/* ((Triforce) 11-20) The Mods screen's music (host/mods_music.c). HOS_OPT_MODS_MUSIC, option: 0 none; 1 the title's list
|
||||
is there with its Mods entry (the tune is read from the game's files ahead of its use); 2 the Mods screen is open from
|
||||
the title's list: the game's shop theme plays quietly and the game's own sound, the title's music, is faded out
|
||||
under it, and back when the screen shuts */
|
||||
enum { HOS_OPT_MODS_MUSIC = 161 };
|
||||
/* ((Triforce) 11-20) The Mods entry with the game's own buttons, not by a clock of its own. HOS_OPT_MODS_ALPHA,
|
||||
option_set: how much of the list's last button the game shows this frame, in thousandths (0 to 1000: the button's own
|
||||
alpha as the game animates it, host/oot_menu.c), told before that frame's HOS_OPT_MODS_LIST. A HOS_OPT_MODS_LIST without
|
||||
HOS_MODS_LIST_SHOWN says that the list is there no longer for the entry (another screen has the game's cursor): the
|
||||
entry goes at once, as the game's buttons did */
|
||||
enum { HOS_OPT_MODS_ALPHA = 162 };
|
||||
/* (Triforce) Achievements (host/achievements.c keeps them, on unless HOS_ACHIEVEMENTS=0; gpu/vk/vk_menu.cpp and
|
||||
gpu/vk/achievement_rows.h show them: the System tab's Achievements page, and a notice when one is earned; 165 to 169 are
|
||||
theirs). HOS_OPT_ACH_ASK, option: 1 from a renderer that shows them. Once it answers the host names its list, in the list's
|
||||
order: HOS_OPT_ACH_TEXT, option_set: the next byte of an achievement's name, UTF-8, then a line feed and the line about it,
|
||||
then a line feed and the day it was earned when it was; HOS_OPT_ACH_ADD, option_set: the bytes sent so far are the next
|
||||
achievement's, the value its icon (0 to 255, host/achievements.h ach_icon) and HOS_ACH_EARNED when it is earned;
|
||||
HOS_OPT_ACH_COUNT, option_set: the list is whole (0: none). HOS_OPT_ACH_EARNED, option_set: the achievement at that place
|
||||
in the list was earned now, the bytes sent so far the day: the notice shows it. One at a time, the host waits between two */
|
||||
enum { HOS_OPT_ACH_TEXT = 165, HOS_OPT_ACH_ADD, HOS_OPT_ACH_COUNT, HOS_OPT_ACH_EARNED, HOS_OPT_ACH_ASK };
|
||||
enum { HOS_ACH_EARNED = 256 };
|
||||
|
||||
/* (hookshot) the mouse over the window while it is free (not captured): the Insert menu's (host/display.c,
|
||||
host/posix/display_sdl.c). x and y: the cursor in the window's pixels, of a window width x height pixels (the
|
||||
|
||||
@@ -29,6 +29,8 @@
|
||||
#endif
|
||||
|
||||
#include "dlss_checks.h"
|
||||
#include "dlss_feature.h" // ((Triforce, the upscaler audit's UPS-01) the render size the feature is made for)
|
||||
#include "lab_ab.h" // ((Triforce, 11-22, PRESENT-38) the lab's "dlssstrict" phases)
|
||||
#include "upscaler_rows.h"
|
||||
#include "vk_menu.h"
|
||||
#include "vk_paths.h"
|
||||
@@ -76,7 +78,13 @@ struct State {
|
||||
NVSDK_NGX_Handle* feature = nullptr;
|
||||
uint32_t feature_rw = 0, feature_rh = 0, feature_ow = 0, feature_oh = 0; // (its largest render size, its output)
|
||||
uint32_t min_rw = 0, min_rh = 0;
|
||||
// ((Triforce, 11-21, the upscaler audit's UPS-01) made for the one render size it is given (dlss_feature.h: a fixed share),
|
||||
// not for the largest of its mode's range; and whether the render size was seen changing under one output in this run
|
||||
// (the game's dynamic resolution), after which every feature is made for the range, as all were before)
|
||||
bool feature_exact = false;
|
||||
bool range_seen = false;
|
||||
unsigned feature_preset = ~0u; // (the preset it was made with: another one from the menu makes it again)
|
||||
dlss_checks::PresetSettle settle; // ((Triforce, 11-21, PRESENT-37) the row's preset, taken once it has held: dlss_checks.h)
|
||||
bool feature_hdr = false; // (made for linear HDR colour, NGX's IsHDR: the other kind of colour makes it again)
|
||||
uint64_t features_made = 0; // (how many times: a feature made again in play costs a stall, so each says why)
|
||||
// (a feature not made for want of video memory: tried again from retry_frame on, kMemoryRetries times before it is given
|
||||
@@ -287,7 +295,17 @@ bool ngx_init() {
|
||||
/// DLSS's preset: the DLSS model row's, RX_VK_DLSS_PRESET over it (upscaler_rows.h dlss_preset)
|
||||
unsigned wanted_preset() {
|
||||
static const char* const env = getenv("RX_VK_DLSS_PRESET");
|
||||
return upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr);
|
||||
// ((Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) with RX_VK_DLSS_CNN_VRAM=1 the
|
||||
// K row on a card under 7 GB asks for preset E, NVIDIA's CNN model (upscaler_rows.h cnn_for_vram); said once)
|
||||
const uint64_t vram = dlss_cnn_vram();
|
||||
const unsigned preset = upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr, vram);
|
||||
static bool told;
|
||||
if (!told && vram && preset != upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr)) {
|
||||
told = true;
|
||||
logf("vk: NVIDIA DLSS: %llu MB of video memory: the K row draws with preset E (RX_VK_DLSS_CNN_VRAM=1)",
|
||||
(unsigned long long)(vram >> 20));
|
||||
}
|
||||
return preset;
|
||||
}
|
||||
|
||||
/// DLSS's feature for this output size and preset, made when either changes (cmd: its creation's work): the engine's
|
||||
@@ -295,10 +313,17 @@ unsigned wanted_preset() {
|
||||
int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh, bool hdr, uint32_t* in_w,
|
||||
uint32_t* in_h, const char** fail) {
|
||||
State& s = st();
|
||||
const unsigned preset = wanted_preset();
|
||||
if (s.feature && s.feature_ow == ow && s.feature_oh == oh && rw <= s.feature_rw && rh <= s.feature_rh &&
|
||||
rw >= s.min_rw && rh >= s.min_rh && s.feature_preset == preset && s.feature_hdr == hdr)
|
||||
// ((Triforce, 11-21, the performance audit's PRESENT-37, after Source A's) the DLSS model row's preset once the row has
|
||||
// held it for RX_VK_DLSS_MODEL_SETTLE frames (30; 0: at once, as before): no feature for a preset stepped past)
|
||||
static const uint64_t settle_frames = dlss_checks::settle_frames(getenv("RX_VK_DLSS_MODEL_SETTLE"));
|
||||
const unsigned preset = s.settle.pick(wanted_preset(), s.feature_preset, s.feature != nullptr, texture_frame_no(), settle_frames);
|
||||
// ((Triforce, UPS-01) the feature there is takes this frame: its one render size, or any of its range (dlss_feature.h takes)
|
||||
const bool same_feature = s.feature && s.feature_ow == ow && s.feature_oh == oh && s.feature_preset == preset && s.feature_hdr == hdr;
|
||||
if (same_feature && dlss_feature::takes(s.feature_exact, s.feature_rw, s.feature_rh, s.min_rw, s.min_rh, rw, rh))
|
||||
return *in_w = s.feature_rw, *in_h = s.feature_rh, 1;
|
||||
// (a feature of one render size, and a frame of another under the same output: the game changes the render size, so
|
||||
// from here on the features of this run are made for their mode's whole range)
|
||||
const bool found_range = dlss_feature::found_range(same_feature, s.feature_exact, s.feature_rw, s.feature_rh, rw, rh);
|
||||
static const char* const kNoMemory = "no video memory for its feature: tried again in a while";
|
||||
if (s.retry_frame && texture_frame_no() < s.retry_frame) return *fail = kNoMemory, 0;
|
||||
// (why it is made: in the log with its time, so a stall in play can be matched with one)
|
||||
@@ -311,50 +336,90 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t
|
||||
snprintf(why, sizeof why, "preset %c, was %c (the DLSS model row)", upscaler_rows::preset_letter(preset),
|
||||
upscaler_rows::preset_letter(s.feature_preset));
|
||||
else if (s.feature_hdr != hdr) snprintf(why, sizeof why, "the colour is %s now", hdr ? "HDR" : "LDR");
|
||||
else if (found_range)
|
||||
snprintf(why, sizeof why, "the render size went from %ux%u to %ux%u: for its mode's whole range from now on", s.feature_rw,
|
||||
s.feature_rh, rw, rh);
|
||||
else
|
||||
snprintf(why, sizeof why, "the render size %ux%u left its range, %ux%u to %ux%u", rw, rh, s.min_rw, s.min_rh,
|
||||
s.feature_rw, s.feature_rh);
|
||||
const uint64_t key = (uint64_t)ow << 44 ^ (uint64_t)oh << 24 ^ (uint64_t)preset << 1 ^ (hdr ? 1 : 0) ^ 1ull << 63;
|
||||
if (s.refused == key) return *fail = "its feature was not made", 0; // (until a DLSS setting changes: upstream #2550)
|
||||
if (s.feature) {
|
||||
NVSDK_NGX_Handle* old = s.feature;
|
||||
temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });
|
||||
s.feature = nullptr;
|
||||
}
|
||||
if (found_range) s.range_seen = true;
|
||||
// (the quality mode whose range of render sizes holds this one: the network's 3/4 is DLSS's MaxQuality range)
|
||||
// (DLAA's when the render size is the output's)
|
||||
static const NVSDK_NGX_PerfQuality_Value scaled[] = {NVSDK_NGX_PerfQuality_Value_MaxQuality, NVSDK_NGX_PerfQuality_Value_DLAA,
|
||||
NVSDK_NGX_PerfQuality_Value_Balanced, NVSDK_NGX_PerfQuality_Value_MaxPerf,
|
||||
NVSDK_NGX_PerfQuality_Value_UltraPerformance};
|
||||
static const NVSDK_NGX_PerfQuality_Value same[] = {NVSDK_NGX_PerfQuality_Value_DLAA, NVSDK_NGX_PerfQuality_Value_MaxQuality};
|
||||
const bool aa = rw == ow && rh == oh;
|
||||
// ((Triforce, 11-22, the performance audit's PRESENT-32, after Source A's) with RX_VK_DLSS_PREWARM=1 the one-size modes
|
||||
// only for NVIDIA DLAA: DLSS at the output's size (the Render resolution row's Auto, before the game lowers it) takes
|
||||
// the scaled mode whose range holds it, and the smaller sizes after it draw with the same feature instead of making a
|
||||
// second one in play (dlss_checks.h dlaa_feature). Off by default: as before, one size whenever the two are equal
|
||||
static const bool dlaa_rule = dlss_checks::prewarm_rule(getenv("RX_VK_DLSS_PREWARM"));
|
||||
const bool aa = dlss_checks::dlaa_feature(dlaa_rule, temporal_mode() == upscaler_rows::DLAA, rw, rh, ow, oh);
|
||||
const NVSDK_NGX_PerfQuality_Value* modes = aa ? same : scaled;
|
||||
const size_t nmodes = aa ? 2 : 5;
|
||||
NVSDK_NGX_PerfQuality_Value mode = modes[0];
|
||||
unsigned max_w = 0, max_h = 0, min_w = 0, min_h = 0;
|
||||
bool found = aa;
|
||||
if (aa) max_w = min_w = ow, max_h = min_h = oh; // (DLAA: the render size is the output's, one size)
|
||||
for (size_t i = 0; i < nmodes && !found; i++) {
|
||||
const NVSDK_NGX_PerfQuality_Value m = modes[i];
|
||||
unsigned w = 0, h = 0, xw = 0, xh = 0, nw = 0, nh = 0;
|
||||
float sharp = 0;
|
||||
if (NVSDK_NGX_FAILED(NGX_DLSS_GET_OPTIMAL_SETTINGS(s.caps, ow, oh, m, &w, &h, &xw, &xh, &nw, &nh, &sharp)) || !w) continue;
|
||||
if (rw >= nw && rh >= nh && rw <= xw && rh <= xh) {
|
||||
mode = m, max_w = xw, max_h = xh, min_w = nw, min_h = nh, found = true;
|
||||
break;
|
||||
// ((Triforce, 11-22, the performance audit's PRESENT-38, after Source A's) every mode's range first, then the one that
|
||||
// takes this size (dlss_checks.h mode_for): the first that holds it, else the first it is outside of by a rounding. The
|
||||
// Render resolution row's shares are rounded up and NGX's sizes down, so at 2560x1440 the row's 33% (854x480) was in
|
||||
// no range (Ultra Performance is 853x480), DLSS was refused every frame and the game's own upscale ran with DLSS
|
||||
// chosen. RX_VK_DLSS_MODE_SLACK=0, or the lab's "dlssstrict" phases: only a range that holds the size, as before
|
||||
if (!found) {
|
||||
dlss_checks::ModeRange ranges[5];
|
||||
for (size_t i = 0; i < nmodes; i++) {
|
||||
unsigned w = 0, h = 0, xw = 0, xh = 0, nw = 0, nh = 0;
|
||||
float sharp = 0;
|
||||
if (NVSDK_NGX_FAILED(NGX_DLSS_GET_OPTIMAL_SETTINGS(s.caps, ow, oh, modes[i], &w, &h, &xw, &xh, &nw, &nh, &sharp)) || !w) continue;
|
||||
ranges[i] = {nw, nh, xw, xh, true};
|
||||
}
|
||||
static const bool slack_env = !(getenv("RX_VK_DLSS_MODE_SLACK") && atoi(getenv("RX_VK_DLSS_MODE_SLACK")) == 0);
|
||||
bool widened = false;
|
||||
const int at = dlss_checks::mode_for(ranges, nmodes, rw, rh, ow, oh, slack_env && !lab_ab::on("dlssstrict"), &widened);
|
||||
if (at >= 0) {
|
||||
const dlss_checks::ModeRange& r = ranges[at];
|
||||
// (a size taken by the slack is in the range from here on: the feature's reuse test takes it next frame)
|
||||
mode = modes[at], found = true;
|
||||
min_w = std::min<unsigned>(r.min_w, rw), min_h = std::min<unsigned>(r.min_h, rh);
|
||||
max_w = std::max<unsigned>(r.max_w, rw), max_h = std::max<unsigned>(r.max_h, rh);
|
||||
static bool told;
|
||||
if (widened && !told) {
|
||||
told = true;
|
||||
logf("vk: NVIDIA DLSS: %ux%u is a rounding outside mode %d's sizes for %ux%u (%ux%u to %ux%u): taken as that "
|
||||
"mode's (RX_VK_DLSS_MODE_SLACK=0: refused, the game's upscale runs)", rw, rh, (int)mode, ow, oh, r.min_w,
|
||||
r.min_h, r.max_w, r.max_h);
|
||||
}
|
||||
}
|
||||
}
|
||||
// (a size no mode takes: the feature there is stays, so the next size one takes does not make it again)
|
||||
if (!found) return *fail = "no DLSS mode takes this render size", 0;
|
||||
if (s.feature) { // (only now is the old one let go: a mode takes this size, and a feature is made for it)
|
||||
NVSDK_NGX_Handle* old = s.feature;
|
||||
temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });
|
||||
s.feature = nullptr;
|
||||
}
|
||||
// (the DLSS model row's preset for every mode, K by default: NVIDIA's transformer model. 0: NVIDIA's own per mode)
|
||||
static_assert(NVSDK_NGX_DLSS_Hint_Render_Preset_K == 11 && NVSDK_NGX_DLSS_Hint_Render_Preset_L == 12 &&
|
||||
NVSDK_NGX_DLSS_Hint_Render_Preset_M == 13 && NVSDK_NGX_DLSS_Hint_Render_Preset_Default == 0);
|
||||
NVSDK_NGX_DLSS_Hint_Render_Preset_M == 13 && NVSDK_NGX_DLSS_Hint_Render_Preset_Default == 0 &&
|
||||
NVSDK_NGX_DLSS_Hint_Render_Preset_E == 5); // ((Triforce, PRESENT-36) E: upscaler_rows.h kPresetE)
|
||||
for (const char* k : {NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_DLAA, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Quality,
|
||||
NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Balanced, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Performance,
|
||||
NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_UltraPerformance,
|
||||
NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_UltraQuality})
|
||||
NVSDK_NGX_Parameter_SetUI(s.params, k, preset);
|
||||
NVSDK_NGX_DLSS_Create_Params cp{};
|
||||
cp.Feature.InWidth = max_w, cp.Feature.InHeight = max_h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh;
|
||||
// ((Triforce, 11-21, the upscaler audit's UPS-01) a fixed share's feature for its own render size, not the largest its
|
||||
// mode allows (NGX answers the output's own size for that: 2.25 times the pixels at Quality, 9 at Ultra Performance, for
|
||||
// a feature whose cost and memory follow the size it is made for; NVIDIA's guide has the render size passed to the
|
||||
// creation). For the mode's whole range where the game changes the render size: the Render resolution row's Auto, or a
|
||||
// size seen changing under one output in this run. RX_VK_DLSS_FEATURE_MAX=1: always the largest, as before
|
||||
static const bool whole_range = dlss_feature::whole_range(getenv("RX_VK_DLSS_FEATURE_MAX"));
|
||||
const dlss_feature::Size made =
|
||||
dlss_feature::size(aa, whole_range || s.range_seen || opts.resolution == 0, rw, rh, max_w, max_h);
|
||||
cp.Feature.InWidth = made.w, cp.Feature.InHeight = made.h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh;
|
||||
cp.Feature.InPerfQualityValue = mode;
|
||||
// (LDR colour, already tonemapped: no exposure; motion at the render size, not jittered; depth not reversed)
|
||||
cp.InFeatureCreateFlags = NVSDK_NGX_DLSS_Feature_Flags_MVLowRes;
|
||||
@@ -367,7 +432,7 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t
|
||||
const double made_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
||||
note_work(cmd); // (NGX's own commands, which vk_work.h does not see: the next full_barrier is not left out)
|
||||
if (NVSDK_NGX_FAILED(r) || !s.feature) {
|
||||
logf("vk: NVIDIA DLSS: its feature was not made (%#x) for %ux%u -> %ux%u", (unsigned)r, max_w, max_h, ow, oh);
|
||||
logf("vk: NVIDIA DLSS: its feature was not made (%#x) for %ux%u -> %ux%u", (unsigned)r, made.w, made.h, ow, oh);
|
||||
s.feature = nullptr;
|
||||
// (out of video memory passes, when the game lets an area's textures go: not the end of DLSS for the run, as any
|
||||
// failure was before. Anything else is)
|
||||
@@ -387,13 +452,15 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t
|
||||
return *fail = "its feature was not made", 0;
|
||||
}
|
||||
s.retries = 0, s.retry_frame = 0, s.refused = 0;
|
||||
s.feature_rw = max_w, s.feature_rh = max_h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h;
|
||||
s.feature_rw = made.w, s.feature_rh = made.h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h;
|
||||
s.feature_exact = made.exact;
|
||||
s.feature_preset = preset, s.feature_hdr = hdr;
|
||||
s.features_made++;
|
||||
logf("vk: NVIDIA DLSS: feature %llu made (%s; frame %llu, %.1f ms of CPU): %ux%u (%ux%u to %ux%u) -> %ux%u, mode %d, "
|
||||
"preset %c%s", (unsigned long long)s.features_made, why, (unsigned long long)texture_frame_no(), made_ms, rw, rh,
|
||||
min_w, min_h, max_w, max_h, ow, oh, (int)mode, upscaler_rows::preset_letter(preset),
|
||||
preset ? "" : " (NVIDIA's default)");
|
||||
"preset %c%s; made for %ux%u%s", (unsigned long long)s.features_made, why, (unsigned long long)texture_frame_no(),
|
||||
made_ms, rw, rh, min_w, min_h, max_w, max_h, ow, oh, (int)mode, upscaler_rows::preset_letter(preset),
|
||||
preset ? "" : " (NVIDIA's default)", made.w, made.h,
|
||||
made.exact ? ", this render size alone (RX_VK_DLSS_FEATURE_MAX=1: its mode's largest)" : ", the largest of its range");
|
||||
*in_w = s.feature_rw, *in_h = s.feature_rh;
|
||||
return 2; // (a new history)
|
||||
}
|
||||
@@ -428,6 +495,7 @@ void release() {
|
||||
temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });
|
||||
s.feature = nullptr;
|
||||
s.feature_rw = s.feature_rh = s.feature_ow = s.feature_oh = s.min_rw = s.min_rh = 0;
|
||||
s.feature_exact = false;
|
||||
}
|
||||
|
||||
uint64_t memory() {
|
||||
@@ -485,4 +553,13 @@ void dlss_device_extensions(std::vector<const char*>&, const std::vector<VkExten
|
||||
|
||||
#endif
|
||||
|
||||
// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The card's video memory for the
|
||||
// DLSS model row's CNN rule, 0 while the rule is off: RX_VK_DLSS_CNN_VRAM=1 turns it on (Source A has it on for every card
|
||||
// under 7 GB; Source B has the same rule behind a switch that is off, for preset E's ghosting and because NVIDIA marks it
|
||||
// deprecated: here off, as there, until the owner has looked at the picture). RX_VK_TEST_CARD_MB plays a smaller card
|
||||
uint64_t dlss_cnn_vram() {
|
||||
static const bool on = env_switch("RX_VK_DLSS_CNN_VRAM") == 1;
|
||||
return on ? device_vram_bytes() : 0;
|
||||
}
|
||||
|
||||
} // namespace mkwvk
|
||||
|
||||
@@ -26,5 +26,10 @@ void dlss_instance_extensions(std::vector<const char*>& want, const std::vector<
|
||||
void dlss_device_extensions(std::vector<const char*>& want, const std::vector<VkExtensionProperties>& all);
|
||||
/// DLSS as the temporal upscaler's engine (DLSS and DLAA on the Upscaler row); nullptr in a build without it
|
||||
const TemporalEngine* dlss_engine();
|
||||
/// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The card's video memory for the
|
||||
/// DLSS model row's CNN rule (upscaler_rows.h cnn_for_vram: the K row draws with preset E under 7 GB), in bytes; 0 unless
|
||||
/// RX_VK_DLSS_CNN_VRAM=1 (off by default: the picture is the owner's call). DLSS's preset and the menu's row both ask
|
||||
/// this, so they agree. In every build
|
||||
uint64_t dlss_cnn_vram();
|
||||
|
||||
} // namespace mkwvk
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user