Triforce v0.7.0 "Sage Of Courage"

The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
This commit is contained in:
ruckus7382sfgsgdf
2026-11-22 11:59:06 -06:00
parent 70c210cee1
commit e6f2f248f2
529 changed files with 55958 additions and 1898 deletions

View File

@@ -15,6 +15,7 @@
#include <span>
#include "common/alignment.h"
#include "../vk/cpu_count.h" // ((Triforce, 11-22, OS-49) the threads to size by: HOS_SIM_CPU
#include "common/div_ceil.h"
#include "video_core/texture_cache/format_lookup_table.h"
#include "video_core/textures/astc.h"
@@ -766,7 +767,7 @@ std::unique_ptr<Image> decode(const TICEntry& tic, const TexInfo& info_in, uint6
copy_layers >= 4 && level_size.depth == 1 && (size_t)level_size.width * level_size.height * 4 <= (8u << 20)) {
const Extent3D block = AdjustMipBlockSize(tiles, li.block, level, li.num_levels);
const u32 stride_alignment = StrideAlignment(tiles, info.block, gob, li.bpp_log2);
const unsigned nthreads = std::max(1u, std::min<unsigned>(copy_layers, std::thread::hardware_concurrency() - 1));
const unsigned nthreads = std::max(1u, std::min<unsigned>(copy_layers, cpu_count() - 1));
std::atomic<u32> next_layer{0};
static std::atomic<int64_t> t_swz, t_dec, t_up;
auto work = [&] {

145
gpu/vk/achievement_rows.h Normal file
View File

@@ -0,0 +1,145 @@
// (Triforce) Achievements on the menu's side: the list the host names (host/achievements.c; vk_api.h HOS_OPT_ACH_*), which of
// them are earned, the one a notice shows, and the Achievements page's choice and pages. vk_menu.cpp draws them.
//
// The achievements are the host's: it reads the lists, asks the save data and keeps what was earned. The menu is told
// each one's name, the line about it, its icon and the day it was earned, a byte a call as the save rows' lines are, and
// later each one earned as it is. An icon is drawn, not a picture file: a shape and a tone (host/achievements.h has the
// same in the same order, ACH_SHAPES and ACH_TONES; tests/test_achievements.py holds the two together).
//
// The host's calls (text, add, done, earn) come on its threads, one at a time (its lock); the list is drawn on the
// renderer's: names, lines and icons are written before count says they are there and never after, a day before its
// achievement's earned says so.
//
// No ImGui and no Vulkan: tests/test_achievements.py compiles it alone.
#pragma once
#include <atomic>
#include <cstdint>
#include <cstring>
namespace achievement_rows {
constexpr int kSlots = 128; // (host/achievements.h ACH_MAX)
constexpr int kName = 41, kAbout = 161, kDay = 16;
constexpr int kEarned = 256; // (vk_api.h HOS_ACH_EARNED)
enum Shape { kStar, kSword, kGem, kMedal, kNote, kToken, kShield, kShapes };
enum Tone { kGold, kGreen, kRed, kBlue, kPurple, kOrange, kYellow, kSilver, kTones };
constexpr int shape_of(int icon) { return (icon & 15) < kShapes ? icon & 15 : kStar; }
constexpr int tone_of(int icon) { return (icon >> 4 & 15) < kTones ? icon >> 4 & 15 : kGold; }
struct List {
char names[kSlots][kName] = {};
char abouts[kSlots][kAbout] = {};
char days[kSlots][kDay] = {};
unsigned char icons[kSlots] = {};
std::atomic<int> earned[kSlots] = {};
std::atomic<int> count{0}; // (the achievements shown: 0 until the host's list is whole)
std::atomic<int> fresh{-1}; // (the one earned now, until the notice takes it)
// the host's side
char in[kName + kAbout + kDay + 8] = {};
int in_n = 0, added = 0;
/// HOS_OPT_ACH_TEXT: the next byte of what is being named
void text(int byte) {
if (byte > 0 && byte < 256 && in_n + 1 < (int)sizeof in) in[in_n++] = (char)byte;
}
/// `from` copied into a field of `cap` bytes, cut at a whole letter
static void put(char* to, int cap, const char* from) {
int n = (int)strlen(from);
if (n > cap - 1) {
n = cap - 1;
while (n && ((unsigned char)from[n] & 0xC0) == 0x80) n--;
}
memcpy(to, from, (size_t)n);
to[n] = 0;
}
/// HOS_OPT_ACH_ADD: the bytes so far are the next achievement's name and, each after a line feed, the line about it and
/// the day it was earned; bits: its icon, and kEarned
void add(int bits) {
in[in_n] = 0;
in_n = 0;
if (count.load(std::memory_order_acquire) || added >= kSlots) return; // (shown already, or no room left)
const int slot = added++;
char* about = strchr(in, '\n');
char* day = about ? strchr(about + 1, '\n') : nullptr;
if (about) *about++ = 0;
if (day) *day++ = 0;
put(names[slot], kName, in[0] ? in : "(no name)");
put(abouts[slot], kAbout, about ? about : "");
put(days[slot], kDay, day ? day : "");
icons[slot] = (unsigned char)(bits & 255);
earned[slot].store((bits & kEarned) != 0, std::memory_order_relaxed);
}
/// HOS_OPT_ACH_COUNT: the list is whole: shown from now on
void done(int n) {
in_n = 0;
if (count.load(std::memory_order_acquire)) return;
count.store(n < 0 ? 0 : n < added ? n : added, std::memory_order_release);
}
/// HOS_OPT_ACH_EARNED: the achievement at that place was earned now, the bytes so far the day. false: no such place
bool earn(int slot) {
in[in_n] = 0;
in_n = 0;
if (slot < 0 || slot >= shown()) return false;
if (!earned[slot].load(std::memory_order_relaxed)) put(days[slot], kDay, in);
earned[slot].store(1, std::memory_order_release);
fresh.store(slot, std::memory_order_release);
return true;
}
int shown() const { return count.load(std::memory_order_acquire); }
bool is_earned(int slot) const { return slot >= 0 && slot < kSlots && earned[slot].load(std::memory_order_acquire) != 0; }
int earned_count() const {
int n = 0;
for (int i = 0, end = shown(); i < end; i++) n += is_earned(i);
return n;
}
/// The one earned since the last ask, for the notice; -1 none
int take_fresh() { return fresh.load(std::memory_order_relaxed) < 0 ? -1 : fresh.exchange(-1, std::memory_order_acq_rel); }
};
/// The page's cards: `columns` to a row, `capacity` to a page (both 1 at least), n of them
struct View {
int choice = 0, page = 0;
static int pages(int n, int capacity) { return n <= 0 ? 1 : (n + (capacity < 1 ? 1 : capacity) - 1) / (capacity < 1 ? 1 : capacity); }
/// The choice kept to the list, and the page shown to the pages there are
void keep(int n, int capacity) {
choice = n <= 0 ? 0 : choice < 0 ? 0 : choice >= n ? n - 1 : choice;
const int last = pages(n, capacity) - 1;
page = page < 0 ? 0 : page > last ? last : page;
}
/// The page that shows the choice
void reveal(int capacity) { page = choice / (capacity < 1 ? 1 : capacity); }
/// The arrows (cards: Left and Right, rows: Up and Down), Page Up and Down (turns), Home and End. true: the choice moved
bool move(int cards, int rows, int turns, bool home, bool end, int n, int columns, int capacity) {
if (n <= 0) return false;
const int was = choice;
const int step = cards + rows * (columns < 1 ? 1 : columns) + turns * (capacity < 1 ? 1 : capacity);
choice = home ? 0 : end ? n - 1 : choice + step;
keep(n, capacity);
if (choice == was) return false;
reveal(capacity);
return true;
}
/// The wheel, or a page arrow that leaves the choice where it is: the page shown turned. true: it changed
bool turn(int by, int n, int capacity) {
const int was = page;
page += by;
keep(n, capacity);
return page != was;
}
/// A click on a card. true: the choice changed
bool choose(int slot, int n) {
if (slot < 0 || slot >= n || slot == choice) return false;
choice = slot;
return true;
}
};
/// How far a notice has come in, 0 (out of sight) to 1 (in place): in over the first 250 ms of its `shown_ms`, out over
/// the last 300
inline float notice_in(int64_t age_ms, int64_t shown_ms) {
if (age_ms < 0 || age_ms >= shown_ms) return 0;
const float in = (float)age_ms / 250.0f, out = (float)(shown_ms - age_ms) / 300.0f;
const float k = in < out ? in : out;
return k > 1 ? 1 : k;
}
} // namespace achievement_rows

View File

@@ -13,4 +13,12 @@ inline bool all_by_vram(uint64_t allowed) {
return allowed && allowed < (8704ull << 20);
}
// (Triforce, 11-22, the performance audit's TEX-12, after Source A's) And every integrated GPU, whatever it is allowed. Its
// allowance is a share of the system's RAM (a 16 GB handheld reported 10,361 MB), so by the allowance alone it kept the
// fine-block textures of 2 to 64 MB as RGBA8, in RAM the game needs too, inside a texture budget capped at 2,560 MB.
// Source A measured the difference on a discrete card (RX_VK_ASTC_BC_ALL 0 against 1, Kokiri Forest: decoded ASTC
// 2,496-2,532 MB against 1,579-1,587, the GPU's frame the same); it was not run on an integrated GPU there either.
// The shape of tex_detail::by_vram(allowed, integrated). RX_VK_ASTC_BC_ALL=0: the finer ones as RGBA8, as before
inline bool all(uint64_t allowed, bool integrated) { return integrated || all_by_vram(allowed); }
} // namespace astc_bc_policy

View File

@@ -18,12 +18,28 @@
// 65535 / 65535 = 1. Modes 4, 5, 10, 12 and 13 carry alpha, and 2, 3, 7, 11, 14 and 15 are the HDR ones, which take the
// decoder's other path: no for all of those, without decoding them (such a block may be opaque all the same).
// One partition has its mode in bits 13 to 16. Several partitions share one mode when bits 23 and 24 are zero (the mode is
// bits 25 to 28); when they do not, the modes' other bits sit below the weights and are not looked for: no.
// bits 25 to 28).
// - (Triforce, 11-21, the performance audit's TEX-50, after Source B's) When bits 23 and 24 are not zero each partition has a
// mode of its own, read as the decoder reads it: 2, 5 or 8 more bits for 2, 3 or 4 partitions sit right below the weights
// (at 128 less the weights' bits less their own count); with bits 25 to 28 under them they are a class bit a partition,
// then two mode bits a partition, and a partition's mode is (bits 23 and 24 as a number, less 1, plus its class bit) * 4
// plus its two bits. Yes only when every partition's mode is one of 0, 1, 6, 8 and 9. Those bits count in the decoder's
// last error too: 96 bits of weights, the 29 read and 5 or 8 more wrap, and the block is an error. Such blocks were
// refused unread, and a texture with one of them stayed BC3. parts_too = false (RX_VK_TEX_BC1_PARTS=0): refused, as before.
#pragma once
#include <atomic>
#include <cstdint>
namespace astc_opaque {
/// Whether a block with a mode a partition is looked at (above); false: refused unread, as before. Set once, at the start
inline bool parts_too = true;
/// How many such blocks were found opaque (the log's)
inline std::atomic<uint64_t> parts_passed{0};
/// Whether the decoder gives both endpoints of a colour endpoint mode the alpha 255: the modes without alpha, low dynamic range
constexpr bool opaque_mode(unsigned cem) { return cem == 0 || cem == 1 || cem == 6 || cem == 8 || cem == 9; }
/// `n` bits (16 at most) of a block from bit `at`, the lowest first, as the decoder's StreamBits reads them
inline uint32_t bits(const uint8_t* b, unsigned at, unsigned n) {
uint32_t v = 0;
@@ -88,17 +104,32 @@ inline bool block(const uint8_t b[16], unsigned bw, unsigned bh) {
range = range - 2 + (layout != 9 && (mode & 0x200) ? 6 : 0);
const unsigned wbits = weight_bits(weights, range);
if (wbits < 24 || wbits > 96) return false;
unsigned read, cem;
unsigned read, cem = 0, extra = 0;
uint32_t base = 0;
if (parts == 1) {
cem = bits(b, 13, 4), read = 17;
} else {
const uint32_t base = bits(b, 23, 6);
if (base & 3) return false; // (a mode a partition: not looked for)
cem = base >> 2, read = 29;
base = bits(b, 23, 6);
read = 29;
if (base & 3) { // (a mode a partition: its other bits are below the weights)
if (!parts_too) return false;
extra = parts == 2 ? 2 : parts == 3 ? 5 : 8;
} else {
cem = base >> 2;
}
}
// (the decoder's last error, its count of the remaining bits wrapping. With one shared mode it cannot: 96 + 29 + 2)
if (wbits + read + (dual ? 2u : 0u) > 128) return false;
return cem == 0 || cem == 1 || cem == 6 || cem == 8 || cem == 9;
// (the decoder's last error, its count of the remaining bits wrapping. With one shared mode it cannot: 96 + 29 + 2. With a
// mode a partition it can: 96 + 29 + 8)
if (wbits + read + extra + (dual ? 2u : 0u) > 128) return false;
if (!extra) return opaque_mode(cem);
// (the decoder's own order: (extra << 6 | base) >> 2 is a class bit a partition, then two bits a partition)
const uint32_t all = ((bits(b, 128 - wbits - extra, extra) << 6) | base) >> 2;
for (unsigned i = 0; i < parts; i++) {
const unsigned cls = (base & 3) - 1 + ((all >> i) & 1);
if (!opaque_mode((cls << 2) | ((all >> (parts + 2 * i)) & 3))) return false;
}
parts_passed.fetch_add(1, std::memory_order_relaxed);
return true;
}
/// Whether every one of `n` blocks (16 bytes each, in any order) is one the decoder writes opaque

133
gpu/vk/astc_reuse.h Normal file
View File

@@ -0,0 +1,133 @@
// (Triforce, 11-21, the performance audit's TEX-14, after Source A's) Decoded ASTC used again.
//
// The game's textures are ASTC, decoded on the GPU and for the most part encoded again as BC3 or BC1: tens of ms of GPU
// time and the decoder's scratch for each. When the game leaves an area it finalizes its textures, and when it comes back
// it makes them again, mostly at other addresses: the same blocks are decoded and encoded again while the image made of
// them the first time may still be in VRAM, kept until the cache needs its room. Source A hashes an ASTC texture's blocks
// at its first whole upload, lists a texture the game finalizes under that hash, and gives a new texture with the same
// blocks and the same shape that image by a GPU copy, the old one going after it. Its tours of two areas four times at an
// 8 GB budget: peak decoded ASTC 3,603 to 2,562 MB, evictions 23,378 to 16,109, the slowest frame 637 to 487 ms.
//
// What keeps a hit right: 128 bits of hash over the blocks as the decoder is given them; every field of the image that
// the decode made and of the storage it was decoded from (Shape); and a source that was uploaded from memory and never
// drawn into, still listed (used again at its own address or gone, it is unlisted).
//
// RX_VK_ASTC_REUSE: 0 or not set, off (the audit: off until measured in play); 2: the blocks are hashed, the finalized
// textures listed and each hit counted, and every texture decoded as before: what a reuse would save, for no risk;
// 1: the hit's image is copied.
//
// No Vulkan here: tests/test_astc_reuse.py compiles it.
#pragma once
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <unordered_map>
#include "upload_same.h" // (content_hash: the first of the two lanes)
namespace astc_reuse {
enum Mode { kOff = 0, kCopy = 1, kCount = 2 };
/// RX_VK_ASTC_REUSE's mode: 1 and 2 as they are, anything else off
inline Mode mode(const char* asked) {
if (!asked || !*asked) return kOff;
const int v = atoi(asked);
return v == 1 ? kCopy : v == 2 ? kCount : kOff;
}
/// A second hash of n bytes, of other constants and another mix than upload_same::content_hash's: eight chains over 64
/// bytes a step, as there, so that both read at memory's speed
inline uint64_t lane2(const void* p, uint64_t n) {
const uint8_t* b = static_cast<const uint8_t*>(p);
const auto rotl = [](uint64_t x, int r) { return (x << r) | (x >> (64 - r)); };
uint64_t h = 0xD6E8FEB86659FD93ull + n * 0x9FB21C651E98DF25ull, i = 0;
if (n >= 512) {
uint64_t c[8];
for (int k = 0; k < 8; k++) c[k] = h + 0xA0761D6478BD642Full * (uint64_t)(2 * k + 1);
for (; i + 64 <= n; i += 64) {
uint64_t w[8];
std::memcpy(w, b + i, 64);
for (int k = 0; k < 8; k++) c[k] = rotl(c[k] + w[k] * 0xE7037ED1A0B428DBull, 29) * 0x8EBC6AF09C88C6E3ull;
}
for (int k = 0; k < 8; k++) h = rotl(h ^ c[k], 23) * 0x589965CC75374CC3ull + 0x1D8E4E27C47D124Full;
}
for (; i + 8 <= n; i += 8) {
uint64_t w;
std::memcpy(&w, b + i, 8);
h = rotl(h + w * 0xE7037ED1A0B428DBull, 29) * 0x8EBC6AF09C88C6E3ull;
}
for (; i < n; i++) h = rotl(h ^ b[i], 11) * 0x589965CC75374CC3ull;
h ^= h >> 32, h *= 0xD6E8FEB86659FD93ull, h ^= h >> 32;
return h;
}
/// The blocks' hash, 128 bits, made of pieces given in order (a big texture's storage is read 64 MB at a time): each piece
/// goes through both lanes kStep bytes at a time, while they are in the cache, and is folded in with its length
struct Hash {
uint64_t a = 0, b = 0;
bool operator==(const Hash& o) const { return a == o.a && b == o.b; }
explicit operator bool() const { return (a | b) != 0; }
};
inline constexpr uint64_t kStep = 256u << 10;
struct Hasher {
Hash h;
uint64_t bytes = 0;
void add(const uint8_t* p, uint64_t n) {
for (uint64_t at = 0; at < n; at += kStep) {
const uint64_t m = n - at < kStep ? n - at : kStep;
h.a = (h.a ^ upload_same::content_hash(p + at, m)) * 0x9E3779B97F4A7C15ull + m;
h.b = (h.b + lane2(p + at, m)) * 0xC2B2AE3D27D4EB4Full ^ (m << 1 | 1);
}
bytes += n;
}
/// The hash, never all zero (zero is "not hashed")
Hash done() const { return (h.a | h.b) ? h : Hash{1, 0}; }
};
/// What has to be the same besides the blocks for one image to stand for another: the image the decode made (its format,
/// size, levels and layers, the top levels left out and whether by the card's texture detail, kept as BC3 or BC1 or
/// RGBA8, BC1 given up for BC3, resized for AMD) and the storage it was decoded from (the NVN format, the key's layout)
struct Shape {
uint32_t format, width, height, depth, levels, layers, skip, nvn_format, layout;
bool bc, bc_resized, bc1, bc1_no, detail_skip;
bool operator==(const Shape& o) const {
return format == o.format && width == o.width && height == o.height && depth == o.depth && levels == o.levels &&
layers == o.layers && skip == o.skip && nvn_format == o.nvn_format && layout == o.layout && bc == o.bc &&
bc_resized == o.bc_resized && bc1 == o.bc1 && bc1_no == o.bc1_no && detail_skip == o.detail_skip;
}
};
/// The finalized textures whose image may serve again, by their blocks' hash. T is the cache's texture; one is in the list
/// at most once: listed and unlisted by its pointer
template <class T>
class Freed {
public:
void list(const Hash& hash, T* t) { by_hash.emplace(hash.a, Entry{hash.b, t}); }
void unlist(const Hash& hash, T* t) {
auto [a, b] = by_hash.equal_range(hash.a);
for (auto it = a; it != b; ++it)
if (it->second.t == t) {
by_hash.erase(it);
return;
}
}
/// One listed with this hash, both lanes, whose shape is `shape` (shape_of(T*) gives each one's) and that is not
/// `self`; null: none
template <class ShapeOf>
T* find(const Hash& hash, const Shape& shape, const T* self, ShapeOf shape_of) const {
auto [a, b] = by_hash.equal_range(hash.a);
for (auto it = a; it != b; ++it)
if (it->second.b == hash.b && it->second.t != self && shape_of(it->second.t) == shape) return it->second.t;
return nullptr;
}
size_t size() const { return by_hash.size(); }
private:
struct Entry {
uint64_t b;
T* t;
};
std::unordered_multimap<uint64_t, Entry> by_hash;
};
} // namespace astc_reuse

View File

@@ -71,6 +71,28 @@ inline bool split_in_pass(const char* env, uint32_t vendor, bool every_draw) {
return false; // helper threads record most frames there, and the rule stands aside for them) and no gain (90.6 / 89.2 fps
// against 92.8 / 90.3 with the old barriers)
}
/// ((Triforce, 11-22, the performance audit's REPLAY-14, after Source A's) Two workarounds for Intel's drivers at a submission's
/// splits (vk_replay.cpp maybe_split and the dry pass's points). No other vendor has either unless its switch asks.
/// RX_VK_SPLIT_BEFORE_DRAW: a part ends only right before a plain draw. On an Arc B580 a command buffer that begins with mesh
/// draws drew its later plain draws as exploded triangles (Source A: 40 shard frames of 1,121 before, 0 of 1,101 after). Unset:
/// on for Intel's GPUs (vendor 0x8086); 0 off; anything else on, on any GPU
inline bool split_before_draw(const char* env, uint32_t vendor) {
if (env && *env) return std::strcmp(env, "0") != 0;
return vendor == 0x8086;
}
/// RX_VK_SPLIT_UBFLUSH: whether a part's end writes the submission's uniform updates so far into the game's memory. Without
/// mirrors they land in the memory later parts' other reads and the game's CPU see, mid-submission, where an unsplit submission
/// writes them once at its end: white blocks and flashes on Intel's integrated GPUs (Source A: 1 run of 3 before, 0 of 3
/// after). Unset: on, as before, but off on Intel's GPUs without mirrors; 0 off; anything else on
inline bool split_ub_flush(const char* env, uint32_t vendor, bool mirrors) {
if (env && *env) return std::strcmp(env, "0") != 0;
return !(vendor == 0x8086 && !mirrors);
}
/// A part may end right before this op: a plain draw (NOP_DRAW, _IDX, _INDIRECT: not a mesh draw, not a dispatch) that will
/// draw (rendering on, no rasterizer discard)
constexpr bool cut_ok(bool plain_draw, bool render_enabled, bool raster_discard) {
return plain_draw && render_enabled && !raster_discard;
}
/// Whether a part's split goes on in the same pass: the rule on, a pass open at the split, nothing recorded at the part's end (quiet:
/// the work count as it was after the pass's end) and no event's write held then (written behind a full barrier before the part's
/// write-backs, which copy its word to the game's memory: the GPU drained there all the same). ((Triforce, P4) no parallel replay's
@@ -85,6 +107,25 @@ inline bool split_start_full(bool carry, bool recorded_between) {
return !carry || recorded_between;
}
/// ((Triforce, 11-22, the performance audit's SYNC-66, after Source B's) RX_VK_CHUNK: how many draws and dispatches a part of a
/// long submission has (vk_replay.cpp maybe_split). 120 on every card since the parts were made: the GPU starts early, and each
/// split costs a queue submit, two full barriers and the part's bookkeeping, about 27 times a frame in a serial replay. Source B
/// cuts at 128 where the GPU idles while the CPU records and at 4096 on desktop drivers; what pays here is not measured on any
/// driver, so the default by driver is 120 for each until the lab picks one (S1 and S2 with RX_VK_PARALLEL=0 and the lab knob
/// chunk:<n>). vendor: the card's PCI vendor; radv: Mesa's RADV
constexpr int64_t kPartDraws = 120;
inline int64_t part_draws_default(uint32_t vendor, bool radv) {
(void)vendor, (void)radv;
return kPartDraws;
}
/// The size in force: RX_VK_CHUNK's number when the variable is set (0 or less: whole submissions, never split); else
/// knob_or_default, the lab knob chunk:<n>'s number in its phases and the driver's default outside them
inline int64_t part_draws(bool env_set, int64_t env_value, int64_t knob_or_default) {
return env_set ? env_value : knob_or_default;
}
/// As the size was read before: the variable's number, 120 unset
inline int64_t part_draws_before(const char* env) { return env ? std::atoll(env) : 120; }
/// RX_VK_EVENT_BARRIER: "late" the rule on any card; "full" (or any other value) as before; unset or empty: with the pass barrier's
/// game mode (pass_barrier::game: NVIDIA's driver unless RX_VK_PASS_BARRIER says otherwise)
inline bool event_late(const char* env, bool pass_game) {
@@ -139,5 +180,18 @@ inline bool batch_flush(Batch m, bool added_held, size_t held, size_t cap) {
if (m == Batch::Parts && !added_held) return true;
return held >= cap;
}
/// ((Triforce, 11-22, the performance audit's SYNC-09, after Source A's) RX_VK_BATCH_IDLE: a submit that joins the held batch
/// sends it at once when the GPU has finished every batch sent. A held batch otherwise waits for a wait, a present, the 2 ms
/// poll or the cap of 32 while the GPU sits idle beside it, until the game waits mid-frame at a capture's late pages; while the
/// GPU is busy, submits still group. Source A has it on wherever batching is (RADV), off where guest memory stayed host-pointer
/// imports (each submit there pays amdgpu's page check, 15-45 ms). HERE: on only with a number other than 0; unset it is OFF,
/// on every driver, until a Linux RADV machine has run it (the audit's lab proof; with NVIDIA's driver and AMD's and Intel's on
/// Windows nothing is held). No guard for host-pointer imports is built: with the switch off unless asked there is nothing for it to guard
inline bool batch_idle_on(const char* env) { return env && std::atoi(env) != 0; }
/// Whether a held batch is looked at for an idle GPU after a submit joined it: the switch on, Hold mode only (Parts sends by
/// its own rule, when the game's submit returns), and something held (the cap's flush may just have sent it all)
inline bool batch_idle(Batch m, bool held, bool on) {
return on && m == Batch::Hold && held;
}
} // namespace mkwvk::barrier_rules

66
gpu/vk/bc_scratch.h Normal file
View File

@@ -0,0 +1,66 @@
// (Triforce, 11-21, the performance audit's TEX-10, after Source A's) How many layers the BC3 scratch has.
//
// Big ASTC textures are decoded into a 2048 x 2048 RGBA8 scratch and encoded again as BC3 on the GPU, into an output
// buffer. Both were made with 8 layers, 128 + 32 = 160 MB (and the second scratch of the textures encoded at another
// size, 128 MB more), whatever a run used; and a run decodes one layer (RX_VK_BC_LAYERS, the dark-line fix), so 140 MB
// were held and never written, for good once made (RX_VK_BC_IDLE_S is 0). Sized by the layers one run uses they are
// 16 + 4 = 20 MB. Source A measured 139 MiB less GPU memory on a Steam Deck, the median of 7 pairs, at the same fps.
// RX_VK_BC_SCRATCH_FULL=1: all 8 layers, as before.
//
// The rule alone, no Vulkan: tests/test_bc_scratch_layers.py compiles it.
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdlib>
namespace bc_scratch {
inline constexpr uint32_t kDim = 2048, kMost = 8; // (the largest level a BC3 texture may have; the most layers a run may take)
/// The layers one run decodes and encodes: RX_VK_BC_LAYERS's value, 1 to kMost; not set, or not a number, 1
inline uint32_t run_layers(const char* asked) {
return asked ? std::clamp<uint32_t>((uint32_t)std::max(atoi(asked), 0), 1u, kMost) : 1u;
}
/// The scratch's layers: a run's, or all of them when full (RX_VK_BC_SCRATCH_FULL)
inline uint32_t layers(uint32_t run, bool full) { return full ? kMost : std::clamp(run, 1u, kMost); }
/// The output buffer's bytes: BC3's 16 bytes a block of 4 x 4, every layer
inline uint64_t out_bytes(uint32_t layers) { return (uint64_t)(kDim / 4) * (kDim / 4) * layers * 16; }
/// What the scratch holds, in MB: the image and its output buffer, and the second image when there is one
inline uint64_t mb(uint32_t layers, bool second) {
const uint64_t image = (uint64_t)kDim * kDim * 4 * layers;
return (image + out_bytes(layers) + (second ? image : 0)) >> 20;
}
// ---- (Triforce, 11-21, the performance audit's TEX-09, after Source A's) the small scratch ----
// A run is a decode, an encode, a copy and three full barriers, and one layer a run made a 1365 x 1365 x 90 terrain array
// of 11 levels about 990 of them: the GPU spent most of that time between the runs (Source A, Hyrule Field's entry on an
// RX 9070 XT: 50 ms of GPU an array at one layer a run, 16-17 ms at 8; the lab's own array took 114.8 ms of an RTX 5090 in
// one upload command buffer). Levels of kSmallDim or less each way, the arrays' second level down, go small_layers() layers
// a run through a scratch of their own, kSmallDim x kSmallDim x 8: 32 MB, and the output buffer 8 MB for 4. One layer a
// run was for dark lines between Hyrule Field's ground tiles on an RX 6600, layers after a run's first wrong at the small
// levels, cause never found: so on AMD's cards the default stays 1 until the picture is checked there, and each layer of a
// run is copied by a region of its own, at the encoder's own layer stride. A level of a texture encoded at another size
// keeps the first scratch. RX_VK_BC_SMALL_LAYERS=<n>: the layers a run, 1 to 8; 1: no small scratch, as before.
inline constexpr uint32_t kSmallDim = 1024;
/// The small scratch's layers, a run's of a small level: RX_VK_BC_SMALL_LAYERS, 1 to kMost; not set, kMost, and 1 on AMD's
/// cards (vendor 0x1002)
inline uint32_t small_layers(const char* asked, uint32_t vendor) {
if (asked && *asked) return std::clamp<uint32_t>((uint32_t)std::max(atoi(asked), 0), 1u, kMost);
return vendor == 0x1002 ? 1u : kMost;
}
/// Whether a level decoded at w x h goes through the small scratch: there is one, the level fits it, and the texture is not
/// one encoded at another size (which has the second scratch)
inline bool small_level(uint32_t w, uint32_t h, bool have_small, bool resized) {
return have_small && !resized && w <= kSmallDim && h <= kSmallDim;
}
/// The small scratch's part of the output buffer, and its image, in bytes
inline uint64_t small_out_bytes(uint32_t layers) { return (uint64_t)(kSmallDim / 4) * (kSmallDim / 4) * layers * 16; }
inline uint64_t small_image_bytes(uint32_t layers) { return (uint64_t)kSmallDim * kSmallDim * 4 * layers; }
/// Where layer k of a run begins in the encoder's output: host_shaders/bc_encoder.comp writes block
/// (z * blocks_y + y) * blocks_x + x, 16 bytes a block, 8 where only the colour half is written (an image kept as BC1)
inline uint64_t layer_offset(uint32_t k, uint32_t w, uint32_t h, bool bc1) {
return (uint64_t)k * ((w + 3) / 4) * ((h + 3) / 4) * (bc1 ? 8 : 16);
}
} // namespace bc_scratch

164
gpu/vk/built_check.h Normal file
View File

@@ -0,0 +1,164 @@
// Whether the caches still have what the light precompile's record says it built (no game, no GPU: the rules; vk_draw.cpp
// built_check_cold, precompile_run, built_play_build). After lib_check.h, which asks the same of the stage libraries, and
// predict_check.h, which asks it of the predicted pipelines.
//
// (Triforce, 11-22, the performance audit's PIPE-21, after Source A's and Source B's.) A light precompile (little RAM:
// precompile_keeps_all) builds the noted pipelines for the caches, lets them go, and records their keys in <notes>.built,
// the compute programs' as one key. A later start trusts that record on its stamp alone: what it lists is left to its first
// draw or dispatch, to be built from the caches there. When the caches lost it (the pipeline cache file deleted or refused,
// the play file lost, a driver's own cache cleared or past its cap), each of those first draws compiles in play while the
// record still says built: a draw left out or waiting for a whole build, a dispatch waiting 50 ms and then left out. Source A
// measured a cleared NVIDIA cache: 702,312 draws left out and 21.3 s of draw waits in 140 s of play, against 456 and none
// once everything was built again before the title.
//
// The check. Before the record is trusted, up to kGraphics of the recorded notes (spread over them from another place each
// start) are asked of Triforce's own caches as a draw asks, and up to kComputes of the listed compute programs as a dispatch
// asks, where the device can ask without compiling (FAIL_ON_PIPELINE_COMPILE_REQUIRED): the answer is the caches' own.
// Not a scratch cache of the check's own: AMD's Windows driver keeps no cache of its own (pcache_bound.h), so a question
// put to the driver alone would always be cold there. Without the flag, a pipeline is built twice and timed, as lib_check
// does: the second comes from the cache by construction, and a first over kRatio times it and kSlackMs more was compiled.
// Under kAnswers answers the sample says nothing (the record is trusted, as before); at half of them or more cold it is cold.
//
// Cold: the record is not trusted this start, everything noted and every listed compute program is built again before the
// title (as for a fresh record), and the record is written anew after it. <notes>.built.check keeps, for the stamp, that the
// check built them again (kStrikes 0): cold again right after that, the caches do not keep them on this PC (often a driver's
// shader cache that is too small), and they are left to their first draws, as before this check, with a hint in the log. A
// warm sample forgets that.
//
// In play, a counter of what the check is for: recorded pipelines whose first build took kSlowUs or more, and recorded
// compute programs the dispatch's ask did not find, logged at most every kSayMs while it grows.
//
// RX_VK_BUILT_CHECK: 1 by default; 0 no check and no counter, as before; 2 always cold (the lab's and the tests').
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>
namespace built_check {
inline constexpr int kGraphics = 24; // (recorded notes asked: Source A asks 48, Source B six computes)
inline constexpr int kLook = 2 * kGraphics; // (notes looked at for them: one may have lost its program)
inline constexpr int kComputes = 6; // (listed compute programs asked)
inline constexpr int kAnswers = 16; // (fewer answers say nothing)
inline constexpr double kRatio = 4.0; // (without the flag: a first build against the second, compiled above)
inline constexpr double kSlackMs = 3.0; // (and this much more)
inline constexpr int kStrikes = 0; // (built again once, and cold again right after: left to the draws)
inline constexpr int64_t kSlowUs = 40000; // (a play build of a recorded pipeline over this was compiled)
inline constexpr int64_t kSayMs = 10000; // (the counter's line: at most this often)
/// RX_VK_BUILT_CHECK: 0 off, 1 on (unset too), 2 always cold
inline int mode(const char* env) {
if (!env || !*env) return 1;
const int v = std::atoi(env);
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
}
/// One pipeline asked. asked: asked of the caches only; answered_cold: they said a compile is required; made: it was made
/// (both times, not asked). One the driver refuses says nothing of the caches
struct Probe {
bool asked = false, answered_cold = false, made = false;
double first_ms = 0, again_ms = 0;
};
inline bool counts(const Probe& p) { return p.answered_cold || p.made; }
inline bool cold(const Probe& p) {
return p.answered_cold || (!p.asked && p.made && p.first_ms > kRatio * p.again_ms + kSlackMs);
}
struct Tally {
int got = 0, cold = 0, answered = 0;
};
inline Tally tally(const std::vector<Probe>& probes) {
Tally t;
for (const Probe& p : probes)
if (counts(p)) t.got++, t.cold += cold(p), t.answered += p.answered_cold;
return t;
}
enum class Verdict { Unknown, Warm, Cold };
/// Under kAnswers answers nothing; half of them or more cold: cold
inline Verdict verdict(const Tally& t) {
if (t.got < kAnswers) return Verdict::Unknown;
return t.cold * 2 >= t.got ? Verdict::Cold : Verdict::Warm;
}
/// k places spread over n, the first at seed's place in the first step: each start asks other pipelines
inline std::vector<size_t> spots(size_t n, size_t k, uint64_t seed) {
std::vector<size_t> at;
if (!n || !k) return at;
const size_t step = std::max<size_t>(1, n / k);
for (size_t i = (size_t)(seed % step); i < n && at.size() < k; i += step) at.push_back(i);
return at;
}
// ---- <notes>.built.check: "built-check <stamp> <strikes>"
struct State {
bool found = false;
uint64_t stamp = 0;
int strikes = 0;
};
inline State state_read(const std::string& path) {
State s;
FILE* f = fopen(path.c_str(), "rb");
if (!f) return s;
char line[96]{};
const size_t n = fread(line, 1, sizeof line - 1, f);
fclose(f);
line[n] = 0;
unsigned long long stamp = 0;
int strikes = 0;
if (sscanf(line, "built-check %16llx %d", &stamp, &strikes) == 2 && strikes >= 0)
s.found = true, s.stamp = stamp, s.strikes = strikes;
return s;
}
/// Written whole beside the old one, then put in place (a start that ends in the middle leaves the old file)
inline bool state_write(const std::string& path, uint64_t stamp, int strikes) {
char line[96];
const int n = snprintf(line, sizeof line, "built-check %016llx %d\n", (unsigned long long)stamp, strikes);
const std::string tmp = path + ".tmp";
FILE* f = fopen(tmp.c_str(), "wb");
if (!f) return false;
bool ok = fwrite(line, 1, (size_t)n, f) == (size_t)n;
ok = fclose(f) == 0 && ok;
if (ok) {
remove(path.c_str()); // (rename does not replace on Windows)
ok = rename(tmp.c_str(), path.c_str()) == 0;
}
if (!ok) remove(tmp.c_str());
return ok;
}
enum class Do { Nothing, Again, Stop };
/// What a start does with the verdict, the record of the last starts and the stamp now. always: mode 2, built again at every
/// start whatever was recorded. *next: the strikes to record once they are built again (Again only)
inline Do decide(Verdict v, const State& had, uint64_t stamp, bool always, int* next) {
if (always) return *next = 0, Do::Again;
if (v != Verdict::Cold) return Do::Nothing;
const bool same = had.found && had.stamp == stamp; // (built again for this stamp, and cold again since)
if (same && had.strikes >= kStrikes) return Do::Stop;
*next = same ? had.strikes + 1 : 0;
return Do::Again;
}
// ---- the counter: play builds of recorded pipelines the caches did not have
/// Whether a build of that many microseconds was compiled rather than found
inline bool slow(int64_t us) { return us >= kSlowUs; }
/// Whether the counter's line is due: said_ms, when it was last written (0: never)
inline bool say(int64_t now_ms, int64_t said_ms) { return !said_ms || now_ms - said_ms >= kSayMs; }
/// keys sorted without repeats, for has()
inline std::vector<uint64_t> sorted(std::vector<uint64_t> keys) {
std::sort(keys.begin(), keys.end());
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
return keys;
}
inline bool has(const std::vector<uint64_t>& sorted_keys, uint64_t key) {
return std::binary_search(sorted_keys.begin(), sorted_keys.end(), key);
}
} // namespace built_check

64
gpu/vk/cap_spare.h Normal file
View File

@@ -0,0 +1,64 @@
// (Triforce, 11-22, the performance audit's MEM-33, after Source A's) Which spare capture buffer a capture takes, how
// much the spares and the ones being made ahead may hold together, and a capture's size in steps.
//
// A capture's buffer comes from the spares (retired buffers, and the 32 to 192 MB ones made ahead on a worker), or is
// made new on the game's submitting thread. Three things were so before:
// - the new buffer was made with cap_free_mu held (AMD's driver: about 0.8 ms a MB), and the replay's take, holding
// pools_mu, waits on that mutex to retire a buffer;
// - any spare that fit was taken, so a 2 MB capture could sit in a load's 192 MB spare for the 24 submissions a capture
// is kept, and the load then made its own on the game's thread;
// - a spare made ahead was made once: the one a load took was not made again.
// The rules here are the decisions alone (no Vulkan, no lock): tests/test_capture_spare_reuse.py compiles them, and
// vk_memory.cpp's cap_buf_get, cap_prewarm_class and cap_buf_free ask them. RX_VK_CAP_SPARE_FIT=0: as before.
#pragma once
#include <cstddef>
#include <cstdint>
namespace cap_spare {
constexpr uint64_t kMB = 1ull << 20;
constexpr uint64_t kBig = 32 * kMB; // (a load's spare: the classes made ahead start here)
constexpr uint64_t kHeld = 768 * kMB; // (the most the spares hold, whatever the card)
constexpr size_t kMost = 12; // (and the most there are)
/// A capture of size class `cls` leaves the spare of `size` bytes alone: a big one (32 MB or more) over twice its class.
/// any_size: no spare is left alone (spares capped at 32 MB, the switch off, or the fallback after a failed allocation)
constexpr bool skips(uint64_t size, uint64_t cls, bool any_size) { return !any_size && size >= kBig && size > 2 * cls; }
/// The spare a capture of `need` bytes (class `cls`) takes of n spares, size_of(i) the i-th one's bytes: the smallest that
/// fits and is not left alone, the first of equals. n: none
template <class SizeOf>
constexpr size_t pick(size_t n, const SizeOf& size_of, uint64_t need, uint64_t cls, bool any_size) {
size_t best = n;
for (size_t i = 0; i < n; i++) {
const uint64_t s = size_of(i);
if (s >= need && !skips(s, cls, any_size) && (best == n || s < size_of(best))) best = i;
}
return best;
}
/// Room for one more spare of `size` bytes to be made ahead: with what the spares hold and what is being made (reserved)
/// it stays within what spares may hold (spare_max: the card's, cap_spare_max) and 768 MB
constexpr bool room(uint64_t held, uint64_t reserved, uint64_t size, uint64_t spare_max) {
const uint64_t limit = spare_max < kHeld ? spare_max : kHeld;
return held + reserved <= limit && size <= limit - (held + reserved);
}
/// A buffer of `size` bytes given back is kept as a spare: fewer than 12 with the ones being made (workers), none over 256
/// MB (keep_old, RX_VK_CAPTURE_KEEP_OLD: 64), and with the bytes being made (reserved) within the limit. With no workers
/// and nothing reserved this is the rule as it was
constexpr bool keeps(size_t spares, size_t workers, uint64_t held, uint64_t reserved, uint64_t size, uint64_t spare_max, bool keep_old) {
const uint64_t limit = keep_old ? spare_max : (spare_max < kHeld ? spare_max : kHeld);
return spares + workers < kMost && size <= (keep_old ? 64 * kMB : 256 * kMB) && held + reserved <= limit &&
size <= limit - (held + reserved);
}
/// (RX_VK_CAP_STEP_MB=<n>, 0: off) a capture over 16 and up to 32 MB in steps of step_mb MB, not the whole 32 MB class,
/// where spares are capped at 32 MB (a card of 6.5 GB or less). cls: its class as it is (cap_buf_size)
constexpr uint64_t stepped(uint64_t need, uint64_t cls, uint64_t step_mb, bool capped) {
if (!step_mb || !capped || cls != kBig || need <= 16 * kMB || need > kBig) return cls;
const uint64_t step = step_mb * kMB, v = (need + step - 1) / step * step;
return v < cls ? v : cls;
}
} // namespace cap_spare

110
gpu/vk/capture_tables.h Normal file
View File

@@ -0,0 +1,110 @@
// (Triforce, 11-21, the upscaler audit's UPS-02) The stored draws' two tables, kept from one frame to the next.
//
// While NVIDIA DLSS, AMD FSR 4 or AMD FSR 3.1 upscales, every plain draw into the scene stores its positions and is drawn
// again into the motion vectors (vk_temporal.cpp capture_motion). Two tables go with that, both filled again every frame:
// the draws of a model so far this frame (which of them a draw is: the n-th this frame is the n-th the frame before), and
// the frame before's draws by their key (where each one's positions of that frame are). Both were std::unordered_map,
// emptied and filled each frame: a node from the heap for every entry and back to it a frame later, about 1,400 of each
// a frame in the owner's log of 11-21 ("captured motion ... of 1444 stored a frame ... 0.650 ms CPU a frame"), on the
// thread that records the frame. Here they keep their memory: the count in one open-addressed array (flat_map.h) that is
// emptied in place, the frame before's draws in two lists that take turns, found through such an array. A frame of about
// the size of the one before takes nothing from the heap.
//
// RX_VK_CAPTURE_TABLES=0: std::unordered_map again, a node an entry, as before (kept()).
//
// No Vulkan: tests/test_capture_tables.py compiles this and counts the heap's calls.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <unordered_map>
#include <utility>
#include <vector>
#include "flat_map.h"
namespace capture_tables {
/// RX_VK_CAPTURE_TABLES (value, null when unset): kept unless it is 0
inline bool kept(const char* value) { return !value || !*value || std::atoi(value) != 0; }
/// An array emptied for the next frame: in place, so the frame fills it without the heap; given back when it is far
/// larger than what the frame put in it (a frame of many draws once, then few), so emptying it does not walk all of that
template <class K, class V>
inline void empty_for_next(FlatMap<K, V>& m) {
if (m.capacity() > 8 * (m.size() + 512)) m.clear();
else m.reset();
}
/// The room a frame's list is given at its start: the frame before's count (kept), so it grows once and not a dozen times
/// from nothing; 0, as before, with the switch off
inline size_t room(size_t taken, bool keep) { return keep ? taken : 0; }
/// How many of a model's draws came so far this frame
class Nth {
public:
explicit Nth(bool keep = true) : keep_(keep) {}
/// The draws of `model` before this one, which is counted
int next(uint64_t model) { return keep_ ? flat_[model]++ : node_[model]++; }
void clear() {
if (keep_) empty_for_next(flat_);
else node_.clear();
}
size_t size() const { return keep_ ? flat_.size() : node_.size(); }
private:
bool keep_;
FlatMap<uint64_t, int> flat_;
std::unordered_map<uint64_t, int> node_;
};
/// A frame's entries by key, and the frame before's: turn() makes what was put since the last turn the frame before's,
/// put() notes one of this frame's (the last with a key stands), before() finds the frame before's
template <class V>
class Before {
public:
explicit Before(bool keep = true) : keep_(keep) {}
void turn() {
if (!keep_) {
node_old_ = std::move(node_now_);
node_now_.clear();
return;
}
std::swap(now_, old_), std::swap(now_at_, old_at_);
now_.clear(); // (its memory kept)
empty_for_next(now_at_);
}
void put(uint64_t key, const V& v) {
if (!keep_) {
node_now_[key] = v;
return;
}
uint32_t& at = now_at_[key]; // (the entry's place in now_, plus one: 0 is none yet)
if (at) now_[at - 1] = v;
else now_.push_back(v), at = (uint32_t)now_.size();
}
/// The frame before's entry of `key`, null when it had none: good until the next turn() or clear()
const V* before(uint64_t key) {
if (!keep_) {
const auto it = node_old_.find(key);
return it == node_old_.end() ? nullptr : &it->second;
}
const auto* s = old_at_.find(key);
return s ? &old_[s->second - 1] : nullptr;
}
/// Nothing of either frame (the chain let go: its memory too)
void clear() {
std::vector<V>().swap(now_), std::vector<V>().swap(old_);
now_at_.clear(), old_at_.clear();
node_now_.clear(), node_old_.clear();
}
size_t size() const { return keep_ ? now_.size() : node_now_.size(); }
private:
bool keep_;
std::vector<V> now_, old_;
FlatMap<uint64_t, uint32_t> now_at_, old_at_;
std::unordered_map<uint64_t, V> node_now_, node_old_;
};
} // namespace capture_tables

View File

@@ -23,6 +23,7 @@ inline const char* name(char c) {
case 'u': return "upload of texture";
case 'g': return "GPU unswizzle of texture";
case 'w': return "write-back diff pass, chunks";
case 's': return "mirror syncs' pass, chunks"; // ((Triforce, 11-21, MEM-48))
case 'r': return "recording begun, value";
case 'v': return "uploads of the recording, value";
case 'z': return "recording ended, value";
@@ -45,6 +46,9 @@ inline const char* gap(char c) {
return "that pool's write-back copies (its mirror -> guest memory, and the shadow's fill), then the write-back diff pass if "
"one follows (w): with no w after it, the end of the recording (z)";
case 'u': case 'g': return "that texture's upload or unswizzle itself";
case 's':
return "the mirror syncs' pass (one compute dispatch in the recording's uploads, reading a capture's buffer and writing "
"the pool's VRAM mirror by device address; RX_VK_MSYNC_PASS=0: copy regions), then the sync's copies from guest memory";
case 'w':
return "the write-back diff pass (one compute dispatch over every pool's chunks, reading mirrors and shadows and writing guest "
"memory by device address), then its barrier and the end of the recording (z)";

120
gpu/vk/commit_room.h Normal file
View File

@@ -0,0 +1,120 @@
// (Triforce, 11-22, the performance audit's OS-05, after Source A's) The textures give commit back near Windows' commit limit,
// and the page files' room to grow counts as commit still to come.
//
// Windows charges every block of a discrete card's memory to the process's commit, as it does the game's own memory. Where the
// system's commit runs out (RAM and the page files full) an allocation is refused, and the game ends unless the refusal passes
// (OS-04's wait, HOS_COMMIT_WAIT_MS, helps only if commit comes back). Source A takes the shortfall under a reserve of free
// commit (2,560 MB) off the texture budget on any card, so that eviction and compaction give blocks back. Its first rule fired
// while a page file Windows manages could still grow (292,526 textures retired in 90 minutes, black textures); the fix counts
// each page file's room to grow, capped by its drive's free space, as commit still free.
//
// The rules here, apart from Windows so that tests/test_commit_room.py can run them:
// - reserve_of: RX_VK_TEX_COMMIT_RESERVE_MB, the reserve. Off (0) unless set: the entry's 2560 is not measured on this tree
// - growth_on: RX_VK_TEX_COMMIT_GROWTH, whether the growth is counted: 0 never, another number always, unset where a reserve is
// - low_line, read_growth: the growth is read only while less is free than the higher of the reserve and the line of
// tex_pressure::commit_low (whose fall-back to the fixed size then gets the growth too: vk_texture.cpp commit_now)
// - cap: the textures at most what they hold less the shortfall under the reserve, never under 1 GB (Source A has no floor,
// and its false cut went to 0)
// - growth: the page files' room to grow, from the records of NtQuerySystemInformation's class 0x90 (undocumented: each
// record is checked against the buffer, and one that is not inside it ends the walk), a drive's page files capped together
// by its free space. vk_texture.cpp page_file_growth asks Windows, at most once a second, and answers 0 on any failure.
// Written from the audit's description: Source A's file is not on this PC.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
namespace commit_room {
inline constexpr uint64_t kFloor = 1ull << 30; // (the textures are never capped under 1 GB)
inline constexpr int64_t kMostMb = 65536; // (the reserve at most)
/// RX_VK_TEX_COMMIT_RESERVE_MB in bytes: 0 (unset, empty, no number, 0 or less): no cut, as before; at most kMostMb MB
inline uint64_t reserve_of(const char* env) {
if (!env || !*env) return 0;
const long long mb = std::atoll(env);
return mb <= 0 ? 0 : (uint64_t)std::min<long long>(mb, kMostMb) << 20;
}
/// RX_VK_TEX_COMMIT_GROWTH: unset or empty, counted where a reserve is set; 0 (or no number), never; another number, always
inline bool growth_on(const char* env, uint64_t reserve) {
if (!env || !*env) return reserve != 0;
return std::atoi(env) != 0;
}
/// tex_pressure::commit_low's line in bytes: want_mb MB; -1 (less than 0) the larger of 4 GB and a tenth of the limit; 0, or a
/// limit not known (0), none
inline uint64_t low_line(uint64_t limit, int64_t want_mb) {
if (!limit || !want_mb) return 0;
return want_mb > 0 ? (uint64_t)want_mb << 20 : std::max<uint64_t>(4096ull << 20, limit / 10);
}
/// Whether the page files' growth is read at this look: counted (on), and less free (avail) than the higher of the reserve and
/// commit_low's line (low)
inline bool read_growth(bool on, uint64_t avail, uint64_t reserve, uint64_t low) { return on && avail < std::max(reserve, low); }
/// The textures' cap: what they hold (total) less the shortfall of what is free (avail, the growth in it) under the reserve,
/// never under kFloor. UINT64_MAX (no cap) with no reserve, or with the reserve free
inline uint64_t cap(uint64_t total, uint64_t avail, uint64_t reserve) {
if (!reserve || avail >= reserve) return UINT64_MAX;
const uint64_t shortfall = reserve - avail;
return std::max(kFloor, total > shortfall ? total - shortfall : 0);
}
/// NtQuerySystemInformation's class for the page files' records (SystemPageFileInformationEx)
inline constexpr unsigned kPageFileClass = 0x90;
/// One record of that class as 64-bit Windows lays it out; sizes in pages
struct PageFileRecord {
uint32_t next; // (bytes from this record to the next; 0: the last)
uint32_t total_pages; // (its size now)
uint32_t in_use_pages, peak_pages;
uint16_t name_bytes, name_room; // (the name: UTF-16, such as \??\C:\pagefile.sys, in the same buffer)
const uint16_t* name;
uint32_t min_pages, max_pages; // (the size it may grow to)
};
static_assert(sizeof(void*) != 8 || sizeof(PageFileRecord) == 40, "the record as 64-bit Windows lays it out");
/// The drive letter of a page file's name ('C' for \??\C:\pagefile.sys), 0 where it has none
inline char drive_of(const uint16_t* name, size_t chars) {
for (size_t i = 0; i + 1 < chars; i++) {
const uint16_t c = name[i];
if (name[i + 1] == ':' && ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) return (char)(c >= 'a' ? c - 32 : c);
}
return 0;
}
/// The page files' room to grow in bytes, from the records in buf[0, len): a record's (max - size) pages of `page` bytes, the
/// records of one drive summed and capped by that drive's free space (free_of('C'), bytes). A record that is not wholly inside
/// the buffer, whose name is not, or whose next one would overlap it, ends the walk: what came before it counts
template <class FreeOf>
inline uint64_t growth(const uint8_t* buf, size_t len, uint64_t page, FreeOf free_of) {
uint64_t by_drive[26] = {};
const uintptr_t base = (uintptr_t)buf;
for (size_t at = 0; len >= sizeof(PageFileRecord) && at <= len - sizeof(PageFileRecord);) {
PageFileRecord r;
std::memcpy(&r, buf + at, sizeof r);
char drive = 0;
if (r.name_bytes) {
const uintptr_t n = (uintptr_t)r.name;
if (n < base || n - base > len || r.name_bytes > len - (n - base)) break;
uint16_t chars[64] = {};
const size_t count = std::min<size_t>(r.name_bytes / 2, 64);
std::memcpy(chars, buf + (n - base), count * 2);
drive = drive_of(chars, count);
}
if (drive && r.max_pages > r.total_pages) by_drive[drive - 'A'] += (uint64_t)(r.max_pages - r.total_pages) * page;
if (!r.next) break;
if (r.next < sizeof r || r.next > len - at) break;
at += r.next;
}
uint64_t sum = 0;
for (int d = 0; d < 26; d++)
if (by_drive[d]) sum += std::min<uint64_t>(by_drive[d], free_of((char)('A' + d)));
return sum;
}
} // namespace commit_room

125
gpu/vk/compute_pin.h Normal file
View File

@@ -0,0 +1,125 @@
// Which compute programs are held to 32 lanes (requiredSubgroupSize), by what the program can tell (no game, no GPU: a rule).
//
// (Triforce, 11-22, the performance audit's PIPE-52, after Source A's and Source B's.) The console's programs are written
// for warps of 32 lanes, so every compute pipeline asked for subgroups of 32 (vk_draw.cpp pin_subgroup). Most programs
// cannot tell how many lanes run beside them: they never vote, never read a lane mask or their lane's index, never
// shuffle between lanes. Mesa's Intel driver compiles a pinned program for 32 lanes only (SIMD32), which on its large
// programs is minutes and gigabytes: Source A measured a 730 KB program at 0.8-1 GB over 23-25 s pinned and at most 300 MB
// unpinned (an Iris Xe, Mesa 26.2.4), with 271 of 1046 variants still pinned by this rule.
//
// A program is pinned when it can see its lanes:
// - a vote, a lane mask, the lane's index, a shuffle, FSWZADD or a matrix multiply (Shader::Info's flags);
// - a barrier of Subgroup scope in its module: this translator's own case, the barrier its lowering puts between a
// warp's shared-memory store and load, which takes the subgroup to be the warp. No flag records it, so the module is
// looked at (subgroup_barrier below), and a module nobody looked at counts as having one;
// - shared memory in a workgroup of 32 invocations or fewer, where the workgroup is one warp and its lanes run in step.
// Everything else gets the driver's own choice: no required size, and no full-subgroups flag either.
//
// RX_VK_COMPUTE_PIN, read once:
// all every compute program pinned, as it was (the default on every driver but Mesa's Intel one)
// lanes the rule above (the default on Mesa's Intel driver)
// shared the rule above, and any shared memory pins, whatever the workgroup: for a program whose warps keep
// shared memory in step in a workgroup over 32, which nothing else here sees
// A doubt pins: an unknown value of the switch is the driver's default, a program without a stage is pinned.
// The graphics stages and the mesh fallback's compute stages are not asked here: they stay with pin_subgroup.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstring>
namespace compute_pin {
enum class Mode { All, Lanes, Shared };
/// What one compute program is, as the rule needs it
struct Facts {
bool has_stage = false; // (false: nothing known, pinned)
bool vote = false; // Shader::Info::uses_subgroup_vote
bool mask = false; // uses_subgroup_mask
bool lane_index = false; // uses_subgroup_invocation_id
bool shuffles = false; // uses_subgroup_shuffles
bool fswzadd = false; // uses_fswzadd
bool matrices = false; // uses_cooperative_matrix
bool subgroup_barrier = true; // its module has an OpControlBarrier of Subgroup scope (true until a look says no)
uint32_t shared_memory = 0; // bytes of shared memory
uint64_t invocations = 0; // the workgroup's x * y * z
};
/// The program can tell how many lanes run in its subgroup, or which one it is
constexpr bool sees_lanes(const Facts& f) {
return f.vote || f.mask || f.lane_index || f.shuffles || f.fswzadd || f.matrices || f.subgroup_barrier;
}
/// RX_VK_COMPUTE_PIN's value (nullptr: unset) on this driver
inline Mode mode_for(const char* env, bool mesa_intel) {
if (env && !std::strcmp(env, "all")) return Mode::All;
if (env && !std::strcmp(env, "lanes")) return Mode::Lanes;
if (env && !std::strcmp(env, "shared")) return Mode::Shared;
return mesa_intel ? Mode::Lanes : Mode::All;
}
inline const char* name(Mode m) {
return m == Mode::All ? "all" : m == Mode::Lanes ? "lanes" : "shared";
}
/// The program's pipeline asks for subgroups of 32
constexpr bool needed(Mode m, const Facts& f) {
if (m == Mode::All || !f.has_stage) return true;
if (sees_lanes(f)) return true;
if (f.shared_memory) return m == Mode::Shared || f.invocations <= 32;
return false;
}
/// Whether a SPIR-V module's entry point is a compute shader's (OpEntryPoint with the GLCompute model): the only modules
/// the rule is asked about, so the only ones worth a note. One that does not read as SPIR-V: no
inline bool compute_entry(const uint32_t* code, size_t words) {
constexpr uint32_t kMagic = 0x07230203, kOpEntryPoint = 15, kOpFunction = 54, kGLCompute = 5;
if (!code || words < 5 || code[0] != kMagic) return false;
for (size_t at = 5; at < words;) {
const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu;
if (!count || at + count > words || op == kOpFunction) return false;
if (op == kOpEntryPoint && count >= 2) return code[at + 1] == kGLCompute;
at += count;
}
return false;
}
/// Whether a SPIR-V module has an OpControlBarrier whose execution scope is the constant Subgroup (3). A module that does
/// not read as SPIR-V counts as having one. words: the module's 32-bit words
inline bool subgroup_barrier(const uint32_t* code, size_t words) {
constexpr uint32_t kMagic = 0x07230203, kOpConstant = 43, kOpControlBarrier = 224, kScopeSubgroup = 3;
if (!code || words < 5 || code[0] != kMagic) return true;
// (the scopes are constants, and constants come before the functions: one pass for the barriers' scope ids, kept small,
// and their values looked up at the end)
constexpr size_t kMost = 64;
uint32_t scopes[kMost];
size_t n_scopes = 0;
for (size_t at = 5; at < words;) {
const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu;
if (!count || at + count > words) return true;
if (op == kOpControlBarrier && count >= 4) {
const uint32_t id = code[at + 1];
bool seen = false;
for (size_t i = 0; i < n_scopes; i++) seen = seen || scopes[i] == id;
if (!seen) {
if (n_scopes == kMost) return true; // (more scope ids than a module has: a doubt)
scopes[n_scopes++] = id;
}
}
at += count;
}
if (!n_scopes) return false;
size_t resolved = 0;
for (size_t at = 5; at < words;) {
const uint32_t count = code[at] >> 16, op = code[at] & 0xffffu;
if (op == kOpConstant && count >= 4) {
for (size_t i = 0; i < n_scopes; i++)
if (scopes[i] == code[at + 2]) {
if (code[at + 3] == kScopeSubgroup) return true;
resolved++;
}
}
at += count;
}
return resolved != n_scopes; // (a scope that is no plain constant: a doubt)
}
} // namespace compute_pin

41
gpu/vk/cpu_count.h Normal file
View File

@@ -0,0 +1,41 @@
// (Triforce, 11-22, the performance audit's OS-49, after Source B's) The logical processors the renderer sizes its pools and
// its threads' counts by: std::thread::hardware_concurrency, or, while the lab's HOS_SIM_CPU=<cores>/<threads> holds the
// process to fewer of the PC's cores (host/hos_sim_cpu.h), its <threads>: an affinity mask alone does not change what
// hardware_concurrency answers, and the pools would be a 24-thread PC's on 6 cores. The host clears the variable when it did
// not apply it, so the two agree. Windows only, as the host's part; unset, hardware_concurrency as before. Every such count in
// gpu/ goes through cpu_count() but vk_shader.cpp's (a file the shader cache's stamp is made from):
// tests/test_sim_cores.py checks that, and the rule.
#pragma once
#include <cstdlib>
#include <thread>
namespace cpu_count_rule {
/// The threads HOS_SIM_CPU asks for ("6/12": 12; "6": 6), when fewer than the PC's hw (0: not known); else 0: not set, not
/// "<cores>[/<threads>]" with each from 1 to 256 and threads not under cores, or not fewer
inline unsigned sim_threads(const char* e, unsigned hw) {
if (!e || !*e) return 0;
char* end = nullptr;
const long c = std::strtol(e, &end, 10);
if (end == e) return 0;
long t = c;
if (*end == '/') {
const char* s = end + 1;
t = std::strtol(s, &end, 10);
if (end == s) return 0;
}
if (*end || c < 1 || c > 256 || t < c || t > 256) return 0;
return hw && (unsigned)t >= hw ? 0 : (unsigned)t;
}
} // namespace cpu_count_rule
/// The logical processors to size by (read once)
inline unsigned cpu_count() {
static const unsigned n = [] {
const unsigned hw = std::thread::hardware_concurrency();
#ifdef _WIN32
if (const unsigned sim = cpu_count_rule::sim_threads(std::getenv("HOS_SIM_CPU"), hw)) return sim;
#endif
return hw;
}();
return n;
}

39
gpu/vk/dead_count.h Normal file
View File

@@ -0,0 +1,39 @@
// (Triforce, 11-21, the performance audit's THREAD-21, its first step) What destroying the dead images costs a present.
//
// A texture let go waits in a list until the GPU is done with it, and texture_frame then destroys every one that is, in
// the middle of the present being recorded: a render target's image goes to the pool (cheap), every other image is
// destroyed with its views. The audit's port bounds that to a share a present, after the present's submit, as Source B
// does on its GPU thread's idle moments. Its own proof asks for the cost first: "if no present spends 3 ms in the loop,
// stop". This is that count, and nothing else changes: a line for a present whose loop took kSayMs or more, and every
// 600 presents the totals and the worst present.
//
// No Vulkan here: tests/test_dead_count.py compiles it.
#pragma once
#include <cstdint>
namespace dead_count {
inline constexpr double kSayMs = 2.0; // (a present's loop that gets a line of its own)
inline constexpr double kWorthMs = 3.0; // (the audit's line: under it in every present, the bounded step is not built)
inline constexpr uint64_t kReport = 600; // (presents between the totals' lines)
struct Counts {
uint64_t presents = 0, destroyed = 0, pooled = 0, over_say = 0, over_worth = 0;
double ms = 0, worst_ms = 0;
uint64_t worst_destroyed = 0, worst_left = 0;
/// A present's loop: n_destroyed images destroyed and n_pooled put in the pool in loop_ms, left still waiting. True
/// when the present gets a line of its own
bool note(uint64_t n_destroyed, uint64_t n_pooled, double loop_ms, uint64_t left) {
presents++, destroyed += n_destroyed, pooled += n_pooled, ms += loop_ms;
if (loop_ms > worst_ms) worst_ms = loop_ms, worst_destroyed = n_destroyed, worst_left = left;
if (loop_ms >= kWorthMs) over_worth++;
if (loop_ms < kSayMs) return false;
over_say++;
return true;
}
/// Whether the totals' line is due, at present `present`
bool due(uint64_t present) const { return presents != 0 && present % kReport == 0; }
void reset() { *this = Counts{}; }
};
} // namespace dead_count

67
gpu/vk/desc_counts.h Normal file
View File

@@ -0,0 +1,67 @@
// A program's descriptor counts kept with the program (no game, no GPU: a rule; vk_shader.h Program::desc_n, vk_draw.cpp
// build_descriptors).
//
// (Triforce, 11-22, the performance audit's DESC-22, after Source A's.) build_descriptors sums the program's buffers,
// images and texel views at every build, six lists a stage, only to size three reserve() calls that keep the pointers
// it hands the writes stable. The sums are fixed once the program is translated: its stages are filled before it is
// ready and never after. So the three are packed into one word kept on the program, stored at its first build once
// ready and read at every later one; a word that is not a count walks as before.
//
// The word is 32 bits because that is what the program has room for: the padding after Program::ready. The struct must
// not grow: sizeof(Program) is in the program cache's stamp, and a change of it throws every player's caches away. So
// three counts of 10 bits; a program with more than 1023 of a kind is not kept (kUnset is what its pack gives) and walks
// at every build, as before.
//
// A short count would let a vector move in the middle of a build and leave the writes with stale pointers. It cannot
// come from the walk that stores it, but the build checks all the same that none of the three moved, and a program
// whose did is marked kNever, walked from then on, and built again at once.
//
// RX_VK_DESC_COUNTS=0: the walk at every build, as before. The lab's knob "descwalk" does the same in its phases.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
namespace mkwvk::desc_counts {
inline constexpr uint32_t kUnset = ~0u; // not counted yet, or too many of a kind to pack
inline constexpr uint32_t kNever = 0xFFFFFFFEu; // its kept counts were short once: walked at every build from then on
inline constexpr uint32_t kMost = 1023; // the most of a kind a word holds (10 bits)
/// RX_VK_DESC_COUNTS: on unless set to 0
inline bool on(const char* env) {
return !env || std::atoi(env) != 0;
}
/// The three counts as one word, or kUnset where one does not fit
constexpr uint32_t pack(size_t bufs, size_t imgs, size_t views) {
if (bufs > kMost || imgs > kMost || views > kMost) return kUnset;
return (uint32_t)bufs | (uint32_t)imgs << 10 | (uint32_t)views << 20;
}
/// Whether the word holds counts (its two top bits are clear: kUnset and kNever have them set)
constexpr bool holds(uint32_t word) {
return (word >> 30) == 0;
}
/// The counts of a word that holds them; false, and nothing written, for one that does not (walk)
constexpr bool unpack(uint32_t word, size_t* bufs, size_t* imgs, size_t* views) {
if (!holds(word)) return false;
*bufs = word & kMost, *imgs = (word >> 10) & kMost, *views = (word >> 20) & kMost;
return true;
}
/// Whether a build that walked stores its counts: the rule on, the program translated (its stages do not change after),
/// and the word still unset (never over kNever)
constexpr bool stores(bool on, bool ready, uint32_t word) {
return on && ready && word == kUnset;
}
/// The guard at a build's end: whether one of the three vectors' storage moved since the reserve (before/after: their
/// data() pointers), in a build that went by kept counts. A build that walked reserved exactly, as it always did
constexpr bool moved(bool walked, const void* bufs0, const void* bufs1, const void* imgs0, const void* imgs1, const void* views0,
const void* views1) {
return !walked && (bufs0 != bufs1 || imgs0 != imgs1 || views0 != views1);
}
} // namespace mkwvk::desc_counts

97
gpu/vk/desc_pool_grow.h Normal file
View File

@@ -0,0 +1,97 @@
// A recording slot's descriptor pool, made at half size and made whole once a recording runs out of it (no game, no GPU:
// a rule; vk_draw.cpp make_pool, slot_reset_locked, draw_begin_recording).
//
// (Triforce, 11-22, the performance audit's DESC-33, after Source A's.) Each of the recording slots (48 at most) keeps a
// descriptor pool of 8192 sets and up to 32768 descriptors a type, and the driver may keep host memory for all of that,
// 48 times. Few recordings come near it: Source A saw one take at most 289 sets on a desktop card, and no pool at half
// size run out in three scenes on a handheld. So a slot's pool starts at half (4096 sets, 16384 uniform buffers and
// combined image samplers, and so on down the types).
//
// A recording that runs out is given a whole extra pool, as it always was (bind_descriptors), and that is the mark: at
// the slot's next reset, where the extra pools are destroyed anyway, a slot that has any and whose own pool is not whole
// has it destroyed and made again whole. From then on the slot is as it was before this rule. A busy slot so pays one
// extra pool, once.
//
// RX_VK_DESC_POOL_GROW=0: whole pools from the start, as before. RX_VK_DESC_POOL_DIV=<n> still divides both sizes.
// What the recordings took (sets and descriptors a type, the most one recording took of each) and how many pools were
// added and slots made whole go into the log's "recording streams" line, so a size can be judged from a run's log.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
namespace mkwvk::desc_pool_grow {
/// RX_VK_DESC_POOL_GROW: on unless set to 0
inline bool on(const char* env) {
return !env || std::atoi(env) != 0;
}
/// A whole pool's sets, and its descriptors by type in the order of Took::n
inline constexpr uint32_t kSets = 8192;
enum Kind { kUniform, kStorage, kSampled, kStorageImage, kUniformTexel, kStorageTexel, kKinds };
inline constexpr uint32_t kWhole[kKinds] = {32768, 16384, 32768, 4096, 2048, 2048};
/// The shift a new slot's pool is made with: half with the rule on, whole without it
constexpr uint32_t first_shift(bool on) {
return on ? 1u : 0u;
}
/// A count of a pool made at `shift` (0 whole, 1 half) and divided by RX_VK_DESC_POOL_DIV's n (1..32): never 0
constexpr uint32_t count(uint32_t whole, uint32_t shift, uint32_t div) {
const uint32_t n = (whole >> (shift > 8 ? 8 : shift)) / (div ? div : 1);
return n ? n : 1;
}
/// Whether a slot's own pool is made again whole at its reset: its last recording ran out (it has extra pools) and the
/// pool is not whole yet
constexpr bool remake(uint32_t shift, size_t extra_pools) {
return shift != 0 && extra_pools != 0;
}
/// Which of the six a Vulkan descriptor type is (VkDescriptorType's numbers, so that this header needs no Vulkan):
/// kKinds for any other
constexpr int kind(uint32_t vk_type) {
switch (vk_type) {
case 6: return kUniform; // VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER
case 7: return kStorage; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER
case 1: return kSampled; // VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER
case 3: return kStorageImage; // VK_DESCRIPTOR_TYPE_STORAGE_IMAGE
case 4: return kUniformTexel; // VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER
case 5: return kStorageTexel; // VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER
default: return kKinds;
}
}
/// What one recording took of its slot's pools: sets, and descriptors by type
struct Took {
uint32_t sets = 0;
uint32_t n[kKinds] = {};
};
/// One more set allocated, with these writes (W: VkWriteDescriptorSet, or a test's own with the same two fields)
template <class W>
inline void took(Took& t, const W* writes, size_t count_of) {
t.sets++;
for (size_t i = 0; i < count_of; i++) {
const int k = kind((uint32_t)writes[i].descriptorType);
if (k < kKinds) t.n[k] += (uint32_t)writes[i].descriptorCount;
}
}
/// The most any recording took of each, raised by one recording's
inline void peak(Took& most, const Took& t) {
if (t.sets > most.sets) most.sets = t.sets;
for (int k = 0; k < kKinds; k++)
if (t.n[k] > most.n[k]) most.n[k] = t.n[k];
}
/// Whether what a recording took fits a pool made at `shift` and divided by div (the tests', and the log's reader's)
constexpr bool fits(const Took& t, uint32_t shift, uint32_t div) {
if (t.sets > count(kSets, shift, div)) return false;
for (int k = 0; k < kKinds; k++)
if (t.n[k] > count(kWhole[k], shift, div)) return false;
return true;
}
} // namespace mkwvk::desc_pool_grow

View File

@@ -4,6 +4,7 @@
#pragma once
#include <cstdint>
#include <cstdlib>
#include <string>
#include <string_view>
@@ -16,6 +17,50 @@ inline bool scene_size_fits(uint32_t w, uint32_t h, uint32_t ow, uint32_t oh) {
return w && h && ow && oh && w <= ow && h <= oh && (uint64_t)w * 3 >= ow && (uint64_t)h * 3 >= oh;
}
/// ((Triforce, 11-22, the performance audit's PRESENT-38, after Source A's) One quality mode's range of render sizes for
/// an output, as NGX answers it (ok: it answered one).
struct ModeRange {
uint32_t min_w = 0, min_h = 0, max_w = 0, max_h = 0;
bool ok = false;
};
/// How far outside a range a render size may be and still be that mode's: a 64th of the range's largest size on that
/// axis, 2 pixels at least. The Render resolution row's shares are twelfths of the output rounded up, NGX's own sizes
/// are rounded down: 854x480 against NGX's 853x480 for Ultra Performance at 2560x1440.
constexpr uint32_t mode_slack(uint32_t largest) { return largest / 64 > 2 ? largest / 64 : 2; }
/// The mode (an index into ranges) that takes a render size of w x h for an output of ow x oh, or -1. The first range
/// that holds the size; else, with slack on, the first it is outside of by no more than mode_slack on each axis, and
/// only for a size DLSS scales at all (scene_size_fits: the output 1 to 3 times it). *widened: taken by the slack.
inline int mode_for(const ModeRange* ranges, size_t n, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh, bool slack, bool* widened) {
*widened = false;
for (size_t i = 0; i < n; i++) {
const ModeRange& r = ranges[i];
if (r.ok && w >= r.min_w && h >= r.min_h && w <= r.max_w && h <= r.max_h) return (int)i;
}
if (!slack || !scene_size_fits(w, h, ow, oh)) return -1;
for (size_t i = 0; i < n; i++) {
const ModeRange& r = ranges[i];
if (!r.ok) continue;
const uint32_t sw = mode_slack(r.max_w), sh = mode_slack(r.max_h);
if (w + sw >= r.min_w && h + sh >= r.min_h && w <= r.max_w + sw && h <= r.max_h + sh) return *widened = true, (int)i;
}
return -1;
}
/// ((Triforce, 11-22, the performance audit's PRESENT-32, after Source A's) Whether a feature is made in a one-size mode
/// (NGX's DLAA mode: vk_dlss.cpp prepare's `same` list): its render size is the output's. Before, whenever the two sizes
/// were equal. NVIDIA DLSS with the Render resolution row at Auto starts at the output's size, so it got a one-size
/// feature, and the game's dynamic resolution left that size some frames later: a second feature made in play (Source
/// A's save load-in: 119 ms, then 29.5 ms 17 frames on). With the rule (rule: RX_VK_DLSS_PREWARM=1), one-size only where
/// NVIDIA DLAA is the Upscaler's choice (dlaa); DLSS at the output's size takes the scaled mode whose range holds it
/// (mode_for: Quality's reaches the output's size), and Auto's smaller sizes then draw with that same feature. Off by
/// default: DLSS at full size then draws with the scaled mode's network, not DLAA's, a picture nobody has compared yet
inline bool dlaa_feature(bool rule, bool dlaa, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh) {
const bool full = rw == ow && rh == oh;
return rule ? full && dlaa : full;
}
/// RX_VK_DLSS_PREWARM (value, null when unset): the rule above is on only when it is 1
inline bool prewarm_rule(const char* text) { return text && *text && std::atoi(text) == 1; }
/// Whether s is all ASCII: NGX's Linux library takes only such paths (vk_dlss.cpp wide())
inline bool ascii(std::string_view s) {
for (const char c : s)
@@ -84,4 +129,28 @@ inline std::wstring utf8_to_wide(std::string_view s, bool* bad = nullptr) {
return out;
}
/// ((Triforce, 11-21, the performance audit's PRESENT-37, after Source A's) The preset DLSS draws with while the DLSS model
/// row is stepped through (vk_dlss.cpp prepare). The row goes K, M, L, NVIDIA's default and applies live, so K to L passes
/// M: a feature was made for each preset passed, tens of ms of CPU each, and NGX keeps a released feature's video memory for
/// some seconds: Source A's 16 GB card at 3440x1440 held 1419 MB of DLSS's for K, M and L at once against 813 MB for L
/// alone, and where memory was short NGX refused the next feature. A new preset is taken once the row has held it for
/// `frames`; until then the feature already made draws on with its own. With no feature (the first, or one let go) the
/// row's preset is taken at once
struct PresetSettle {
unsigned wanted = ~0u; // (the row's preset, as last seen)
uint64_t since = 0; // (the frame it was first seen in)
/// The preset for frame `now`: want, the row's; have, the feature's; has: there is a feature
unsigned pick(unsigned want, unsigned have, bool has, uint64_t now, uint64_t frames) {
if (want != wanted) wanted = want, since = now;
if (!has || want == have) return want;
return now - since >= frames ? want : have;
}
};
/// RX_VK_DLSS_MODEL_SETTLE's frames: 30 when not set; 0 (and what is no number, or less): at once, as before
inline uint64_t settle_frames(const char* text) {
if (!text || !*text) return 30;
const long v = std::atol(text);
return v > 0 ? (uint64_t)v : 0;
}
} // namespace mkwvk::dlss_checks

58
gpu/vk/dlss_feature.h Normal file
View File

@@ -0,0 +1,58 @@
// (Triforce, 11-21, the upscaler audit's UPS-01) The render size NVIDIA DLSS's feature is made for.
//
// vk_dlss.cpp made every feature for the largest render size its quality mode allows, so that the game's dynamic
// resolution could change the render size under it without the feature being made again. NGX answers that largest size as
// the output's own (the owner's log of 11-21: "feature 1 made ...: 2944x1243 (2208x932 to 4416x1864) -> 4416x1864, mode
// 2, preset M"), so with the Render resolution row at a fixed share, where the render size never changes, the feature was
// still made for 2.25 times the pixels it was ever given at Quality, 4 times at Performance and 9 at Ultra Performance,
// and the chain's three input images with it. NVIDIA's guide (DLSS Programming Guide 310.6.0, 3.2 and 5.2.8) has the
// optimal render size passed to the creation "exactly as given", and says of the second generation transformer's presets
// L and M that their cost follows the size the feature is initialized for, which is why it is to be the dynamic range's
// largest only where there is a dynamic range. The same log has DLSS's own video memory at 1069 MB for that feature, where
// NVIDIA's table has 456 MB for preset M at 4K.
//
// So: a fixed share gets a feature of its own render size (exact), as AMD FSR 4 gets a context of its own
// (temporal_rules.h plan_context). Where the render size is the game's to change (the row's Auto), or once it has changed
// under one output in this run, the feature is made for the mode's whole range, as before. DLAA's is the output's either
// way.
//
// RX_VK_DLSS_FEATURE_MAX=1: every feature for its mode's largest render size, as before (whole_range()).
//
// No NGX, no Vulkan: tests/test_dlss_feature.py compiles this.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace dlss_feature {
/// RX_VK_DLSS_FEATURE_MAX (value, null when unset): not 0, every feature for its mode's largest render size, as before
inline bool whole_range(const char* value) { return value && *value && std::atoi(value) != 0; }
/// What a feature is made for: w x h, and whether that is the one render size it takes (exact) or the largest of a range
struct Size {
uint32_t w, h;
bool exact;
};
/// The size to make a feature for, for a frame of rw x rh in a mode whose render sizes go up to max_w x max_h. aa: DLAA,
/// the render size the output's (its range is that one size). range: the render size may change under this output (the
/// row's Auto, the size seen changing, or the switch)
inline Size size(bool aa, bool range, uint32_t rw, uint32_t rh, uint32_t max_w, uint32_t max_h) {
if (aa || range || rw > max_w || rh > max_h) return {max_w, max_h, false};
return {rw, rh, true};
}
/// Whether a feature made for fw x fh takes a frame of rw x rh: its one size (exact), or any from min_w x min_h up to its own
inline bool takes(bool exact, uint32_t fw, uint32_t fh, uint32_t min_w, uint32_t min_h, uint32_t rw, uint32_t rh) {
if (exact) return rw == fw && rh == fh;
return rw <= fw && rh <= fh && rw >= min_w && rh >= min_h;
}
/// Whether this frame shows that the render size changes under one output: a feature of one size (exact) for the same
/// output, preset and colour (same), and a frame of another size. The caller keeps it for the run: the next feature, and
/// every one after it, is made for the mode's whole range
inline bool found_range(bool same, bool exact, uint32_t fw, uint32_t fh, uint32_t rw, uint32_t rh) {
return same && exact && (rw != fw || rh != fh);
}
} // namespace dlss_feature

View File

@@ -93,6 +93,36 @@ inline void unjitter(float m[16], const float jitter[2]) {
}
}
/// ((Triforce, 11-20) the owner's "DLSS flicker": the whole picture and the shadows, while the camera moves, with NVIDIA
/// DLSS frame generation on, and not without it.) Streamline's camera matrices are in the clip space of the depth buffer it
/// is given: its depth value is that clip space's z over w, 1 at the near plane with depthInverted. The game's projection
/// is NVN's, clip z from -1 at the near plane to 1 at the far one, and the depth frame generation gets is FSR's copy
/// (vk_fsr3_inputs.cpp): d = (1 - ndc z) / 2 where the game's buffer has 0 at the near plane (flipped: Camera::flip_depth),
/// (ndc z + 1) / 2 where it is reversed already. Given the game's own matrices with that depth, a depth read as clip z put
/// what is far a fifth of a metre from the camera and what is near at the far plane, so whatever Streamline works out from
/// the depth and the camera's move was wrong as soon as the camera moved, and right while it stood. Only DLSS frame
/// generation takes matrices (FSR takes the near and far planes and the field of view), so nothing else showed it.
/// The depth's clip space is the game's with z' = (w - z) / 2 (flipped) or (w + z) / 2: Z * clip, Z the identity but for
/// its z row, (0 0 -1/2 1/2) or (0 0 1/2 1/2).
/// A projection (or a view-projection) into the depth's clip space: Z * m, row major (clip = M * position)
inline void depth_clip_projection(const float m[16], bool flipped, float out[16]) {
const float s = flipped ? -0.5f : 0.5f;
for (int i = 0; i < 16; i++) out[i] = m[i];
for (int c = 0; c < 4; c++) out[2 * 4 + c] = s * m[2 * 4 + c] + 0.5f * m[3 * 4 + c];
}
/// A clip-to-clip matrix of the game's (this frame's clip space to the frame before's: reprojection) between the depth's
/// clip spaces: Z * m * inverse(Z), inverse(Z)'s z row (0 0 -2 1) or (0 0 2 -1). The identity stays the identity
inline void depth_clip_reprojection(const float m[16], bool flipped, float out[16]) {
const float k = flipped ? -2.0f : 2.0f, t = flipped ? 1.0f : -1.0f;
float r[16];
for (int row = 0; row < 4; row++) { // (m * inverse(Z): the z column scaled, and added to the w column)
r[row * 4 + 0] = m[row * 4 + 0], r[row * 4 + 1] = m[row * 4 + 1];
r[row * 4 + 2] = k * m[row * 4 + 2];
r[row * 4 + 3] = m[row * 4 + 3] + t * m[row * 4 + 2];
}
depth_clip_projection(r, flipped, out); // (Z * that)
}
/// A jitter in clip space as DLSS takes it: in render pixels, x right and y down (clip y is up)
inline void jitter_to_pixels(const float clip[2], unsigned width, unsigned height, float out[2]) {
out[0] = clip[0] * 0.5f * (float)width;

38
gpu/vk/dmabuf_gate.h Normal file
View File

@@ -0,0 +1,38 @@
// (Triforce, 11-22, the performance audit's OS-08, after Source A's) Whether guest memory is offered to the driver as
// udmabuf dma-bufs at all, and what the start's line says (vk_memory.cpp import_dma_buf asks at its first call; plain code:
// tests/test_dmabuf_driver_gate.py runs it anywhere).
//
// On Linux the guest window is a memfd and each pool is first offered to the driver as a dma-buf made by udmabuf. NVIDIA's
// proprietary driver refuses every one (VK_ERROR_OUT_OF_DEVICE_MEMORY) and the pool ends as a host-pointer import anyway:
// 49-50 refused imports a run and a kernel message at each start on Source A's RTX 4090, the first refusal 2.5-3.7 ms, one
// of a 64 MB piece 198 ms on the presentation thread during a save load. So with that driver /dev/udmabuf is not opened
// and the pools are host-pointer imports from the start. NVK and RADV have other driver ids and keep their dma-bufs.
// RX_VK_DMABUF_NVIDIA=1: tried on NVIDIA's driver too, as before (for a later driver that takes them).
#pragma once
namespace dmabuf_gate {
/// What the start found. fn: the host gave its memfd function (false also with RX_VK_NO_DMABUF). memfd: the window is a
/// memfd. ext: the device imports dma-bufs (VK_EXT_external_memory_dma_buf and its fd properties call). nvidia: NVIDIA's
/// proprietary driver, and RX_VK_DMABUF_NVIDIA not set
struct Facts {
bool fn = false, memfd = false, ext = false, nvidia = false;
};
/// Whether /dev/udmabuf is opened: only then can a pool be offered as a dma-buf
inline bool try_udmabuf(const Facts& f) { return f.fn && f.memfd && f.ext && !f.nvidia; }
/// As it was: NVIDIA's driver was offered them too
inline bool try_udmabuf_before(const Facts& f) { return f.fn && f.memfd && f.ext; }
/// The start's "vk: guest memory as dma-bufs: ..." answer. opened: /dev/udmabuf opened (asked only after try_udmabuf)
inline const char* answer(const Facts& f, bool opened) {
if (try_udmabuf(f) && opened) return "yes";
if (!f.fn) return "no (no memfd window, or RX_VK_NO_DMABUF)";
if (!f.memfd) return "no (the window is anonymous memory)";
if (!f.ext) return "no (VK_EXT_external_memory_dma_buf)";
if (f.nvidia) return "no (NVIDIA's driver refuses them: host pointers)";
return "no (/dev/udmabuf cannot be opened)";
}
} // namespace dmabuf_gate

41
gpu/vk/drawn_page_map.h Normal file
View File

@@ -0,0 +1,41 @@
// (Triforce, 11-21, the performance audit's TEX-21, after Source A's) A texture the GPU drew gets its page map at its first
// draw.
//
// A texture in a virtual pool (its storage mapped 64 KB a page) is checked through its mapping: texture_dirty keeps each
// page's guest address (page_map) and says which pages were mapped anew or written since. The map is recorded at the
// texture's first check, which for a texture uploaded from memory is its first upload. One the GPU drew first is marked
// uploaded by the draw and never checked, so it has no map; once the game finalizes it, its first check finds none, takes
// that as "everything was mapped since", and the whole storage is uploaded from memory over what the GPU drew: a HUD
// label or a render target gone blank after an area change, and an upload of up to 15 MB each. Source A's sessions from
// Hyrule Field to Castle Town: such whole uploads of HUD labels 6, other HUD 4 to 6, render targets 4 to 9, all to 0.
// Here: the map is recorded at the first draw that says who drew it, by one walk of the mapping (the storage's size / 64 KB
// lookups, once a texture). The later check then gives only the pages mapped anew or written by the CPU.
// Not for a write that gives no reason ("?": a clear, the movie upscaler): those keep the path that brings a font cache's
// letters back from memory. RX_VK_DRAWN_PAGE_MAP=0: no map at the draw, as before.
//
// The rule alone: tests/test_drawn_page_map.py compiles it, and texture_dirty cut from the source.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
namespace drawn_page_map {
inline constexpr uint64_t kVirtual = 1ull << 40; // (a virtual pool's addresses start here)
inline constexpr uint64_t kPage = 1ull << 16; // (and its pages are 64 KB)
/// RX_VK_DRAWN_PAGE_MAP: off only with 0
inline bool on(const char* asked) { return !asked || !*asked || atoi(asked) != 0; }
/// The pages of a storage of guest_size bytes
constexpr size_t pages(uint64_t guest_size) { return (size_t)((guest_size + kPage - 1) / kPage); }
/// Whether a write by `why` into a texture at gpu, of guest_size bytes of storage, whose map holds map_pages pages now,
/// records its map first
inline bool seeds(bool switch_on, const char* why, uint64_t gpu, uint64_t guest_size, size_t map_pages) {
if (!switch_on || !why || !std::strcmp(why, "?")) return false;
return gpu >= kVirtual && guest_size != 0 && map_pages != pages(guest_size);
}
} // namespace drawn_page_map

140
gpu/vk/driver_faults.h Normal file
View File

@@ -0,0 +1,140 @@
// Programs the driver faulted on, kept out of it, and one build of a program at a time (no game, no GPU: the rules and the
// gate; vk_draw.cpp fault_quarantine, build_graphics, guarded_graphics). Apart from Vulkan so that
// tests/test_driver_faults.py runs it alone, on many made-up PCs.
//
// (Triforce, 11-22, the performance audit's PIPE-57, after Source A's.) The guards catch a fault inside the driver's
// compiler (vk_draw.cpp guarded_graphics), and a fault during the start's own shader work starts the game again
// (driver_fault_restart). Nothing remembered the program: the next start asked the driver for it again, faulted again and
// restarted again, and in play a second thread could ask for the same program while the driver was still inside the
// first. Source A saw that on AMD's Windows driver (an RX 6800 XT): in a freeze, three threads sat in the driver on two
// programs' builds while the presenting thread waited for one of them; a preparation stuck at "6625 of 6627, nothing
// finished for 30 s" took 32.5 s, and 4.2 s with three faulted programs kept out on the first run, 2.3 s after. Triforce
// has the same class of report: "two of 4926 on an RX 5700 XT, stuck at 99%".
//
// What. On AMD's own Windows driver (or everywhere with RX_VK_DRIVER_FAULTS=2):
// - a program whose pipeline build faulted in the driver is not asked of the driver again this run, and its key goes to
// triforce_driver_faults.txt beside the executable, one line under the vendor and the driver's version, written by the
// fault's handler before the restart can end the process (so a start that faults does not fault again); later starts
// with that driver skip those programs, and say so once (delete the file to try again; another driver version ignores
// the lines);
// - one build of a program at a time: a second waits at the gate, outside the driver, and a probe that asks the caches
// only is told a compile is required.
// Elsewhere (NVIDIA's, Intel's, Mesa's drivers, AMD's on Linux) nothing of it runs, as before: the NVIDIA fault of 11-13
// was a race in the driver, not a bad program. RX_VK_DRIVER_FAULTS=0: nowhere.
#pragma once
#include <condition_variable>
#include <cstdint>
#include <cstdlib>
#include <mutex>
#include <string>
#include <unordered_set>
#include <vector>
namespace driver_faults {
/// RX_VK_DRIVER_FAULTS: 0 off, 1 (unset) AMD's Windows driver, 2 every driver
inline int mode(const char* env) {
if (!env || !*env) return 1;
const int v = std::atoi(env);
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
}
/// Whether the quarantine, its file and the gate are on: mode 2 anywhere, mode 1 on Windows with AMD's own driver
/// (VK_DRIVER_ID_AMD_PROPRIETARY)
inline bool on(int mode, bool windows, bool amd_proprietary) {
return mode >= 2 || (mode == 1 && windows && amd_proprietary);
}
inline constexpr int kLine = 35; // ("vvvvvvvv dddddddd kkkkkkkkkkkkkkkk\n")
/// One line of the file into out (kLine bytes and a 0): made without the C library, as the fault's handler makes it
inline int line(char* out, uint32_t vendor, uint32_t driver, uint64_t key) {
static const char hex[] = "0123456789abcdef";
int n = 0;
for (int i = 7; i >= 0; i--) out[n++] = hex[(vendor >> (4 * i)) & 0xF];
out[n++] = ' ';
for (int i = 7; i >= 0; i--) out[n++] = hex[(driver >> (4 * i)) & 0xF];
out[n++] = ' ';
for (int i = 15; i >= 0; i--) out[n++] = hex[(key >> (4 * i)) & 0xF];
out[n++] = '\n';
out[n] = 0;
return n;
}
/// The keys of the file's lines for this vendor and driver version; another's, and a broken line, are passed over
inline std::vector<uint64_t> parse(const std::string& text, uint32_t vendor, uint32_t driver) {
std::vector<uint64_t> keys;
const auto hexval = [](const std::string& s, size_t at, int digits, uint64_t* v) {
if (at + (size_t)digits > s.size()) return false;
uint64_t x = 0;
for (int i = 0; i < digits; i++) {
const char c = s[at + i];
const int d = c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1;
if (d < 0) return false;
x = x << 4 | (uint64_t)d;
}
*v = x;
return true;
};
size_t at = 0;
while (at < text.size()) {
size_t end = text.find('\n', at);
if (end == std::string::npos) end = text.size();
std::string l = text.substr(at, end - at);
if (!l.empty() && l.back() == '\r') l.pop_back();
uint64_t v = 0, d = 0, k = 0;
if (l.size() == (size_t)kLine - 1 && l[8] == ' ' && l[17] == ' ' && hexval(l, 0, 8, &v) && hexval(l, 9, 8, &d) &&
hexval(l, 18, 16, &k) && v == vendor && d == driver)
keys.push_back(k);
at = end + 1;
}
return keys;
}
/// The run's quarantine and the gate: begin before a program's build, end after it (with whether it faulted)
class Quarantine {
public:
enum class Begin { Go, Busy, Faulted };
void load(const std::vector<uint64_t>& keys) {
std::lock_guard lk(mu_);
bad_.insert(keys.begin(), keys.end());
}
/// Faulted: not asked of the driver. Busy: another build of it is inside the driver and the caller may not wait (a
/// probe of the caches). Go: this build may go in; the caller ends it with end(). may_wait: wait at the gate instead
Begin begin(uint64_t key, bool may_wait) {
std::unique_lock lk(mu_);
for (;;) {
if (bad_.count(key)) return Begin::Faulted;
if (!busy_.count(key)) break;
if (!may_wait) return Begin::Busy;
cv_.wait(lk);
}
busy_.insert(key);
return Begin::Go;
}
/// True when it faulted now for the first time
bool end(uint64_t key, bool faulted) {
bool fresh = false;
{
std::lock_guard lk(mu_);
busy_.erase(key);
if (faulted) fresh = bad_.insert(key).second;
}
cv_.notify_all();
return fresh;
}
bool faulted(uint64_t key) const {
std::lock_guard lk(mu_);
return bad_.count(key) != 0;
}
size_t count() const {
std::lock_guard lk(mu_);
return bad_.size();
}
private:
mutable std::mutex mu_;
std::condition_variable cv_;
std::unordered_set<uint64_t> bad_, busy_;
};
} // namespace driver_faults

65
gpu/vk/evict_reason.h Normal file
View File

@@ -0,0 +1,65 @@
// Whether the write-back of a texture the eviction chose is stale: another image was drawn over its storage since its own
// last drawing (no game, no GPU: a rule). tests/test_evict_reason.py
//
// (Triforce, 11-22, the performance audit's TEX-R404, after Source A's.) Over the budget, the eviction writes each chosen
// texture the GPU drew back into the game's memory before it goes, so that one made again over that storage is uploaded
// with what the GPU drew. The game draws its HUD labels (110 x 110, 46 x 46, 140 x 140) inside the storage of a
// 1920 x 1080 target it no longer uses. The eviction's own tests do not see that: drawn_over looks only at images of the
// same start, and target_overlap.h leaves out an image inside the target. So when the target went, its old texels landed
// over the labels, the pages counted as written, and the labels, finalized, were uploaded from them as noise (Source A:
// in 3 of 3 runs of two minutes in Hyrule Field at a 6 GB allowance, none with the rule).
//
// The rule: a write-back is left out when another image the GPU drew overlaps the texture's storage and was drawn in a
// later frame than the texture's own last drawing. The texture still goes. One drawn in the same frame keeps the
// write-back (a frame's targets drawn side by side), and so does one that only touches it. What is given up: the parts
// of the texture the newer image does not cover are not written back either; made again there and sampled undrawn, it
// starts from what the memory holds, as before the write-back existed. RX_VK_EVICT_STALE_FLUSH=1: written back, as before.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <vector>
namespace evict_reason {
/// RX_VK_EVICT_STALE_FLUSH: unset, empty or 0 leaves a stale write-back out; anything else writes it back, as before
inline bool flush_stale(const char* env) { return env && *env && std::atoi(env) != 0; }
/// Whether the storage [lo, hi) and [olo, ohi) share a byte (ranges that touch do not)
constexpr bool overlaps(uint64_t lo, uint64_t hi, uint64_t olo, uint64_t ohi) {
return lo < hi && olo < ohi && lo < ohi && olo < hi;
}
/// Whether a texture last drawn in frame `last` holds texels older than what is in its storage: `other` is the frame of
/// the newest drawing of another image over that storage (0: none)
constexpr bool write_back_stale(uint64_t last, uint64_t other) { return other > last; }
/// A texture whose write-back the eviction is about to make
struct Candidate {
uint64_t lo = 0, hi = 0; // its storage
size_t index = 0; // the caller's own: its place in the eviction's list
const void* self = nullptr; // itself, so that it is not taken for another image over its own storage
uint64_t newest = 0; // the frame of the newest drawing of another image over its storage (0: none)
};
/// Puts the candidates in the order of their starts, for note(); returns the longest one's size
inline uint64_t by_start(std::vector<Candidate>& c) {
std::sort(c.begin(), c.end(), [](const Candidate& a, const Candidate& b) { return a.lo < b.lo; });
uint64_t longest = 0;
for (const Candidate& k : c) longest = std::max(longest, k.hi > k.lo ? k.hi - k.lo : 0);
return longest;
}
/// One image the GPU drew, [olo, ohi), last drawn in frame `o_written`: every candidate it overlaps (but itself) takes
/// that frame when it is newer than what the candidate has. `c` is in by_start's order, `longest` what by_start gave
inline void note(std::vector<Candidate>& c, uint64_t longest, uint64_t olo, uint64_t ohi, uint64_t o_written, const void* other) {
if (ohi <= olo) return;
// (a candidate over [olo, ohi) starts before ohi and after olo less its size: the search begins at olo less the longest)
auto it = std::lower_bound(c.begin(), c.end(), olo > longest ? olo - longest : 0,
[](const Candidate& k, uint64_t g) { return k.lo < g; });
for (; it != c.end() && it->lo < ohi; ++it)
if (it->self != other && overlaps(it->lo, it->hi, olo, ohi) && o_written > it->newest) it->newest = o_written;
}
} // namespace evict_reason

View File

@@ -0,0 +1,73 @@
// Mesh-fallback compute stages too big for two Mesa drivers are not handed to them (no game, no GPU: a rule).
//
// (Triforce, 11-22, the performance audit's PIPE-56, after Source A's.) A mesh program on a GPU without mesh shaders runs
// as compute stages (vk_draw.cpp fallback_compute_pipelines). Every stage went to the driver, and a refusal was only
// remembered after the build. Two drivers do not get that far on the largest stages:
// - Mesa's RADV on GCN 1 to 4 (GFX6-8: vk_vendor.h radv_gcn). Source A's numbers under drm-shim with a 6 GB cap: the
// 79,947-phi stage not built after 45-90 s and killed (25 GB at 150 s), where GFX9 and Navi built it in 13-14 s at
// 2.9 GB. Limit: 50,000 OpPhi in one stage (2 of about 800 modules are over it);
// - Mesa's Intel driver. An Iris Xe: a 251-call stage hit a 14 GB cap unfinished after 27 minutes. Limits: 20,000 OpPhi
// or 200 OpFunctionCall in one stage (10 of 376 stages left out in Source A's full preparation).
// A program with a stage over its driver's limit goes into the failed set before anything is built, as a program the
// driver refused does: its draws are left out. Which objects those are is not known. No other driver has a limit, and the
// switches set none there: NVIDIA, Intel's Windows driver, AMD's own drivers and RADV from Vega on build every stage.
//
// RX_VK_FALLBACK_PHI_LIMIT=<n>, RX_VK_FALLBACK_CALL_LIMIT=<n>: the two limits on those two drivers; 0: no limit.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
namespace fallback_size {
constexpr uint32_t kOpFunctionCall = 57, kOpPhi = 245;
constexpr uint32_t kGcnPhis = 50000, kIntelPhis = 20000, kIntelCalls = 200;
struct Size {
uint32_t phis = 0, calls = 0;
};
struct Limits {
uint32_t phis = 0, calls = 0; // (0: none)
};
enum class Driver { Other, RadvGcn, MesaIntel };
/// A module's OpPhi and OpFunctionCall instructions. One that does not read as SPIR-V counts 0 of both
inline Size count(const uint32_t* code, size_t words) {
Size s;
if (!code || words < 5 || code[0] != 0x07230203u) return s;
for (size_t at = 5; at < words;) {
const uint32_t n = code[at] >> 16, op = code[at] & 0xffffu;
if (!n || at + n > words) return Size{};
s.phis += op == kOpPhi, s.calls += op == kOpFunctionCall;
at += n;
}
return s;
}
inline uint32_t spirv_phis(const uint32_t* code, size_t words) { return count(code, words).phis; }
inline uint32_t spirv_calls(const uint32_t* code, size_t words) { return count(code, words).calls; }
/// A limit's switch: unset, empty or not a number, the driver's own; 0, none; else the number
inline uint32_t parse_limit(const char* env, uint32_t own) {
if (!env || !*env) return own;
char* end = nullptr;
const unsigned long long v = std::strtoull(env, &end, 10);
if (end == env || *end) return own;
return v > 0xffffffffull ? 0xffffffffu : (uint32_t)v;
}
/// The driver: RADV on a GCN 1 to 4 GPU first (it is never Mesa's Intel driver), then Mesa's Intel driver
constexpr Driver driver_of(bool radv_gcn, bool mesa_intel) {
return radv_gcn ? Driver::RadvGcn : mesa_intel ? Driver::MesaIntel : Driver::Other;
}
/// The limits on that driver, with the two switches' values (nullptr: unset). None on any other driver, whatever is set
inline Limits limits(Driver d, const char* phi_env, const char* call_env) {
if (d == Driver::Other) return Limits{};
const Limits own = d == Driver::RadvGcn ? Limits{kGcnPhis, 0} : Limits{kIntelPhis, kIntelCalls};
return Limits{parse_limit(phi_env, own.phis), parse_limit(call_env, own.calls)};
}
/// The stage is over a limit: its program is not built
constexpr bool refused(const Limits& l, const Size& s) {
return (l.phis && s.phis > l.phis) || (l.calls && s.calls > l.calls);
}
} // namespace fallback_size

38
gpu/vk/fg_motion.h Normal file
View File

@@ -0,0 +1,38 @@
// (Triforce, 11-22, the owner's DLSS frame generation flicker, again) Whose motion vectors frame generation is given.
//
// Since FG-4 (vk_fsr3_inputs.cpp chain_motion), while NVIDIA DLSS or AMD FSR 4 upscales, frame generation's motion is the
// upscaler chain's: the camera's from the depth, with every draw of the scene that stored its positions drawn again over
// it (vk_temporal.cpp capture_motion). With NVIDIA DLSS frame generation that showed as the owner's flicker of 11-20 again
// (the whole picture while the camera moved), on a build with the clip-space fix of that day (dlss_math.h
// depth_clip_projection) in it. The owner's runs of 11-22 (DLSS at 50%, 2x frame generation, HDR, before the bloom):
// - RX_VK_FG_OWN_MOTION=0, the camera's motion alone: no flicker;
// - the chain's motion with the upscaler audit's commits all set back (RX_VK_CAPTURE_TABLES=0, RX_VK_REDRAW_STATE=0,
// RX_VK_DLSS_FEATURE_MAX=1): the flicker, so it is not theirs;
// - RX_VK_DLSS_PROBE=1 in that run: the redraw's motion of the scene's largest draw within 0.03 px of the camera's.
// What in the chain's motion Streamline takes for a flicker is not known: its made frames are not in any log or shot. DLSS
// frame generation finds what moves by itself with its own optical flow, and FSR's inputs give it the depth and the
// camera, so it is given the camera's motion alone: what the first run showed. AMD's frame generation and FSR 3.1's
// interpolation keep the chain's, as since FG-4 (no flicker of theirs is reported).
//
// RX_VK_DLSSG_OWN_MOTION=1: DLSS frame generation is given the chain's motion again, as before. RX_VK_FG_OWN_MOTION=0 still
// gives every generator the camera's alone.
//
// No Vulkan: tests/test_fg_motion.py compiles this.
#pragma once
#include <cstdlib>
namespace fg_motion {
/// A switch read as the renderer reads its others: set and not 0 (value null when unset)
inline bool set_on(const char* value) { return value && *value && std::atoi(value) != 0; }
/// RX_VK_FG_OWN_MOTION=0 (value null when unset): the camera's motion alone, for every generator
inline bool camera_alone(const char* value) { return value && *value && std::atoi(value) == 0; }
/// Whether frame generation is given the upscaler chain's motion this frame. dlssg: NVIDIA DLSS frame generation runs in
/// this session (vk_dlssg.h active). fg_own: RX_VK_FG_OWN_MOTION; dlssg_own: RX_VK_DLSSG_OWN_MOTION (null when unset)
inline bool chain(bool dlssg, const char* fg_own, const char* dlssg_own) {
if (camera_alone(fg_own)) return false;
return !dlssg || set_on(dlssg_own);
}
} // namespace fg_motion

View File

@@ -0,0 +1,42 @@
// (Triforce, 11-21, the performance audit's TEX-43, after Source A's) A finalized texture the GPU drew whose storage the
// game has since written over.
//
// A texture the GPU drew in one frame is kept after the game finalizes it (texture_sweep keep_finalized, up to
// RX_VK_KEEP_FINALIZED_MB): the game may make a texture over that storage again and sample it without drawing it, as the
// terrain does with its 260 x 260 tiles. Within the budget only a draw over one let it go, and the game's own CPU writes
// to that memory never did: in players' logs about 200 render target images an hour stayed for good, most of them such
// tiles, with the blocks they pin. Source A's rule: once the CPU has written three quarters or more of the storage since
// the release, the game has put other data there, the image holds nothing that memory holds on the console, and the
// sweep lets it go whatever the budget says. Only one of 32 KB or more (a small one shares its pages with what the game
// keeps beside it, and a write there says nothing of it) and not one in a virtual pool. Its laps of Hyrule Field at a
// 6 GB allowance: render target images 129, 166, 221 against 128, 131, 129; the kept ones 131 against 41.
// The answer is kept with the texture: once rewritten it stays so until it is released again, and one not yet rewritten
// is looked at again at most every kRecheck presents (the look walks its pages' write generations).
// RX_VK_KEEP_OBSOLETE=1: kept, as before.
//
// The rule alone: tests/test_finalized_obsolete.py compiles it.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace finalized_obsolete {
inline constexpr uint64_t kMinBytes = 32u << 10;
inline constexpr uint64_t kVirtual = 1ull << 40;
inline constexpr uint64_t kRecheck = 60;
/// RX_VK_KEEP_OBSOLETE: not 0 keeps them, as before
inline bool keep(const char* asked) { return asked && atoi(asked) != 0; }
/// Whether a texture can be one: finalized after the GPU wrote it, in no virtual pool, of 32 KB of storage or more
constexpr bool candidate(bool released, bool gpu_written, uint64_t gpu, uint64_t guest_size) {
return released && gpu_written && gpu < kVirtual && guest_size >= kMinBytes;
}
/// Whether `written` bytes of a storage of guest_size, written by the CPU since the release, make it rewritten
constexpr bool rewritten(uint64_t written, uint64_t guest_size) { return guest_size != 0 && written * 4 >= guest_size * 3; }
/// Whether one not rewritten when last looked at, in present `checked` (0: never), is looked at again in present `frame`
constexpr bool due(uint64_t frame, uint64_t checked) { return !checked || frame - checked >= kRecheck; }
} // namespace finalized_obsolete

74
gpu/vk/first_start.h Normal file
View File

@@ -0,0 +1,74 @@
// (Triforce, 11-22, the performance audit's PRESENT-46, after Source A's) A first start sized to the card: the Render
// resolution row set once on a weak GPU.
//
// With no options file every card started the same: the Render resolution at Auto, the game's dynamic resolution. Source A
// reads the card and puts it in a tier; its tier 1 is an integrated GPU (a handheld's is one), a card without mesh shaders,
// or a card with 4.5 GB of video memory or less, and it draws the 3D at 50% there, or at 33% on an integrated GPU beside
// 4 processor threads or fewer (its note: an i3-1115G4's own GPU took 60 to 220 ms a frame at 50%).
//
// This is that test, for a first start only (vk_menu.cpp menu_first_start): no options file, RX_VK_LOW_SPEC not set,
// nothing saved by it, so a start after the player has saved anything is the player's own. Left out, as the audit's
// entry says: Source A's preset row, its fingerprint that chooses again on new hardware (it would overwrite the player's
// rows), its tuner in play, any change of the upscaler or of the output's size, and its test of the SteamDeck variable,
// which took desktop PCs started from that launcher for handhelds: a handheld is found by its integrated GPU alone.
//
// RX_VK_FIRST_START_AUTO: not set or 0, off (as before; the default until the owner has decided what a first start on a
// weak card gets). 1: on. A larger number: on, the card taken as having that many MB of video memory, for the lab.
//
// No Vulkan, no menu: tests/test_first_start.py compiles this.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace first_start {
/// What is read of the card
struct Card {
bool integrated = false; // (VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU)
bool mesh_shaders = true; // (VK_EXT_mesh_shader, as the device was made)
uint64_t vram_mb = 0; // (its video memory; 0: not known)
unsigned threads = 0; // (the processor's threads; 0: not known)
};
constexpr uint64_t kSmallVramMb = 4608; // 4.5 GB: tier 1 up to here
constexpr unsigned kFewThreads = 4; // an integrated GPU beside this many threads or fewer: 33%
constexpr int kRowHalf = 4, kRowThird = 6; // opts.resolution's 50% and 33% (upscaler_rows.h)
enum Why { kKeep = 0, kIntegrated, kNoMeshShaders, kSmallVram };
/// The Render resolution row for the card's first start: row 0, the row left as it is (not tier 1)
struct Choice {
int row = 0;
Why why = kKeep;
};
inline Choice choose(const Card& c) {
if (c.integrated) return {c.threads && c.threads <= kFewThreads ? kRowThird : kRowHalf, kIntegrated};
if (!c.mesh_shaders) return {kRowHalf, kNoMeshShaders};
if (c.vram_mb && c.vram_mb <= kSmallVramMb) return {kRowHalf, kSmallVram};
return {};
}
inline const char* why_text(Why w) {
switch (w) {
case kIntegrated: return "an integrated GPU";
case kNoMeshShaders: return "no mesh shaders";
case kSmallVram: return "4.5 GB of video memory or less";
default: return "not a weak card";
}
}
/// RX_VK_FIRST_START_AUTO (value, null when unset): -1 off; 0 on, the card's own video memory; otherwise on, the card
/// taken as having that many MB
inline int64_t switch_mb(const char* text) {
if (!text || !*text) return -1;
const long long v = std::atoll(text);
return v <= 0 ? -1 : v == 1 ? 0 : (int64_t)v;
}
/// Whether the first start's choice is made at all: the switch on, no options file, RX_VK_LOW_SPEC not asked (it sets the
/// row itself), and no switch that sets the Render resolution for this run (HOS_DYNRES)
inline bool applies(bool has_options, bool low_spec, bool resolution_env, int64_t switch_value) {
return switch_value >= 0 && !has_options && !low_spec && !resolution_env;
}
} // namespace first_start

View File

@@ -28,6 +28,7 @@ struct Split {
double wb_wait = 0; // (the waits for write-backs still on the GPU, wherever they were: memory sync's capture, a read)
double wb_held = 0; // (of them, the part after the writer's recording was done: the timeline held, not the GPU)
double qs_bg = 0; // (the queue submits made while a background import ran)
double gpu_waits = 0; // ((Triforce, 11-22, SYNC-27) every thread's time in calls that block on the GPU: gpu_wait_kinds.h)
};
/// Outside the replay and the present: the game's own update, its waits for the replay and the GPU among it. As the stall line's
@@ -35,12 +36,13 @@ inline double outside(const Split& s) { return s.interval - s.replay - s.present
/// The columns, after the trace's first ones (their names; the line's values in this order: line)
constexpr const char* kColumns =
"replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms,game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms";
"replay_ms,present_ms,dispatch_ms,qsubmit_ms,par_wait_ms,outside_ms,game_wait_ms,captures_ms,wb_wait_ms,wb_held_ms,qs_bg_ms,"
"gpu_waits_ms";
/// The values for kColumns, comma-separated, no line end: snprintf's count
inline int line(const Split& s, char* out, size_t cap) {
return std::snprintf(out, cap, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f", s.replay, s.present, s.dispatch,
s.qsubmit, s.par_wait, outside(s), s.game_wait, s.captures, s.wb_wait, s.wb_held, s.qs_bg);
return std::snprintf(out, cap, "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f", s.replay, s.present, s.dispatch,
s.qsubmit, s.par_wait, outside(s), s.game_wait, s.captures, s.wb_wait, s.wb_held, s.qs_bg, s.gpu_waits);
}
} // namespace mkwvk::frame_trace

49
gpu/vk/framegen_idle.h Normal file
View File

@@ -0,0 +1,49 @@
// (Triforce, 11-21, the performance audit's PRESENT-18, after Source A's) When frame generation's images are let go.
//
// The built-in interpolation's frames, lumas, flows and made-frame sets (and, for AMD FSR 4's frame generation, the four
// images of the picture without the HUD) are made at the first frame that needs them and were kept until a resize: turned
// off, or left for NVIDIA's own frame generation, they stayed in VRAM for the run. Source A measured 150 to 160 MiB given
// back at 2560 x 1440 with 2x after its switch away. They are let go after kPresents presents in a row with frame
// generation off, about two seconds at 60: a row stepped through on the way to another choice keeps them. A present that
// recorded made frames counts as on whatever the row says by then: its set is busy until the display thread has shown
// it, and a drain then would wait for a job that is only queued afterwards. So the count is taken after the present's
// queue (vk_replay.cpp, after framegen_queue). They are made again at the next frame that needs them, as after a resize.
// RX_VK_FG_KEEP=1: kept, as before.
//
// The rule alone, no Vulkan: tests/test_framegen_idle.py compiles it.
#pragma once
#include <cstdint>
namespace framegen_idle {
inline constexpr uint32_t kPresents = 120;
/// The presents in a row off, after this one: on now, or used at this present, starts again
inline uint32_t next_off(uint32_t off, bool on_now, bool used) { return on_now || used ? 0 : off < UINT32_MAX ? off + 1 : off; }
/// Whether this present lets them go: there are some, and they have been off long enough
inline bool due(uint32_t off, bool have) { return have && off >= kPresents; }
/// One present, as framegen_idle sees it after the present's queue
struct Present {
bool on; // frame generation on by the row, read again (framegen_on)
bool recorded; // this present recorded made frames (framegen_record's true)
bool hudless_wanted; // AMD FSR 4's frame generation is the row's (fsr4fg::frames_wanted)
bool have_images; // the shared images are there (fg.w)
bool have_hudless; // the images of the picture without the HUD are there
};
struct Counts {
uint32_t off = 0, hudless_off = 0;
};
struct Release {
bool shared, hudless;
};
/// The two counts after this present, and what it lets go. keep: RX_VK_FG_KEEP, nothing ever
inline Release step(Counts& c, const Present& p, bool keep) {
c.off = next_off(c.off, p.on, p.recorded);
c.hudless_off = next_off(c.hudless_off, p.on && p.hudless_wanted, false);
if (keep) return {false, false};
return {due(c.off, p.have_images), due(c.hudless_off, p.have_hudless)};
}
} // namespace framegen_idle

40
gpu/vk/framegen_refresh.h Normal file
View File

@@ -0,0 +1,40 @@
// (Triforce, 11-21, the upscaler audit's UPS-05) A fixed multiplier's frames, no more than the display can show.
//
// The Frame generation row's 2x to 8x make that many frames for each of the game's whatever the display's refresh rate
// is; only the row's Auto goes by it. The owner's log of 11-20 has 5x on a 165 Hz display with the game near 66 frames a
// second: 1,655 frames shown in every 5 s (331 a second, two for each refresh), each present call 2.2 to 3.0 ms where it
// takes 0.03 ms without frame generation, and the display thread's line "0 frames shown on time, 1655 late". Half of the
// made frames were never on the screen (the swapchain is mailbox: the newest at each refresh), and each was a pass over
// the whole picture and a present the display thread could not keep up with.
//
// With the switch, a fixed multiplier makes no more frames than refreshes fall in one frame of the game: the game's
// recent frame time over the display's period, rounded up, a tenth of a refresh forgiven so a game at a whole fraction
// of the refresh rate is not given one more for its frames' jitter. Never more than the row asks, never fewer than the
// game's own frame. The row's Auto and AMD FSR's own are not touched.
//
// RX_VK_FRAMEGEN_REFRESH=1: on. Off unless set (the row's number is the frames made, as before): it changes what a row
// the player set shows, so it is the owner's to turn on.
//
// No Vulkan: tests/test_framegen_refresh.py compiles this.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace framegen_refresh {
/// RX_VK_FRAMEGEN_REFRESH (value, null when unset): on when it is not 0; unset, off
inline bool on(const char* value) { return value && *value && std::atoi(value) != 0; }
/// The frames shown for each of the game's (the game's own among them) with a fixed multiplier m, the game's frames game_ns
/// apart, the display refreshing every period_ns. An unknown frame time or period: m
inline int frames(int m, int64_t game_ns, int64_t period_ns) {
if (m <= 1 || game_ns <= 0 || period_ns <= 0) return m;
// (the refreshes in a frame of the game, rounded up, a tenth of one forgiven: ceil(game / period - 0.1))
const int64_t most = (game_ns * 10 + period_ns * 9 - 1) / (period_ns * 10);
return most < 1 ? 1 : most < m ? (int)most : m;
}
/// The display's period for a refresh rate in mHz (vk_main.cpp display_refresh_mhz); 0 when it is not known
inline int64_t period_ns(int refresh_mhz) { return refresh_mhz > 0 ? (int64_t)(1000000000000LL / refresh_mhz) : 0; }
} // namespace framegen_refresh

View File

@@ -232,4 +232,24 @@ template <class Ops> void teardown(Ops& ops, bool exiting) {
ops.destroy_images();
}
// ---- (Triforce, 11-20) What a present writes into a set of frames: never past the frames the set was made with ----
// The owner's "upscaling crash": 0xc0000005 reading 0x590 in NVIDIA's driver, inside a vkCmdCopyImage the recorder thread
// ran, in three runs of 11-20, each with frame interpolation on and the Frame generation row stepped up in the options
// (Auto in the first, 3x at the next start, 5x at the one after). The sets' frames are made for the row's multiplier when
// a present begins (vk_framegen.cpp ensure_size); the present then waits for the display thread to let a set go, and
// while frames are made the display thread is the one that runs the menu (present_image, menu_tick), so a step of the row
// lands in that wait. The multiplier read again after it was more than the frames made, and the game's frame was copied
// into an image that was never made (VK_NULL_HANDLE). A step down only left frames unused. The row's new multiplier takes
// effect at the next present, which makes the sets anew.
/// The multiplier a present records with: the row's, and no more than the frames its sets were made with
constexpr int set_multiplier(int row_multiplier, int made_with) {
const int most = made_with < 1 ? 1 : made_with;
return row_multiplier < 1 ? 1 : row_multiplier > most ? most : row_multiplier;
}
/// The game's own frame's place in a set, after `made` made ones: the set's last frame at most
constexpr int game_slot(int made, int made_with) {
const int last = (made_with < 1 ? 1 : made_with) - 1;
return made < 0 ? 0 : made > last ? last : made;
}
} // namespace mkwvk::framegen_rows

View File

@@ -46,4 +46,24 @@ inline int pick(const std::string& want, const std::vector<std::string>& names,
return menu_device(menu, (int)names.size());
}
/// ((Triforce) 11-22, tickets "igpu targeted for shader cache", "Launcher \"GPU device\" setting is non-functional" (RX 6600
/// with a Ryzen 5600G's graphics) and on 0.6.0 "still picking IGPU for building shaders") The choice that pick takes by its
/// number alone (no saved name, or a name no card has here) when that number is the processor's graphics while a card of its
/// own is listed: not taken, Auto instead. Such a number comes from an options file written before 11-20, when the launcher
/// numbered the cards in Windows' order (the display class's driver keys: the processor graphics' driver first on a 5600G)
/// and saved no name with it; the update to 0.6.0 still prepared its shaders there and the game started there, as no name
/// came with the number until the row was chosen again. A choice saved with its name (the launcher and the menu since 11-20)
/// is taken as it is, the processor graphics too. types: the usable devices' VkPhysicalDeviceType (1 integrated, 2 discrete),
/// in the game's order, as names
inline bool unnamed_integrated(const std::string& want, const std::vector<std::string>& names, const std::vector<int>& types,
int menu) {
const int k = pick(want, names, menu);
if (k < 0 || (size_t)k >= types.size() || types[(size_t)k] != 1) return false;
const std::string key = name_key(want);
if (!key.empty() && name_key(names[(size_t)k]) == key) return false; // (chosen by its name)
for (const int t : types)
if (t == 2) return true;
return false;
}
} // namespace mkwvk::gpu_choice

57
gpu/vk/gpu_clock.h Normal file
View File

@@ -0,0 +1,57 @@
// (Triforce) Whether the renderer holds an AMD GPU at amdgpu's stable power state while the game runs, and which one (Linux:
// vk_gpu_clock.cpp; RX_VK_GPU_CLOCK). Plain code with no system calls: tests/test_gpu_clock.py runs it on made-up devices.
//
// The Steam Deck's automatic GPU clock drops while the frame waits on the CPU or a readback (200 MHz for 27% of a Kokiri run), so
// the GPU work after the wait runs slow. amdgpu lets any process that opens the GPU's render node ask for a stable power state
// through a context of its own (DRM_IOCTL_AMDGPU_CTX, AMDGPU_CTX_OP_SET_STABLE_PSTATE, Linux 5.18): no root and no sysfs write.
// The kernel gives the level back when that context goes, at the process's end too, a crash or kill -9 included. Measured on a
// Deck OLED (SteamOS 3.9.5, plugged in): Kokiri handheld 11.1 -> 16.2 fps, 1% low 6.8 -> 12.1, GPU time 58 -> 32 ms at the same
// 15 W; Hyrule Field +19% for 4 W more. It costs battery where the APU was under its power cap.
//
// - Unset: the peak state, on the Deck's APU only (its Vulkan name, or SteamDeck=1 from Steam on an integrated AMD GPU), and
// only while the level is amdgpu's own (power_dpm_force_performance_level "auto"): a level the player set is left alone.
// - RX_VK_GPU_CLOCK=0: nothing opened. =peak (or 1), =standard: that state on any AMD GPU, whatever the level.
#pragma once
#include <cstdint>
#include <cstring>
namespace mkwvk::gpu_clock {
enum Want { kNone, kPeak, kStandard };
// (uapi/drm/amdgpu_drm.h, written out: no libdrm and no kernel headers needed to build)
constexpr uint32_t kCtxOpAlloc = 1, kCtxOpSetStablePstate = 6;
constexpr uint32_t kPstateStandard = 1, kPstatePeak = 4;
constexpr uint32_t kCtxArgBytes = 16; // (union drm_amdgpu_ctx)
/// DRM_IOCTL_AMDGPU_CTX: DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDGPU_CTX, union drm_amdgpu_ctx) = _IOWR('d', 0x40 + 0x02, 16 bytes)
constexpr unsigned long kCtxIoctl = (3ul << 30) | ((unsigned long)kCtxArgBytes << 16) | ((unsigned long)'d' << 8) | 0x42ul;
/// The Deck's APU: "AMD Custom GPU 0405" (LCD) or "0932" (OLED) in its Vulkan name; or SteamDeck=1 (Steam sets it there) on an
/// integrated GPU
inline bool deck_apu(const char* device_name, bool integrated, const char* steam_deck_env) {
if (device_name && (std::strstr(device_name, "Custom GPU 0405") || std::strstr(device_name, "Custom GPU 0932"))) return true;
return integrated && steam_deck_env && !std::strcmp(steam_deck_env, "1");
}
struct Choice {
Want want;
const char* why;
};
/// switch_value: RX_VK_GPU_CLOCK (nullptr unset); level: power_dpm_force_performance_level's text, first word (nullptr: unread)
inline Choice choose(const char* switch_value, uint32_t vendor_id, bool integrated, const char* device_name,
const char* steam_deck_env, const char* level) {
if (vendor_id != 0x1002) return {kNone, "not an AMD GPU"};
if (switch_value && *switch_value) {
if (!std::strcmp(switch_value, "0")) return {kNone, "RX_VK_GPU_CLOCK=0"};
if (!std::strcmp(switch_value, "peak") || !std::strcmp(switch_value, "1")) return {kPeak, "RX_VK_GPU_CLOCK=peak"};
if (!std::strcmp(switch_value, "standard")) return {kStandard, "RX_VK_GPU_CLOCK=standard"};
}
if (!deck_apu(device_name, integrated, steam_deck_env)) return {kNone, "not a Steam Deck (RX_VK_GPU_CLOCK=peak: any AMD GPU)"};
if (!level) return {kNone, "the GPU's performance level could not be read"};
if (std::strcmp(level, "auto")) return {kNone, "the GPU's performance level is set by hand"};
return {kPeak, "the Steam Deck's GPU, its level amdgpu's own"};
}
} // namespace mkwvk::gpu_clock

86
gpu/vk/gpu_wait_kinds.h Normal file
View File

@@ -0,0 +1,86 @@
// (Triforce, 11-22, the performance audit's SYNC-27, after Source A's) The host's time in calls that block on the GPU,
// counted by kind where the Vulkan calls are wrapped (vk_work.h).
//
// The log's "GPU-bound wait" counts only the waits for a recording slot: it reads about 0 while threads wait on the GPU
// elsewhere (a fence of the game's, a write-back, the present's frame fence, a device idle, a query read that waits).
// Those were timed apart or not at all. Here every blocking wait is timed in one place and summed by kind:
// fences vkWaitForFences with a timeout above 0
// timelines vkWaitSemaphores with a timeout above 0
// idle vkQueueWaitIdle, vkDeviceWaitIdle
// queries vkGetQueryPoolResults with VK_QUERY_RESULT_WAIT_BIT
// A poll (timeout 0, no wait bit) counts nothing. A wait made inside another counts once, as the outer one (a depth of
// the thread's). A wait inside an Exclude scope counts nothing: pacing (the driver's low-latency sleep), which is not a
// wait for the GPU's work. The totals only grow; each reader keeps its own last values (Reader), so the log's line and
// the frame trace's column each see every nanosecond.
//
// The cost is two clock reads a blocking wait. RX_VK_GPU_WAITS=0, or the lab knob nogpuwaits in its phases: the calls
// straight through, nothing counted. No Vulkan here: tests/test_gpu_waits.py compiles it.
#pragma once
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdlib>
#include "lab_ab.h"
namespace gpu_wait_kinds {
enum Kind : int { FENCE = 0, TIMELINE = 1, IDLE = 2, QUERY = 3, KINDS = 4 };
struct Totals {
std::atomic<int64_t> ns[KINDS]{}, n[KINDS]{};
};
inline Totals totals; // (every thread's, since the start: they only grow)
inline thread_local int depth = 0; // (this thread is inside a counted wait)
inline thread_local int excluded = 0; // (this thread is inside an Exclude scope)
/// Counted at all (RX_VK_GPU_WAITS, read once; on unless 0), and not in the lab knob's phases
inline bool on() {
static const bool env = [] {
const char* e = getenv("RX_VK_GPU_WAITS");
return !(e && *e && atoi(e) == 0);
}();
return env && !mkwvk::lab_ab::on("nogpuwaits");
}
/// The waits made on this thread while one of these lives count nothing
struct Exclude {
Exclude() { excluded++; }
~Exclude() { excluded--; }
Exclude(const Exclude&) = delete;
Exclude& operator=(const Exclude&) = delete;
};
/// call(), and its time under `kind` when it blocks (blocks: a timeout above 0, the wait bit), unless this thread is
/// inside a counted wait or an Exclude scope, or the count is off. What call() returns
template <class Call>
inline auto timed(Kind kind, bool blocks, Call&& call) -> decltype(call()) {
if (!blocks || depth || excluded || !on()) return call();
depth++;
const auto t0 = std::chrono::steady_clock::now();
const auto result = call();
const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
depth--;
totals.ns[kind].fetch_add(ns, std::memory_order_relaxed);
totals.n[kind].fetch_add(1, std::memory_order_relaxed);
return result;
}
/// One reader's last values: take() gives what was counted since its last take (ns and waits by kind, either may be
/// null) and returns the nanoseconds of all kinds together
struct Reader {
int64_t last_ns[KINDS]{}, last_n[KINDS]{};
int64_t take(int64_t* ns_by_kind, int64_t* n_by_kind) {
int64_t all = 0;
for (int k = 0; k < KINDS; k++) {
const int64_t ns = totals.ns[k].load(std::memory_order_relaxed), n = totals.n[k].load(std::memory_order_relaxed);
if (ns_by_kind) ns_by_kind[k] = ns - last_ns[k];
if (n_by_kind) n_by_kind[k] = n - last_n[k];
all += ns - last_ns[k];
last_ns[k] = ns, last_n[k] = n;
}
return all;
}
};
} // namespace gpu_wait_kinds

112
gpu/vk/guest_holes.h Normal file
View File

@@ -0,0 +1,112 @@
// (Triforce) The all-zero upload check's reads (vk_texture.cpp upload_all_zero): which bytes of a storage it reads, split between its
// threads. tests/test_guest_holes.py.
//
// On Linux, where /dev/udmabuf is usable, the guest window is a memfd's (host/hos_main.c). A page the game never wrote is a hole
// there: it reads as zero, but a read through the shared mapping makes the kernel allocate and zero it. The check read every byte
// of a 32 MB+ storage, so the never-written 288 MB shadow map (12288 x 12288 D16) became RAM at each Hyrule Field arrival, and
// took 200-300 ms of the replay thread on a Steam Deck. fd_data_extents asks the memfd where its data is (lseek SEEK_DATA and
// SEEK_HOLE): a hole is zero by definition, and SEEK_DATA may call a hole data, never the other way round (a page in swap is
// data), so reading only the data says the same. Anywhere else, or on any failure, one extent of the whole storage.
#pragma once
#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstring>
#include <thread>
#include <vector>
#if defined(__linux__) || defined(GUEST_HOLES_TEST) // (the test: its own lseek)
#include <cerrno>
#include <unistd.h>
#endif
namespace mkwvk::guest_holes {
struct Extent {
uint64_t lo, hi; // (bytes from the storage's start)
};
inline uint64_t bytes(const std::vector<Extent>& ex) {
uint64_t n = 0;
for (const Extent& e : ex) n += e.hi - e.lo;
return n;
}
/// Thread t's part of n: the extents' bytes, not their range, split into n runs of whole 4 KB (the last takes the rest); every
/// byte of the extents in exactly one part
inline std::vector<Extent> share(const std::vector<Extent>& ex, uint32_t t, uint32_t n) {
std::vector<Extent> out;
const uint64_t total = bytes(ex);
if (!n || t >= n || !total) return out;
const uint64_t per = ((total / n) + 4095) & ~4095ull;
const uint64_t a = std::min<uint64_t>(total, (uint64_t)t * per), b = t + 1 == n ? total : std::min<uint64_t>(total, a + per);
uint64_t at = 0; // (bytes of the extents before e)
for (const Extent& e : ex) {
const uint64_t len = e.hi - e.lo, lo = std::max(a, at), hi = std::min(b, at + len);
if (lo < hi) out.push_back({e.lo + (lo - at), e.lo + (hi - at)});
at += len;
if (at >= b) break;
}
return out;
}
/// Whether any byte of p's extents is not 0, read on nthreads threads (this one among them), each stopping once one found it
inline bool any_set(const uint8_t* p, const std::vector<Extent>& ex, uint32_t nthreads) {
std::atomic<bool> found{false};
if (ex.empty()) return false;
nthreads = std::max(nthreads, 1u);
auto part = [&](uint32_t t) {
for (const Extent& e : share(ex, t, nthreads))
for (uint64_t at = e.lo; at < e.hi && !found.load(std::memory_order_relaxed); at += 65536) {
const uint64_t end = std::min<uint64_t>(e.hi, at + 65536);
uint64_t acc = 0;
uint64_t k = at;
for (; k + 8 <= end; k += 8) {
uint64_t w;
std::memcpy(&w, p + k, 8);
acc |= w;
}
for (; k < end; k++) acc |= p[k];
if (acc) found.store(true, std::memory_order_relaxed);
}
};
if (nthreads > 1) {
std::vector<std::thread> th;
for (uint32_t t = 1; t < nthreads; t++) th.emplace_back(part, t);
part(0);
for (auto& x : th) x.join();
} else {
part(0);
}
return found.load();
}
#if defined(__linux__) || defined(GUEST_HOLES_TEST) // (the test: its own lseek)
/// The data of [off, off + len) in fd, as extents from off (none: all holes). At most max extents: past them, the rest as one.
/// False when the file cannot say (a seek failed): the caller reads it all
inline bool fd_data_extents(int fd, uint64_t off, uint64_t len, std::vector<Extent>* out, size_t max = 1024) {
out->clear();
const uint64_t end = off + len;
uint64_t pos = off;
while (pos < end) {
if (out->size() + 1 >= max) { // (the last one: the rest, whole)
out->push_back({pos - off, len});
return true;
}
const off_t d = lseek(fd, (off_t)pos, SEEK_DATA);
if (d < 0) {
if (errno == ENXIO) return true; // (no data from pos to the file's end)
return out->clear(), false;
}
if ((uint64_t)d >= end) return true;
const off_t h = lseek(fd, d, SEEK_HOLE);
if (h < 0 || (uint64_t)h <= (uint64_t)d) return out->clear(), false;
const uint64_t hi = std::min<uint64_t>(end, (uint64_t)h);
out->push_back({(uint64_t)d - off, hi - off});
pos = hi;
}
return true;
}
#endif
} // namespace mkwvk::guest_holes

View File

@@ -7,7 +7,7 @@
// what it waits for comes within it, it never decides anything. The audit's trace: the helpers began a segment 0.262 ms after
// they were given it, and the replay thread waited 1.74 ms a frame for them (r22-trace). tests/test_helper_spin.py
//
// RX_VK_HELPER_SPIN_US=<us>: the spin's bound (100, NX2navi's; at most 1000000); 0: no spin, straight to the wait as before
// RX_VK_HELPER_SPIN_US=<us>: the spin's bound (20 since 11-21; 100 before, NX2navi's; at most 1000000); 0: no spin, straight to the wait as before
#pragma once
#include <chrono>
#include <cstdint>
@@ -20,7 +20,8 @@ namespace helper_spin {
/// The bound in nanoseconds that RX_VK_HELPER_SPIN_US's text gives (null: unset)
inline int64_t bound_ns(const char* text) {
if (!text || !*text) return 100000;
if (!text || !*text) return 20000; // ((Triforce, 11-21) 20 us, was 100 (NX2navi's): with the game's threads the frame's limit, a
// helper spinning 100 us took its SMT sibling from them. Lab A/B 79.3 -> 80.0, 1% low +2.0)
const long long us = std::atoll(text);
return us <= 0 ? 0 : us >= 1000000 ? 1000000000LL : us * 1000;
}

53
gpu/vk/import_budget.h Normal file
View File

@@ -0,0 +1,53 @@
// What the idle imports' valve measures the system heap against (no game, no GPU: a rule).
//
// (Triforce, 11-22, the performance audit's MEM-22, after Source A's.) An import is a pool of the game's memory the driver
// pins and counts in the system heap. vk_memory.cpp imports_give_back gives idle ones back past RX_VK_IMPORT_HIGH percent
// (85) of that heap's budget, down to RX_VK_IMPORT_LOW percent (75), but by default only where the budget is under 12 GB:
// on a PC with more, live imports pile up for the whole session (2.0 GB at a fresh Market, 6.5 GB after ten areas), and
// on Linux's amdgpu every submission pays for each one. Source A's numbers (Arch, RX 9070 XT, RADV, 32 GB, a 15,434 MB
// budget): uncapped after about two hours 221 live imports (7,392 MB), queue submit 1.11-1.14 ms, read-back waits
// 6.29-7.03 ms, 1,511 MB of swap, 44-48 fps; capped, 50-64 live (2,256-2,720 MB), 0.32-0.73 ms, 0.64-1.14 ms, no swap,
// 63-68 fps (with other settings changed in the same runs: the cap's own share is not known).
//
// So with Mesa's RADV and a budget of 12 GB or more the valve is on and measures the heap against 4,500 MB, not the whole
// budget, and a pass of it gives back 1 GB at most. Everything else is as it was:
// - a budget under 12 GB: the whole budget, on (any driver);
// - 12 GB or more on any other driver: off unless RX_VK_IMPORT_HIGH is set, then the whole budget;
// - RX_VK_IMPORT_HIGH or RX_VK_IMPORT_LOW set by the player: the whole budget, as before, RADV too.
// RX_VK_IMPORT_CAP_MB=<n>: the cap on any driver and any budget (it turns the valve on); =0: no cap, which is RADV's
// behaviour before. A given-back import costs 5-45 ms when its pool is bound again, which is why only pools idle for 600
// submissions go.
#pragma once
#include <cstdint>
namespace import_budget {
constexpr uint64_t kMB = 1ull << 20;
constexpr uint64_t kBigBudget = 12ull << 30; // from here the valve was off by default
constexpr uint64_t kCapMb = 4500; // RADV's cap at a big budget
constexpr uint64_t kPassMost = 1ull << 30; // a capped pass gives back this much at most
struct Rule {
bool on = false; // the heap's trigger runs
uint64_t cap = 0; // bytes the heap is measured against at most (0: the whole budget)
};
/// budget: the system heap's budget when the rule is first asked. high_set, low_set: the player set RX_VK_IMPORT_HIGH,
/// RX_VK_IMPORT_LOW. cap_mb: RX_VK_IMPORT_CAP_MB (-1: unset). radv: the driver is Mesa's RADV
constexpr Rule rule(uint64_t budget, bool high_set, bool low_set, int64_t cap_mb, bool radv) {
const bool before = high_set || budget < kBigBudget; // (as it was: on where the budget is small, or asked for)
if (cap_mb > 0) return Rule{true, (uint64_t)cap_mb * kMB};
if (cap_mb == 0 || high_set || low_set || !radv || budget < kBigBudget) return Rule{before, 0};
return Rule{true, kCapMb * kMB};
}
/// What the heap's use is measured against now (the budget moves; the cap does not)
constexpr uint64_t measured(const Rule& r, uint64_t budget) {
return r.cap && r.cap < budget ? r.cap : budget;
}
/// How much a pass by the heap's trigger wants back: down to the target, and 1 GB at most where the cap is what measures
constexpr uint64_t pass_want(const Rule& r, uint64_t budget, uint64_t used, uint64_t target) {
const uint64_t want = used > target ? used - target : 0;
return r.cap && r.cap < budget && want > kPassMost ? kPassMost : want;
}
} // namespace import_budget

30
gpu/vk/index_rebase.h Normal file
View File

@@ -0,0 +1,30 @@
// (Triforce, 11-21, the performance audit's DESC-12, after Source A's) An indexed draw's index buffer bound at offset 0,
// with the draw's offset given as its first index.
//
// The game's draws share one buffer at different offsets, so bind_index's skip (the same buffer, offset and type as the
// one on the command buffer) rarely held: nearly every indexed draw called vkCmdBindIndexBuffer, about 2,900 calls a frame
// in the Market. Bound at 0 with the offset as the draw's firstIndex, the skip holds until the buffer or the index type
// changes. Only where the offset is a whole number of indices that fits a first index; otherwise the draw binds at its
// offset as before. Indirect indexed draws are not touched: their first index is in the GPU's records.
// RX_VK_INDEX_REBASE=0: every draw binds at its own offset, as before.
//
// The rule alone, no Vulkan: tests/test_index_rebase.py compiles it.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace index_rebase {
/// The switch's value: on unless it is set to 0
inline bool on(const char* asked) { return !(asked && *asked && atoi(asked) == 0); }
/// Whether a draw whose indices start offset bytes into its buffer, unit bytes an index, can be drawn from the buffer bound
/// at 0: *first is then its first index. Not when the offset falls inside an index, nor past what a first index the draw's
/// own range is added to can hold
inline bool rebase(uint64_t offset, uint32_t unit, uint32_t* first) {
if (!unit || offset % unit || offset / unit > 0x7fffffffull) return false;
*first = (uint32_t)(offset / unit);
return true;
}
} // namespace index_rebase

View File

@@ -10,12 +10,21 @@
A knob named d_<site> is a delay: a busy wait of <value> microseconds at that site, at most once a frame (delay(), below):
what the frame's time answers to a part of it taking longer, which is whether that part is on the frame's critical path
(1 ms in for 1 ms of delay) or has slack (nothing). Any other name is a switch the code reads with on("<name>").
(Triforce, 11-22, the performance audit's REPLAY-51, after Source B's) A knob named e_<event> is a delay at every
occurrence of an event, not once a frame: a busy wait of <value> NANOseconds at each draw (e_draw) or dispatch
(e_dispatch), first in vk_draw.cpp's draw() and dispatch() so that the profiles leave it out, on whichever thread
records, counted (each(), below). The frame's change against the time spun says how much of that path is on the frame's
critical path: 200 ns at ~3,050 draws a frame is 0.6 ms of work. One line a phase gives each knob's spins and ms a second
(each_report). d_present_after is d_present's after: the same wait just after the present's drain(0), where d_present is
just before it. Pin RX_VK_PARALLEL for e_ knobs: Automatic moves draws between threads (the replay warns).
Not for players: nothing reads these without the variable. */
#pragma once
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
@@ -32,6 +41,17 @@ inline std::atomic<uint64_t> frame_no{0}; // (presents counted here)
inline std::atomic<uint64_t> fired[16]; // (the frame each delay last waited in)
inline std::atomic<uint32_t> pres_thread{0}; // (the thread that presents: the game's Presentation thread, in the plain mode)
inline int64_t phase_frames = 300, skip_frames = 30;
// ((Triforce, 11-22, REPLAY-51) the e_ knobs: each event's wait in this phase (ns, 0: none), and what was spun)
enum Each { E_DRAW = 0, E_DISPATCH = 1, E_COUNT = 2 };
inline const char* const each_names[E_COUNT] = {"e_draw", "e_dispatch"};
inline std::atomic<int64_t> each_ns[E_COUNT];
inline std::atomic<uint64_t> each_spins[E_COUNT], each_spun_ns[E_COUNT];
struct EachPhase { // (the presenting thread's: the phase being counted)
int knob = 0;
std::chrono::steady_clock::time_point since{};
};
inline EachPhase each_phase;
inline char each_line[256]; // (the line of a phase just over, for the replay to log; empty: none)
inline void init() {
if (n_knobs >= 0) return;
@@ -53,6 +73,53 @@ inline void init() {
if (const char* s = getenv("RX_VK_LAB_AB_SKIP")) skip_frames = atoll(s);
n_knobs = n;
}
/// ((Triforce, 11-22, REPLAY-51) Whether any knob is an e_ one (the replay warns once unless RX_VK_PARALLEL is pinned)
inline bool any_each() {
for (int i = 0; i < n_knobs; i++)
for (const char* name : each_names)
if (!strcmp(knobs[i].name, name)) return true;
return false;
}
/// (phase_tick, the presenting thread) The phase is knob k (0: none): at a change, the line of the phase that ended (each_line,
/// when it had an e_ knob: its spins and ms a second), and the events' waits for this one
inline void each_tick(int k) {
if (k == each_phase.knob && each_phase.since.time_since_epoch().count()) return;
const auto now = std::chrono::steady_clock::now();
each_line[0] = 0;
const int was = each_phase.knob;
if (was > 0 && each_phase.since.time_since_epoch().count()) {
const double s = std::chrono::duration<double>(now - each_phase.since).count();
for (int e = 0; e < E_COUNT; e++) {
if (strcmp(knobs[was - 1].name, each_names[e])) continue;
const uint64_t spins = each_spins[e].exchange(0, std::memory_order_relaxed);
const uint64_t ns = each_spun_ns[e].exchange(0, std::memory_order_relaxed);
snprintf(each_line, sizeof each_line, "vk: lab A/B: phase %s:%lld over: %.0f waits a second, %.2f ms a second spun "
"(%llu waits in %.1f s)", each_names[e], (long long)knobs[was - 1].value, s > 0 ? spins / s : 0.0,
s > 0 ? ns / 1e6 / s : 0.0, (unsigned long long)spins, s);
}
}
for (int e = 0; e < E_COUNT; e++)
each_ns[e].store(k > 0 && !strcmp(knobs[k - 1].name, each_names[e]) ? std::max<int64_t>(0, knobs[k - 1].value) : 0,
std::memory_order_relaxed);
each_phase.knob = k, each_phase.since = now;
}
/// At every occurrence of an event (first in draw() and dispatch()): one relaxed load without its knob; with it, a busy wait
/// of the knob's nanoseconds, counted
inline void each(int event) {
const int64_t ns = each_ns[event].load(std::memory_order_relaxed);
if (!ns) return;
const auto t0 = std::chrono::steady_clock::now(), until = t0 + std::chrono::nanoseconds(ns);
auto now = t0;
while (now < until) {
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
__builtin_ia32_pause();
#endif
now = std::chrono::steady_clock::now();
}
each_spins[event].fetch_add(1, std::memory_order_relaxed);
each_spun_ns[event].fetch_add((uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(now - t0).count(),
std::memory_order_relaxed);
}
/// Once a present, on the presenting thread: the phase this frame belongs to (the trace's column)
inline int phase_tick() {
init();
@@ -61,6 +128,7 @@ inline int phase_tick() {
const uint64_t phase = f / (uint64_t)phase_frames; // (0 none, 1 knob 1, 2 none, 3 knob 2, ...)
const int k = (phase & 1) ? (int)((phase / 2) % (uint64_t)n_knobs) + 1 : 0;
active.store(k, std::memory_order_relaxed);
each_tick(k); // ((Triforce, 11-22, REPLAY-51) the e_ knobs' waits for this phase, and the line of the one before)
const bool settling = (int64_t)(f % (uint64_t)phase_frames) < skip_frames;
return settling ? -(k + 1) : k; // (-1: none, settling; -(k + 1): knob k, settling)
}

49
gpu/vk/late_inline.h Normal file
View File

@@ -0,0 +1,49 @@
// (Triforce, 11-22, the performance audit's MEM-80, after Source B's) Which of take_capture's no-wait rules for late pages run.
//
// A late page is one the CPU wrote while a submission still on the GPU writes it back (vk_memory.cpp capture_make). By default
// every late page is waited for (RX_VK_LATE_QUEUE: at the queue's next submit) and taken again. RX_VK_LATE_NEW=1 (the lab
// knob latenew) turns four rules on that take the wait away from some of them: pages of pools with no mirror skipped, pages
// with the bytes they were last synced with left as they are, pages written back in part synced in their other parts (the
// partial rule, late_partial), pages written back whole put off. Each brought brightness steps back at the Castle Town save.
//
// RX_VK_LATE_INLINE=1 (the lab knob lateinline), OFF unless set: the partial rule ALONE, Source B's analogue (the CPU's
// bytes go in, the GPU's ranges stay the GPU's, nothing waits); the other three stay off, and the pages they would take
// (no mirror, texture storage, whole write-backs, unknown ranges) are waited for as before.
//
// And the partial rule asks which ranges are still on the GPU against the GPU's progress capture_make read before its
// copies, not a second read at the take: a write-back that landed between the copy and the take is then still counted in
// flight, so its range is not synced from the older copy (the bug Source B fixed with the same change). This is so under
// RX_VK_LATE_NEW too.
//
// The rule alone: no Vulkan, no game. tests/test_late_inline.py.
#pragma once
#include <cstdlib>
namespace late_inline {
/// RX_VK_LATE_INLINE: a number other than 0 on, 0 off; unset: the lab knob lateinline (on only in its phases)
inline bool on(const char* env, bool knob) {
if (env) return std::atoi(env) != 0;
return knob;
}
struct Rules {
bool any = false; // (any rule: the late pages looked at one by one; else every one waited for, as by default)
bool skip = false; // (a page of a pool with no mirror: not waited for)
bool same = false; // (a page with the bytes it was last synced with: left as it is)
bool part = false; // (a page written back in part: its other parts synced, no wait)
bool defer = false; // (a page written back whole: put off)
};
/// late_new: RX_VK_LATE_NEW or its knob; inline_on: on() above; same_on, part_on, defer_on: RX_VK_LATE_SAME, RX_VK_LATE_PART,
/// RX_VK_LATE_DEFER as read today (on, on, off unless set)
inline Rules rules(bool late_new, bool inline_on, bool same_on, bool part_on, bool defer_on) {
Rules r;
r.any = late_new || inline_on;
r.skip = late_new;
r.same = late_new && same_on;
r.part = r.any && part_on;
r.defer = late_new && defer_on;
return r;
}
} // namespace late_inline

153
gpu/vk/linux_pressure.h Normal file
View File

@@ -0,0 +1,153 @@
// (Triforce, 11-22, the performance audit's MEM-141, after Source A's) A memory pressure hold for Linux and the Steam Deck that
// counts the swap too (vk_memory.cpp steps it from memory_ram_watch, off Windows; this is the rule, plain code:
// tests/test_linux_pressure.py runs it anywhere).
//
// The RAM rules looked at MemAvailable alone. A Linux PC with swap keeps MemAvailable up by pushing the game's pages out:
// on a 16 GB Steam Deck Source A saw the swap fill (4,036 MB at the peak, 0 MB free at the lowest) while MemAvailable said
// little, and the stalls were the pages coming back. So the swap left counts: the PC is short when MemAvailable is under
// 1536 MB (the imports' own line), or when MemAvailable and SwapFree together are under the reserve (RX_VK_SWAP_RESERVE_MB).
// While it is short a hold is on and the renderer gives memory back sooner:
// - a capture buffer is kept as a spare only up to 32 MB of spares (not 768);
// - the staging arenas the GPU is done with are freed when the hold begins, then after 5 s unused (not 30);
// - malloc_trim is asked every 600 presents from 64 MB free (not every 1800 from 192);
// - idle imports are given back, in all of one episode at most as many bytes as the PC was short at its worst (not up to
// 1 GB a pass: the thrash of imports given back and made again was seen here before), and still only after 3,600 idle
// submissions;
// - on an integrated GPU, whose "VRAM" is the same RAM, the texture budget is less by what the PC is short (never under
// 512 MB), and comes back 256 MB a sample.
// The hold ends once MemAvailable and the sum are both 256 MB clear of their lines. A PC without swap, or one whose swap
// cannot be read, is never short by the sum: not known is not short. The program modules are let go by the program pack
// already (vk_shader.cpp), so the hold has no valve for them.
//
// RX_VK_SWAP_RESERVE_MB=<MB>: the reserve; 0 or not set: no hold at all, as before. Source A's is 2560; it is off here until
// a Steam Deck run has shown it. RX_VK_TEST_SWAP_MB=<n> plays that much swap free, as RX_VK_TEST_AVAIL_MB plays MemAvailable.
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <cstring>
namespace linux_pressure {
constexpr uint64_t kMB = 1ull << 20;
constexpr uint64_t kAvailLine = 1536 * kMB; // (short under this much MemAvailable, whatever the swap)
constexpr uint64_t kClear = 256 * kMB; // (the hold ends once both are this far clear of their lines)
constexpr uint64_t kSourceReserve = 2560 * kMB; // (Source A's reserve: MemAvailable + SwapFree)
constexpr uint64_t kSampleEvery = 30; // (presents between two looks)
constexpr uint64_t kSpareCap = 32 * kMB; // (capture spares while held)
constexpr int64_t kArenaIdleS = 5; // (a staging arena unused this long goes, while held)
constexpr uint64_t kTrimEvery = 600; // (presents between two malloc_trim looks, while held)
constexpr uint64_t kTrimMB = 64; // (free MB in the arenas from which malloc_trim is asked, while held)
constexpr uint64_t kBudgetFloor = 512 * kMB; // (an integrated GPU's textures are never cut under this)
constexpr uint64_t kBudgetBack = 256 * kMB; // (the cut comes back this much a sample)
/// RX_VK_SWAP_RESERVE_MB's text as bytes; 0 (not set, empty, 0, not a number): no hold
inline uint64_t reserve_of(const char* e) {
const long long mb = e && *e ? atoll(e) : 0;
return mb > 0 ? (uint64_t)mb * kMB : 0;
}
/// What /proc/meminfo said. A field that is not there is not known (has_*), never 0
struct Meminfo {
uint64_t total = 0, avail = 0, swap_total = 0, swap_free = 0;
bool has_total = false, has_avail = false, has_swap_total = false, has_swap_free = false;
};
/// /proc/meminfo's text ("Name: <n> kB" a line)
inline Meminfo parse(const char* text) {
Meminfo m;
auto field = [text](const char* name, uint64_t* out) {
const size_t len = strlen(name);
for (const char* p = text; p && *p; p = strchr(p, '\n'), p = p ? p + 1 : p) {
if (strncmp(p, name, len) || p[len] != ':') continue;
const char* v = p + len + 1;
while (*v == ' ' || *v == '\t') v++;
if (*v < '0' || *v > '9') return false;
*out = strtoull(v, nullptr, 10) << 10; // (kB)
return true;
}
return false;
};
m.has_total = field("MemTotal", &m.total);
m.has_avail = field("MemAvailable", &m.avail);
m.has_swap_total = field("SwapTotal", &m.swap_total);
m.has_swap_free = field("SwapFree", &m.swap_free);
return m;
}
/// One look at the PC's memory. swap_known: there is swap and what is free of it was read
struct Sample {
bool avail_known = false, swap_known = false;
uint64_t avail = 0, swap_free = 0;
};
/// The sample of a meminfo, with the lab's played values over it (negative: not played). Swapless (SwapTotal 0), or SwapFree
/// or SwapTotal unreadable: the swap is not known. A played swap counts as known
inline Sample sample_of(const Meminfo& m, long long test_avail_mb = -1, long long test_swap_mb = -1) {
Sample s;
s.avail_known = m.has_avail, s.avail = m.avail;
s.swap_known = m.has_swap_total && m.has_swap_free && m.swap_total > 0, s.swap_free = s.swap_known ? m.swap_free : 0;
if (test_avail_mb >= 0) s.avail_known = true, s.avail = (uint64_t)test_avail_mb * kMB;
if (test_swap_mb >= 0) s.swap_known = true, s.swap_free = (uint64_t)test_swap_mb * kMB;
return s;
}
/// How many bytes the PC is short: the larger of what MemAvailable lacks to its line and what the sum lacks to the reserve.
/// 0: not short, or not known
inline uint64_t shortfall(const Sample& s, uint64_t reserve) {
if (!reserve || !s.avail_known) return 0;
const uint64_t by_avail = s.avail < kAvailLine ? kAvailLine - s.avail : 0;
const uint64_t sum = s.avail + s.swap_free;
const uint64_t by_sum = s.swap_known && sum < reserve ? reserve - sum : 0;
return std::max(by_avail, by_sum);
}
/// Both lines cleared by kClear (the hold may end). Not known: clear
inline bool clear(const Sample& s, uint64_t reserve) {
if (!reserve || !s.avail_known) return true;
if (s.avail < kAvailLine + kClear) return false;
return !s.swap_known || s.avail + s.swap_free >= reserve + kClear;
}
/// The hold: on from a short sample until a clear one. One episode is one stretch of it
struct Hold {
bool held = false;
uint64_t deepest = 0; // (the episode's largest shortfall)
uint64_t given = 0; // (the bytes of imports given back in this episode)
uint64_t now = 0; // (the last sample's shortfall)
uint64_t cut = 0; // (an integrated GPU's texture budget is less by this)
uint64_t episodes = 0;
enum Step { Same, Entered, Left };
/// One sample. reserve 0: never held
Step step(const Sample& s, uint64_t reserve) {
const uint64_t lacking = shortfall(s, reserve);
Step r = Same;
if (!held && lacking) held = true, deepest = 0, given = 0, episodes++, r = Entered;
else if (held && clear(s, reserve)) held = false, r = Left;
now = held ? lacking : 0;
if (held) deepest = std::max(deepest, lacking);
// (the budget's cut: down at once to what is lacking, back up kBudgetBack a sample)
cut = now > cut ? now : std::max(now, cut > kBudgetBack ? cut - kBudgetBack : 0);
return r;
}
/// The bytes of idle imports this episode may still give back: its largest shortfall, less what it gave
uint64_t episode_want() const { return held && deepest > given ? deepest - given : 0; }
void gave(uint64_t bytes) { given += bytes; }
};
/// The valves' sizes, held or not
inline uint64_t spare_cap(bool held, uint64_t usual) { return held ? std::min(usual, kSpareCap) : usual; }
/// (idle_s 0: the player keeps the arenas: kept while held too)
inline int64_t arena_idle_s(bool held, int64_t usual) { return held && usual > kArenaIdleS ? kArenaIdleS : usual; }
inline uint64_t trim_every(bool held, uint64_t usual) { return held ? std::min(usual, kTrimEvery) : usual; }
/// (trim_mb 0: never, held or not)
inline uint64_t trim_mb(bool held, uint64_t usual) { return held && usual ? std::min(usual, kTrimMB) : usual; }
/// An integrated GPU's texture budget less the cut: never under the floor, never more than it was
inline uint64_t cut_budget(uint64_t budget, uint64_t cut) {
if (!cut || budget <= kBudgetFloor) return budget;
return std::max(kBudgetFloor, budget > cut ? budget - cut : 0);
}
} // namespace linux_pressure

213
gpu/vk/lock_census.h Normal file
View File

@@ -0,0 +1,213 @@
// (Triforce, 11-22, the performance audit's THREAD-68, after Source B's) The lock census: who holds the lock a thread waits for.
//
// The log has totals of the waits for the renderer's locks (the memory sync's lock wait, the virtual cores' line, the order lock's
// slow takes), none of which names the holder. With RX_VK_LOCK_CENSUS=1 a lock that is busy when a thread comes for it notes the
// waiter, the thread that held it, and how long the wait was; every 10 s a thread of the census's own logs the 12 pairs that
// waited longest:
// vk: lock census: Presentation Thread <- MainThread (replay_mu) 17.2 ms/s (2900/s, 6 us each)
// Counted: replay_mu (DrainingMutex), sync_order_mu (OrderMutex), held_mu, and in vk_memory.cpp pools_mu, sync_mu and cap_mu.
// Only the counting is ported: Source B's also samples the holders' stacks, which disturbs a millisecond or two a second.
//
// Off (unset, or 0): every lock is taken exactly as before, behind one branch on a value read once. On: a free lock is one
// try_lock and a store of its holder; only a busy lock reads the clock, twice. The holder is read without the lock, so a holder
// that lets go at that moment can be misnamed, never more. The census's own mutex is taken last, with the counted lock held, and
// nothing that holds it ever takes a counted lock. With RX_VK_LAB_AB naming a knob lockcensus it counts only in that knob's phases.
//
// No windows.h here (vk_recorder.h, which includes this, has none): a thread's name is read through g_thread_name, which
// vk_replay.cpp sets. tests/test_lock_census.py cuts DrainingMutex and OrderMutex out and compiles them with this.
#pragma once
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <map>
#include <mutex>
#include <string>
#include <thread>
#include <tuple>
#include <vector>
namespace mkwvk {
void logf(const char* fmt, ...); // (vk_main.cpp)
namespace lock_census {
/// RX_VK_LOCK_CENSUS, read once: 1 (any number but 0) counts
inline bool enabled() {
static const bool v = [] {
const char* e = getenv("RX_VK_LOCK_CENSUS");
return e && atoi(e) != 0;
}();
return v;
}
inline std::atomic<bool> phase_on{true}; // (lab: false outside the knob's phases)
inline bool counting() { return enabled() && phase_on.load(std::memory_order_relaxed); }
/// The clock a busy lock reads (a test counts the reads through it)
inline int64_t steady_ns() {
return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();
}
inline int64_t (*now_ns)() = steady_ns;
inline std::atomic<int> period_ms{10000}; // (between two reports)
constexpr int kTop = 12;
struct Pair {
int64_t ns = 0, n = 0;
};
/// The census's own state. Names are kept as pointers to strings that are never freed, one per distinct name
struct Book {
std::mutex mu;
std::map<std::string, const char*> names;
std::map<std::tuple<const char*, const char*, const char*>, Pair> pairs; // (waiter, holder, lock)
bool reporter = false;
int64_t since_ns = 0;
};
inline Book& book() {
static Book* b = new Book; // (never destroyed: a thread may count while the process ends)
return *b;
}
/// the one pointer for a name (book().mu held)
inline const char* intern(Book& b, const char* name) {
auto it = b.names.find(name);
if (it == b.names.end()) {
char* kept = new char[strlen(name) + 1];
strcpy(kept, name);
it = b.names.emplace(name, kept).first;
}
return it->second;
}
/// A thread as the census names it: its description (the game's threads and the renderer's own are named), else its number. The
/// slot is never freed, so a lock's note of its holder can be read after that thread has ended
struct Slot {
std::atomic<const char*> name{"?"};
int64_t named_ns = 0; // (its own thread's: when the name was last read)
};
/// (set by vk_replay.cpp, which has windows.h: the calling thread's description into out, left empty when it has none; and what
/// the reporting thread does first, which names it)
inline void (*g_thread_name)(char* out, size_t cap) = nullptr;
inline void (*g_report_start)() = nullptr;
inline void name_slot(Slot* s) {
char text[96] = "";
if (g_thread_name) g_thread_name(text, sizeof text);
if (!text[0]) snprintf(text, sizeof text, "thread %zu", std::hash<std::thread::id>()(std::this_thread::get_id()) % 100000);
Book& b = book();
std::lock_guard lk(b.mu);
s->name.store(intern(b, text), std::memory_order_relaxed);
}
/// the calling thread's slot; named at its first use, and again by `again_at` (a waiter's clock read: a thread named after its
/// first lock is not "thread 1234" for the whole session) when a second has passed
inline const Slot* me(int64_t again_at = 0) {
thread_local Slot* mine = nullptr;
if (!mine) {
mine = new Slot;
mine->named_ns = again_at; // (0 on a free lock, which reads no clock: named again at this thread's first wait)
name_slot(mine);
} else if (again_at && again_at - mine->named_ns > 1000000000) {
mine->named_ns = again_at;
name_slot(mine);
}
return mine;
}
/// The pairs counted since the last call, the longest waited first, kTop at most, as the log's lines; all counts start again
inline std::vector<std::string> report(int64_t now) {
Book& b = book();
std::vector<std::pair<std::tuple<const char*, const char*, const char*>, Pair>> rows;
double seconds = 0;
{
std::lock_guard lk(b.mu);
rows.assign(b.pairs.begin(), b.pairs.end());
b.pairs.clear();
seconds = (double)(now - b.since_ns) / 1e9;
b.since_ns = now;
}
if (seconds <= 0) seconds = 1e-9;
std::sort(rows.begin(), rows.end(), [](const auto& x, const auto& y) { return x.second.ns > y.second.ns; });
std::vector<std::string> out;
for (size_t i = 0; i < rows.size() && i < (size_t)kTop; i++) {
const auto& [key, p] = rows[i];
char line[320];
snprintf(line, sizeof line, "vk: lock census: %s <- %s (%s) %.1f ms/s (%.0f/s, %.0f us each)", std::get<0>(key), std::get<1>(key),
std::get<2>(key), (double)p.ns / 1e6 / seconds, (double)p.n / seconds, p.n ? (double)p.ns / 1e3 / (double)p.n : 0.0);
out.emplace_back(line);
}
return out;
}
inline void report_main() {
if (g_report_start) g_report_start();
for (;;) {
// (slept in steps, the period read at each: a test shortens it while this thread sleeps)
for (int slept = 0; slept < period_ms.load(std::memory_order_relaxed); slept += 50)
std::this_thread::sleep_for(std::chrono::milliseconds(50));
for (const std::string& line : report(now_ns())) logf("%s", line.c_str());
}
}
/// A wait is over: self waited ns for the lock `lock`, which `holder` held when it came (null: not known). The first one starts
/// the reporting thread
inline void waited(const Slot* self, const Slot* holder, const char* lock, int64_t ns, int64_t now) {
Book& b = book();
bool start = false;
{
std::lock_guard lk(b.mu);
if (!b.since_ns) b.since_ns = now - ns;
Pair& p = b.pairs[{self->name.load(std::memory_order_relaxed), holder ? holder->name.load(std::memory_order_relaxed) : intern(b, "?"),
intern(b, lock)}];
p.ns += ns, p.n++;
if (!b.reporter) b.reporter = start = true;
}
if (start) std::thread(report_main).detach();
}
/// m.lock() for a counted lock. owner: where that lock keeps its holder (set here, cleared by unlocking())
template <class M>
inline void lock(M& m, std::atomic<const Slot*>& owner, const char* name) {
if (!counting()) { // (off: the lock as it was)
m.lock();
return;
}
if (m.try_lock()) { // (free: no clock read)
owner.store(me(), std::memory_order_relaxed);
return;
}
const Slot* holder = owner.load(std::memory_order_relaxed);
const int64_t t0 = now_ns();
m.lock();
const int64_t t1 = now_ns();
const Slot* self = me(t1);
owner.store(self, std::memory_order_relaxed);
waited(self, holder, name, t1 - t0, t1);
}
/// m.try_lock() for a counted lock
template <class M>
inline bool try_lock(M& m, std::atomic<const Slot*>& owner) {
if (!m.try_lock()) return false;
if (counting()) owner.store(me(), std::memory_order_relaxed);
return true;
}
/// just before a counted lock's unlock: it has no holder
inline void unlocking(std::atomic<const Slot*>& owner) {
if (owner.load(std::memory_order_relaxed)) owner.store(nullptr, std::memory_order_relaxed);
}
/// A std::mutex the census counts, for std::lock_guard and std::unique_lock
struct CensusMutex {
const char* name;
std::mutex m;
std::atomic<const Slot*> owner{nullptr};
explicit CensusMutex(const char* n) : name(n) {}
void lock() { lock_census::lock(m, owner, name); }
bool try_lock() { return lock_census::try_lock(m, owner); }
void unlock() {
unlocking(owner);
m.unlock();
}
};
} // namespace lock_census
} // namespace mkwvk

View File

@@ -18,6 +18,7 @@
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <string>
#include <vector>
@@ -210,5 +211,30 @@ inline uint64_t own_for_keep(uint64_t ws, uint64_t cache_read, bool amd) {
if (amd) return ws;
return ws > cache_read ? ws - cache_read : 0;
}
/// (11-22, after Casual Gamer's fix) Whether the game is told its mapped pages are used (host/hos_mem.c query_used), from the
/// host's switches as it reads them: HOS_MEM_QUERY_USED not "0", and the census on (hos_mem_census_on: HOS_MEM_CENSUS not 0 or
/// less, or HOS_MEM_GIVE_BACK above 0)
inline bool query_used_on(const char* query_used, const char* census, const char* give_back) {
const bool census_on = !(census && atoi(census) <= 0) || (give_back && atoi(give_back) > 0);
return !(query_used && query_used[0] == '0' && !query_used[1]) && census_on;
}
/// (11-22) Whether the precompile keeps every pipeline for a PC of under 20 GB (vk_draw.cpp precompile_keeps_all): on a discrete
/// GPU with the game told its mapped pages are used (query_on), which lets old areas go; total 0: not known
inline bool keep_all_small_pc(bool discrete, uint64_t total, bool query_on) {
return discrete && query_on && total && total < (20ull << 30);
}
/// (11-22, after Casual Gamer's fix) Whether the retired imports kept for good (vk_memory.cpp retired_imports_give_back: Windows,
/// AMD's and Intel's drivers) are given back all the same: the system heap past high percent of its budget (used of budget;
/// a budget of 0: not known), or an import refused for want of room within the last kRefusedHoldMs (refused_ms: when, as now_ms
/// counts; 0: never)
constexpr int64_t kRefusedHoldMs = 60000;
inline bool retired_pressed(uint64_t used, uint64_t budget, int high, int64_t now_ms, int64_t refused_ms) {
return (budget && high > 0 && used > budget / 100 * (uint64_t)high) || (refused_ms && now_ms - refused_ms < kRefusedHoldMs);
}
/// (11-22, after Casual Gamer's fix) Whether the capture's spare buffers are held to 32 MB (vk_memory.cpp cap_spare_max): on a
/// card the driver allows 6.5 GB or less (small_card), or on a PC of under 20 GB of RAM (ram_total; 0: not known)
constexpr uint64_t kSpareRamBytes = 20ull << 30;
inline bool spare_capped(bool small_card, uint64_t ram_total) { return small_card || (ram_total && ram_total < kSpareRamBytes); }
} // namespace mem_parts

73
gpu/vk/memory_priority.h Normal file
View File

@@ -0,0 +1,73 @@
// VK_EXT_memory_priority: which device memory the system should keep in video memory when the card is short (no game, no
// GPU: the switch's reading and the tiers). tests/test_memory_priority.py
//
// (Triforce, 11-22, the performance audit's TEX-47, after Source A's.) When a card is short of video memory, Windows moves
// some of a program's device memory to system memory, which the GPU then reads over the bus, and it picks what goes. With
// this extension a program ranks its memory from 0 to 1: the lower goes first. Source A gives the render targets 1.0,
// the sampled textures 0.25 and the rest 0.5, and measured it on an RTX 4090 with the video memory taken by other
// programs and the textures held to an 8 GB card's blocks: with 1.3 GB moved out, the GPU's median frame was 109 to
// 179 ms unset and 24 to 29 ms on. The textures' low rank is the lever: with them at 0.5 (`1,0.5`) it ended at 68 ms.
//
// Off unless RX_VK_MEMORY_PRIORITY is set: no small card has run it, drivers may ignore it, and in Source A's runs the
// wrong tiers were worse than none. Unset, the extension is not asked for and nothing here is used.
// RX_VK_MEMORY_PRIORITY unset, empty, 0, off, no: off
// 1: render targets 1.0, textures 0.25, the rest 0.5
// <targets>,<textures>: those two, each from 0 to 1; the rest 0.5
// anything else (one number that is not 0 or 1, three numbers, a number outside 0 to 1): off
// With the extension on, the allocator (VMA) gives every pool and every allocation of its own the priority its create
// info names, and one that names none gets 0, below the textures: so every one in vk_vma.cpp names its tier.
#pragma once
#include <cctype>
#include <cmath>
#include <cstdlib>
#include <cstring>
namespace memory_priority {
inline constexpr float kDefault = 0.5f; // (what a driver gives memory that names no priority)
struct Tiers {
bool on = false;
float targets = kDefault, textures = kDefault, other = kDefault;
};
/// RX_VK_MEMORY_PRIORITY as written (see the top of the file); what cannot be read is off
inline Tiers parse(const char* env) {
Tiers t;
if (!env) return t;
while (*env == ' ' || *env == '\t') env++;
char word[32] = {};
size_t n = std::strlen(env);
while (n && (env[n - 1] == ' ' || env[n - 1] == '\t')) n--;
if (!n || n >= sizeof word) return t;
for (size_t i = 0; i < n; i++) word[i] = (char)std::tolower((unsigned char)env[i]);
if (!std::strcmp(word, "0") || !std::strcmp(word, "off") || !std::strcmp(word, "no")) return t;
if (!std::strcmp(word, "1")) {
t.on = true, t.targets = 1.0f, t.textures = 0.25f;
return t;
}
// (two numbers and a comma between them, nothing else)
char* end = nullptr;
const double a = std::strtod(word, &end);
if (end == word || *end != ',') return t;
const char* second = end + 1;
const double b = std::strtod(second, &end);
if (end == second || *end) return t;
if (!std::isfinite(a) || !std::isfinite(b) || a < 0 || a > 1 || b < 0 || b > 1) return t;
t.on = true, t.targets = (float)a, t.textures = (float)b;
return t;
}
/// The tiers a run uses: the switch's where the device has the extension and its feature, else none (all the default)
inline Tiers active(const char* env, bool device_has_it) { return device_has_it ? parse(env) : Tiers{}; }
/// What a piece of device memory is for. Held: the lab's ballast (RX_VK_TEST_CARD_MB), which must stay on the card
enum class Kind { Target, Texture, Other, Held };
/// Its priority under the tiers; the default for everything while they are off
inline float of(const Tiers& t, Kind k) {
if (!t.on) return kDefault;
return k == Kind::Target ? t.targets : k == Kind::Texture ? t.textures : k == Kind::Held ? 1.0f : t.other;
}
} // namespace memory_priority

38
gpu/vk/mesa_spilling.h Normal file
View File

@@ -0,0 +1,38 @@
// Mesa's Intel shader compiler told to spill registers at rate 2, before the driver is loaded (no game, no GPU: a rule).
//
// (Triforce, 11-22, the performance audit's PIPE-55, after Source A's.) Mesa's drivers read their options (driconf) from
// the environment by name. shader_spilling_rate is read by the Intel compiler alone: after a register allocation that
// failed, it spills that many times more registers at once, so a program that does not fit is built in fewer, larger
// steps. A program that does not spill builds the same. Source A's numbers on an Iris Xe (i9-13900H, 32 GB, Mesa 26.2.4):
// one compute program not built after 20 minutes and 13 GB at the driver's own rate, 32 s and 1.7 GB at rate 2; another
// 531 s and 7.6 GB against 87 s and 4 GB; rate 3 took 165 s there and rate 4 did not finish in 8 minutes. The driver's
// own estimate for a spilling program's speed at rate 2 is from 12% better to 53% worse.
//
// The Linux build sets it in create_instance (vk_main.cpp) without overwriting, so a value the player set wins.
// RX_VK_NO_SPILLING_RATE=1 leaves the driver as it is. The Windows build sets nothing: under Proton the driver reads the
// Linux environment, which the Windows build cannot write, and those players put shader_spilling_rate=2 in the launch
// options.
// Nothing here is read by NVIDIA's or AMD's drivers, RADV included.
#pragma once
#include <cstdlib>
namespace mesa_spilling {
constexpr const char* kName = "shader_spilling_rate";
constexpr const char* kRate = "2";
/// The build sets the rate: off Windows, unless RX_VK_NO_SPILLING_RATE is set to something other than 0. off: its value
inline bool wanted(bool windows_build, const char* off) {
return !windows_build && !(off && std::atoi(off) != 0);
}
/// What the driver will read once the build has done its part: the player's own value where there is one (set without
/// overwriting), else the rate where it is wanted, else nullptr (the driver's own rate)
inline const char* effective(const char* players, bool is_wanted) {
return players ? players : is_wanted ? kRate : nullptr;
}
/// The log names the rate on Mesa's Intel driver alone
constexpr bool told(bool mesa_intel) {
return mesa_intel;
}
} // namespace mesa_spilling

27
gpu/vk/mirror_budget.h Normal file
View File

@@ -0,0 +1,27 @@
// (Triforce) The VRAM mirrors' budget by the card (vk_memory.cpp mirror_budget; RX_VK_MIRROR_MB=<MB> goes over it). Plain code with
// no Vulkan calls: tests/test_mirror_budget.py runs it on made-up cards.
//
// - A discrete card over 4.5 GB: a quarter of its video memory, at most 3 GB. On a 6 GB card (RTX 3060 Laptop, Hyrule Field,
// whole mirrors) 512 MB of mirrors gave 17.6 fps, 1024 MB 30.7 and a quarter (1498 MB) 39.2: the pools left in system memory
// were read over PCIe each frame. The eighth those cards had (768 MB) sat between two sizes measured as losses. Textures on
// cards of 8.5 GB or less are 45% of the allowance and give memory back at 90% and 97% of it (vk_texture.cpp), the room a
// quarter needs; the eighth was set for a 6 GB laptop before that rule, when a quarter and the textures went past the card.
// - 4.5 GB or less, and an integrated GPU up to 6.5 GB: an eighth (no smaller size was measured on a real small card, and an
// integrated GPU's "video memory" is the system's).
// - Video memory not known: 3 GB.
#pragma once
#include <algorithm>
#include <cstdint>
namespace mkwvk::mirror_size {
constexpr uint64_t kMB = 1ull << 20;
constexpr uint64_t bytes(uint64_t vram, bool discrete) {
if (!vram) return 3072 * kMB;
if (vram <= 4608 * kMB || (!discrete && vram <= 6656 * kMB)) return vram / 8;
return std::min<uint64_t>(vram / 4, 3072 * kMB);
}
} // namespace mkwvk::mirror_size

42
gpu/vk/mirror_spare.h Normal file
View File

@@ -0,0 +1,42 @@
// (Triforce, 11-21, the performance audit's MEM-27, after Source A's: its first step) What the mirrors' sparse buffers cost.
//
// A mirror in pieces is a sparse buffer: address space, not memory, yet the driver charges by its size. Source A measured
// about 0.06 ms a MB to create one and 0.07 to destroy it (NVIDIA 617.42), 52 to 64 ms batches of destructions at an area
// change, and the game frees its pools there and makes them again at the same sizes. Its answer is to keep a freed
// mirror's buffer, unbound, for the next mirror of exactly that size. Before that is built here, this counts: the creates
// and destroys, their time, the longest, and how many creates a buffer freed earlier, of exactly their size, could have
// served. The line "vk: mirrors' sparse buffers" every 1200 submissions has them (vk_memory.cpp).
//
// The count alone, no Vulkan: tests/test_mirror_spare.py compiles it.
#pragma once
#include <cstdint>
#include <map>
namespace mirror_spare {
struct Counts {
uint64_t creates = 0, destroys = 0, servable = 0; // (servable: creates a freed buffer of their size was there for)
double create_ms = 0, destroy_ms = 0, longest_create_ms = 0, longest_destroy_ms = 0;
uint64_t create_bytes = 0, destroy_bytes = 0, over = 0; // (over: calls of more than kLongMs)
std::map<uint64_t, uint32_t> freed; // (sizes freed and not yet matched by a create)
};
inline constexpr double kLongMs = 4.0;
/// A sparse buffer of size bytes was destroyed in ms
inline void destroyed(Counts& c, uint64_t size, double ms) {
c.destroys++, c.destroy_ms += ms, c.destroy_bytes += size, c.over += ms > kLongMs;
if (ms > c.longest_destroy_ms) c.longest_destroy_ms = ms;
c.freed[size]++;
}
/// A sparse buffer of size bytes was created in ms; true when one freed before, of exactly that size, could have served
inline bool created(Counts& c, uint64_t size, double ms) {
c.creates++, c.create_ms += ms, c.create_bytes += size, c.over += ms > kLongMs;
if (ms > c.longest_create_ms) c.longest_create_ms = ms;
auto it = c.freed.find(size);
if (it == c.freed.end()) return false;
if (!--it->second) c.freed.erase(it);
c.servable++;
return true;
}
} // namespace mirror_spare

View File

@@ -16,6 +16,7 @@
// No ImGui and no Vulkan: tests/test_mods_entry_anim.py compiles it alone.
#pragma once
#include <algorithm>
#include <cmath>
#include <cstdint>
namespace mods_entry {
@@ -69,6 +70,16 @@ inline int64_t in_at(float amount) {
return (int64_t)((double)hi * (double)kFrameNs);
}
/// ((Triforce, 11-20) the game's own alpha of its buttons as the amount the entry is drawn with. The game blends its
/// buttons in linear light; the overlay blends in the window's code values unless the window is HDR. A white word at alpha
/// a over a dark picture is a^(1/2.2) of the way to white in code values when blended in linear light: drawn at a itself
/// in code values, the entry was dimmer than the game's buttons all through a fade (a quarter of the way where they were
/// half way), though it began and ended with them. linear: the overlay blends in linear light too (HDR): a itself)
inline float list_amount(float game_alpha, bool linear) {
const float a = std::clamp(game_alpha, 0.0f, 1.0f);
return linear ? a : std::pow(a, 1.0f / 2.2f);
}
/// The entry's state from one drawing to the next: how chosen it looks (focus) and how much of it shows with the list
struct Anim {
bool sel = false; // (it had the cursor at the last step)
@@ -78,8 +89,12 @@ struct Anim {
int64_t list_ns = 0; // (when the list came)
float gone_from = 0; // (shown when the list went)
float focus = 0, shown = 0;
/// One drawing at now: whether the entry has the cursor, how long since the host last told of the list
void step(int64_t now, bool selected, int64_t list_age_ns) {
/// One drawing at now: whether the entry has the cursor, how long since the host last told of the list.
/// ((Triforce, 11-20) list_alpha: how much of its own buttons the game shows now (0 to 1, the host's word each frame:
/// HOS_OPT_MODS_ALPHA); the entry shows as much while the list is there, so it comes and goes with them, frame for
/// frame, whatever animation the game plays (the title's opening, a second long, after a second and a half of
/// nothing; the 15 frames as the list comes back). Under 0: not told, the list's In by this clock, as before)
void step(int64_t now, bool selected, int64_t list_age_ns, float list_alpha = -1.0f) {
if (!since_ns || selected != sel) from = since_ns ? focus : (selected ? 1.0f : 0.0f), sel = selected, since_ns = now;
const float f = frames(now - since_ns);
focus = sel ? std::max(from, curve(kSelect, f)) : from * curve(kUnSelect, f);
@@ -87,7 +102,8 @@ struct Anim {
if (seen && !there) list_ns = now - in_at(shown); // ((Triforce, 11-19) from where it is: no blink to nothing)
if (!seen && there) gone_from = shown;
there = seen;
shown = seen ? curve(kListIn, frames(now - list_ns)) : gone_from * curve(kUnSelect, frames(list_age_ns - kListGoneNs));
shown = !seen ? gone_from * curve(kUnSelect, frames(list_age_ns - kListGoneNs))
: list_alpha >= 0 ? list_alpha : curve(kListIn, frames(now - list_ns));
shown = std::clamp(shown, 0.0f, 1.0f), focus = std::clamp(focus, 0.0f, 1.0f);
}
/// LoopRandom's value now (it runs from the cursor's coming)

48
gpu/vk/mods_page_anim.h Normal file
View File

@@ -0,0 +1,48 @@
// (Triforce, 11-20) The Mods screen as it opens. The owner: "Mod menu fade in". The entry that opens it is animated as the
// game's own entries are (mods_entry_anim.h on the title, focus_anim.h under the pause menu's System list), and Quit's
// dialog opens as the game's SystemWindow_00; the screen itself came at once, whole, on the frame after the press. It
// fades in now as the game's lists and windows do: the alpha of Title_00's list and of SystemWindow_00's N_In_00 (the
// same keys in both layouts: 0 to 1 over 15 frames, slow to start, slopes 0.02 and 0.14), with no rise, as a whole screen
// has none. It takes input once that has played, as the game's own screens and Quit's dialog do. Back is a cut, as the
// game's windows go in 2 frames.
// No ImGui and no Vulkan: tests/test_mods_page_anim.py compiles it alone.
#pragma once
#include <algorithm>
#include <cstdint>
#include "mods_entry_anim.h" // (the layouts' curves: Key, curve, frames)
namespace mods_page {
using mods_entry::curve;
using mods_entry::frames;
using mods_entry::Key;
using mods_entry::kFrameNs;
constexpr Key kInAlpha[] = {{0, 0, 0.02f}, {15, 1, 0.14f}};
constexpr int64_t kInNs = 15 * kFrameNs;
/// How much of the screen shows t ns after it opened (0 to 1)
inline float in_alpha(int64_t t) {
if (t <= 0) return 0;
return std::clamp(curve(kInAlpha, frames(t)), 0.0f, 1.0f);
}
/// The screen from one tick to the next: whether it is shown, and since when
struct Page {
bool shown = false; // (it was the page shown at the last step)
int64_t since_ns = 0; // (when it came; 0 while it is not shown)
/// One tick at now: whether the Mods screen is the page shown. A screen that comes starts its In; one that goes has none
void step(int64_t now, bool on) {
if (on && !shown) since_ns = now;
if (!on) since_ns = 0;
shown = on;
}
/// How much of it shows at now. settle: as it stands once its In is done (a still, a picture that compares). A screen
/// asked for and not stepped yet (the page was set this tick, the step comes at the tick's end) shows nothing: drawn in
/// between at full, it flashed whole for a frame before its In began
float alpha(int64_t now, bool settle = false) const { return settle ? 1.0f : !shown ? 0.0f : in_alpha(now - since_ns); }
/// (it takes input once its In has played, as the game's)
bool ready(int64_t now) const { return shown && now - since_ns >= kInNs; }
};
} // namespace mods_page

95
gpu/vk/msync_pass.h Normal file
View File

@@ -0,0 +1,95 @@
// (Triforce, 11-21, the performance audit's MEM-48, after Source A's) A mirror sync's pages from the captures by one compute
// dispatch: the rules.
//
// A pool the GPU reads has a mirror in VRAM, and a sync brings the pages the CPU wrote into it. Those that come from a
// capture's buffer (host memory) went as copy regions, a page or a few each, 700 to 1,300 a frame: the GPU reads each
// region over PCIe after the one before. The write-back diff's pass (shaders/wb_diff.comp) reads a source, compares it
// with a shadow and writes a destination where they differ; with the shadow and the destination both the mirror it is a
// copy, a workgroup a chunk, every chunk of a sync in one dispatch. Source A's RTX 5080 in Hyrule Field: the syncs' GPU
// time 1.41 to 0.69 ms a frame, 79.7 to 83.1 fps where the GPU bounds the frame.
//
// What keeps it right:
// - a run goes as chunks only in whole words at word-aligned addresses (words); any other run of a sync is copied;
// - the chunk tables are host memory in a ring of blocks, each table from an atom of its own (the flush covers whole
// atoms), a block taken again only once the GPU is done with the recording that last read it (Ring); a sync whose
// table finds no room is copied, as before;
// - copies into one buffer are in order among themselves, a dispatch is not: where a mirror is written twice in one
// upload command buffer and one of the two is the pass, a full barrier goes between (barrier_first). The pass's own
// chunks and a sync's other copies never share a page.
// RX_VK_MSYNC_PASS: on for NVIDIA when not set (measured there; no gain on an RX 5600 XT), 1: on every GPU, 0: copies.
//
// No Vulkan here: tests/test_msync_pass.py compiles it.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace msync_pass {
inline constexpr int kBlocks = 16;
inline constexpr uint64_t kBlockBytes = 256u << 10; // (8192 chunks of 32 bytes: a frame's syncs took about 1,300)
inline constexpr uint64_t kChunkBytes = 64u << 10; // (the pass's chunk at most)
/// Whether the pass is used: asked, RX_VK_MSYNC_PASS's value; not set: on NVIDIA's GPUs
inline bool on(const char* asked, bool nvidia) {
if (!asked || !*asked) return nvidia;
return atoi(asked) != 0;
}
/// Whether a run from src to dst (addresses) of size bytes can go as chunks
constexpr bool words(uint64_t src, uint64_t dst, uint64_t size) { return size != 0 && !((src | dst | size) & 3); }
/// A run's chunks: emit(source, destination, count, vectors), the count in 16-byte vectors or in words. split(at, size,
/// e(at, bytes, vectors)) is the write-back diff's (wbd_split: the words before the first 16-byte line, the lines 64 KB
/// at a time, the words after). Addresses not alike within 16 bytes go in words alone
template <class Split, class Emit>
void run_chunks(uint64_t src, uint64_t dst, uint64_t size, Split&& split, Emit&& emit) {
if ((src ^ dst) & 15) {
for (uint64_t at = 0; at < size; at += kChunkBytes) {
const uint64_t len = size - at < kChunkBytes ? size - at : kChunkBytes;
emit(src + at, dst + at, (uint32_t)(len / 4), false);
}
return;
}
split(dst, size, [&](uint64_t a, uint64_t len, bool vecs) { emit(src + (a - dst), a, (uint32_t)(len / (vecs ? 16 : 4)), vecs); });
}
/// What wrote a mirror so far in an upload command buffer
enum : uint8_t { kCopies = 1, kPass = 2 };
/// Whether a full barrier goes before a write of kind `now` into a mirror that `before` wrote earlier in the same
/// upload command buffer: the pass after anything, and anything after the pass
constexpr bool barrier_first(uint8_t before, uint8_t now) { return now == kPass ? before != 0 : (before & kPass) != 0; }
/// The tables' blocks: which are made, the newest recording that reads each, and how far each is filled
struct Ring {
struct Block {
bool made = false;
uint64_t value = 0, fill = 0;
};
Block b[kBlocks];
int cur = -1;
/// Room for a table of `bytes` read by the recording rec_value, the GPU done up to `completed`: the block's index
/// and *at, the table's offset in it, a multiple of align; -1: none (too big, or the next block is still read).
/// A block returned with made false is the caller's to make before it writes
int get(uint64_t bytes, uint64_t align, uint64_t rec_value, uint64_t completed, uint64_t* at) {
if (!bytes || bytes > kBlockBytes || !align) return -1;
if (cur >= 0) {
Block& c = b[cur];
const uint64_t start = (c.fill + align - 1) / align * align;
if (start + bytes <= kBlockBytes) {
c.fill = start + bytes;
if (rec_value > c.value) c.value = rec_value;
*at = start;
return cur;
}
}
const int next = (cur + 1) % kBlocks;
Block& n = b[next];
if (n.made && n.value > completed) return -1;
cur = next;
n.fill = bytes, n.value = rec_value;
*at = 0;
return next;
}
};
} // namespace msync_pass

49
gpu/vk/new_wait.h Normal file
View File

@@ -0,0 +1,49 @@
// (Triforce, 11-22, the performance audit's OS-04, after Source A's and Source B's) A `new` of the renderer's that fails at
// Windows' commit limit waits before it throws.
//
// A refused allocation in the renderer was an uncaught std::bad_alloc: the game ended. At the commit limit (RAM and the page
// file full) a refusal often ends in moments, as the page file grows or commit is given back, so the renderer's new handler
// (vk_main.cpp new_refused, set in mkwvk_init) sleeps 50 ms and lets the allocation be tried again, for up to 5 s a thread
// (HOS_COMMIT_WAIT_MS, the host's switch for the same wait: 0, none), and throws after that. Only while the system has less
// than 4 GiB of commit left: a `new` refused with more than that free was not refused for the limit (a size no memory could
// hold), and throws at once, as before. Source A's is the wait; the 4 GiB rule is Source B's, written anew here.
//
// A `new (std::nothrow)` whose caller has its own answer to a refusal (a staging block, a copy that is only a saving, the
// recorder's ring) is not held up: NoWait around it makes the handler throw at once, and the nothrow new answers null.
//
// The rule alone, no system calls: tests/test_commit_limit.py compiles it.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace mkwvk::new_wait {
constexpr uint64_t kNear = 4ull << 30; // (the commit left under which a refusal is taken for the limit's)
constexpr int kStepMs = 50, kWaitMs = 5000, kMostMs = 60000;
/// HOS_COMMIT_WAIT_MS in ms: not set (or empty) 5000; a number that many, 0 and less none, 60000 at most
inline int wait_ms(const char* env) {
if (!env || !*env) return kWaitMs;
const long v = std::atol(env);
return v <= 0 ? 0 : v > kMostMs ? kMostMs : (int)v;
}
inline thread_local int t_waited_ms = 0; // (what this thread has waited in all: its 5 s are not given twice)
inline thread_local int t_no_wait = 0; // (NoWait scopes open on this thread)
/// Around a new (std::nothrow) that must answer at once
struct NoWait {
NoWait() { t_no_wait++; }
~NoWait() { t_no_wait--; }
NoWait(const NoWait&) = delete;
NoWait& operator=(const NoWait&) = delete;
};
/// A `new` was refused. true: sleep a step and let it be tried again; false: throw. known and avail: the system's commit left,
/// when it could be read; waited_ms: this thread's waits so far; max_ms: wait_ms's; no_wait: NoWait scopes open
inline bool waits(bool known, uint64_t avail, int waited_ms, int max_ms, int no_wait) {
return !no_wait && known && avail < kNear && waited_ms < max_ms;
}
} // namespace mkwvk::new_wait

View File

@@ -15,6 +15,8 @@
// The game also predicates draws on these words (render conditions, NOP_RENDER_COND): as NX2navi decides them
// (runtime/vk_exec_xfer.c cond_enabled), a word whose newest report is a samples-passed one with its count still to come is
// unknown, and the condition renders; a word whose count was written is read from memory, where a real 0 can skip the draw.
// (11-22) A count over a draw that such a skip left out is marked (Counter::gate, Writers::kGatedSrc, the line counts them);
// RX_VK_OCC_GATE=1: a condition on it renders (cond_decide), so a skip's 0 cannot hide the same object again and again.
//
// Counter one counting, from a reset to its report, over the queries its draws began: the replay's (one per NVN
// queue's state, under replay_mu) or a parallel replay's segment's (the helper thread's: a segment starts with
@@ -95,6 +97,7 @@ struct Rep {
uint32_t first = 0, n = 0;
uint64_t pos = 0;
int64_t present = 0;
bool gated = false; // (a draw of its count left out by a condition that read a written count: Counter::gate)
};
/// One samples-passed counting. The replay calls reset() at a SAMPLES_PASSED reset, draw_enter() as a draw goes into draw(),
@@ -113,8 +116,10 @@ struct Counter {
int open = -1; // (the query begun and not ended, its index in the block)
bool lost = false; // (a draw in another recording, or the block full)
uint32_t ops = 0, issued = 0; // (draws that went into draw() while counting, and those it recorded)
enum Why { kNone, kSkipped, kOtherCmd, kFull };
enum Why { kNone, kSkipped, kOtherCmd, kFull, kColour };
Why why = kNone;
bool gated = false; // (since the reset a draw was left out by a render condition that read a written count)
bool colour = false; // ((Triforce, 11-22, GPUWORK-01) a draw that writes colour came in since the reset)
/// A parallel replay's segment starts (or ends), or the replay's counting is given up: nothing is counting
void segment() { *this = Counter{}; }
@@ -126,7 +131,35 @@ struct Counter {
state = kCounting, slot = none ? -1 : slot_, serial = none ? 0 : serial_, first = next, open = -1, lost = none;
ops = issued = 0;
why = none ? kOtherCmd : kNone;
gated = false;
colour = false;
}
/// (11-22) A draw the replay passes with rendering off because a condition read a written count (a 0 of an earlier
/// report): while counting, this count is gated, NVN counts nothing for that draw and neither do the queries. Such a count
/// is a skip's result, not the object's visibility (Writers::kGatedSrc)
void gate(bool keep) {
if (counting(keep)) gated = true;
}
/// ((Triforce, 11-22, the performance audit's GPUWORK-01, after Source A's) A draw that writes colour comes in, its pass
/// open, before begin(). The game's occlusion tests draw depth only (a bounding box against the depth buffer); a query
/// that stays open over colour draws makes the GPU wait at its end for the work it encloses. What the caller does with
/// the answer: first, this is the window's first such draw, and open, a query is open over it (both counted, whether or
/// not the rule is on); lose, the rule is on (depth_only) and the count is given up: the caller ends the open query and
/// calls lose_colour(). Nothing for a count not counting or lost already
struct Colour {
bool first = false, open = false, lose = false;
};
Colour colour_draw(bool keep, bool depth_only) {
Colour r;
if (!counting(keep) || lost) return r;
r.first = !colour, colour = true;
r.open = open >= 0;
r.lose = depth_only;
return r;
}
/// The count given up for a colour draw (its open query ended by the caller): no query begins until the next reset,
/// its report is incomplete, so the record keeps the placeholder and its render condition renders
void lose_colour() { lost = true, why = kColour; }
void draw_enter(bool keep) {
if (counting(keep)) ops++;
}
@@ -162,7 +195,7 @@ struct Counter {
if (!complete && !lost) why = kSkipped;
state = kReported;
if (!complete) return kIncomplete;
rep->first = first, rep->n = next - first;
rep->first = first, rep->n = next - first, rep->gated = gated;
return kMeasured;
}
};
@@ -170,57 +203,55 @@ struct Counter {
/// A map from 64-bit keys, open addressing with linear probes and backward-shift deletion (no tombstones): no allocation per
/// entry. The replay placed one entry for every samples-passed report (~2,600 a frame at the Market) in a std::unordered_map,
/// a node allocated and freed each time: 0.76 ms a frame of the Presentation Thread (all-thread sample, 2026-11-20)
// ((Triforce, 11-21) The keys, the used marks and the values in three arrays: a probe reads 9 bytes a slot, not a 48-byte slot.
// The game's report addresses rotate through ~49,000 a 600-present period, ~27,000 live at once (kKeepAge): the table of slots was
// ~3 MB, past the core's L2, and its probes and the prune's walk over it were 5% of the Presentation Thread (exact profile of a
// line-table build, 2026-11-21). Same entries, same order of probes and shifts as before)
template <class V>
class FlatU64 {
public:
struct Slot {
uint64_t key = 0;
V val{};
bool used = false;
};
V* find(uint64_t k) {
if (!n_) return nullptr;
for (size_t i = idx(k);; i = (i + 1) & mask_) {
Slot& s = s_[i];
if (!s.used) return nullptr;
if (s.key == k) return &s.val;
if (!used_[i]) return nullptr;
if (keys_[i] == k) return &vals_[i];
}
}
const V* find(uint64_t k) const { return const_cast<FlatU64*>(this)->find(k); }
V& operator[](uint64_t k) {
if (V* v = find(k)) return *v;
if ((n_ + 1) * 10 > s_.size() * 7) grow();
if ((n_ + 1) * 10 > keys_.size() * 7) grow();
size_t i = idx(k);
while (s_[i].used) i = (i + 1) & mask_;
s_[i].key = k, s_[i].val = V{}, s_[i].used = true;
while (used_[i]) i = (i + 1) & mask_;
keys_[i] = k, vals_[i] = V{}, used_[i] = 1;
n_++;
return s_[i].val;
return vals_[i];
}
size_t count(uint64_t k) const { return find(k) ? 1 : 0; }
size_t erase(uint64_t k) {
if (!n_) return 0;
size_t i = idx(k);
for (;; i = (i + 1) & mask_) {
if (!s_[i].used) return 0;
if (s_[i].key == k) break;
if (!used_[i]) return 0;
if (keys_[i] == k) break;
}
// backward shift: the entries after the hole that would be found from a slot at or before it move into it
for (size_t j = (i + 1) & mask_;; j = (j + 1) & mask_) {
if (!s_[j].used) break;
const size_t home = idx(s_[j].key);
if (!used_[j]) break;
const size_t home = idx(keys_[j]);
const bool moves = i <= j ? (home <= i || home > j) : (home <= i && home > j);
if (moves) s_[i] = s_[j], i = j;
if (moves) keys_[i] = keys_[j], vals_[i] = vals_[j], i = j;
}
s_[i] = Slot{};
keys_[i] = 0, vals_[i] = V{}, used_[i] = 0;
n_--;
return 1;
}
template <class F>
void erase_if(F pred) { // (the keys to go first, then each erased: a shift during the walk would skip or revisit entries)
std::vector<uint64_t> gone;
for (const Slot& s : s_)
if (s.used && pred(s.key, s.val)) gone.push_back(s.key);
for (uint64_t k : gone) erase(k);
gone_.clear();
for (size_t i = 0; i < keys_.size(); i++)
if (used_[i] && pred(keys_[i], vals_[i])) gone_.push_back(keys_[i]);
for (uint64_t k : gone_) erase(k);
}
bool empty() const { return n_ == 0; }
size_t size() const { return n_; }
@@ -228,18 +259,23 @@ public:
private:
size_t idx(uint64_t k) const { return (size_t)((k * 0x9E3779B97F4A7C15ull) >> shift_) & mask_; }
void grow() {
std::vector<Slot> old;
old.swap(s_);
const size_t cap = old.empty() ? 1024 : old.size() * 2;
s_.assign(cap, Slot{});
std::vector<uint64_t> ok;
std::vector<uint8_t> ou;
std::vector<V> ov;
ok.swap(keys_), ou.swap(used_), ov.swap(vals_);
const size_t cap = ok.empty() ? 1024 : ok.size() * 2;
keys_.assign(cap, 0), used_.assign(cap, 0), vals_.assign(cap, V{});
mask_ = cap - 1;
shift_ = 64;
for (size_t c = cap; c > 1; c >>= 1) shift_--;
n_ = 0;
for (const Slot& s : old)
if (s.used) (*this)[s.key] = s.val;
for (size_t i = 0; i < ok.size(); i++)
if (ou[i]) (*this)[ok[i]] = ov[i];
}
std::vector<Slot> s_;
std::vector<uint64_t> keys_;
std::vector<uint8_t> used_;
std::vector<V> vals_;
std::vector<uint64_t> gone_; // (erase_if's list, kept: no allocation a prune)
size_t n_ = 0, mask_ = 0;
int shift_ = 64;
};
@@ -259,12 +295,15 @@ public:
kWritten, // (its count is in the record: completion wrote it)
kFinal // (no count comes: skip list, knob off, a clear flag, not counted whole, lost, late, the CPU wrote it)
};
enum Src { kNone, kPendingSrc, kWrittenSrc, kFinalSrc }; // (what a render condition reading the word finds)
// (what a render condition reading the word finds. kGatedSrc: written, a count over a draw a written count's condition
// left out, see Counter::gate)
enum Src { kNone, kPendingSrc, kWrittenSrc, kFinalSrc, kGatedSrc };
struct Placed {
uint64_t pos = 0;
int64_t present = 0;
uint64_t value = 0; // (what the replay, or a count written since, put in the word: a count is written only over it)
State state = kFinal;
bool gated = false; // (its count, written, was over a draw a written count's condition left out)
};
/// A report's placeholder `value` written at the replay, at pos in the stream (the newest there from now on). write:
/// its count is to be written once read back (not on RX_VK_OCC_QUERY_SKIP, the lab knob on, not a clear flag)
@@ -287,10 +326,11 @@ public:
const auto* it = m_.find(addr);
return !it ? 0 : it->value;
}
/// The count of the report at (addr, pos) is in its record now
void wrote(uint64_t addr, uint64_t pos, uint64_t count) {
/// The count of the report at (addr, pos) is in its record now (gated: counted over a draw a written count's condition
/// left out)
void wrote(uint64_t addr, uint64_t pos, uint64_t count, bool gated = false) {
auto* it = m_.find(addr);
if (it && it->pos == pos) it->state = kWritten, it->value = count;
if (it && it->pos == pos) it->state = kWritten, it->value = count, it->gated = gated;
}
/// No count comes for the report at (addr, pos) (not counted whole, lost, late, never sent, the CPU wrote the record):
/// a condition reads its word from memory from now on, as before
@@ -317,7 +357,7 @@ public:
if (!it) return kNone;
switch (it->state) {
case kPending: return now - it->present > kLateAge ? kFinalSrc : kPendingSrc;
case kWritten: return kWrittenSrc;
case kWritten: return it->gated ? kGatedSrc : kWrittenSrc;
case kFinal: break;
}
return kFinalSrc;
@@ -339,23 +379,33 @@ private:
/// they differ) over the words' sources (Writers::word), after NX2navi's cond_enabled: a samples-passed count not done yet
/// renders; a written one, as any other word, is read from memory (base_enabled: the replay's answer from memory). active:
/// RX_VK_OCC_QUERY=1 or the lab knob's phase (else the answer is the base one, whatever the table says); cond_on: false
/// with RX_VK_OCC_COND=0, every condition on a samples-passed word renders
/// with RX_VK_OCC_COND=0, every condition on a samples-passed word renders.
/// (11-22) gate_on (RX_VK_OCC_GATE=1): a word whose count was written over a draw that a written count's condition left
/// out (Writers::kGatedSrc) is unknown, and the condition renders. The count is a skip's result, not the object's: on the
/// console the game's CPU reads a 0 before it records the next frame and draws the object's proxy instead; here the count
/// often lands after the CPU has read the placeholder (4096: visible), the game goes on drawing the object under the
/// condition, the condition reads the skip's 0, the next count is 0 again, and the object stays hidden for good. Off: as a
/// written count (read from memory)
struct Cond {
enum Kind { kNotSamples, kBase, kUnknown, kForced, kWrittenOn, kWrittenOff, kMemory };
enum Kind { kNotSamples, kBase, kUnknown, kForced, kWrittenOn, kWrittenOff, kMemory, kGated };
bool enabled;
Kind kind;
};
inline Cond cond_decide(bool base_enabled, Writers::Src sx, Writers::Src sy, bool active, bool cond_on) {
inline Cond cond_decide(bool base_enabled, Writers::Src sx, Writers::Src sy, bool active, bool cond_on, bool gate_on = false) {
if (sx == Writers::kNone && sy == Writers::kNone) return {base_enabled, Cond::kNotSamples};
if (!active) return {base_enabled, Cond::kBase};
if (!cond_on) return {true, Cond::kForced};
if (sx == Writers::kPendingSrc || sy == Writers::kPendingSrc) return {true, Cond::kUnknown};
if (sx == Writers::kWrittenSrc || sy == Writers::kWrittenSrc)
const bool gated = sx == Writers::kGatedSrc || sy == Writers::kGatedSrc;
if (gated && gate_on) return {true, Cond::kGated};
if (sx == Writers::kWrittenSrc || sy == Writers::kWrittenSrc || gated)
return {base_enabled, base_enabled ? Cond::kWrittenOn : Cond::kWrittenOff};
return {base_enabled, Cond::kMemory};
}
inline const char* src_name(Writers::Src s) {
return s == Writers::kPendingSrc ? "pending" : s == Writers::kWrittenSrc ? "written" : s == Writers::kFinalSrc ? "memory, no count to come" : "memory";
return s == Writers::kPendingSrc ? "pending" : s == Writers::kWrittenSrc ? "written"
: s == Writers::kGatedSrc ? "written, over a draw a written count skipped"
: s == Writers::kFinalSrc ? "memory, no count to come" : "memory";
}
/// What happened since the last line (under the caller's lock)
@@ -370,9 +420,16 @@ struct Stats {
int64_t rewritten = 0; // records rewritten in the stream after their report (a small clear, another counter type)
int64_t measured = 0, no_reset = 0, incomplete = 0, doubles = 0; // counted whole by queries, and those not
int64_t inc_skipped = 0, inc_other_cmd = 0, inc_full = 0; // why not whole
// ((Triforce, 11-22, GPUWORK-01) and of the incomplete ones, those given up for a colour draw in their window
// (RX_VK_OCC_DEPTH_ONLY); the windows that met a colour draw, the colour draws under an open query, the counts given up
// (colour_line, a line of its own: line() is at the log's length already)
int64_t inc_colour = 0, colour_windows = 0, colour_open = 0, colour_lost = 0;
// render conditions that read a samples-passed report's word (Cond::Kind), and the work they skipped
int64_t conds = 0, cond_base = 0, cond_unknown = 0, cond_forced = 0, cond_written_on = 0, cond_written_off = 0, cond_memory = 0;
int64_t draws_skipped = 0, clears_skipped = 0;
// (11-22) counts over a draw a written count's condition left out (Counter::gate): counted whole, written (of them 0),
// the conditions that read one, and those RX_VK_OCC_GATE=1 rendered
int64_t gated = 0, gated_written = 0, gated_zeros = 0, cond_gated_read = 0, cond_gated = 0;
// at completion
int64_t written = 0, zeros = 0; // counts written into their records (of them 0: the object hidden)
int64_t lost = 0; // a query not available once its recording was done: 4096 stays
@@ -389,7 +446,17 @@ struct Stats {
if (k == Cond::kNotSamples) return;
conds++;
(k == Cond::kBase ? cond_base : k == Cond::kUnknown ? cond_unknown : k == Cond::kForced ? cond_forced
: k == Cond::kWrittenOn ? cond_written_on : k == Cond::kWrittenOff ? cond_written_off : cond_memory)++;
: k == Cond::kWrittenOn ? cond_written_on : k == Cond::kWrittenOff ? cond_written_off : k == Cond::kGated ? cond_gated
: cond_memory)++;
}
/// ((Triforce, 11-22, the performance audit's GPUWORK-01) The colour draws' line, into out. on: the rule is on
int colour_line(char* out, size_t size, int64_t presents, bool on) const {
return snprintf(out, size,
"vk: occlusion queries (RX_VK_OCC_QUERY), %lld presents (3): %lld counting windows met a draw that "
"writes colour, %lld such draws were recorded under an open query; depth-only counting "
"(RX_VK_OCC_DEPTH_ONLY) is %s: %lld counts given up at a colour draw, %lld reports kept 4096 for it",
(long long)presents, (long long)colour_windows, (long long)colour_open, on ? "on" : "off",
(long long)colour_lost, (long long)inc_colour);
}
/// The line, into out
int line(char* out, size_t size, int64_t presents, size_t tracked) const {
@@ -407,7 +474,8 @@ struct Stats {
"%lld (%lld as before: knob occq off; %lld rendered: a count pending (unknown); %lld rendered: "
"RX_VK_OCC_COND=0; %lld rendered from a written count; %lld skipped from a written count; %lld from "
"memory, no count to come); skipped by them: %lld draws and dispatches, %lld clears; %lld addresses "
"this period, %zu tracked",
"this period, %zu tracked; counts over a draw such a skip left out: %lld counted whole, %lld written "
"(%lld of them 0), %lld conditions read one (%lld rendered: RX_VK_OCC_GATE=1)",
(long long)presents, (long long)reports, (long long)in_segments, (long long)skipped, (long long)fixed,
(long long)flagged, (long long)rewritten, (long long)measured, (long long)no_reset,
(long long)incomplete, (long long)inc_skipped, (long long)inc_other_cmd, (long long)inc_full,
@@ -417,7 +485,8 @@ struct Stats {
(long long)at[kAtWait], (long long)at[kAtBegin], (long long)at[kAtPresent], (long long)at[kAtReuse],
(long long)conds, (long long)cond_base, (long long)cond_unknown, (long long)cond_forced,
(long long)cond_written_on, (long long)cond_written_off, (long long)cond_memory,
(long long)draws_skipped, (long long)clears_skipped, (long long)addrs, tracked);
(long long)draws_skipped, (long long)clears_skipped, (long long)addrs, tracked, (long long)gated,
(long long)gated_written, (long long)gated_zeros, (long long)cond_gated_read, (long long)cond_gated);
}
};
@@ -458,9 +527,10 @@ void complete(const std::vector<Rep>& reps, const uint64_t* pairs, uint32_t used
w.settle(p.addr, p.pos);
continue;
}
w.wrote(p.addr, p.pos, sum);
w.wrote(p.addr, p.pos, sum, p.gated);
const int64_t lat = std::max<int64_t>(0, now - p.present);
st.written++, st.zeros += sum ? 0 : 1, st.lat_sum += lat, st.lat_ge1 += lat >= 1, st.lat_ge2 += lat >= 2;
if (p.gated) st.gated_written++, st.gated_zeros += sum ? 0 : 1;
}
}
/// A recording's reports that never went to the GPU: no count comes for them

82
gpu/vk/occ_lean.h Normal file
View File

@@ -0,0 +1,82 @@
// (Triforce, 11-22, the performance audit's DESC-82, after Source B's: its first step, the census) The game's occlusion tests
// draw proxies that write no colour, and each such draw runs its whole fragment shader and fills its fragment descriptors.
// Source B makes a sibling pipeline without the fragment stage for a counted draw that writes no colour and whose fragment
// shader has no effect but colour (no depth or sample-mask output, no demote, no stores, image writes, atomics or global
// memory) and whose state has neither alpha to coverage nor an alpha test. It was never on by default nor measured; its
// census at Kakariko counted about 2,570 colour-off draws of about 3,420 counted. The entry's own order is the census first,
// and the lean pipelines only if the census finds more than one fragment descriptor a proxy to save.
//
// This is the census alone. RX_VK_OCC_LEAN=3: each draw under an open occlusion query that writes no colour is classed
// (classify: lean, or the first refusal), and its fragment stage's descriptors added to its class; every 600 presents one
// line gives each class's draws and descriptors a frame (vk_replay.cpp status_tick). No pipeline, key or descriptor changes.
// 1 (the lean pipelines) and 2 (their check) are not built: they read as off. Unset: nothing of it runs.
//
// The rules alone: tests/test_occ_lean.py compiles them.
#pragma once
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
namespace occ_lean {
/// RX_VK_OCC_LEAN: 3 for the census; anything else is off (1 and 2: the steps after the census, not built)
inline int mode_of(const char* e) { return e && *e && std::atoi(e) == 3 ? 3 : 0; }
enum Class { kLean, kNoFragment, kDepth, kSampleMask, kDemote, kStores, kImageWrites, kAtomics, kGlobal, kA2c, kAlphaTest, kClasses };
inline const char* const kNames[kClasses] = {"lean", "no fragment stage", "depth out", "sample mask", "demote", "stores",
"image writes", "atomics", "global memory", "alpha to coverage", "alpha test"};
/// What the class reads of a draw: its fragment stage's (Shader::Info) and its state's
struct Facts {
bool has_fragment = true;
bool depth_out = false, sample_mask = false, demote = false, stores = false, image_writes = false, atomics = false,
global = false;
bool a2c = false, alpha_test = false;
};
/// The draw's class: no fragment stage (nothing to leave out), the first refusal of the shader's then of the state's, or lean
inline Class classify(const Facts& f) {
if (!f.has_fragment) return kNoFragment;
if (f.depth_out) return kDepth;
if (f.sample_mask) return kSampleMask;
if (f.demote) return kDemote;
if (f.stores) return kStores;
if (f.image_writes) return kImageWrites;
if (f.atomics) return kAtomics;
if (f.global) return kGlobal;
if (f.a2c) return kA2c;
if (f.alpha_test) return kAlphaTest;
return kLean;
}
/// The counts, any recording thread (relaxed)
struct Census {
std::atomic<uint64_t> draws[kClasses]{};
std::atomic<uint64_t> descriptors[kClasses]{};
void note(Class c, uint32_t frag_descriptors) {
draws[c].fetch_add(1, std::memory_order_relaxed);
descriptors[c].fetch_add(frag_descriptors, std::memory_order_relaxed);
}
/// The line for `presents` presents (draws and fragment descriptors a frame, the lean ones' descriptors a draw, then each
/// refusal that met any), and the counts set back. Its length
int line(char* out, size_t n, uint64_t presents) {
uint64_t d[kClasses], s[kClasses], all = 0;
for (int i = 0; i < kClasses; i++)
d[i] = draws[i].exchange(0, std::memory_order_relaxed), s[i] = descriptors[i].exchange(0, std::memory_order_relaxed),
all += d[i];
const double p = presents ? (double)presents : 1.0;
int at = std::snprintf(out, n, "vk: occlusion proxies (RX_VK_OCC_LEAN=3, %llu presents): %.0f colour-off counted draws a frame; "
"lean %.0f a frame with %.2f fragment descriptors each", (unsigned long long)presents, all / p,
d[kLean] / p, d[kLean] ? (double)s[kLean] / (double)d[kLean] : 0.0);
for (int i = 1; i < kClasses && at > 0 && (size_t)at < n; i++)
if (d[i])
at += std::snprintf(out + at, n - (size_t)at, "; %s %.0f (%.2f descriptors each)", kNames[i], d[i] / p,
(double)s[i] / (double)d[i]);
return at;
}
};
inline Census census;
} // namespace occ_lean

100
gpu/vk/page_marks.h Normal file
View File

@@ -0,0 +1,100 @@
// (Triforce, 11-22, the performance audit's THREAD-25, after Source A's) Pages that something ever happened to, asked without a lock;
// and the question an invalidate asks of them first.
//
// The game invalidates each shape's 64-byte occlusion slot, per view: tens of thousands of nvnBufferInvalidateMappedRange calls in a
// few seconds. The host gives the calling thread's virtual core up before every one and queues for it again after, as the call can
// wait for the replay or for a write-back still on the GPU; nearly always it has nothing to wait for. What it would wait for is
// known from three things: a write-back ever made into the range's pages, a texture the GPU wrote ever listed there, and a live
// texture's storage there. The first two are tables under their own locks; PageMarks keeps, beside each, one bit a 4 KB page that is
// only ever set, set before the table changes and under the table's lock, and read with no lock. A range with no mark in either and
// no texture storage is one invalidate() returns from without waiting, so the host may keep the core and not make the call
// (invalidate_quick in vk_replay.cpp, asked through mkwvk_invalidate_quick by host/rx_backend.c).
//
// A mark is never taken back: a page written back once declines for good, and waits as before. Addresses at or past 512 GB are
// not kept and answer as marked. The header is written from the audit's description of Source A's, whose file is not here.
//
// RX_VK_INVALIDATE_QUICK=1 turns the question on; off unless set (see docs/development.md for why). tests/test_invalidate_quick.py
#pragma once
#include <atomic>
#include <cstdint>
#include <cstdlib>
namespace mkwvk {
struct PageMarks {
static constexpr uint64_t kSpan = 1ull << 39; // (512 GB: the guest window, and the GPU's addresses the pools lie in)
static constexpr uint64_t kPageShift = 12, kPages = kSpan >> kPageShift;
static constexpr uint64_t kChunkPages = 1ull << 18; // (a chunk: 1 GB of addresses, 32 KB of bits, made at its first mark)
static constexpr uint64_t kChunks = kPages / kChunkPages, kChunkWords = kChunkPages / 64;
std::atomic<std::atomic<uint64_t>*> chunks[kChunks] = {};
/// every page with a byte in [lo, hi) marked (what lies at or past 512 GB is not kept: any() answers true there)
void mark(uint64_t lo, uint64_t hi) {
if (hi > kSpan) hi = kSpan;
if (hi <= lo) return;
const uint64_t last = (hi - 1) >> kPageShift;
for (uint64_t p = lo >> kPageShift; p <= last;) {
std::atomic<uint64_t>* c = chunks[p / kChunkPages].load(std::memory_order_acquire);
if (!c) {
std::atomic<uint64_t>* made = new std::atomic<uint64_t>[kChunkWords];
for (uint64_t i = 0; i < kChunkWords; i++) made[i].store(0, std::memory_order_relaxed);
if (chunks[p / kChunkPages].compare_exchange_strong(c, made, std::memory_order_acq_rel)) c = made;
else delete[] made; // (another thread made it first: c is its chunk)
}
// (the pages of this word from p to the word's end, or to `last`)
const uint64_t in_chunk = p % kChunkPages, bit = in_chunk % 64;
const uint64_t word_last = p - bit + 63, to = word_last < last ? word_last : last;
const uint64_t n = to - p + 1;
const uint64_t mask = (n == 64 ? ~0ull : ((1ull << n) - 1)) << bit;
c[in_chunk / 64].fetch_or(mask, std::memory_order_seq_cst);
p = to + 1;
}
}
/// whether a page with a byte in [lo, lo + size) was ever marked; true for a range that reaches 512 GB or wraps
bool any(uint64_t lo, uint64_t size) const {
if (!size) return false;
if (lo >= kSpan || size > kSpan - lo) return true;
const uint64_t last = (lo + size - 1) >> kPageShift;
for (uint64_t p = lo >> kPageShift; p <= last;) {
const std::atomic<uint64_t>* c = chunks[p / kChunkPages].load(std::memory_order_seq_cst);
if (!c) { // (a chunk never marked: skipped whole)
p = (p / kChunkPages + 1) * kChunkPages;
continue;
}
const uint64_t in_chunk = p % kChunkPages, bit = in_chunk % 64;
const uint64_t word_last = p - bit + 63, to = word_last < last ? word_last : last;
const uint64_t n = to - p + 1;
const uint64_t mask = (n == 64 ? ~0ull : ((1ull << n) - 1)) << bit;
if (c[in_chunk / 64].load(std::memory_order_seq_cst) & mask) return true;
p = to + 1;
}
return false;
}
};
namespace inv_quick { // (the rule of vk_replay.cpp's invalidate_quick)
/// Whether the question is asked at all. quick: RX_VK_INVALIDATE_QUICK (a number other than 0 turns it on; off unless set). The
/// three others are switches that make invalidate() log every call or wait for every range: with one set, every call is made
inline bool on(const char* quick, const char* log_invalidate, const char* log_readback, const char* wait_all) {
return quick && std::atoi(quick) != 0 && !log_invalidate && !log_readback && !wait_all;
}
/// What is known of a range when the question is asked
struct Look {
uint64_t gpu = 0, size = 0;
bool ticket_pending = false; // (a fast fence poll's order ticket not yet left: invalidate() waits for that section first)
bool tex_storage = false; // (a live texture's storage in the range's pages: tex_pages_any)
bool vw_marked = false; // (a texture the GPU wrote ever listed there)
bool wb_marked = false; // (a write-back ever made into its pages in the game's memory)
};
/// true: invalidate(gpu, size) would return without waiting for anything and without work: the call may be left out
inline bool nothing_to_wait_for(const Look& l) {
if (!l.size) return true;
if (l.gpu >= PageMarks::kSpan || l.size > PageMarks::kSpan - l.gpu) return false;
return !l.ticket_pending && !l.tex_storage && !l.vw_marked && !l.wb_marked;
}
} // namespace inv_quick
} // namespace mkwvk

View File

@@ -25,6 +25,7 @@
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <new>
#include <stdexcept>
#include <vector>
@@ -87,6 +88,7 @@ enum class Read {
OverBound, // (longer than the bound: not a byte of it read)
NoMemory, // (no block of its size)
Cut, // (the read ended early: nothing of it is used)
Foreign, // ((Triforce) its header is another GPU's or driver's: not read past it)
};
inline const char* why(Read r) {
@@ -96,13 +98,19 @@ inline const char* why(Read r) {
case Read::OverBound: return "it is over the bound";
case Read::NoMemory: return "there is no memory for a block of its size";
case Read::Cut: return "reading it ended early";
case Read::Foreign: return "it is another GPU's or driver's";
default: return "";
}
}
/// The whole file into out, or nothing: out is empty unless the answer is Ok. *len: the file's length whenever it could be
/// asked, read or not. least: a file of at most this many bytes is refused; bound: one longer than this too (0: none)
inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<uint8_t>& out, uint64_t* len) {
/// asked, read or not. least: a file of at most this many bytes is refused; bound: one longer than this too (0: none).
/// ((Triforce) head_ok: when given, the file's first kHead bytes (a pipeline cache's header) are read alone and handed to it
/// with the file's length; refused, the rest is not read (Foreign). The file's stamp names only the translated shaders, so
/// after a driver update a file of 0.65-1.6 GB was read whole only for its header to be refused)
constexpr size_t kHead = 32;
inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<uint8_t>& out, uint64_t* len,
bool (*head_ok)(const uint8_t* head, uint64_t size) = nullptr) {
out.clear();
*len = 0;
FILE* f = fopen(path, "rb");
@@ -120,7 +128,14 @@ inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<u
else if ((uint64_t)n <= least) r = Read::Short;
else if (!within((uint64_t)n, bound)) r = Read::OverBound;
else if ((uint64_t)n > SIZE_MAX) r = Read::NoMemory;
else {
uint8_t head[kHead];
size_t got = 0;
if (r == Read::Ok && head_ok && (uint64_t)n >= kHead) {
got = fread(head, 1, kHead, f);
if (got != kHead) r = Read::Cut;
else if (!head_ok(head, (uint64_t)n)) r = Read::Foreign;
}
if (r == Read::Ok) {
try {
out.resize((size_t)n);
} catch (const std::bad_alloc&) {
@@ -128,7 +143,7 @@ inline Read read(const char* path, uint64_t least, uint64_t bound, std::vector<u
} catch (const std::length_error&) {
r = Read::NoMemory;
}
size_t got = 0;
if (r == Read::Ok && got) std::memcpy(out.data(), head, got);
while (r == Read::Ok && got < out.size()) {
const size_t piece = std::min(out.size() - got, kPiece);
const size_t had = fread(out.data() + got, 1, piece, f);

107
gpu/vk/pcache_libs.h Normal file
View File

@@ -0,0 +1,107 @@
// A pipeline cache file of stage libraries and compute pipelines alone, on NVIDIA PCs of little RAM (no game, no GPU: the
// rule, the driver's cache folder and its size, the record; vk_draw.cpp pcache_libs_decide, pcache_init, build_graphics,
// pcache_play_open). Apart from Vulkan so that tests/test_pcache_libs.py runs it on many made-up PCs.
//
// (Triforce, 11-22, the performance audit's PIPE-78, after Source A's and Source B's.) mkw_vk_pipelines.bin is read whole at
// every start and held by the driver all session: about 650 MB fresh, 1.1 to 1.6 GB with the play's pipelines on a 16 GB PC
// (pcache_bound.h), where paging is what hitches. Source A leaves its file unread on NVIDIA's own driver with 20 GB of RAM or
// less and 512 MB or more in the driver's own disk cache (GLCache), whole pipelines then coming from that cache: on a 16 GB
// RTX 3070 Ti, peak memory 12,481 -> 12,029 and 12,638 -> 11,822 MB over a tour, fps 54.3 -> 54.6, a pipeline's load 1.9 ->
// 5.0 ms. Triforce cannot drop the file: a stage library asked under the compile-required flag must be in the cache it hands
// the driver (21 of 21 cold otherwise). So the file keeps the libraries and the compute pipelines, and only whole graphics
// pipelines are left to the driver's cache, with no play file.
//
// Decided only when a fresh file begins (none read for these shaders), and kept in <file>.lean ("lean <stamp>") for that
// file's life: a full file stays full and a lean one lean until the next fresh one. Not in a start that only prepares.
//
// RX_VK_PCACHE_LIBS_ONLY: unset or 0, off (every pipeline in the file, as before); 1, by the rule; 2, at every fresh file
// whatever the PC (the lab's and the tests'). Off by default: the driver's cache is shared and capped, and on a 32 GB PC
// Source A measured p99 17.23 -> 18.58 ms.
#pragma once
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <string>
#include <system_error>
namespace pcache_libs {
inline constexpr uint64_t kRamMost = 20ull << 30; // (20 GB of RAM or less)
inline constexpr uint64_t kGlCacheLeast = 512ull << 20; // (512 MB or more in the driver's own cache)
/// RX_VK_PCACHE_LIBS_ONLY: 0 off (unset too), 1 by the rule, 2 forced
inline int mode(const char* env) {
if (!env || !*env) return 0;
const int v = std::atoi(env);
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
}
struct Pc {
bool nvidia_proprietary = false; // (VK_DRIVER_ID_NVIDIA_PROPRIETARY)
uint64_t ram = 0; // (the PC's RAM, bytes; 0 unknown)
uint64_t glcache = 0; // (bytes in the driver's own cache folder)
bool gpl = false; // (the stage libraries are wanted and kept in our cache: gpl_wanted, gpl_libs_cached)
bool prepare_only = false; // (a start that only prepares)
};
/// Whether the fresh file is a lean one
inline bool choose(int mode, const Pc& pc) {
if (mode <= 0 || pc.prepare_only) return false;
if (mode >= 2) return true;
return pc.nvidia_proprietary && pc.gpl && pc.ram && pc.ram <= kRamMost && pc.glcache >= kGlCacheLeast;
}
/// The folder NVIDIA's driver keeps its Vulkan and OpenGL cache in: on Windows <LOCALAPPDATA>\NVIDIA\GLCache; elsewhere
/// __GL_SHADER_DISK_CACHE_PATH, else <XDG_CACHE_HOME>/nvidia/GLCache, else <HOME>/.cache/nvidia/GLCache. Empty: unknown
inline std::string nvidia_cache_dir(bool windows, const char* localappdata, const char* gl_path, const char* xdg, const char* home) {
if (windows) return localappdata && *localappdata ? std::string(localappdata) + "\\NVIDIA\\GLCache" : std::string();
if (gl_path && *gl_path) return gl_path;
if (xdg && *xdg) return std::string(xdg) + "/nvidia/GLCache";
if (home && *home) return std::string(home) + "/.cache/nvidia/GLCache";
return {};
}
/// The bytes of the files under a folder, at most most_entries looked at (a folder that is not there: 0)
inline uint64_t folder_bytes(const std::string& dir, size_t most_entries = 200000) {
uint64_t sum = 0;
if (dir.empty()) return 0;
std::error_code ec;
const std::filesystem::path root(std::u8string(reinterpret_cast<const char8_t*>(dir.data()), dir.size())); // (UTF-8)
std::filesystem::recursive_directory_iterator it(root, std::filesystem::directory_options::skip_permission_denied, ec);
for (size_t n = 0; !ec && it != std::filesystem::recursive_directory_iterator() && n < most_entries; it.increment(ec), n++) {
std::error_code e2;
if (it->is_regular_file(e2)) {
const uint64_t s = it->file_size(e2);
if (!e2) sum += s;
}
}
return sum;
}
// ---- <file>.lean: "lean <stamp>"
inline bool lean_read(const std::string& path, const char* stamp16) {
FILE* f = fopen(path.c_str(), "rb");
if (!f) return false;
char line[40]{};
const size_t n = fread(line, 1, sizeof line - 1, f);
fclose(f);
line[n] = 0;
return n >= 21 && !memcmp(line, "lean ", 5) && !memcmp(line + 5, stamp16, 16);
}
inline bool lean_write(const std::string& path, const char* stamp16) {
FILE* f = fopen(path.c_str(), "wb");
if (!f) return false;
const bool ok = fprintf(f, "lean %.16s\n", stamp16) > 0;
return fclose(f) == 0 && ok;
}
/// What a start does: fresh (no file read for these shaders), whether the rule says lean now, and the record of the file
/// read. Lean for the session: a fresh file the rule makes lean, or a file read whose record says lean (and the switch on)
inline bool lean_now(int mode, bool fresh, bool chose, bool recorded) {
if (mode <= 0) return false;
return fresh ? chose : recorded;
}
} // namespace pcache_libs

View File

@@ -22,6 +22,7 @@
#pragma once
#include <cstdint>
#include <cstdlib>
namespace pcache_play {
@@ -33,6 +34,13 @@ inline bool routes(bool main_cache, bool play_cache, bool preparing) {
return main_cache && play_cache && !preparing;
}
/// ((Triforce, 11-22, the performance audit's PIPE-95, after Source A's) RX_VK_PCACHE_OWN_PASSES: whether the back end's own
/// compute passes (FSR 3.1's, the anti-aliasing's, the texture converters': vk_draw.cpp shared_cache_compute) are made
/// through the play cache as the game's are, on unless set to 0 (0: bare into the main cache, as before)
inline bool own_passes(const char* env) {
return !env || std::atoi(env) != 0;
}
/// At the start, the play file read: whether it is folded into the main cache now (fold_bytes 0: never)
inline bool fold_due(uint64_t play_bytes, uint64_t fold_bytes) {
return fold_bytes && play_bytes >= fold_bytes;

62
gpu/vk/piece_idle.h Normal file
View File

@@ -0,0 +1,62 @@
// (Triforce, 11-22, the performance audit's MEM-26, its first step, after Source A's) What a mirror in pieces holds that
// no game pool covers any more, and how much of that no binding has reached for a while: counted, nothing released.
//
// A mirror in pieces gives a piece its memory when a binding first reaches it, and keeps it until the whole mirror goes.
// The mirror is an import's (a chunk of guest memory around what was bound), and the game's pools inside it come and go:
// a long-lived import keeps the memory of every piece any area reached, also where no game pool is left. Source A gives
// such memory back after two waits of 600 submissions. Before that is ported (the unbind and the free must wait for the
// GPU and for the bind timeline: memory freed under a compute pass that reads it lost devices before) this counts what
// it would find, every 1200 submissions, in the "mirrors in pieces" line: under about 100 MB at the end of a tour on a
// 6 GB card, the release is not worth having.
//
// Counted by piece. A release frees whole runs of pieces (one allocation a run), so this is the most it could give back.
// The rule alone, no Vulkan: tests/test_mirror_piece_idle.py compiles it against a page-by-page model.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <utility>
#include <vector>
namespace piece_idle {
using Ranges = std::vector<std::pair<uint64_t, uint64_t>>; // ([first, second))
/// The ranges sorted and merged: disjoint, ascending, none empty (touching ones joined)
inline void merge(Ranges& r) {
std::sort(r.begin(), r.end());
size_t m = 0;
for (const auto& e : r) {
if (e.second <= e.first) continue;
if (m && e.first <= r[m - 1].second) r[m - 1].second = std::max(r[m - 1].second, e.second);
else r[m++] = e;
}
r.resize(m);
}
struct Count {
uint64_t held = 0; // (bytes of the pieces that have memory)
uint64_t uncovered = 0; // (of them, the pieces no game pool touches)
uint64_t idle = 0; // (of those, the ones no binding reached for idle_for submissions; 0 when that is not tracked)
};
/// A mirror at guest address lo of `size` bytes, in pieces of 1 << shift bytes: resident(i) says piece i has memory,
/// last(i) the submission a binding last reached it at. pools: the game's pools, merged (merge above). now: this
/// submission; idle_for: 0, or how many submissions without a binding make a piece idle. Adds to out
template <class Resident, class Last>
inline void count(uint64_t lo, uint64_t size, uint32_t shift, const Resident& resident, const Last& last, const Ranges& pools, uint64_t now,
uint64_t idle_for, Count& out) {
const uint64_t piece = 1ull << shift, n = (size + piece - 1) >> shift;
size_t at = 0; // (the first pool that ends after the piece's start: the pieces come ascending)
for (uint64_t i = 0; i < n; i++) {
if (!resident(i)) continue;
const uint64_t a = lo + (i << shift), b = lo + std::min(size, (i + 1) << shift);
out.held += b - a;
while (at < pools.size() && pools[at].second <= a) at++;
if (at < pools.size() && pools[at].first < b) continue; // (a game pool touches it)
out.uncovered += b - a;
if (idle_for && now - last(i) >= idle_for) out.idle += b - a;
}
}
} // namespace piece_idle

48
gpu/vk/pipe_ahead.h Normal file
View File

@@ -0,0 +1,48 @@
// Where the pipelines the draws first need stand in the last sessions' order, against where the draws are (no game, no GPU:
// the rules; vk_draw.cpp order_met, graphics_pipeline, draw_left_out_take). Apart from Vulkan so that tests/test_pipe_ahead.py
// runs them.
//
// (Triforce, 11-22, the performance audit's PIPE-81, after Source B's: its first step.) Where the precompile keeps no
// pipelines (light mode: precompile_keeps_all), every pipeline of a load is new at its first draw: linked there from its
// stage libraries, or by the warm a moment earlier (pipe_warm.h, pipe_order.h). The entry proposes building whole pipelines
// just ahead of the draws on light frames, from the order's next places, and keeping them; and it says to count first:
// each pipeline first drawn, its place ahead of where the draws are (pipe_order.h Front) or none, and whether the warm had
// linked it. Under a third within the window: stop, the builder is not worth its RAM and its compiles.
//
// This is that count, and only it: nothing is built or kept. A line every 600 presents while it grows (with the order used):
// within the window ahead, further ahead, behind, not in the order, and how many of those within the window the warm had
// linked; and whether the builder would be worth it by the entry's rule.
//
// RX_VK_PIPE_AHEAD=<places>: the window, 1024 unset (1 to 4096); 0: not counted.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace pipe_ahead {
inline constexpr int64_t kWindow = 1024;
inline constexpr int64_t kMost = 4096;
/// RX_VK_PIPE_AHEAD: the window in places, kWindow unset; 0 off; clamped to 1..kMost
inline int64_t window(const char* env) {
if (!env || !*env) return kWindow;
const long long v = std::atoll(env);
return v <= 0 ? 0 : v > kMost ? kMost : (int64_t)v;
}
enum class Where { None, Near, Far, Behind };
/// A pipeline first drawn at place in the order (-1: not in it), the draws at front (Front::furthest; -1: not yet placed,
/// counted from the order's start)
inline Where where(int64_t place, int64_t front, int64_t window_places) {
if (place < 0) return Where::None;
const int64_t from = front < 0 ? 0 : front;
if (place < from) return Where::Behind;
return place - from <= window_places ? Where::Near : Where::Far;
}
/// The entry's rule for the builder: worth it when a third or more of the pipelines first drawn were within the window
inline bool worth(int64_t within, int64_t all) {
return all > 0 && within * 3 >= all;
}
} // namespace pipe_ahead

View File

@@ -32,6 +32,7 @@
#pragma once
#include <algorithm>
#include <climits>
#include <cstdint>
#include <deque>
#include <iterator>
@@ -45,7 +46,11 @@
namespace mkwvk::pipe_order {
constexpr size_t kKeysMax = program_order::kKeysMax; // (pipelines a session's order keeps, and the file)
// ((Triforce, 11-21) 16384, was the programs' 4096: one ten-area tour needs 3918 pipelines, so an adult tour pushed half of the
// child tour's out and its next tour linked 828 of 2274 ahead where it had linked 1455 of 2012 (pipeline stalls 1.3 -> 2.6 s,
// the worst 174 -> 464 ms). The file only: what the warm holds stays bounded by kLinksMax and kLibsMax below, and the programs'
// read-ahead (its RAM) by program_order's own 4096. 8 bytes a pipeline: 128 KB at most)
constexpr size_t kKeysMax = 16384; // (pipelines a session's order keeps, and the file)
constexpr size_t kPerVariant = 32; // (the order's pipelines made for a program variant, at most: as the notes')
constexpr size_t kLinksMax = 1024; // (links held for draws that have not come: a load's, beside the title's taken)
constexpr size_t kLibsMax = 2048; // (stage libraries the warm made that no draw has had: two a program variant of 1024)
@@ -58,9 +63,13 @@ inline bool on(const char* env) { return !(env && env[0] == '0' && !env[1]); }
/// The file: program_order.h's form, its own magic
inline std::vector<uint8_t> write(const std::vector<uint64_t>& keys) { return program_order::write(keys, kMagic); }
inline std::vector<uint64_t> read(const uint8_t* data, size_t n) { return program_order::read(data, n, kMagic); }
using program_order::merge;
using Record = program_order::Record;
inline std::vector<uint64_t> read(const uint8_t* data, size_t n) { return program_order::read(data, n, kMagic, kKeysMax); }
inline std::vector<uint64_t> merge(const std::vector<uint64_t>& now, const std::vector<uint64_t>& before, size_t max = kKeysMax) {
return program_order::merge(now, before, max);
}
struct Record : program_order::Record {
Record() { max = kKeysMax; }
};
/// One pipeline of the order: its place, its note's record in the notes file
struct Entry {
@@ -154,12 +163,19 @@ public:
by_place_.emplace(std::make_pair(place, key), o);
place_of_.emplace(key, place);
while (by_place_.size() > max_) {
const auto last = std::prev(by_place_.end());
gone.push_back(last->second);
place_of_.erase(last->first.second);
by_place_.erase(last);
// ((Triforce, 11-21, soft) one behind the draws first (the farthest behind), then the farthest ahead as before
const bool back = soft_ && by_place_.begin()->first.first < behind_at_;
const auto out = back ? by_place_.begin() : std::prev(by_place_.end());
gone.push_back(out->second);
place_of_.erase(out->first.second);
by_place_.erase(out);
}
}
/// ((Triforce, 11-21) RX_VK_ORDER_SOFT_BEHIND) Soft: behind() lets nothing go, it marks where behind begins; those held there
/// stay for a draw that comes back to them (a tour's shared pipelines sit in another session's region of the merged order,
/// and the draws went back and forth: 1265 of 3048 links let go as behind in a child tour after an adult one) and go first
/// when a new link needs the room. The bound (max) is the same
void soft(bool on) { soft_ = on; }
/// The link held for key, taken out (a draw's from now on): false when none is
bool take(uint64_t key, One& out) {
const auto it = place_of_.find(key);
@@ -172,6 +188,10 @@ public:
}
/// Those placed before at, taken out into gone
void behind(int64_t at, std::vector<One>& gone) {
if (soft_) {
behind_at_ = std::max(behind_at_, at);
return;
}
const auto end = by_place_.lower_bound({at, 0});
for (auto it = by_place_.begin(); it != end;) {
gone.push_back(it->second);
@@ -193,7 +213,8 @@ public:
}
/// Whether a link at place would stay held: room left, or nearer than the farthest held (made, not let go at once)
bool room(int64_t place) const {
return by_place_.size() < max_ || (!by_place_.empty() && place < std::prev(by_place_.end())->first.first);
return by_place_.size() < max_ || (!by_place_.empty() && place < std::prev(by_place_.end())->first.first) ||
(soft_ && !by_place_.empty() && by_place_.begin()->first.first < behind_at_);
}
/// Whether a link held was linked from library lib
bool uses(uint64_t lib) const {
@@ -216,6 +237,8 @@ private:
return false;
}
size_t max_;
bool soft_ = false;
int64_t behind_at_ = INT64_MIN;
std::map<std::pair<int64_t, uint64_t>, One> by_place_; // ((place, key): the nearest first)
std::unordered_map<uint64_t, int64_t> place_of_;
};

View File

@@ -39,6 +39,13 @@
// libraries after it, so the draws of such a pipeline waited for the whole build. The same objects are made either way.
// A draw that waits runs the first job itself (vk_jobs.cpp jobs_wait): the libraries and the link, not the whole build.
// RX_VK_PIPE_FAST_WORKERS=0: the whole build first, as before.
// - choose(), depth_new ((Triforce, 11-22, the performance audit's PIPE-29, after Source B's): a depth-tested draw into a
// pass drawn once whose shaders write no memory, outside a samples-passed count and the item icon capture's kept targets):
// from the frame's wait pool, then left out, and a pipeline that ran out of time is left out at once until it is built
// (vk_draw.cpp). Source B froze 11-13 s twice on an Arc B580 compiling some fifty new pipelines for such passes, and made
// them wait at most 50 ms a frame. RX_VK_PIPE_DEPTH_BUDGET=1; off by default on every driver, Intel's too, where the
// entry would have it on: a render into a new depth target that is left out can stay missing until it is drawn again,
// and that trade is the owner's to make (depth_budget).
// A wait from a budget is vk_jobs.cpp jobs_wait_for: the pipeline's job goes to the front of the queue, and is not run on the
// draw's thread (a compile there would outlast the bound).
#pragma once
@@ -46,6 +53,7 @@
#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstdlib>
namespace pipe_wait {
@@ -139,7 +147,12 @@ private:
// ---- the draw's choice
enum class Act { Skip, Wait, WaitAll };
enum Kind { K_NONE, K_ONESHOT, K_FULL, K_LAST };
enum Kind { K_NONE, K_ONESHOT, K_FULL, K_LAST, K_DEPTH };
/// ((Triforce, 11-22, PIPE-29) RX_VK_PIPE_DEPTH_BUDGET: on when set to anything but 0; unset, off on every driver)
inline bool depth_budget(const char* env) {
return env && *env && std::atoi(env) != 0;
}
struct Rules {
bool always_wait = false; // (RX_VK_PIPE_SKIP=0: every pipeline waited for)
@@ -155,9 +168,16 @@ struct Choice {
/// A draw whose pipeline is not ready. kept: its pass (kept()); writes: its shaders write memory; full: full_screen(); held:
/// the hold of the last picture would take this frame if it lacks a draw (present_hold.h DEFERS, the pipelines counted);
/// pool_ns, last_ns: what the frame's wait pool and its last wait have left
inline Choice choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns) {
/// pool_ns, last_ns: what the frame's wait pool and its last wait have left. depth_new ((Triforce, 11-22, PIPE-29)
/// RX_VK_PIPE_DEPTH_BUDGET, the caller's: a depth-tested draw into a pass drawn once whose shaders write no memory): from the
/// pool, then left out
inline Choice choose(const Rules& r, bool kept, bool writes, bool full, bool held, int64_t pool_ns, int64_t last_ns,
bool depth_new = false) {
if (r.always_wait) return {Act::WaitAll, 0, K_NONE};
if (!kept && !writes && depth_new) {
if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_DEPTH};
return {Act::Skip, 0, K_NONE};
}
if (!kept || writes) {
if (!r.oneshot_budget) return {Act::WaitAll, 0, K_NONE};
if (pool_ns >= kMinWaitNs) return {Act::Wait, pool_ns, K_ONESHOT};

92
gpu/vk/precompile_save.h Normal file
View File

@@ -0,0 +1,92 @@
// The pipeline cache saved before the title after a long precompile, and after a preparation without stage libraries
// (no game, no GPU: a rule; vk_draw.cpp precompile_run, prepare_run, prepare_compute_pipeline, pipeline_cache_save).
//
// (Triforce, 11-22, the performance audit's PIPE-23, after Source A's.) A precompile that keeps every pipeline (enough
// RAM: precompile_keeps_all) ended without a save: what it compiled was first written in play, at present 300 and only
// after 32 more builds, and on Intel only at a clean close. A crash, a kill or a power cut before that cost the whole
// build again at the next start, minutes of it when the driver's own cache did not have them either. Source A wrote its
// cache before the title once the precompile had built for 30 s or more (after a 1,454 s compile on a handheld: 439 MB
// in 821 ms, and the next start's precompile took 13 s).
//
// Here the same, with two differences that are this project's:
// - the save is the play file's alone while there is a play cache (pcache_play.h): the precompile's compiles went into
// it. The main cache's hits count as builds too (cached_create), and on NVIDIA the main cache grows by the driver's
// copies, so an ordinary forced save would write the whole main file, 2 to 3 GB, before the title. With no play
// cache (a start that folded the play file, or no pipelineCreationCacheControl) the main file is what there is, and
// it is saved;
// - not where something else saves or the workers still build: a precompile the game went on from (its workers build
// behind the game: no fetch while they do, Intel's driver crashed there), a stopped one (the stop's own save), a
// start that only prepares (its end saves), and the light precompile (it saves before its record, as before).
//
// The second gap: a preparation without stage libraries compiled its compute pipelines into the main cache, counted
// them nowhere the save looks (pcache_main_built) and ended without a save, where the one with libraries saves after
// them (warm_libraries). Now they are counted, and that preparation saves before its record says it is made.
//
// RX_VK_PRECOMPILE_SAVE_S=<seconds>: the build time from which the precompile's end saves, 30 by default, at most a day;
// 0: never, and the preparation's count and save are off too, all as before.
#pragma once
#include <cstdlib>
namespace mkwvk::precompile_save {
inline constexpr int kAfterS = 30; // Source A's
inline constexpr int kMostS = 86400;
/// RX_VK_PRECOMPILE_SAVE_S: the seconds of building from which the precompile's end saves (0: never)
inline int after_s(const char* env) {
if (!env || !*env) return kAfterS;
const long v = std::atol(env);
return v <= 0 ? 0 : v > kMostS ? kMostS : (int)v;
}
/// Whether the rule is on at all (the preparation's count and its save go by this too)
constexpr bool on(int save_s) {
return save_s > 0;
}
/// Whether a precompile that built for build_s seconds is a long one: only those say anything
constexpr bool long_build(double build_s, int save_s) {
return save_s > 0 && build_s >= (double)save_s;
}
/// Whether the keep-all precompile's end saves now. gone_on: the game went on while workers still build; stopped: a stop
/// was asked; prepare_only: a start that only prepares
constexpr bool due(double build_s, int save_s, bool gone_on, bool stopped, bool prepare_only) {
return long_build(build_s, save_s) && !gone_on && !stopped && !prepare_only;
}
enum class Outcome {
Saved, // (written: the line gives its size and time)
Nothing, // (the cache holds nothing its files lack)
GoneOn,
Stopped,
PrepareOnly,
NoCache, // (no pipeline cache this session)
};
/// Why a long precompile's end does not save (only asked when due() said no, or with no cache)
constexpr Outcome why_not(bool gone_on, bool stopped, bool prepare_only, bool cache) {
return !cache ? Outcome::NoCache : stopped ? Outcome::Stopped : gone_on ? Outcome::GoneOn :
prepare_only ? Outcome::PrepareOnly : Outcome::Nothing;
}
/// The log line's words after "vk: precompile: N s of building: "
constexpr const char* said(Outcome o) {
switch (o) {
case Outcome::Saved: return "the pipeline cache saved before the title";
case Outcome::Nothing: return "the pipeline cache holds nothing new to save before the title";
case Outcome::GoneOn: return "the pipeline cache is not saved before the title: the game went on and the workers still build";
case Outcome::Stopped: return "the pipeline cache is not saved here: the stop's own save writes it";
case Outcome::PrepareOnly: return "the pipeline cache is not saved here: this start only prepares and saves at its end";
case Outcome::NoCache: return "there is no pipeline cache to save";
}
return "";
}
/// Whether a preparation without stage libraries saves the cache before its record is written: the rule on, not behind
/// the game, and compute pipelines built into the main cache
constexpr bool prepare_saves(int save_s, bool libraries_warmed, bool background, unsigned long long computes_built) {
return on(save_s) && !libraries_warmed && !background && computes_built > 0;
}
} // namespace mkwvk::precompile_save

211
gpu/vk/predict_check.h Normal file
View File

@@ -0,0 +1,211 @@
// Whether the driver still has the predicted pipelines, at a start whose record says they were all built (no game, no
// GPU: the rules; vk_draw.cpp predict_check_cold, predict_background, predicted_play_build). After lib_check.h, which
// asks the same of the stage libraries.
//
// (Triforce, 11-22, the performance audit's PIPE-68, after Source B's and Source A's.) The predicted pipelines (the ones
// the notes predict for programs no note names) are built once for the driver's own cache and let go, and a record
// (<notes>.predicted) says which programs are done; once it says all, no start looks again. But the driver's cache is
// the driver's: NVIDIA's has a size cap, AMD's Windows driver kept 30 to 44% of what Source B had prepared, and a cache
// folder can be cleared. Source B found 105 dispatches waiting 115 s in all on prepared pipelines the driver had dropped,
// while its record still said built. Then each of those programs' first draw compiles in play: the freezes on arrival
// the prediction is there to take away.
//
// The check. At a start whose record says all built, on the builder's idle thread after its delay and, as the builder,
// only while the frame shown is a light one: up to kSample predicted pipelines from the plan, spread over it from
// another place each start, each built for the driver's cache alone and timed, then built once more at once. The second
// comes from the driver's memory by construction, so it is this driver's warm time on this PC and no fixed time is
// needed: a first build over kRatio times the second and kSlackMs more was compiled. Not asked with
// FAIL_ON_PIPELINE_COMPILE_REQUIRED: under that flag a driver answers only from the cache it is handed, and these are in
// none of ours. The sample ends once its majority is decided.
//
// Cold (more than half of a sample): the record is removed, so the builder builds them all again behind the game at
// light frames, this session and the next, and <notes>.predicted.check takes a strike for this stamp. Cold again at a
// start after they were built again: this driver does not keep them. That is said once and recorded, and no later start
// checks for this stamp (the compiles of the check itself are not free), so the pipelines are left to their first draw
// as before the prediction. Warm: the strike is forgotten. Another stamp (a new driver, new shaders) begins anew.
//
// The counter, for what the check is worth: builds in play of programs the plan predicts that took over 20 ms, which a
// driver that had them does not need. Logged at most every 10 s while it grows.
//
// RX_VK_PREDICT_CHECK: 1 by default; 0: no check and no counter, as before; 2: checked at every start, a recorded
// "does not keep them" too (the lab's).
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <unordered_map>
#include <vector>
namespace predict_check {
inline constexpr int kSample = 12; // (pipelines sampled, as lib_check's)
inline constexpr int kLook = 2 * kSample; // (plan entries looked at for them: one may have lost its notes or its program)
inline constexpr double kRatio = 4.0; // (the first build against the second: compiled above)
inline constexpr double kSlackMs = 3.0; // (and this much more: lookups of a few milliseconds say nothing)
inline constexpr int kStrikes = 1; // (the record removed once; cold again after that: stop)
inline constexpr int64_t kSlowUs = 20000; // (a play build of a predicted program over this was compiled)
inline constexpr int64_t kSayMs = 10000; // (the counter's line: at most this often)
/// RX_VK_PREDICT_CHECK: 0 off, 1 on (unset too), 2 every start
inline int mode(const char* env) {
if (!env || !*env) return 1;
const int v = std::atoi(env);
return v <= 0 ? 0 : v >= 2 ? 2 : 1;
}
/// One pipeline sampled: made both times, and what each took (ms). One the driver refuses says nothing of its cache
struct Probe {
bool made = false;
double first_ms = 0, again_ms = 0;
};
inline bool cold(const Probe& p) {
return p.made && p.first_ms > kRatio * p.again_ms + kSlackMs;
}
struct Tally {
int got = 0, cold = 0;
double first_median = 0, again_median = 0;
};
inline double median(std::vector<double> v) {
if (v.empty()) return 0;
std::sort(v.begin(), v.end());
return v[v.size() / 2];
}
inline Tally tally(const std::vector<Probe>& probes) {
Tally t;
std::vector<double> first, again;
for (const Probe& p : probes) {
if (!p.made) continue;
t.got++, t.cold += cold(p);
first.push_back(p.first_ms), again.push_back(p.again_ms);
}
t.first_median = median(first), t.again_median = median(again);
return t;
}
enum class Verdict { Unknown, Warm, Cold };
/// Cold when more than half of the sample is; under half a sample says nothing
inline Verdict verdict(const Tally& t) {
if (t.got * 2 < kSample) return Verdict::Unknown;
return t.cold * 2 > t.got ? Verdict::Cold : Verdict::Warm;
}
/// Whether the sample needs no more probes: it is full, or what is still to come cannot change its verdict
inline bool decided(const Tally& t) {
return t.got >= kSample || t.cold * 2 > kSample || (t.got - t.cold) * 2 >= kSample;
}
/// k places spread over n, the first at seed's place in the first step: each start samples other pipelines
inline std::vector<size_t> spots(size_t n, size_t k, uint64_t seed) {
std::vector<size_t> at;
if (!n || !k) return at;
const size_t step = std::max<size_t>(1, n / k);
for (size_t i = (size_t)(seed % step); i < n && at.size() < k; i += step) at.push_back(i);
return at;
}
/// The records of some notes in the notes file f, read from its start: a head of {stamp, rec_bytes}, then records of
/// rec_bytes whose first 8 bytes are the key (0: the first after the head). Only the keys are read, each record's rest
/// stepped over, and the walk ends once every wanted key is found. A key twice keeps its first record. Empty when the
/// head is another build's
inline std::unordered_map<uint64_t, uint32_t> find(FILE* f, uint64_t stamp, uint64_t rec_bytes, const std::vector<uint64_t>& keys) {
std::unordered_map<uint64_t, uint32_t> at;
uint64_t head[2] = {0, 0};
if (!f || rec_bytes < 8 || rec_bytes > (1u << 30) || fread(head, 8, 2, f) != 2 || head[0] != stamp || head[1] != rec_bytes) return at;
std::unordered_map<uint64_t, bool> want;
for (const uint64_t k : keys) want.emplace(k, false);
size_t left = want.size();
uint64_t key = 0;
for (uint32_t i = 0; left && fread(&key, 8, 1, f) == 1; i++) {
const auto w = want.find(key);
if (w != want.end() && !w->second) w->second = true, at.emplace(key, i), left--;
if (fseek(f, (long)(rec_bytes - 8), SEEK_CUR) != 0) break;
}
return at;
}
// ---- <notes>.predicted.check: "predict-check <stamp> <strikes>"
struct State {
bool found = false;
uint64_t stamp = 0;
int strikes = 0;
};
inline State state_read(const std::string& path) {
State s;
FILE* f = fopen(path.c_str(), "rb");
if (!f) return s;
char line[96]{};
const size_t n = fread(line, 1, sizeof line - 1, f);
fclose(f);
line[n] = 0;
unsigned long long stamp = 0;
int strikes = 0;
if (sscanf(line, "predict-check %16llx %d", &stamp, &strikes) == 2 && strikes >= 0)
s.found = true, s.stamp = stamp, s.strikes = strikes;
return s;
}
/// Written whole beside the old one, then put in place (a start that ends in the middle leaves the old file)
inline bool state_write(const std::string& path, uint64_t stamp, int strikes) {
char line[96];
const int n = snprintf(line, sizeof line, "predict-check %016llx %d\n", (unsigned long long)stamp, strikes);
const std::string tmp = path + ".tmp";
FILE* f = fopen(tmp.c_str(), "wb");
if (!f) return false;
bool ok = fwrite(line, 1, (size_t)n, f) == (size_t)n;
ok = fclose(f) == 0 && ok;
if (ok) {
remove(path.c_str()); // (rename does not replace on Windows)
ok = rename(tmp.c_str(), path.c_str()) == 0;
}
if (!ok) remove(tmp.c_str());
return ok;
}
/// Whether the record says this driver does not keep them, for this stamp: cold again after they were built again
inline bool stopped(const State& had, uint64_t stamp) {
return had.found && had.stamp == stamp && had.strikes > kStrikes;
}
/// Whether a start checks at all: the switch, and (but for mode 2) not once the record says stop for this stamp
inline bool runs(int mode, const State& had, uint64_t stamp) {
return mode >= 2 || (mode == 1 && !stopped(had, stamp));
}
enum class Do { Nothing, Again, Stop };
/// What a start does with a sample's verdict, the record of the last starts and the stamp now. *next: the strikes to
/// record (Again and Stop). Again: the record of what is built is removed, and the builder builds them again
inline Do decide(Verdict v, const State& had, uint64_t stamp, int* next) {
if (v != Verdict::Cold) return Do::Nothing;
const bool same = had.found && had.stamp == stamp; // (found cold for this stamp before, and built again since)
*next = same ? had.strikes + 1 : 1;
return same && had.strikes >= kStrikes ? Do::Stop : Do::Again;
}
// ---- the counter: play builds of predicted programs the driver compiled
/// Whether a build of that many microseconds was compiled rather than found
inline bool slow(int64_t us) {
return us > kSlowUs;
}
/// Whether the counter's line is due: said_ms, when it was last written (0: never)
inline bool say(int64_t now_ms, int64_t said_ms) {
return !said_ms || now_ms - said_ms >= kSayMs;
}
/// The plan's programs as a sorted list without repeats, for has() (Plan: entries with .prog)
template <class Plan>
std::vector<uint64_t> programs(const std::vector<Plan>& plan) {
std::vector<uint64_t> keys;
keys.reserve(plan.size());
for (const Plan& p : plan) keys.push_back(p.prog);
std::sort(keys.begin(), keys.end());
keys.erase(std::unique(keys.begin(), keys.end()), keys.end());
return keys;
}
inline bool has(const std::vector<uint64_t>& sorted, uint64_t prog) {
return std::binary_search(sorted.begin(), sorted.end(), prog);
}
} // namespace predict_check

74
gpu/vk/predict_stream.h Normal file
View File

@@ -0,0 +1,74 @@
// (Triforce, 11-21, the performance audit's PIPE-83, after Source B's) The predicted builder's plan as record numbers, its
// notes read when their program's turn comes.
//
// predict_background (vk_draw.cpp) builds, behind the game, the predicted pipelines the start did not get to: on the starts
// after a translator or driver update that is most of them, and it rests while the game is in play, so it runs for much of
// a session. It read every note of the notes file (a PipeNote is ~27 KB), made a full note of every predicted pipeline, and
// held both until it ended: about 1 GB with the game's 14,535 notes and 24,182 predicted pipelines. The draws, the
// read-ahead and the warm read a note by its record already (pipe_note_at). Here: the notes file is gone through once into
// one record's buffer for each key's record number, the plan becomes {variant, the vertex note's record, the output note's
// record} for each program still to build, 16 bytes a pipeline, and a worker reads the two notes through its own handle
// when it builds that pipeline. The same programs and pipelines; a note gone since drops its pipeline, as before.
// RX_VK_PREDICT_BG_STREAM=0: every note and predicted note held, as before.
//
// No Vulkan and no PipeNote here: tests/test_predict_stream.py compiles it with a file of its own.
#pragma once
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace predict_stream {
/// One predicted pipeline: its program's variant, and the records of the note whose vertex layout and of the note whose
/// targets and state it takes
struct Entry {
uint64_t variant;
uint32_t vertex_rec, output_rec;
};
/// A program still to build and its pipelines, in the plan's order
struct Program {
uint64_t prog;
std::vector<Entry> entries;
};
/// Each note's key to its record (0: the first after the head), of the notes file f at its start: a head of {stamp,
/// rec_bytes}, then records of rec_bytes whose first 8 bytes are the key. rec: a buffer of rec_bytes each is read into;
/// sane(): whether the one in it is used (it may mend it). A key twice keeps its first record, as pipe_notes_read. Empty
/// when the head is another build's
template <class Sane>
std::unordered_map<uint64_t, uint32_t> index(FILE* f, uint64_t stamp, uint64_t rec_bytes, void* rec, Sane&& sane) {
std::unordered_map<uint64_t, uint32_t> at;
uint64_t head[2] = {0, 0};
if (!f || rec_bytes < 8 || fread(head, 8, 2, f) != 2 || head[0] != stamp || head[1] != rec_bytes) return at;
for (uint32_t i = 0; fread(rec, (size_t)rec_bytes, 1, f) == 1; i++) {
if (!sane()) continue;
uint64_t key;
std::memcpy(&key, rec, 8);
at.emplace(key, i);
}
return at;
}
/// The plan (entries with prog, variant, vertex_note, output_note: PredictPlan) as the programs still to build: those not
/// in had, in the order the plan first names them, each with its pipelines in the plan's order. A pipeline one of whose
/// notes is not in the index is dropped, and a program left with none is not listed
template <class Plan>
std::vector<Program> group(const std::vector<Plan>& plan, const std::unordered_map<uint64_t, uint32_t>& at,
const std::unordered_set<uint64_t>& had) {
std::vector<Program> out;
std::unordered_map<uint64_t, size_t> where;
for (const Plan& p : plan) {
if (had.count(p.prog)) continue;
const auto v = at.find(p.vertex_note), o = at.find(p.output_note);
if (v == at.end() || o == at.end()) continue;
const auto [it, fresh] = where.try_emplace(p.prog, out.size());
if (fresh) out.push_back(Program{p.prog, {}});
out[it->second].entries.push_back(Entry{p.variant, v->second, o->second});
}
return out;
}
} // namespace predict_stream

96
gpu/vk/predict_types.h Normal file
View File

@@ -0,0 +1,96 @@
// The prediction's checks (no game, no GPU: vk_predict.cpp predict_pipelines and vk_draw.cpp predict_run,
// predict_background): whether a predicted vertex layout can feed a program's vertex inputs, and the switch.
//
// (Triforce, 11-22, the performance audit's PIPE-44, after Source A's.) The predicted pipelines take a program no note names
// and give it the vertex layout of notes of the same vertex module, or else the layout most programs with its vertex inputs
// had. A program's key already says which of its inputs it reads as integers (vk_shader.cpp graphics_key), so a layout
// borrowed from another program can hand an integer input a float attribute, or the other signedness: such a pipeline is
// invalid (VUID-VkGraphicsPipelineCreateInfo-Input-08733), built for nothing or worse. Source A reads each vertex input's
// base type from the module's SPIR-V and drops a predicted layout whose attribute types definitely differ: 8,209 of 64,797
// predicted pipelines on its PC, none by colour. Scaled formats count as integers here (the translator reads them so).
//
// And a program's predictions can give the same pipeline twice (two layouts that differ only in what is dynamic, a stride):
// Source A found 420 built twice. Built once each now, by the pipeline's key, as play keys them: the same pipelines.
//
// RX_VK_PREDICT_TYPES: unset, each pipeline built once (the same pipelines as before, fewer builds); 1, that and the type
// check, which drops pipelines (off by default: it changes which pipelines exist, and a plan is made once for the shaders,
// so it applies to a plan made after it is set); 0, as before.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <unordered_map>
#include <vector>
namespace predict_types {
/// RX_VK_PREDICT_TYPES: 0 as before, 1 (unset) built once, 2 built once and the type check
inline int mode(const char* env) {
if (!env || !*env) return 1;
return std::atoi(env) == 0 ? 0 : std::atoi(env) == 1 ? 2 : 1;
}
inline bool once(int mode) { return mode >= 1; }
inline bool typed(int mode) { return mode >= 2; }
/// A module's vertex inputs by location (under 32): read, read as an integer, read as a signed integer
struct Locations {
uint32_t mask = 0, ints = 0, sign = 0;
};
/// The Input (storage class 1) variables with a Location in a SPIR-V module, and each one's base type: through its
/// pointer type to a vector's or an array's element, an OpTypeInt (signedness) or anything else (float). Built-ins carry
/// no Location and are not counted
inline Locations locations(const std::vector<uint32_t>& w) {
std::unordered_map<uint32_t, uint32_t> loc, pointee, element;
std::unordered_map<uint32_t, int> int_sign; // (OpTypeInt id: 1 signed, 0 unsigned)
std::vector<std::pair<uint32_t, uint32_t>> inputs; // (variable id, its pointer type)
for (size_t i = 5; i < w.size();) {
const uint32_t op = w[i] & 0xFFFF, n = w[i] >> 16;
if (!n || i + n > w.size()) break;
if (op == 71 && n >= 4 && w[i + 2] == 30) loc[w[i + 1]] = w[i + 3]; // OpDecorate Location
else if (op == 59 && n >= 4 && w[i + 3] == 1) inputs.push_back({w[i + 2], w[i + 1]}); // OpVariable Input
else if (op == 32 && n >= 4) pointee[w[i + 1]] = w[i + 3]; // OpTypePointer
else if ((op == 23 || op == 28) && n >= 4) element[w[i + 1]] = w[i + 2]; // OpTypeVector, OpTypeArray
else if (op == 21 && n >= 4) int_sign[w[i + 1]] = w[i + 3] ? 1 : 0; // OpTypeInt
i += n;
}
Locations out;
for (const auto& [var, ptr] : inputs) {
const auto l = loc.find(var);
if (l == loc.end() || l->second >= 32) continue;
const auto p = pointee.find(ptr);
if (p == pointee.end()) continue;
uint32_t t = p->second;
for (int depth = 0; depth < 4; depth++) { // (a vector of, an array of vectors of)
const auto e = element.find(t);
if (e == element.end()) break;
t = e->second;
}
const uint32_t bit = 1u << l->second;
out.mask |= bit;
if (const auto s = int_sign.find(t); s != int_sign.end()) {
out.ints |= bit;
if (s->second) out.sign |= bit;
}
}
return out;
}
/// What an attribute of the layout hands its location: nothing known (unset: it reads zeros, as play's), a float, an
/// unsigned or a signed integer (scaled formats are read as integers)
enum class Kind : uint8_t { Unknown, Float, Uint, Sint };
/// Whether a layout's attributes (kinds by location) can feed the module's inputs: false only where a read location's
/// attribute is definitely of the other kind (integer against float, or the other signedness)
inline bool fits(const Locations& in, const Kind kinds[16]) {
for (int i = 0; i < 16; i++) {
if (!(in.mask >> i & 1) || kinds[i] == Kind::Unknown) continue;
const bool want_int = in.ints >> i & 1, want_sign = in.sign >> i & 1;
if ((kinds[i] != Kind::Float) != want_int) return false;
if (want_int && (kinds[i] == Kind::Sint) != want_sign) return false;
}
return true;
}
} // namespace predict_types

79
gpu/vk/prepare_again.h Normal file
View File

@@ -0,0 +1,79 @@
// (Triforce, 11-22, after Casual Gamer's #dev-fixes post) Two ways a start took its shaders for prepared when they were not, apart
// from Vulkan so that tests/test_prepare_again.py runs them (vk_draw.cpp prepare_run).
//
// The program cache made again for another GPU. triforce_prepared.bin says the shaders were translated for this stamp
// (program_cache_stamp: the translator and what of the GPU and its driver changes a translation). A start on another GPU or
// with other settings that did not prepare (or did not rewrite the record) makes triforce_programs.bin again for its own
// stamp: back here the record still matches, prepare_check says nothing to do, the program cache opens with none of these
// shaders (vk_shader.cpp disk_init throws the other stamp's away), and every program is translated in play. So when the
// record says nothing to do, the program cache's head (kDiskMagic, its stamp) and size are read: a cache of another stamp, or
// of its head alone, holds none of them, and the shaders are prepared again. A cache carried over an update keeps the stamp in
// its head (its second word, the build that wrote it, is another: program_carry.h) and passes.
//
// The mark of the driver's part without stage libraries, <prepared>.warming: "this step began and did not end", an empty file.
// A start that found it left the part out for good, on whatever GPU and driver came next. Now it holds what the part is made
// for (vk_draw.cpp libraries_stamp: the GPU, the driver, the programs), as the libraries' journal does (gpl_rules.h
// journal_left): a mark of another stamp is another driver's, and the part goes on. An empty one (an older build's) is this
// driver's, as before.
// RX_VK_PREPARE_RECHECK=0: neither, as before.
#pragma once
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <string>
#include <system_error>
namespace prepare_again {
inline constexpr uint32_t kProgramsMagic = 0x31505348; // (vk_shader.cpp kDiskMagic: the program cache's head)
inline constexpr uint64_t kProgramsHead = 16; // (the magic, the build that wrote it, the stamp)
/// RX_VK_PREPARE_RECHECK: on unless it is 0
inline bool on(const char* env) { return !(env && env[0] == '0' && !env[1]); }
/// Whether a program cache, by its first n bytes (head) and its size, holds none of the shaders of stamp: no program cache's
/// head, another stamp's, or its head alone
inline bool programs_none(const uint8_t* head, size_t n, uint64_t size, uint64_t stamp) {
if (n < kProgramsHead || size <= kProgramsHead) return true;
uint32_t magic;
uint64_t was;
std::memcpy(&magic, head, 4), std::memcpy(&was, head + 8, 8);
return magic != kProgramsMagic || was != stamp;
}
/// The same for the file at path (none there: none of them). *size: its size, 0 when there is none
inline bool programs_none_at(const std::string& path, uint64_t stamp, uint64_t* size = nullptr) {
std::error_code ec;
const uint64_t n = std::filesystem::is_regular_file(path, ec) && !ec ? (uint64_t)std::filesystem::file_size(path, ec) : 0;
const uint64_t whole = ec ? 0 : n;
if (size) *size = whole;
uint8_t head[kProgramsHead]{};
size_t got = 0;
if (FILE* f = std::fopen(path.c_str(), "rb")) {
got = std::fread(head, 1, sizeof head, f);
std::fclose(f);
}
return programs_none(head, got, whole, stamp);
}
/// What the last start left of the driver's part without stage libraries: no mark, a mark of this stamp (or an empty one, an
/// older build's; or any, with by_stamp false: RX_VK_PREPARE_RECHECK=0), or one of another stamp
enum class Mark { None, Ours, Other };
inline Mark mark_left(const std::string& path, uint64_t stamp, bool by_stamp) {
FILE* f = std::fopen(path.c_str(), "rb");
if (!f) return Mark::None;
uint64_t was = 0;
const bool stamped = std::fread(&was, 8, 1, f) == 1;
std::fclose(f);
return !by_stamp || !stamped || was == stamp ? Mark::Ours : Mark::Other;
}
/// The mark written before the part: the stamp it is made for (empty with by_stamp false, as before). False: not written
inline bool mark_write(const std::string& path, uint64_t stamp, bool by_stamp) {
FILE* f = std::fopen(path.c_str(), "wb");
if (!f) return false;
const bool ok = !by_stamp || std::fwrite(&stamp, 8, 1, f) == 1;
return std::fclose(f) == 0 && ok;
}
} // namespace prepare_again

View File

@@ -58,4 +58,14 @@ inline unsigned translate(unsigned all, bool amd_older, bool asked) {
return amd_older && !asked ? std::min(all, kAmdOlderMost) : all;
}
/// (Triforce) Which heap the process is on (Windows), by the signature word at byte 16 of GetProcessHeap()'s header: the
/// segment heap's (an exe built with HOOKSHOT_SEGMENT_HEAP=1, nx2/win_manifest.py) or the NT heap's; said in the log's line
/// on the preparation's threads, so that a run says which of the two exes it was
inline constexpr uint32_t kSegmentHeapSignature = 0xDDEEDDEEu;
inline constexpr uint32_t kNtHeapSignature = 0xFFEEFFEEu;
inline const char* heap_name(uint32_t signature) {
return signature == kSegmentHeapSignature ? "segment" : signature == kNtHeapSignature ? "NT" : "not known";
}
inline bool segment_heap(uint32_t signature) { return signature == kSegmentHeapSignature; }
} // namespace prepare_threads

View File

@@ -100,11 +100,11 @@ inline std::vector<uint8_t> write(const std::vector<uint64_t>& keys, uint32_t ma
}
/// The keys of a file's bytes; none when they are not a whole file of this version (cut short, another file, a bad check word)
inline std::vector<uint64_t> read(const uint8_t* data, size_t n, uint32_t magic = kMagic) {
inline std::vector<uint64_t> read(const uint8_t* data, size_t n, uint32_t magic = kMagic, size_t max_keys = kKeysMax) {
uint32_t head[4];
if (!data || n < 24) return {};
std::memcpy(head, data, 16);
if (head[0] != magic || head[1] != kVersion || head[2] > kKeysMax || n != 16 + (size_t)head[2] * 8 + 8) return {};
if (head[0] != magic || head[1] != kVersion || head[2] > max_keys || n != 16 + (size_t)head[2] * 8 + 8) return {};
std::vector<uint64_t> keys(head[2]);
if (!keys.empty()) std::memcpy(keys.data(), data + 16, keys.size() * 8);
uint64_t c;
@@ -129,8 +129,9 @@ struct Record {
std::vector<uint64_t> keys;
std::unordered_set<uint64_t> seen;
size_t saved = 0; // (keys.size() at the last write)
size_t max = kKeysMax; // ((Triforce, 11-21) the pipelines' order keeps more: pipe_order.h kKeysMax)
bool add(uint64_t key) {
if (!key || keys.size() >= kKeysMax || !seen.insert(key).second) return false;
if (!key || keys.size() >= max || !seen.insert(key).second) return false;
keys.push_back(key);
return true;
}

37
gpu/vk/range_walk.h Normal file
View File

@@ -0,0 +1,37 @@
// (Triforce, 11-22, the performance audit's MEM-42, after Source A's) Is an address in a sorted list of ranges, asked for
// the pages of a whole loop: a cursor that only moves forward.
//
// The capture and collection loops ask of every page whether it lies in a range list sorted by first address (the
// captures' ranges, the texture storage's), and each asked with a binary search. A loop's pages come ascending, or
// nearly, so one pass over the ranges serves the whole loop. An address below the last one asked searches again, so any
// order gives the answers the search gave: tests/test_range_walk.py compares the two.
//
// A walk lives inside one loop, under the lock of its list: the list must not change while it is used.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <utility>
#include <vector>
namespace range_walk {
using Ranges = std::vector<std::pair<uint64_t, uint64_t>>; // ([first, second), sorted by first)
struct RangeWalk {
const Ranges& r;
bool search = false; // (always search, as before: RX_VK_RANGE_WALK=0, the lab knob norangewalk)
size_t next = 0; // (the first range that begins after the last address asked)
uint64_t last = 0;
/// a lies in one of the ranges: std::upper_bound(r, {a, ~0}) != begin && a < prev->second, as the loops asked
bool in(uint64_t a) {
if (search || a < last) next = (size_t)(std::upper_bound(r.begin(), r.end(), std::make_pair(a, ~uint64_t(0))) - r.begin());
else
while (next < r.size() && r[next].first <= a) next++;
last = a;
return next != 0 && a < r[next - 1].second;
}
};
} // namespace range_walk

123
gpu/vk/readback_late.h Normal file
View File

@@ -0,0 +1,123 @@
// (Triforce, 11-22, the performance audit's SYNC-19, after Source A's) Late readbacks: ranges the CPU reads back without
// the wait for their write-back. The replay's invalidate (vk_replay.cpp) waits for the write-back of every range the game is
// about to read: without it the game read 0 from its exposure meter one frame and twice the sum the next (the white flashes).
// Some ranges the game reads right after it submitted the work that fills them, so the console cannot count on that work
// either: there the wait is the GPU's whole latency for nothing but a value one frame newer. Source A names such ranges and
// reads them from what guest memory holds, the last write-back made:
// a size in hex every range of that size (Source A's default is 3980: the culling camera block, 0x3980 bytes)
// newest a range whose writer is the newest submission handed out (the game cannot have waited for it)
// RX_VK_READBACK_LATE=<rule>[,<rule>...]. Unset: NO rule here until the lab has run it (the audit's lab proof: a handheld or an
// AMD iGPU, where the waits are long); =0: none either. The lab knob rblate (lab_ab.h) turns the rules a later default would
// have (default_rules: 3980) on in its phases when the variable is unset. A late range keeps its note for the CPU's
// write-back (memory_note_readback) and skips the catch-up and the wait, nothing else.
// What it risks: a late read hands the game the frame before's values; with mirrors, when the CPU resets a range read late,
// the pending write-back lands over its reset (the exposure meter's 2x flashes). `newest` can catch that meter, so it is
// never a default. RX_VK_LOG_READBACK_RANGES=1 counts the readbacks by range (RangeLog) to find such ranges on other hardware.
// tests/test_readback_late.py
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <map>
#include <utility>
#include <vector>
namespace mkwvk::readback_late {
constexpr int kMaxSizes = 8;
/// The size Source A reads late by default: the culling camera block
constexpr uint64_t kCameraBlock = 0x3980;
struct Rules {
uint64_t sizes[kMaxSizes] = {};
int n = 0;
bool newest = false;
int bad = 0; // (parts of the text that were neither a hex size nor "newest", or sizes past kMaxSizes: left out)
bool any() const { return n > 0 || newest; }
bool has(uint64_t size) const {
for (int i = 0; i < n; i++)
if (sizes[i] == size) return true;
return false;
}
};
/// The rules a default would have once the lab has passed them (and the knob rblate has now): the camera block's size alone
inline Rules default_rules() {
Rules r;
r.sizes[r.n++] = kCameraBlock;
return r;
}
/// RX_VK_READBACK_LATE's text. Unset or empty: no rule (until the lab passes; then default_rules()). "0": none. Otherwise parts
/// separated by commas: "newest", or a size in hex with or without 0x. A part that is neither is left out and counted in bad
inline Rules parse(const char* text) {
Rules r;
if (!text || !*text) return r;
for (const char* p = text; *p;) {
const char* end = std::strchr(p, ',');
const size_t len = end ? (size_t)(end - p) : std::strlen(p);
if (len == 6 && !std::strncmp(p, "newest", 6)) {
r.newest = true;
} else if (len > 0 && len < 20) {
char part[24] = {};
std::memcpy(part, p, len);
char* stop = nullptr;
const unsigned long long v = std::strtoull(part, &stop, 16);
if (stop == part || *stop) r.bad++;
else if (v == 0) {} // ("0": none, and no rule of its own)
else if (r.has(v)) {}
else if (r.n < kMaxSizes) r.sizes[r.n++] = v;
else r.bad++;
} else if (len) {
r.bad++;
}
if (!end) break;
p = end + 1;
}
return r;
}
/// Whether a readback of `size` bytes is read late. writer_newest: its write-back's value is at or past the newest value
/// handed out (the fence a game's submit got last)
inline bool wanted(const Rules& r, uint64_t size, bool writer_newest) {
return (r.newest && writer_newest) || r.has(size);
}
/// The decision invalidate makes: only a range whose write-back is still ahead of the GPU (pending) can be late; one whose
/// write-back is done waits for nothing anyway
inline bool late(const Rules& r, uint64_t size, uint64_t writer_value, uint64_t gpu_done, uint64_t newest_handed_out) {
return r.any() && writer_value > gpu_done && wanted(r, size, writer_value >= newest_handed_out);
}
/// RX_VK_LOG_READBACK_RANGES: the readbacks by range. Not thread safe: the caller locks
struct RangeLog {
struct Row {
int64_t readbacks = 0, waits = 0, wait_ns = 0, newest = 0, late = 0;
};
std::map<std::pair<uint64_t, uint64_t>, Row> rows; // ((address, size))
size_t cap = 4096; // (the game reads a ring of slots at new addresses: past this, the rest is counted together)
Row other;
void note(uint64_t gpu, uint64_t size, bool waited, int64_t ns, bool writer_newest, bool was_late) {
Row* row = &other;
auto it = rows.find({gpu, size});
if (it != rows.end()) row = &it->second;
else if (rows.size() < cap) row = &rows[{gpu, size}];
row->readbacks++, row->waits += waited, row->wait_ns += waited ? ns : 0, row->newest += writer_newest, row->late += was_late;
}
struct Line {
uint64_t gpu = 0, size = 0;
Row row;
};
/// The n ranges that waited longest in all, longest first (ties: the lower address first)
std::vector<Line> top(size_t n) const {
std::vector<Line> all;
all.reserve(rows.size());
for (const auto& [key, row] : rows) all.push_back({key.first, key.second, row});
std::stable_sort(all.begin(), all.end(), [](const Line& a, const Line& b) { return a.row.wait_ns > b.row.wait_ns; });
if (all.size() > n) all.resize(n);
return all;
}
};
} // namespace mkwvk::readback_late

56
gpu/vk/recorder_auto.h Normal file
View File

@@ -0,0 +1,56 @@
// (Triforce) The recorder thread's default (vk_recorder.h g_auto_thread, RX_VK_RECORDER unset), chosen once the device is known
// (vk_main.cpp pick_device). Plain code with no Vulkan or system calls: tests/test_recorder_auto.py runs it on made-up PCs.
//
// - NVIDIA: on with 12 logical processors or more, as since 11-16 (lab, RTX 4070 SUPER + i7-13700K: Market +11%; held to 8
// logical processors -10%: the recorder needs a core of its own).
// - AMD: on with 6 physical cores or more in the process's affinity. AMD's driver spends about twice NVIDIA's CPU time on a call,
// so the replay thread's vkCmd* calls cost it more; taken out of a renderer of the same per-call design, the densest Market view
// (~2,500 draws) fell 16-25% behind on RX 7900 XTX, 9070 XT and 9060 XT cards. Physical cores, not logical processors: the
// recorder spins on a core while the replay records, and an SMT sibling of a busy core is not one (a 4-core Steam Deck: off).
// Not on AMD's older Windows driver (vk_vendor.h amd_older_compiler, RX 5000 and RX 6000): it lost the device with commands
// recorded beside the replay thread (replay_threads.h), so it waits for a lab of its own.
// - Anything else (Intel, a GPU not named): off, nothing measured.
// RX_VK_RECORDER=thread or =off goes over this either way.
#pragma once
#include <cstdint>
namespace mkwvk::recorder_default {
enum Why : int { kNvidia, kNvidiaFew, kAmd, kAmdFew, kAmdUnknown, kAmdOlder, kOther };
/// Why the recorder is on or off by default. logical: cpu_count() (cpu_count.h: hardware_concurrency, or the lab's HOS_SIM_CPU); physical: physical_cores (0: unknown)
constexpr Why why(uint32_t vendor_id, unsigned logical, unsigned physical, bool amd_older) {
if (vendor_id == 0x10DE) return logical >= 12 ? kNvidia : kNvidiaFew;
if (vendor_id == 0x1002) {
if (amd_older) return kAmdOlder;
if (!physical) return kAmdUnknown;
return physical >= 6 ? kAmd : kAmdFew;
}
return kOther;
}
constexpr bool on(Why w) { return w == kNvidia || w == kAmd; }
constexpr bool rule(uint32_t vendor_id, unsigned logical, unsigned physical, bool amd_older) {
return on(why(vendor_id, logical, physical, amd_older));
}
constexpr const char* text(Why w) {
switch (w) {
case kNvidia: return "NVIDIA, 12 logical processors or more";
case kNvidiaFew: return "NVIDIA, under 12 logical processors";
case kAmd: return "AMD, 6 physical cores or more";
case kAmdFew: return "AMD, under 6 physical cores";
case kAmdUnknown: return "AMD, the physical cores not known";
case kAmdOlder: return "AMD's driver for RX 5000 and RX 6000 (not tried there)";
default: return "not an NVIDIA or AMD GPU (not measured)";
}
}
/// The physical cores with at least one logical processor in affinity: masks[i] is core i's logical processors (one
/// RelationProcessorCore entry each: GetLogicalProcessorInformation, which on Linux reads sysfs's package and core ids)
inline unsigned physical_cores(const uint64_t* masks, int n, uint64_t affinity) {
unsigned cores = 0;
for (int i = 0; i < n; i++) cores += (masks[i] & affinity) != 0;
return cores;
}
} // namespace mkwvk::recorder_default

57
gpu/vk/redraw_state.h Normal file
View File

@@ -0,0 +1,57 @@
// (Triforce, 11-21, the upscaler audit's UPS-03) What a redraw of the stored draws has to set before it draws.
//
// The stored draws are drawn again into the motion vectors in one pass (vk_temporal.cpp capture_motion), about 900 of
// them a frame in the owner's log of 11-21. Each one set the primitive topology, the primitive restart and the index
// buffer before it drew, whatever the draw before had set: three commands a draw beside its push constants and the draw
// itself, and nearly every draw of the scene is a list of triangles without restart. Dynamic state and the index buffer
// stay as they were set until they are set again in the same command buffer, so a command that would set what is set
// already says nothing new. Here is what the pass has set so far, and whether a draw's own differs from it.
//
// A pass starts knowing nothing (a new Pass): its first draw sets everything, as before.
// RX_VK_REDRAW_STATE=0: every draw sets all three, as before (every()).
//
// No Vulkan: tests/test_redraw_state.py compiles this.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace redraw_state {
/// RX_VK_REDRAW_STATE (value, null when unset): 0, every draw sets its state whatever is set, as before
inline bool every(const char* value) { return value && *value && std::atoi(value) == 0; }
/// What one pass has set so far, from its first draw on
struct Pass {
bool all = false; // (every(): each asks true)
bool have_topology = false, have_restart = false, have_index = false;
uint32_t topology = 0;
bool restart = false;
uint64_t index_buffer = 0, index_offset = 0;
uint32_t index_type = 0;
uint64_t set = 0, left = 0; // (commands recorded, and left out: for a count in the log)
};
/// Whether the topology is to be set for a draw of `topology` (and it is then what the pass has set)
inline bool topology(Pass& p, uint32_t topology) {
if (!p.all && p.have_topology && p.topology == topology) return p.left++, false;
p.have_topology = true, p.topology = topology;
return p.set++, true;
}
/// The same for primitive restart
inline bool restart(Pass& p, bool restart) {
if (!p.all && p.have_restart && p.restart == restart) return p.left++, false;
p.have_restart = true, p.restart = restart;
return p.set++, true;
}
/// The same for an indexed draw's index buffer: the buffer's handle as a number, the offset into it and the index type,
/// all three the same as the one bound
inline bool index(Pass& p, uint64_t buffer, uint64_t offset, uint32_t type) {
if (!p.all && p.have_index && p.index_buffer == buffer && p.index_offset == offset && p.index_type == type)
return p.left++, false;
p.have_index = true, p.index_buffer = buffer, p.index_offset = offset, p.index_type = type;
return p.set++, true;
}
} // namespace redraw_state

64
gpu/vk/release_queue.h Normal file
View File

@@ -0,0 +1,64 @@
// (Triforce, 11-22, the performance audit's MEM-85, after Source B's idea) Texture and program releases queued for the
// replaying thread, not waited for.
//
// The game's nvnTextureFinalize and nvnProgramFinalize reach vk_replay.cpp texture_released_of and program_released_locked.
// Without the replay thread (RX_VK_ASYNC=0, the default) a release is held for the replaying thread only while a queue is
// stopped at a wait (PR 430: after the stopped work, which may still use it); otherwise the game's thread takes replay_mu,
// which a replay holds through whole submissions: the finalizing thread waited 45% of its time in Kokiri before PR 430.
//
// RX_VK_RELEASE_QUEUE=1 (OFF unless set; the lab knob relq in its phases while it is unset): every such release is pushed to
// a list under held_mu, stopped queue or not, and applied by the thread that replays next, at the start of a submission's
// replay or a present (replay_mu held there already), when no queue is stopped; with one stopped, update_blocked applies
// them after the stopped work, as PR 430's. A texture's release carries the write generation of its call, so the CPU's
// writes after the call still count as written after the release (vk_texture.cpp release_gen). The RX_VK_ASYNC job path is
// not touched. Unset, a release goes the way it went: way() is the old decision then (tests/test_release_queue.py).
//
// And a count, on by default: the releases that went the old way, their time in sync_drain and the replay's lock, the
// longest; a line every 600 releases when one of them waited 0.5 ms or more.
//
// The rule alone: no Vulkan, no game, no clock.
#pragma once
#include <cstddef>
#include <cstdint>
#include <cstdlib>
namespace release_queue {
/// RX_VK_RELEASE_QUEUE: a number other than 0 on, 0 off; unset: the lab knob relq (on only in its phases)
inline bool on(const char* env, bool knob) {
if (env) return std::atoi(env) != 0;
return knob;
}
/// Where a release without the replay thread goes. Now: sync_drain(1) and the replay's lock on the caller's thread, as
/// before; Held: PR 430's list, while a queue is stopped (hold: RX_VK_RELEASE_DRAIN unset; sync_waits: the event ordering
/// on); Queued: the switch's list, whatever is stopped
enum class Way { Now, Held, Queued };
inline Way way(bool queue_on, bool hold, bool sync_waits, bool blocked) {
if (queue_on) return Way::Queued;
return hold && sync_waits && blocked ? Way::Held : Way::Now;
}
/// The old decision, for the test: held or not
inline bool held_before(bool hold, bool sync_waits, bool blocked) { return hold && sync_waits && blocked; }
/// Whether the queued releases are applied at a submission's replay start or a present: something queued, no queue stopped
inline bool apply_now(size_t queued, int blocked) { return queued != 0 && blocked == 0; }
/// The releases that went the old way: how many, how many waited kWaitNs or more, their time, the longest
struct Waits {
static constexpr int64_t kWaitNs = 500000; // (0.5 ms: a replay met, not the release's own work)
static constexpr int64_t kEvery = 600;
int64_t n = 0, waited = 0, ns = 0, max_ns = 0, queued = 0;
void note(int64_t took_ns) {
n++, ns += took_ns;
if (took_ns >= kWaitNs) waited++;
if (took_ns > max_ns) max_ns = took_ns;
}
void note_queued() { queued++; }
/// Every kEvery releases (either way) the counts start again; a line when one of them waited, or went to the switch's list
bool due() const { return n + queued >= kEvery; }
bool worth_a_line() const { return waited > 0 || queued > 0; }
void reset() { *this = Waits{}; }
};
} // namespace release_queue

77
gpu/vk/replay_pin.h Normal file
View File

@@ -0,0 +1,77 @@
// (Triforce, 11-22, the performance audit's THREAD-09, after Source A's) Where a thread may run while it replays, on a processor with
// cores of more than one kind.
//
// A thread that replays is pinned for that time to the core the host keeps for the replay (vk_replay.cpp ReplayPin,
// hos_reserved_cpus). That core is kept from the game's threads, not from other programs': on a 6+8-core hybrid processor Source A
// saw Windows park one of its two logical processors while another program's thread at priority 15 held the other 30 to 40 ms at a
// time, and the pinned replay, raised only to the highest of the normal levels, waited for it ready to run: 14 to 37 frames over
// 40 ms in a 40 s run. So on a processor of two kinds of core the pin is soft: the thread may run on the reserved core and on the
// fastest-kind cores among its own processors, its ideal processor is the reserved core's last logical processor, and the core it
// is on when the pin starts is left out for that pin, so that it moves at once, to the reserved core when that is free. Widening
// alone would leave it where it is: that is the "only preferring the core" that gained nothing. After the replay it has the
// affinity and the ideal processor it had.
//
// A processor of one kind of core keeps the strict pin: there the pin itself is what was measured to help, and nothing changes.
// RX_VK_REPLAY_PIN_SOFT unset or 1: soft where the cores are of more than one kind; 0: strict everywhere, as before;
// 2: soft everywhere (the lab).
// Plain rules over masks, no system calls: tests/test_replay_pin_soft.py runs them over made-up processors.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace mkwvk::replay_pin {
constexpr int kStrict = 0, kHybrid = 1, kAlways = 2;
/// RX_VK_REPLAY_PIN_SOFT's value: unset (or empty, or 1) kHybrid, 0 (or anything that is no number above 0) kStrict, 2 and more kAlways
inline int mode(const char* env) {
if (!env || !*env) return kHybrid;
const int v = std::atoi(env);
return v <= 0 ? kStrict : v == 1 ? kHybrid : kAlways;
}
struct Kinds {
uint64_t fastest = 0; // (the logical processors of the cores with the highest efficiency class)
bool hybrid = false; // (some core is of a lower class)
};
/// The fastest kind of core among n cores (each core's logical processors and its EfficiencyClass, higher the faster, as Windows
/// lists them; a core with no processors is not one). hybrid only when a listed core is not of that kind
inline Kinds kinds(const uint64_t* masks, const unsigned char* classes, int n) {
Kinds k;
int best = -1;
uint64_t all = 0;
for (int i = 0; i < n; i++) {
if (!masks[i]) continue;
all |= masks[i];
if ((int)classes[i] > best) best = classes[i], k.fastest = masks[i];
else if ((int)classes[i] == best) k.fastest |= masks[i];
}
k.hybrid = k.fastest != all;
return k;
}
/// Whether the pin is soft: by the switch and the processor's kinds
inline bool soft(int mode, bool hybrid) {
return mode >= kAlways || (mode == kHybrid && hybrid);
}
/// The soft pin's processors: the reserved core, and the fastest-kind ones among the thread's own (own: its affinity before the
/// pin, which keeps a thread held to one die, or to anything else, inside what it was held to). Never fewer than the reserved core
inline uint64_t wide(uint64_t reserved, uint64_t fastest, uint64_t own) {
return reserved | (fastest & own);
}
/// The same without the core the thread is on now (now: that core's logical processors, 0 when not known), so that the thread
/// moves at once. Left whole when it is on the reserved core already or when nothing else would be left
inline uint64_t moved(uint64_t wide, uint64_t reserved, uint64_t now) {
if (!now || (now & reserved) || !(wide & ~now)) return wide;
return wide & ~now;
}
/// The ideal processor of a replaying thread: the reserved core's last logical processor (-1: no core reserved)
inline int ideal(uint64_t reserved) {
return reserved ? 63 - __builtin_clzll(reserved) : -1;
}
} // namespace mkwvk::replay_pin

View File

@@ -25,7 +25,8 @@
// they record only those trial frames. (Triforce, 11-15) And its rule (rule below): where the replay is plainly the frame's
// limit they go on without a trial, and where they cannot pay no trial is made. (Triforce, 11-16) And once a way has held,
// trials come ever rarer (up to some 14,400 presents apart) and shorter, and early only when the scene clearly changes (the
// back-off, at Tuner).
// back-off, at Tuner). (Triforce, 11-22) And both judge settled scenes only: the rule no window of a load or an arrival, and a
// trial the scene changed under decides nothing (settled, at Tuner).
//
// The parallel replay was audited for this by reading (nothing could be run). What was found and is fixed, each with a
// check in tests/test_parallel_fixes.py:
@@ -236,6 +237,18 @@ inline Verdict rule(const Frame& m, int threads) {
/// trial whose first and last parts (the same way) moved so had the scene change under it: the windows after it are the
/// new scene's reference, so the back-off starts again there too, the next trial kEvery on.
/// A change of way, by a trial or by the rule, starts the back-off again. tests/test_replay_threads_tuner.py "backoff"
/// (Triforce, 11-22) Settled (settled; vk_replay.cpp par_settled_on, RX_VK_TUNER_SETTLED=0: off, as before), after
/// Kaefergeneral's #dev-fixes post: on a 5900X with an RTX 3090, where the helpers lost in every settled scene (1.2 to 4.5 ms a
/// frame), Automatic was 6 to 10% slower than Off. The rule turned them on in the window of an area's load (long frames, the
/// replay 71 to 82% of them, nearly every draw cuttable), not in the settled Market (the replay 62%), and the first trial after
/// the arrival straddled the load ("parallel 4.76 ms a frame, the other way 24.10, parallel again 17.25: as it was"): the
/// scene's change was seen (moved) but the verdict stood, for kEvery presents or more. So:
/// - the rule judges a window only if the one before it, on the replay thread alone, describes the same scene (scene_moved,
/// the back-off's test): a load's window and the first after it get no verdict, neither "on" nor "no trial". After a trial
/// or a change of way the first window has none before it, so the rule's "on" comes a window later than it did;
/// - a trial whose first and last parts moved so decides nothing, either way (not better, no recheck), and the next one comes
/// kWindow after a window of the way in use like the one before it (settle_look), not kEvery later, and never while the rule
/// says no trial. kHold and the thresholds as they were. tests/test_replay_threads_tuner.py "settled"
class Tuner {
public:
static constexpr int kEvery = 900, kTake = 80, kSettle = 24, kWindow = 120, kHold = 4 * kEvery;
@@ -243,6 +256,7 @@ public:
static constexpr double kClear = 1.10, kSceneDraws = 0.25, kSceneDrawsMin = 200, kScenePart = 0.30, kScenePartMs = 1.0;
bool use_rule = true;
bool backoff = true; // (the back-off above; false: every step as before it)
bool settled = true; // (the rule and the trials only on a settled scene, above; false: as before)
int threads = 4; // (the threads that record segments: rule()'s)
struct Step {
bool flip = false; // change the way in use
@@ -262,6 +276,9 @@ public:
bool moved = false; // (the scene changed under the trial that ended: the back-off starts again)
bool sure = false; // (the trial kept the way, the other way longer than both its parts: the back-off grows)
int64_t next = 0; // (the present the next trial may begin at: set at a trial's end and at a scene change)
// (settled's)
bool unsettled = false; // (a window the rule did not judge, or a trial that decided nothing: the scene moved)
bool settle = false; // (the scene settled after such a trial: the next trial at next)
};
/// f: the frame just presented; on: the helper threads in use for it; order_only: both ways are the replay thread alone
Step step(const Frame& f, bool on, bool order_only) {
@@ -272,31 +289,44 @@ public:
if (f.frame <= 60.0) win_.add(f);
if (++win_frames_ >= kWindow) {
if (win_.n >= kWindow * 3 / 4 && !order_only) {
s.ruled = true, s.window = win_.mean(), s.verdict = rule(s.window, threads);
const bool turn_on = s.verdict.kind == Verdict::kOn && frames_ >= hold_until_;
s.rule_new = s.verdict.kind != last_ || turn_on;
last_ = s.verdict.kind;
if (turn_on) {
s.flip = true;
next_trial_ = frames_ + kEvery;
if (backoff) held_ = 0, fresh_ = true, scene_restart();
} else if (s.verdict.kind == Verdict::kOff) {
next_trial_ = std::max(next_trial_, frames_ + kEvery);
no_trial_until_ = frames_ + kEvery;
// (settled: a window is judged only if the one before it, on the replay thread alone too, describes
// the same scene)
s.window = win_.mean();
const bool same = !settled || (win_prev_set_ && !scene_moved(win_prev_, s.window));
win_prev_ = s.window, win_prev_set_ = true;
if (!same) {
s.unsettled = true;
} else {
s.ruled = true, s.verdict = rule(s.window, threads);
const bool turn_on = s.verdict.kind == Verdict::kOn && frames_ >= hold_until_;
s.rule_new = s.verdict.kind != last_ || turn_on;
last_ = s.verdict.kind;
if (turn_on) {
s.flip = true;
next_trial_ = frames_ + kEvery;
waiting_ = false, win_prev_set_ = false; // (the way changed: a trial kEvery on, as after a change)
if (backoff) held_ = 0, fresh_ = true, scene_restart();
} else if (s.verdict.kind == Verdict::kOff) {
next_trial_ = std::max(next_trial_, frames_ + kEvery);
no_trial_until_ = frames_ + kEvery;
}
}
} else {
win_prev_set_ = false; // (a load, or DLSS / FSR kept in order: the next window has none before it)
}
win_ = Part{}, win_frames_ = 0;
if (s.flip) return s;
}
} else {
win_ = Part{}, win_frames_ = 0;
win_ = Part{}, win_frames_ = 0, win_prev_set_ = false;
}
if (backoff) scene_look(f, s);
if (waiting_) settle_look(f, s);
else if (backoff) scene_look(f, s);
if (frames_ < next_trial_) return s;
if (order_only) return s;
phase_ = 1, n_ = 0;
short_ = backoff && held_ >= 2;
win_ = Part{}, win_frames_ = 0; // (the rule's next window begins after the trial)
win_ = Part{}, win_frames_ = 0, win_prev_set_ = false; // (the rule's next window begins after the trial)
sum_[0] = sum_[1] = sum_[2] = 0, cnt_[0] = cnt_[1] = cnt_[2] = 0;
part_[0] = part_[1] = part_[2] = Part{};
return s;
@@ -318,6 +348,11 @@ public:
s.a = cnt_[0] ? sum_[0] / cnt_[0] : 0, s.b = cnt_[1] ? sum_[1] / cnt_[1] : 0, s.c = cnt_[2] ? sum_[2] / cnt_[2] : 0;
s.better = s.ok && s.b < std::min(s.a, s.c) * 0.96;
s.short_trial = short_;
// (settled: a trial whose first and last parts, the same way, describe two scenes measured the change of scene, not
// the way: it decides nothing, either way, and the next one waits for the scene to settle)
const bool moved = (backoff || settled) && cnt_[0] && cnt_[2] && scene_moved(part_[0].mean(), part_[2].mean());
s.moved = moved;
if (settled && moved) s.better = false, s.unsettled = true;
if (s.better && short_) s.better = false, s.recheck = true; // (a full trial decides: at once, below)
if (s.better) {
s.flip = true, changes_++;
@@ -334,7 +369,6 @@ public:
// put off ever longer; such a trial costs little, its two ways' frames being about as long. The first trial
// after a change of scene or way that is too close to tell: kEvery, a second look soon. The trials' means
// drift with the scene, and right after a change the first is the one most often wrong)
s.moved = cnt_[0] && cnt_[2] && scene_moved(part_[0].mean(), part_[2].mean());
const double top = std::max(s.a, s.c);
s.sure = s.ok && !s.better && !s.recheck && !s.moved && s.b > top;
const bool again = !s.sure && fresh_;
@@ -345,6 +379,10 @@ public:
last_end_ = frames_;
scene_restart();
}
if (s.unsettled) { // (the next trial kWindow after a window like the one before it: settle_look)
waiting_ = true, next_trial_ = INT64_MAX;
settle_win_ = Part{}, settle_frames_ = 0, settle_prev_set_ = false;
}
s.next = next_trial_;
s.trial = true, s.parts = part_;
}
@@ -367,6 +405,22 @@ public:
private:
void scene_restart() { scene_win_ = Part{}, scene_frames_ = 0, ref_set_ = false, strikes_ = 0; }
/// (settled: after a trial the scene moved under, the way's windows each against the one before; the first like it, and
/// the next trial kWindow on, never while the rule says no trial. A load breaks the run)
void settle_look(const Frame& f, Step& s) {
if (f.frame <= 60.0) settle_win_.add(f);
if (++settle_frames_ < kWindow) return;
const bool counted = settle_win_.n >= kWindow * 3 / 4;
const Frame m = settle_win_.mean();
settle_win_ = Part{}, settle_frames_ = 0;
const bool same = counted && settle_prev_set_ && !scene_moved(settle_prev_, m);
settle_prev_ = m, settle_prev_set_ = counted;
if (!same) return;
waiting_ = false, s.settle = true;
next_trial_ = std::max(frames_ + kWindow, no_trial_until_);
s.next = next_trial_;
if (backoff) scene_restart(); // (the settled scene is the back-off's reference)
}
/// (between trials: the scene's windows against the reference, the next trial brought forward when two in a row moved)
void scene_look(const Frame& f, Step& s) {
if (f.frame <= 60.0) scene_win_.add(f);
@@ -406,6 +460,11 @@ private:
int scene_frames_ = 0, strikes_ = 0;
bool ref_set_ = false;
Frame ref_;
// (settled's: the rule's window before, and after a trial the scene moved under, the wait for it to settle and its windows)
Frame win_prev_, settle_prev_;
bool win_prev_set_ = false, waiting_ = false, settle_prev_set_ = false;
Part settle_win_;
int settle_frames_ = 0;
};
} // namespace replay_threads

View File

@@ -0,0 +1,85 @@
// (Triforce, 11-22, the performance audit's MEM-21, after Source A's) When a retired pool's import is given back.
//
// A pool remade larger is retired: never bound again, and once the GPU is past it its import is pinned RAM nothing uses.
// vk_memory.cpp retired_imports_give_back gives such imports back at once on Linux and with NVIDIA's driver on Windows, and
// keeps them for the session with AMD's and Intel's drivers on Windows: given back at once there, an Intel Arc B390, an
// Ally X and an RX 6600 lost the device right after. It never gives back the import of a pool whose mirror was refused
// (RX_VK_NO_MIRROR_POOLS, a pool over 1 GB, VRAM refused), on any OS.
//
// Source A gives them back on Windows AMD and Intel after a wait instead: 30 s after the pool retired, or 2 s while the
// system heap is past 85% of its budget; and the refused mirrors' imports too. Here both are OFF unless asked:
// RX_VK_KEEP_RETIRED=wait the wait, on any driver (RX_VK_RETIRE_SECS, 30; RX_VK_RETIRE_MIN_SECS, 2)
// RX_VK_RETIRED_REFUSED=1 the refused mirrors' imports too, by the mode in force (with `wait`: on unless =0)
// Unset, and every value RX_VK_KEEP_RETIRED took before, decide exactly as before: tests/test_retired_give_back.py compares
// want() with the old condition on random inputs. NVIDIA's driver on Windows keeps its behaviour.
//
// The rule alone, no Vulkan, no clock: the caller passes the times in milliseconds.
#pragma once
#include <cstdint>
#include <cstdlib>
#include <cstring>
namespace retired_give_back {
enum class Mode { AtOnce, Wait, Keep };
/// RX_VK_KEEP_RETIRED: "wait" the wait; any other value as it was read before (a value beginning with 0 gives back at once,
/// anything else keeps, the empty text too); unset: kept on Windows unless the card is NVIDIA's, given back at once elsewhere
inline Mode mode(const char* env, bool windows, bool nvidia) {
if (env) {
if (std::strcmp(env, "wait") == 0) return Mode::Wait;
return env[0] != '0' ? Mode::Keep : Mode::AtOnce;
}
return windows && !nvidia ? Mode::Keep : Mode::AtOnce;
}
/// The old reading, for the test: whether retired imports were kept
inline bool keep_before(const char* env, bool windows, bool nvidia) {
if (windows) return env ? env[0] != '0' : !nvidia;
return env && env[0] != '0';
}
/// RX_VK_RETIRED_REFUSED: a number other than 0 on, 0 off; unset: on with the wait (Source A's rule whole), off otherwise
inline bool refused_too(const char* env, Mode m) {
if (env) return std::atoi(env) != 0;
return m == Mode::Wait;
}
/// RX_VK_RETIRE_SECS, RX_VK_RETIRE_MIN_SECS: whole seconds, 0 or more; unset or not a number of 0 or more: the default
inline int64_t secs(const char* env, int64_t fallback) {
if (!env || !*env) return fallback;
char* end = nullptr;
const long long v = std::strtoll(env, &end, 10);
return end && *end == 0 && v >= 0 ? (int64_t)v : fallback;
}
/// Whether the system heap is short: used past 85% of its budget (no budget known: not short)
inline bool pressure(uint64_t used, uint64_t budget) {
return budget && used > budget / 100 * 85;
}
/// The wait over: now_ms - retired_ms at least min_s seconds while the heap is short, wait_s seconds otherwise
inline bool due(int64_t now_ms, int64_t retired_ms, int64_t wait_s, int64_t min_s, bool short_heap) {
return now_ms - retired_ms >= (short_heap ? min_s : wait_s) * 1000;
}
/// Whether a retired pool's import goes back now. mirror_none, mirror_refused: its mirror's state; inherit: the pool that took
/// it in still copies from its mirror; gpu_past: the GPU is past its retirement and its last use; is_due: due() (the wait only)
inline bool want(Mode m, bool refused, bool mirror_none, bool mirror_refused, bool inherit, bool gpu_past, bool is_due) {
if (m == Mode::Keep) return false;
if (!mirror_none && !(refused && mirror_refused)) return false;
if (inherit || !gpu_past) return false;
return m == Mode::AtOnce || is_due;
}
/// The old condition, for the test
inline bool want_before(bool keep, bool mirror_none, bool inherit, bool gpu_past) {
return !keep && mirror_none && !inherit && gpu_past;
}
/// The system heap's use as the pressure give-back (imports_give_back) should read it with the wait on: less the imports given
/// back and not freed yet (dead_imports, freed once the GPU is past them), which the driver still counts. One overshoot,
/// one valve: without this a burst of retired imports given back made it give back live ones as well
inline uint64_t settled(uint64_t used, uint64_t dead_bytes) {
return used > dead_bytes ? used - dead_bytes : 0;
}
} // namespace retired_give_back

58
gpu/vk/settle_gate.h Normal file
View File

@@ -0,0 +1,58 @@
// (Triforce, 11-22, the performance audit's SYNC-29, after Source A's) A CPU read looks at the stopped queues only after a
// new CPU signal. A read that settles (vk_replay.cpp sync_settle: the stopped queues whose waits are met go on, none forced)
// takes sync_order_mu whenever a queue is stopped, and that lock is held through whole replays and presents: the readback's
// look (RX_VK_RB_SETTLE, off) and the two looks invalidate makes for a range a texture may be in.
//
// What can make a stopped queue's wait met:
// a signal in the stream replayed by a thread that holds sync_order_mu, and every such holder runs schedule() and ends its
// burst (burst_end: finish_recording) before it lets the lock go: it looked itself;
// a signal of the CPU's event_signal_queued: queued, then counted (signals);
// a plain store into the memory of an event nothing has set yet (the wait reads its value from guest memory,
// event_value_in): not seen by anything, so such a wait keeps every look (wait_in_memory).
// So a look is needed only while a queue is stopped and the CPU has signalled since the last look, or a stopped queue waits
// on an event read from memory. A look notes the signals' count at its start (burst_begin) and publishes it once its recording
// is finished (burst_end, after finish_recording): a reader that sees signals == settled finds the work those signals let on
// recorded. Publishing the count as it is at the look's END would cover a signal that came after the look's schedule() and
// was never acted on; counting a signal BEFORE it is queued would let a look publish it without having seen it. The test runs
// both mistakes and finds them.
//
// RX_VK_SETTLE_GATE: unset or 0, no gate (every look as before); 1 the readback's look (the one RX_VK_RB_SETTLE turns on);
// 2 the texture sites' looks too. Those two order the exposure meter's readback: the trial that skipped them outright made
// the Market flicker at once, and Source A left them ungated, so 2 is new ground and needs the lab's flicker count. The lab
// knobs settlegate (1) and texgate (2) in their phases. tests/test_settle_gate.py
#pragma once
#include <cstdint>
namespace mkwvk::settle_gate {
/// The sites that settle
enum Site { kReadback = 0, kTexture = 1, kSites = 2 };
/// RX_VK_SETTLE_GATE's text: "1" and "2"; unset, empty, "0" and anything else: 0, no gate
inline int level(const char* text) {
if (!text || !text[0] || text[1]) return 0;
return text[0] == '1' ? 1 : text[0] == '2' ? 2 : 0;
}
/// The level in force: the variable's, raised by the lab knobs in their phases
inline int level_now(int env, bool knob_settlegate, bool knob_texgate) {
int l = env;
if (knob_settlegate && l < 1) l = 1;
if (knob_texgate && l < 2) l = 2;
return l;
}
/// Whether a site's look goes through the gate at this level
inline bool gated(int level, int site) { return site == kReadback ? level >= 1 : level >= 2; }
/// Whether a gated site looks: a queue stopped, and a CPU signal since the last look or a stopped queue waiting on an event
/// whose value is read from guest memory
inline bool look(bool blocked, uint64_t signals, uint64_t settled, bool wait_in_memory) {
return blocked && (signals != settled || wait_in_memory);
}
/// The whole decision of a site: 0 nothing to do (no queue stopped: as before), 1 look, 2 skipped by the gate
enum What { kNothing = 0, kLook = 1, kSkip = 2 };
inline int decide(int level, int site, bool blocked, uint64_t signals, uint64_t settled, bool wait_in_memory) {
if (!blocked) return kNothing;
if (gated(level, site) && !look(true, signals, settled, wait_in_memory)) return kSkip;
return kLook;
}
} // namespace mkwvk::settle_gate

View File

@@ -0,0 +1,16 @@
#version 460
// (Triforce, 11-22, the performance audit's OS-49, after Source B's) RX_VK_SIM_GPU_SLOW's burn (gpu/vk/sim_gpu.h): one
// invocation runs a chain of dependent multiply-adds `loops` times, so that a dispatch takes a time the loop count sets; the
// result goes to a word nothing reads, so that the chain is not left out.
// glslangValidator -V --target-env vulkan1.3 --vn SIM_GPU_BURN_COMP_SPV -o sim_gpu_burn_comp_spv.h sim_gpu_burn.comp
layout(local_size_x = 1) in;
layout(push_constant) uniform P {
uint loops;
float seed;
} pc;
layout(std430, set = 0, binding = 0) buffer Sink { float v; } sink;
void main() {
float x = pc.seed;
for (uint i = 0u; i < pc.loops; i++) x = x * 0.99999994 + 0.5;
sink.v = x;
}

View File

@@ -0,0 +1,43 @@
// 1116.6.0
#pragma once
const uint32_t SIM_GPU_BURN_COMP_SPV[] = {
0x07230203,0x00010600,0x0008000b,0x00000031,0x00000000,0x00020011,0x00000001,0x0006000b,
0x00000001,0x4c534c47,0x6474732e,0x3035342e,0x00000000,0x0003000e,0x00000000,0x00000001,
0x0007000f,0x00000005,0x00000004,0x6e69616d,0x00000000,0x0000000c,0x0000002a,0x00060010,
0x00000004,0x00000011,0x00000001,0x00000001,0x00000001,0x00030003,0x00000002,0x000001cc,
0x00040005,0x00000004,0x6e69616d,0x00000000,0x00030005,0x00000008,0x00000078,0x00030005,
0x0000000a,0x00000050,0x00050006,0x0000000a,0x00000000,0x706f6f6c,0x00000073,0x00050006,
0x0000000a,0x00000001,0x64656573,0x00000000,0x00030005,0x0000000c,0x00006370,0x00030005,
0x00000013,0x00000069,0x00040005,0x00000028,0x6b6e6953,0x00000000,0x00040006,0x00000028,
0x00000000,0x00000076,0x00040005,0x0000002a,0x6b6e6973,0x00000000,0x00030047,0x0000000a,
0x00000002,0x00050048,0x0000000a,0x00000000,0x00000023,0x00000000,0x00050048,0x0000000a,
0x00000001,0x00000023,0x00000004,0x00030047,0x00000028,0x00000002,0x00050048,0x00000028,
0x00000000,0x00000023,0x00000000,0x00040047,0x0000002a,0x00000021,0x00000000,0x00040047,
0x0000002a,0x00000022,0x00000000,0x00020013,0x00000002,0x00030021,0x00000003,0x00000002,
0x00030016,0x00000006,0x00000020,0x00040020,0x00000007,0x00000007,0x00000006,0x00040015,
0x00000009,0x00000020,0x00000000,0x0004001e,0x0000000a,0x00000009,0x00000006,0x00040020,
0x0000000b,0x00000009,0x0000000a,0x0004003b,0x0000000b,0x0000000c,0x00000009,0x00040015,
0x0000000d,0x00000020,0x00000001,0x0004002b,0x0000000d,0x0000000e,0x00000001,0x00040020,
0x0000000f,0x00000009,0x00000006,0x00040020,0x00000012,0x00000007,0x00000009,0x0004002b,
0x00000009,0x00000014,0x00000000,0x0004002b,0x0000000d,0x0000001b,0x00000000,0x00040020,
0x0000001c,0x00000009,0x00000009,0x00020014,0x0000001f,0x0004002b,0x00000006,0x00000022,
0x3f7fffff,0x0004002b,0x00000006,0x00000024,0x3f000000,0x0003001e,0x00000028,0x00000006,
0x00040020,0x00000029,0x0000000c,0x00000028,0x0004003b,0x00000029,0x0000002a,0x0000000c,
0x00040020,0x0000002c,0x0000000c,0x00000006,0x00040017,0x0000002e,0x00000009,0x00000003,
0x0004002b,0x00000009,0x0000002f,0x00000001,0x0006002c,0x0000002e,0x00000030,0x0000002f,
0x0000002f,0x0000002f,0x00050036,0x00000002,0x00000004,0x00000000,0x00000003,0x000200f8,
0x00000005,0x0004003b,0x00000007,0x00000008,0x00000007,0x0004003b,0x00000012,0x00000013,
0x00000007,0x00050041,0x0000000f,0x00000010,0x0000000c,0x0000000e,0x0004003d,0x00000006,
0x00000011,0x00000010,0x0003003e,0x00000008,0x00000011,0x0003003e,0x00000013,0x00000014,
0x000200f9,0x00000015,0x000200f8,0x00000015,0x000400f6,0x00000017,0x00000018,0x00000000,
0x000200f9,0x00000019,0x000200f8,0x00000019,0x0004003d,0x00000009,0x0000001a,0x00000013,
0x00050041,0x0000001c,0x0000001d,0x0000000c,0x0000001b,0x0004003d,0x00000009,0x0000001e,
0x0000001d,0x000500b0,0x0000001f,0x00000020,0x0000001a,0x0000001e,0x000400fa,0x00000020,
0x00000016,0x00000017,0x000200f8,0x00000016,0x0004003d,0x00000006,0x00000021,0x00000008,
0x00050085,0x00000006,0x00000023,0x00000021,0x00000022,0x00050081,0x00000006,0x00000025,
0x00000023,0x00000024,0x0003003e,0x00000008,0x00000025,0x000200f9,0x00000018,0x000200f8,
0x00000018,0x0004003d,0x00000009,0x00000026,0x00000013,0x00050080,0x00000009,0x00000027,
0x00000026,0x0000000e,0x0003003e,0x00000013,0x00000027,0x000200f9,0x00000015,0x000200f8,
0x00000017,0x0004003d,0x00000006,0x0000002b,0x00000008,0x00050041,0x0000002c,0x0000002d,
0x0000002a,0x0000001b,0x0003003e,0x0000002d,0x0000002b,0x000100fd,0x00010038
};

View File

@@ -53,6 +53,8 @@
#include <utility>
#include <vector>
#include "new_wait.h" // ((Triforce, 11-22, OS-04) a nothrow new that must answer at once: NoWait)
namespace shadow_prev {
inline constexpr uint64_t kPage = 4096;
@@ -217,7 +219,10 @@ inline bool keep_old(std::unique_ptr<uint8_t[]>& b, uint64_t& hash, Vers& v, int
v.n = (uint8_t)(maxn - 1);
} else {
if (alive().load(std::memory_order_relaxed) >= cap) return false;
fresh.reset(new (std::nothrow) uint8_t[kPage]);
{
mkwvk::new_wait::NoWait at_once; // ((Triforce, OS-04) no page: none kept; not a wait at the commit limit)
fresh.reset(new (std::nothrow) uint8_t[kPage]);
}
if (!fresh) return false;
alive().fetch_add(1, std::memory_order_relaxed);
}

56
gpu/vk/sim_gpu.h Normal file
View File

@@ -0,0 +1,56 @@
// (Triforce, 11-22, the performance audit's OS-49, after Source B's) A lab switch: a slower GPU, so that a lab PC's renderer
// meets the GPU-bound frames a weaker card's player has (the write-back waits, the capture's late pages, the recording slots,
// the pacing), on the same 5090. RX_VK_SIM_GPU_SLOW=<factor> (over 1, at most 10; 2.5 is read): for each ms of work the
// recordings measure, (factor - 1) ms more, a one-invocation compute loop (shaders/sim_gpu_burn.comp) at the end of the
// replay's own recordings, between full barriers, inside the span the GPU time is measured over, so that the frame's GPU time
// and Resolution Auto see the slower card (vk_replay.cpp sim_burn). What a burn took is measured by a third timestamp between
// the recording's work and the burn, and the difference is carried: the debt below. It repeats the fast card's relative
// answer only: no bandwidth, no other driver, no upscaler's cost. Unset: nothing of it runs, two timestamps a recording.
//
// The rules alone, no Vulkan: tests/test_sim_cores.py compiles them.
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdlib>
namespace sim_gpu {
inline constexpr int64_t kLeastNs = 1'000'000; // (a burn is planned for 1 ms at least: less owed waits)
inline constexpr int64_t kMostNs = 16'000'000; // (and 16 ms at most: the rest goes to the next recordings)
inline constexpr int64_t kDebtLeast = -50'000'000; // (the time owed is held between these: a burn that ran long is not
inline constexpr int64_t kDebtMost = 100'000'000; // carried for seconds, nor a backlog after a hitch)
inline constexpr double kFirstNsPerLoop = 4.0; // (a loop's time before the first measure)
/// RX_VK_SIM_GPU_SLOW: the factor, over 1 and at most 10; 0 (off) for anything else
inline double factor_of(const char* e) {
if (!e || !*e) return 0;
char* end = nullptr;
const double f = std::strtod(e, &end);
if (end == e || *end || !(f > 1.0) || f > 10.0) return 0;
return f;
}
/// The time owed after a recording measured `work_ns` of work and `added_ns` of burn, before the bounds
inline double owed(int64_t debt, int64_t work_ns, int64_t added_ns, double factor) {
return (double)debt + (double)work_ns * (factor - 1.0) - (double)added_ns;
}
/// And within them
inline int64_t held(double owed_ns) { return (int64_t)std::clamp(owed_ns, (double)kDebtLeast, (double)kDebtMost); }
/// The time the next burn is planned for: what is owed, less the burns planned and not measured yet; 0 under kLeastNs
inline int64_t plan_ns(int64_t debt, int64_t in_flight) {
const int64_t o = debt - in_flight;
return o < kLeastNs ? 0 : std::min(o, kMostNs);
}
/// The loop count for a planned time, at the measured time a loop takes
inline uint32_t loops_for(int64_t ns, double ns_per_loop) {
if (ns <= 0) return 0;
const double l = (double)ns / (ns_per_loop > 0.01 ? ns_per_loop : 0.01);
return (uint32_t)std::clamp(l, 1.0, (double)(1u << 30));
}
/// The time a loop takes after a burn of `loops` took `added_ns`: 0.8 of what it was, 0.2 of what this one measured
inline double calibrated(double ns_per_loop, int64_t added_ns, uint32_t loops) {
if (!loops || added_ns <= 0) return ns_per_loop;
const double m = (double)added_ns / loops;
return ns_per_loop > 0 ? 0.8 * ns_per_loop + 0.2 * m : m;
}
} // namespace sim_gpu

49
gpu/vk/slot_retire.h Normal file
View File

@@ -0,0 +1,49 @@
// (Triforce, 11-21, the performance audit's TEX-76, after Source B's) When a depth target that another size has replaced at its
// start is let go.
//
// The game draws each character's shadow map (D32, one level) at a fixed address, sized from the camera every frame. An
// image is kept by its storage, width and height, so each new size is a new texture, and the one before it has been drawn
// over. The sweep took such a texture only once it was unused for 120 presents (60 over a fixed budget): a pile of dead maps
// at every slot, in VRAM for two seconds each. Source B's rule, as it ended after its earlier ones retired targets the game
// drew into again (after 1 frame; colour targets; depth of the same size): a depth target of one level and layer at the
// same start and of the same format as a newer target of another width or height, the newer one written after its last
// use, goes once it is unused for kWait presents. Its VRAM peak in the night market fell from 5,972 to 5,482 MiB, and no
// retired image was wanted again within 16 frames (29 were with a wait of 1). Nothing is written back: the newer target's
// texels are in that storage. The image goes to the render targets' pool by its create info, as any dead target's.
// RX_VK_SLOT_RETIRE=<presents>: the wait, 16 when not set; 0: only after the sweep's own 120, as before.
//
// The rule alone, no Vulkan: tests/test_slot_retire.py compiles it.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace slot_retire {
inline constexpr uint64_t kWait = 16;
/// The wait in presents: asked, RX_VK_SLOT_RETIRE's value; not set or empty kWait; 0 or less (and what is no number): 0, off
inline uint64_t wait(const char* asked) {
if (!asked || !*asked) return kWait;
const long v = atol(asked);
return v > 0 ? (uint64_t)v : 0;
}
/// What the rule looks at of a texture
struct Target {
bool gpu_written, depth; // (the GPU drew it; its aspect is depth)
uint32_t levels, layers, deep; // (deep: its depth in texels, 1 for a 2D one)
int nvn_format;
uint32_t width, height;
uint64_t last_frame, last_written; // (the present it was last used in; the frame the GPU last wrote it in)
};
/// Whether old, unused for `unused` presents, is let go because newer took its place: both at the same start
inline bool retired_by(const Target& old, const Target& newer, uint64_t unused, uint64_t wait_presents) {
if (!wait_presents || unused < wait_presents) return false;
if (!old.gpu_written || !old.depth || old.levels != 1 || old.layers != 1 || old.deep > 1) return false;
if (newer.nvn_format != old.nvn_format) return false;
if (newer.width == old.width && newer.height == old.height) return false; // (the same size: the game may draw into it again)
return newer.last_written > old.last_frame; // (written after its last use: drawn over)
}
} // namespace slot_retire

41
gpu/vk/staging_keep.h Normal file
View File

@@ -0,0 +1,41 @@
// How much of the idle staging arenas a present may keep (no game, no GPU: rules). tests/test_staging_burst.py
//
// (Triforce, 11-22, the performance audit's TEX-28, after Source A's.) The uploads stage their bytes in host-visible
// arenas of 64 MB (vk_texture.cpp staging()). An arena is taken again once the GPU is done with the submission that read
// it, which texture_frame learns a present later: frames that stage one after another need two arenas in turn. Each
// present gives the idle ones back beyond what is kept, 256 MB, and 64 MB on a card of 6.5 GB or less, where an arena may
// be VRAM behind a resizable BAR. With 64 MB kept, the arena that had its turn was idle at every second present of a load,
// was freed, and was made again the present after: Source A counted 16 arenas made, 225 ms of them on the thread that
// replays, in one arrival on a handheld (15.5 ms each), and 4 with this rule.
//
// The rule: an idle arena that something staged into less than RX_VK_STAGING_BURST_S seconds ago (2) is one of a burst's,
// and while such an arena is looked at the idle ones may add up to 256 MB instead of what is kept. Not while VRAM is over
// 90% of the allowance (the render targets' pool's own flag, pool_pressed): a full card keeps what it kept. Where 256 MB
// are kept anyway, every card over 6.5 GB, nothing changes. The arenas over 64 MB still go at once, and the 30 s rule
// (RX_VK_STAGING_IDLE_S) stays. RX_VK_STAGING_BURST_S=0: as before.
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdlib>
namespace staging_keep {
inline constexpr uint64_t kBurstKeep = 256ull << 20; // (what the idle arenas of a burst may add up to)
inline constexpr int64_t kBurstS = 2; // (how long after its last staging an arena is one of a burst's)
/// RX_VK_STAGING_BURST_S: unset or empty kBurstS; a number of seconds, at most an hour; 0, less or not a number: no rule
inline int64_t burst_s(const char* env) {
if (!env || !*env) return kBurstS;
const long long v = std::atoll(env);
return v > 0 ? (int64_t)std::min<long long>(v, 3600) : 0;
}
/// Whether an arena last staged into `since_used_ms` ago is one of a burst's: `pressed` is VRAM over 90% of the allowance
constexpr bool recent(int64_t burst_s, bool pressed, int64_t since_used_ms) {
return burst_s > 0 && !pressed && since_used_ms >= 0 && since_used_ms < burst_s * 1000;
}
/// What the arenas looked at so far and this one may add up to for this one to stay: `keep` is what a present keeps anyway
constexpr uint64_t allowance(uint64_t keep, bool recent) { return recent ? std::max(keep, kBurstKeep) : keep; }
} // namespace staging_keep

View File

@@ -80,6 +80,9 @@ struct Frame {
double wb_unsent_ms = 0;
int64_t wb_held = 0; // ((Triforce, wp20) how many went on after the writer's recording was done on the GPU, and
double wb_held_ms = 0; // that part of them: the submissions' timeline held by work stopped at a wait, wb_rec_wait.h)
// ((Triforce, 11-22, the performance audit's SYNC-122) that part by caller, ms: a capture's take, a read of a page put off,
// a binding of one, a readback's fallback (wb_rec_wait.h Caller; the readbacks' is in none of the counts above)
double wb_held_by[4] = {0, 0, 0, 0};
double pipe_wait = 0; // (waits for shaders and pipelines)
int64_t pipe_waits = 0, left_out = 0; // (how many; draws left out while theirs were built)
int64_t pipes_first = 0, pipes_ahead = 0, pipes_order = 0; // ((Triforce, wp18) vk_draw.cpp graphics_pipeline: the pipelines the
@@ -186,6 +189,10 @@ inline int stall_line(const Frame& f, char* out, size_t cap) {
(long long)f.wb_held, f.wb_held_ms);
put("%s", ")");
}
// ((Triforce, 11-22, SYNC-122) the held part by caller, when there is any: the reads' side of RX_VK_WB_WAIT_REC)
if (f.wb_held_by[0] + f.wb_held_by[1] + f.wb_held_by[2] + f.wb_held_by[3] > 0)
put("; the timeline held after a writer's recording was done, by caller: takes %.1f, reads %.1f, binds %.1f, readbacks %.1f ms",
f.wb_held_by[0], f.wb_held_by[1], f.wb_held_by[2], f.wb_held_by[3]);
if (f.pipe_waits || f.left_out)
put("; %lld waits for shaders or pipelines (%.1f ms), %lld draws left out", (long long)f.pipe_waits, f.pipe_wait,
(long long)f.left_out);

53
gpu/vk/stub_same_size.h Normal file
View File

@@ -0,0 +1,53 @@
// A gameplay upscale network that cannot run: its input drawn into its output, also where the two are one size (no
// game, no GPU: the rule; vk_draw.cpp upscale_stub).
//
// (Triforce, 11-22, the performance audit's GPUWORK-21, after Source B's and Source A's.) The game draws its 3D through
// a small network every frame: fd029a317a9a4b94 from 3/4 size, 9dc98871eb7fc101 at full size. upscale_stub stands in
// for a network that was left out or did not translate (RX_VK_NO_F16, a card or driver that cannot build it), by
// drawing the network's input into its output. It declined a dispatch whose output is no larger than its input, as not
// an upscale's last pass, so at Render resolution 100% a 9dc98871 that could not run was skipped whole: nothing wrote
// the picture, a black 3D under the HUD. Source B turns its bypass on by itself where its network cannot be built, and
// Source A lets that pass through for the gameplay networks.
//
// So, for the two gameplay networks only:
// - an output as large as its input is accepted too (any other program is still declined: a pass of that shape that
// is no gameplay network is not an upscale's last one);
// - both take the typed upscale first, which reads the scene's shared-exponent picture as what it is and undoes the
// network's square-root encoding (shaders/taau_upscale.comp). It was fd029a317a9a4b94's alone, so a 9dc98871 that
// upscaled went to the blit, which copies the encoded bits as they lie;
// - the blit, where the typed upscale cannot be made, warns of wrong colours for both.
// The stand-in looks like the upscaler Off (GPUWORK-20): softer, none of the network's anti-aliasing. Only the path of
// a network that cannot run changes; where it runs, nothing does.
//
// RX_VK_STUB_SAME_SIZE=0: the decline and the first network alone in the typed upscale, as before.
// RX_VK_NO_UPSCALE_STUB still turns the whole stub off.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace stub_same_size {
/// RX_VK_STUB_SAME_SIZE: on unless set to 0
inline bool on(const char* env) {
return !env || std::atoi(env) != 0;
}
/// Whether the stub takes a dispatch by its sizes: an output larger than its input, or (the rule on) a gameplay
/// network's as large as it. An output smaller than its input never
constexpr bool sized(uint64_t out_area, uint64_t in_area, bool gameplay, bool on) {
return out_area > in_area || (on && gameplay && out_area == in_area && out_area != 0);
}
/// Whether the typed upscale is tried first: any program that is not a gameplay network, the 3/4 network, and (the
/// rule on) the other gameplay network too
constexpr bool typed(bool gameplay, bool taau, bool on) {
return !gameplay || taau || on;
}
/// Whether the blit that follows a typed upscale that could not be made warns of wrong colours: the 3/4 network's, and
/// (the rule on) every gameplay network's
constexpr bool warns(bool gameplay, bool taau, bool on) {
return taau || (on && gameplay);
}
} // namespace stub_same_size

39
gpu/vk/surface_lost.h Normal file
View File

@@ -0,0 +1,39 @@
// (Triforce, 11-22, the performance audit's PRESENT-45, after Source A's) Presentation made again after the window's
// surface is lost.
//
// An acquire, a present or a swapchain's creation can answer VK_ERROR_SURFACE_LOST_KHR: the surface itself is gone (a
// display driver reset, a monitor change, a remote session), and no swapchain can be made for it again. Nothing handled
// that result: every acquire after it failed without a word, so the picture stayed frozen while the game ran on, until a
// restart.
//
// Now the renderer lets go of what hangs on the surface, in the order Vulkan asks (vk_main.cpp invalidate_lost_surface:
// the device idle, the swapchain's views, its semaphores, the swapchain, then the surface), and the next swapchain's
// creation first makes a surface for the window again with the startup's own calls (open_window_surface): only while the
// window has a size (a minimized window waits), one try each kRetryMs at most, and with the presents' queue chosen again
// for the new surface.
//
// RX_VK_SURFACE_RECOVER=0: as before, the result is not handled. Nor is it while NVIDIA DLSS frame generation runs: the
// swapchain is Streamline's there, and that path has not been tried.
//
// No Vulkan: tests/test_present_surface_lost.py compiles this, and runs vk_main.cpp's and vk_replay.cpp's side against a
// model of a driver that loses its surface.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace surface_lost {
/// RX_VK_SURFACE_RECOVER (value, null when unset): on unless it is 0
inline bool on(const char* value) { return !(value && *value && std::atoi(value) == 0); }
/// A Vulkan result is a lost surface that is recovered from: lost (the result is VK_ERROR_SURFACE_LOST_KHR), the switch
/// on, and the swapchain the renderer's own (not Streamline's)
inline bool recovers(bool lost, bool on, bool dlssg_active) { return lost && on && !dlssg_active; }
/// The least time between two tries to make the surface again
constexpr int64_t kRetryMs = 250;
/// A try is due at now_ms: none made yet (last_ms 0), or the last one kRetryMs ago or more
inline bool retry_due(int64_t now_ms, int64_t last_ms) { return !last_ms || now_ms - last_ms >= kRetryMs; }
} // namespace surface_lost

97
gpu/vk/table_stream.h Normal file
View File

@@ -0,0 +1,97 @@
// (Triforce, 11-22, the performance audit's SYNC-04, after Source A's) The guest page table's changes of one recording,
// streamed into transfers of at most 64 KiB.
//
// A change to the table is a run: `count` pages from page `first`, each given `address` plus its offset (or 0 when the
// address is 0: unmapped). The recording's flush made a (page, value) pair for every page of every run, sorted them all
// and kept the last of each page: 16 bytes a page and a sort, on the recording thread, for ranges that are hundreds of
// MB when a pool is imported ahead or merged (150,000 pages: about 8 ms, and 2.4 MB of scratch).
//
// stream() writes the same transfers in the same order without the pairs: runs that come in page order and do not
// overlap, which is nearly always, are appended one after the other; any other set is swept by page with a heap of
// the runs covering the page, in which the newest run (the highest index) owns it. One batch of 8,192 entries is reused
// for every transfer. The transfers are those of the sort it replaces, entry for entry: tests/test_page_table_stream.py
// compares the two on random runs. RX_VK_TABLE_STREAM=0 (vk_memory.cpp memory_table_flush): the sort, as before.
//
// No Vulkan here: emit(first_page, values, n) is the caller's vkCmdUpdateBuffer.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <numeric>
#include <vector>
namespace table_stream {
constexpr size_t kBatch = 8192; // (entries a transfer: 64 KiB, the most vkCmdUpdateBuffer takes)
struct Scratch {
std::vector<uint64_t> batch; // (the transfer being filled, kept from flush to flush)
std::vector<uint32_t> order, heap; // (the sweep's: an index a run)
};
/// The runs' changes (the later run of a page wins) as transfers of consecutive pages, ascending, each of kBatch entries
/// at most: emit(first_page, values, n). Run: {first, count, address}. How many entries were written
template <class Run, class Emit>
uint64_t stream(const Run* runs, size_t n, Scratch& s, Emit&& emit) {
s.batch.clear();
if (s.batch.capacity() < kBatch) s.batch.reserve(kBatch);
uint64_t batch_first = 0, written = 0;
const auto send = [&] {
if (!s.batch.empty()) emit(batch_first, (const uint64_t*)s.batch.data(), s.batch.size());
s.batch.clear();
};
// (count pages from `page`, the first one's value `address`, 0: all zero. A transfer ends at a gap or when it is full)
const auto append = [&](uint64_t page, uint64_t count, uint64_t address) {
written += count;
while (count) {
if (!s.batch.empty() && (page != batch_first + s.batch.size() || s.batch.size() == kBatch)) send();
if (s.batch.empty()) batch_first = page;
const uint64_t room = kBatch - s.batch.size(), take = count < room ? count : room;
const size_t at = s.batch.size();
s.batch.resize(at + (size_t)take);
uint64_t* const out = s.batch.data() + at;
if (address)
for (uint64_t i = 0; i < take; i++) out[i] = address + (i << 12);
else
std::fill(out, out + take, uint64_t(0));
page += take, count -= take;
if (address) address += take << 12;
}
};
// (in page order and apart: each run ends at or before the next one's first page)
const bool in_order = std::adjacent_find(runs, runs + n, [](const Run& a, const Run& b) { return a.first + a.count > b.first; }) == runs + n;
if (in_order) {
for (size_t i = 0; i < n; i++) append(runs[i].first, runs[i].count, runs[i].address);
send();
return written;
}
// (overlapping or out of order: the runs by first page; of those covering a page the newest owns it. A run stays in
// the heap past its end until it is the top, and is dropped there)
s.order.resize(n);
std::iota(s.order.begin(), s.order.end(), uint32_t(0));
std::sort(s.order.begin(), s.order.end(), [&](uint32_t a, uint32_t b) { return runs[a].first != runs[b].first ? runs[a].first < runs[b].first : a < b; });
s.heap.clear();
uint64_t pos = 0;
size_t k = 0;
while (k < n || !s.heap.empty()) {
if (s.heap.empty()) pos = runs[s.order[k]].first;
while (k < n && runs[s.order[k]].first <= pos) {
s.heap.push_back(s.order[k++]);
std::push_heap(s.heap.begin(), s.heap.end());
}
while (!s.heap.empty() && runs[s.heap.front()].first + runs[s.heap.front()].count <= pos) {
std::pop_heap(s.heap.begin(), s.heap.end());
s.heap.pop_back();
}
if (s.heap.empty()) continue;
const Run& r = runs[s.heap.front()];
uint64_t stop = r.first + r.count; // (this run owns the pages up to its end, or to where a run begins that may be newer)
if (k < n && runs[s.order[k]].first < stop) stop = runs[s.order[k]].first;
append(pos, stop - pos, r.address ? r.address + ((pos - r.first) << 12) : 0);
pos = stop;
}
send();
return written;
}
} // namespace table_stream

View File

@@ -17,4 +17,32 @@ inline bool shifted_over(uint64_t lo, uint64_t hi, uint64_t last_used, uint64_t
return b > a && (b - a) * 8 >= hi - lo;
}
/// ((Triforce, 11-21, the performance audit's TEX-30, after Source A's) The same question for all the other render targets
/// together. Judged one by one, an eighth each, a target that two others had drawn over between them, one over its head
/// and one over its tail, each less than an eighth, was kept for good: its texels are as gone from that memory. Here the
/// parts the others cover inside [lo, hi) are added up, each byte once: add() takes the others in the order of their starts
/// (vk_texture.cpp's by_start), so a part never starts before the one before it and one pass merges them. A target with
/// the same start, one inside it and one not written after its last use count nothing, as in shifted_over
struct ShiftedOver {
uint64_t lo, hi, last_used;
uint64_t closed = 0, run_lo = 0, run_hi = 0; // (the parts merged and done with, and the one still growing)
void add(uint64_t olo, uint64_t ohi, uint64_t o_written) {
if (hi <= lo || ohi <= olo || olo == lo || o_written <= last_used) return;
if (olo >= lo && ohi <= hi) return;
const uint64_t a = std::max(lo, olo), b = std::min(hi, ohi);
if (b <= a) return;
if (a <= run_hi) {
run_hi = std::max(run_hi, b);
} else {
closed += run_hi - run_lo;
run_lo = a, run_hi = b;
}
}
/// The bytes of it the others cover
uint64_t covered() const { return closed + (run_hi - run_lo); }
/// Whether that is an eighth of it or more
bool an_eighth() const { return hi > lo && covered() * 8 >= hi - lo; }
};
} // namespace mkwvk::target_overlap

58
gpu/vk/target_pool.h Normal file
View File

@@ -0,0 +1,58 @@
// (Triforce, 11-21, the performance audit's TEX-29, after Source A's and Source B's) The render targets' pool: how much it
// may hold, what goes first, and when.
//
// The pool gives a dead render target's image to the next target of its shape, so that a target the game makes again each
// frame costs no image creation. It held a fixed 256 MB, kept every image 120 presents, was trimmed in the order of its
// hash table, and was emptied whenever VRAM use passed 90% of the allowance. Source A found it starved at 2560 x 1440: of
// 22,000 images it was given in 100 s it destroyed 21,879 for being over the bound, and with a larger bound, an age order
// and tiers the images made new fell from about 20,000 to about 4,300 a run and the 1% low rose from 36 to 44 fps.
// - The bound: a twelfth of what the driver allows, 256 MB at least and 768 at most (RX_VK_TARGET_POOL_MB sets it; 0: no
// pool).
// - Trimmed oldest first, through one list in the order the images came in: over the bound, the image that has waited
// longest goes, whatever its shape.
// - The wait: a shape never taken from the pool goes after RX_VK_POOL_IDLE presents (8): the game's per-character shadow
// maps come in sizes of their own that nothing asks for again, and at 120 presents they filled the bound. A shape that
// was taken before keeps 120.
// - Tiers by the VRAM in use: above 90% of the allowance half the bound, above 97% none. With a floor of blocks kept for
// the targets (vk_vma.cpp target_floor_kept) the tiers do nothing, as the pool was not given back there before either:
// what it gave back went into blocks the floor keeps.
// RX_VK_TARGET_POOL_TRIM=0: no tiers (the pool emptied above 90%), as before; with RX_VK_TARGET_POOL_MB=256 and
// RX_VK_POOL_IDLE=120 the pool is the one it was, but for the order.
//
// The rules alone, no Vulkan: tests/test_target_pool_rules.py compiles them.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
namespace target_pool {
inline constexpr uint64_t kLeastMb = 256, kMostMb = 768;
inline constexpr uint64_t kIdle = 8, kIdleTaken = 120; // (presents: a shape never taken from the pool; one taken before)
inline constexpr size_t kTrim = 16, kTrimPressed = 64; // (images destroyed a present; while a tier holds)
inline constexpr size_t kTakenKeys = 4096; // (the shapes remembered as taken)
/// The bound in MB with no switch: a twelfth of the allowance (bytes), kLeastMb to kMostMb; not known (0): kLeastMb
inline uint64_t default_mb(uint64_t allowed) {
return allowed ? std::clamp<uint64_t>((allowed >> 20) / 12, kLeastMb, kMostMb) : kLeastMb;
}
/// The tier: 0, 1 with the VRAM in use above 90% of the allowance, 2 above 97%. known: the driver gave both numbers
inline int tier(bool known, uint64_t in_use, uint64_t allowed) {
if (!known || !allowed) return 0;
return in_use > allowed / 100 * 97 ? 2 : in_use > allowed / 10 * 9 ? 1 : 0;
}
/// What the pool may hold under a tier. tiers: RX_VK_TARGET_POOL_TRIM (off: any tier empties it, as before). floor_kept: a
/// floor of the targets' blocks is kept, and the pool is not given back at all
inline uint64_t cap_for(uint64_t cap, int tier, bool tiers, bool floor_kept) {
if (tier <= 0 || floor_kept) return cap;
return tiers && tier == 1 ? cap / 2 : 0;
}
/// How many images a present may destroy
inline size_t trim_most(int tier, bool floor_kept) { return tier > 0 && !floor_kept ? kTrimPressed : kTrim; }
/// Whether an image put in at present since has waited out its time at present now. idle: RX_VK_POOL_IDLE; taken_before:
/// its shape was taken from the pool lately
inline bool idle_out(uint64_t now, uint64_t since, uint64_t idle, bool taken_before) {
return now - since > (taken_before ? std::max(idle, kIdleTaken) : idle);
}
} // namespace target_pool

View File

@@ -13,7 +13,9 @@
// same call with the same arguments as BC3's colour half. And BC3's alpha half for such a block is (255, 255, indices 0),
// which decodes to 255, what the BC1 RGB format gives every texel. Which textures are all opaque: astc_opaque.h.
//
// A row on the Graphics tab, Off unless set, because what a test cannot show is whether the GPU decoder agrees with
// ((Triforce, 11-21, TEX-50) On unless the row says off, since options_version 5; and with no options file too, which
// started it at 0 until then: vk_menu.cpp tex_bc1_at_start.) It was a row on the Graphics tab, off unless set, because what
// a test cannot show is whether the GPU decoder agrees with
// astc_opaque.h on every block of the game's textures, and the image changes in vk_texture.cpp have not run in the game.
#pragma once
#include <cstdint>

40
gpu/vk/tex_count.h Normal file
View File

@@ -0,0 +1,40 @@
// What a block-compressed image the GPU samples as it is (BC, or ASTC where the driver samples it) takes: whole blocks
// (no game, no GPU: a rule). tests/test_tex_count_blocks.py
//
// (Triforce, 11-22, the performance audit's TEX-86, after Source A's.) vk_texture.cpp counted such an image by its texels:
// a byte a texel for BC, and for ASTC the 16 bytes of a block for every texel, which is 16 times its size at 4 x 4, 64
// times at 8 x 8 and 144 times at 12 x 12; BC1 and BC4, eight bytes a block, twice. The texture budget then let textures
// go for memory nothing held (Source A's Intel UHD 770, whose driver samples ASTC: 17,608 MB counted against a budget of
// 2,560, 2,241 MB by blocks), and the first size estimate left out the top levels of arrays that fit. A level is
// ceil(w / bw) x ceil(h / bh) blocks a slice, each block its bytes, times the layers. The textures the renderer encodes
// itself (decoded ASTC kept as BC3 or BC1) were counted right already and are not asked here.
// RX_VK_COUNT_BY_TEXELS=1: the count as it was.
#pragma once
#include <cstdint>
#include <cstdlib>
namespace tex_count {
/// RX_VK_COUNT_BY_TEXELS: unset, empty or 0 counts by blocks; anything else brings the count by texels back
inline bool by_texels(const char* env) { return env && *env && std::atoi(env) != 0; }
/// The blocks of one slice of level `l`: the level's size rounded up to whole blocks each way
constexpr uint64_t level_blocks(uint32_t width, uint32_t height, uint32_t l, uint32_t bw, uint32_t bh) {
const uint32_t w = l < 32 && (width >> l) ? (width >> l) : 1, h = l < 32 && (height >> l) ? (height >> l) : 1;
const uint32_t x = bw ? bw : 1, y = bh ? bh : 1;
return (uint64_t)((w + x - 1) / x) * ((h + y - 1) / y);
}
/// The bytes of the levels `first` .. `levels - 1` of an image of blocks of `bw` x `bh` texels and `block_bytes` bytes:
/// a 3D image's level has its depth in slices of blocks (a block is flat), every layer has the whole chain
constexpr uint64_t bytes(uint32_t width, uint32_t height, uint32_t depth, uint32_t first, uint32_t levels, uint32_t layers,
uint32_t bw, uint32_t bh, uint32_t block_bytes) {
uint64_t blocks = 0;
for (uint32_t l = first; l < levels; l++) {
const uint32_t d = l < 32 && (depth >> l) ? (depth >> l) : 1;
blocks += level_blocks(width, height, l, bw, bh) * d;
}
return blocks * block_bytes * (layers ? layers : 1);
}
} // namespace tex_count

View File

@@ -18,6 +18,9 @@
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <unordered_set>
#include <vector>
namespace tex_warm {
@@ -49,4 +52,109 @@ inline bool ripe(uint64_t noted, uint64_t frame) { return frame >= noted + kAgeF
/// Whether the look goes on after a texture: neither the time spent nor the frame's uploads took the budget
inline bool goes_on(int64_t spent_ns, int64_t frame_upload_ns) { return spent_ns < kBudgetNs && frame_upload_ns < kBudgetNs; }
// ---- (Triforce, 11-22, the performance audit's TEX-23, after Source A's) The list of what was drawn ----
// Every registered slot made ahead needs the 40% rule above, and on a card of 8 GB that rule stops the warming at once (40% is
// about 2.8 GB, below the 4 GB a stand in the lab drew from). Source A keeps a list of the textures a draw used within 90 s of
// the game making them, between runs, and makes those ahead with a room of their own: while the textures and 256 MB more fit
// their budget. A texture is named in the list by its place: its shape and eight 64-byte samples of its bytes (the signature),
// mixed with its storage's address, so that a texture at a new address is not taken for one the list has (it is made at its
// draw as before, and learnt then). RX_VK_TEX_WARM_LIST=1: the listed slots first, the others after them while the 40% rule
// allows; 2: the listed slots alone; unset or 0: every slot as above. The list is learnt only while it is on. The file is
// triforce_textures_drawn.bin beside the game, its own magic and version; one cut short or of another version is not read.
inline int list_mode_of(const char* env) {
if (!env || !*env) return 0;
const int v = std::atoi(env);
return v == 1 || v == 2 ? v : 0;
}
inline constexpr int64_t kLearnNs = 90'000'000'000; // (a texture drawn within this of the game making it is learnt)
inline constexpr size_t kListMost = 32768; // (places in the list: past it the list starts again)
inline constexpr uint64_t kListRoom = 256ull << 20; // (listed textures are made while the textures and this fit their budget)
inline constexpr uint64_t kSaveEvery = 300; // (presents between saves of a list that changed)
inline constexpr int kSamples = 8;
inline constexpr uint64_t kSampleBytes = 64;
inline constexpr char kListMagic[8] = {'T', 'F', 'D', 'R', 'A', 'W', 'N', 0};
inline constexpr uint32_t kListVersion = 1;
/// What a texture's signature takes of its shape (its descriptor and its storage's size)
struct Shape {
uint32_t width, height, depth, levels, format, target, layers;
uint64_t size;
};
/// Whether a texture the game made at born_ns (0: not known) and drawn at now_ns is learnt: within kLearnNs of it
inline bool learnable(int64_t born_ns, int64_t now_ns) { return born_ns > 0 && now_ns >= born_ns && now_ns - born_ns <= kLearnNs; }
/// Where sample i (0 to kSamples - 1) of a storage of `size` bytes starts: spread from its start to its end, 64-byte aligned
inline uint64_t sample_at(uint64_t size, int i) {
if (size <= kSampleBytes) return 0;
return ((size - kSampleBytes) * (uint64_t)i / (kSamples - 1)) & ~(kSampleBytes - 1);
}
/// How many bytes each sample reads: kSampleBytes, or the whole of a smaller storage
inline uint64_t sample_len(uint64_t size) { return size < kSampleBytes ? size : kSampleBytes; }
/// The signature: the shape and the samples (kSamples * kSampleBytes bytes, sample i at i * kSampleBytes), FNV-1a
inline uint64_t signature(const Shape& s, const uint8_t* samples) {
uint64_t h = 0xCBF29CE484222325ull;
const auto add = [&h](uint64_t v) {
for (int i = 0; i < 8; i++) h = (h ^ ((v >> (8 * i)) & 0xFF)) * 1099511628211ull;
};
add(s.width), add(s.height), add(s.depth), add(s.levels), add(s.format), add(s.target), add(s.layers), add(s.size);
for (uint64_t i = 0; i < kSamples * kSampleBytes; i++) h = (h ^ samples[i]) * 1099511628211ull;
return h;
}
/// A signature at a storage address: what the list holds (never 0)
inline uint64_t place(uint64_t sig, uint64_t gpu) {
uint64_t h = sig ^ (gpu * 0x9E3779B97F4A7C15ull);
h ^= h >> 31, h *= 0xBF58476D1CE4E5B9ull, h ^= h >> 29, h *= 0x94D049BB133111EBull, h ^= h >> 32;
return h | 1;
}
/// A place learnt: true when the list changed. A full list (kListMost) starts again, so that it follows what the game draws now
inline bool remember(std::unordered_set<uint64_t>& list, uint64_t p) {
if (!p || list.count(p)) return false;
if (list.size() >= kListMost) list.clear();
list.insert(p);
return true;
}
/// Whether a listed texture may be made: the target pool is not being given back (VRAM near the allowance), and the textures
/// with kListRoom more fit their budget (0 or all ones: none is set, no room is known)
inline bool listed_room(bool pressed, uint64_t texture_bytes, uint64_t budget) {
if (pressed || !budget || budget == ~0ull) return false;
return texture_bytes <= budget && budget - texture_bytes >= kListRoom;
}
inline uint64_t list_check(const uint8_t* b, size_t n) {
uint64_t h = 0xCBF29CE484222325ull;
for (size_t i = 0; i < n; i++) h = (h ^ b[i]) * 1099511628211ull;
return h;
}
/// The file's bytes: the magic, the version, the count, the places, and a check of the places
inline std::vector<uint8_t> encode(const std::unordered_set<uint64_t>& list) {
std::vector<uint8_t> o(16 + list.size() * 8 + 8);
const uint32_t head[2] = {kListVersion, (uint32_t)list.size()};
std::memcpy(o.data(), kListMagic, 8), std::memcpy(o.data() + 8, head, 8);
size_t at = 16;
for (const uint64_t p : list) std::memcpy(o.data() + at, &p, 8), at += 8;
const uint64_t check = list_check(o.data() + 16, list.size() * 8);
std::memcpy(o.data() + at, &check, 8);
return o;
}
/// The places of a file's bytes into `out`; false, and `out` left empty, for anything that is not a whole file of this version
inline bool decode(const uint8_t* b, size_t n, std::unordered_set<uint64_t>& out) {
out.clear();
if (!b || n < 24 || std::memcmp(b, kListMagic, 8)) return false;
uint32_t head[2];
std::memcpy(head, b + 8, 8);
if (head[0] != kListVersion || head[1] > kListMost || n != 16 + (size_t)head[1] * 8 + 8) return false;
uint64_t check;
std::memcpy(&check, b + n - 8, 8);
if (check != list_check(b + 16, (size_t)head[1] * 8)) return false;
for (uint32_t i = 0; i < head[1]; i++) {
uint64_t p;
std::memcpy(&p, b + 16 + (size_t)i * 8, 8);
if (p) out.insert(p);
}
return true;
}
} // namespace tex_warm

71
gpu/vk/translate_wait.h Normal file
View File

@@ -0,0 +1,71 @@
// A bounded wait for a draw whose program is still translating (no game, no GPU: the switch, the choice and the notes of
// which programs write memory; vk_draw.cpp draw, translate_note). Apart from Vulkan so that tests/test_translate_wait.py runs
// it alone.
//
// (Triforce, 11-22, the performance audit's PIPE-30, after Source A's.) A draw whose program is still being translated
// waits for it, its thread running the job if it is still queued (jobs_wait); RX_VK_PIPE_SKIP=2 leaves such a draw out
// when its targets hold an earlier frame, with no bound and no regard for what its shaders write. Source A, with its switch
// at 8 ms: a draw into targets that hold an earlier frame waits at most that much in all a frame, never running the job
// itself, then is left out (new targets and compute still wait). On its RTX 4070 SUPER, title to Castle Town to the field
// on a cold bank, fps 44.9/45.3 -> 50.0/47.4, the summed wait 4.94/4.85 -> 1.80/1.70 s, frames over 100 ms 39/34 -> 22/23,
// but 4.4k -> 21-23k draws left out, mostly in black load frames.
//
// With the guard the audit asked for: an untranslated program cannot be asked whether it writes memory, so only a program
// key whose variants translated before are known to write none is bounded (the variants of a key share their stage code).
// A key never translated, or one with a variant that writes, waits as before, and so does a draw into new targets.
//
// RX_VK_TRANSLATE_WAIT_MS=<ms>: 1 to 100 ms a frame in all, digits only (8 to try); unset, 0 or anything else: off, every
// such draw waits, as before. RX_VK_PIPE_SKIP=0 and =2 keep their meaning.
#pragma once
#include <cstdint>
#include <mutex>
#include <unordered_map>
namespace translate_wait {
inline constexpr int64_t kMs = 1000000; // (ns)
inline constexpr int kMostMs = 100;
/// RX_VK_TRANSLATE_WAIT_MS: the milliseconds a frame, strictly: digits only, 1 to kMostMs; anything else 0 (off)
inline int parse(const char* env) {
if (!env || !*env) return 0;
int v = 0;
for (const char* c = env; *c; c++) {
if (*c < '0' || *c > '9') return 0;
v = v * 10 + (*c - '0');
if (v > kMostMs) return 0;
}
return v;
}
enum class Act { AsBefore, Bounded, Skip };
/// A draw whose program is still translating. kept: its targets hold an earlier frame (pass_kept); clean: its program key is
/// known to write no memory; left_ns: what the frame's budget has left. Bounded: wait at most left_ns, then leave it out;
/// Skip: the budget is spent, left out at once; AsBefore: waited for
inline Act choose(int ms, bool kept, bool clean, int64_t left_ns) {
if (ms <= 0 || !kept || !clean) return Act::AsBefore;
return left_ns >= kMs ? Act::Bounded : Act::Skip;
}
/// Which program keys write memory, from their variants once translated: a key is clean only when every variant noted
/// wrote none; one that writes sticks
class Notes {
public:
void note(uint64_t key, bool writes) {
std::lock_guard lk(mu_);
bool& w = by_[key];
w = w || writes;
}
bool clean(uint64_t key) const {
std::lock_guard lk(mu_);
const auto it = by_.find(key);
return it != by_.end() && !it->second;
}
private:
mutable std::mutex mu_;
std::unordered_map<uint64_t, bool> by_; // (key -> some variant writes)
};
} // namespace translate_wait

52
gpu/vk/ub_batch.h Normal file
View File

@@ -0,0 +1,52 @@
// (Triforce, 11-22, the performance audit's SYNC-21, after Source A's) A recording's uniform updates as copies of many
// regions, ordered where they overlap: when a run of them ends.
//
// A recording's uniform updates are written into guest memory at its end, between two full barriers: one
// vkCmdUpdateBuffer an update (about 80 a frame), and nothing between two updates of the same bytes, whose order on the
// GPU is then not defined (transfer writes with no dependency between them; so far a validation finding only). With
// RX_VK_UB_BATCH=1 the updates' bytes go into the recording's stream and each run of them into one buffer is one
// vkCmdCopyBuffer of many regions. The regions of one copy may not overlap, and two writes of the same bytes need an
// order: a run ends where a piece would write bytes the run already wrote, or where the destination buffer changes, and
// the next run starts behind a barrier. This header is that rule alone (no Vulkan): vk_replay.cpp flush_ub_batched asks
// it, tests/test_ub_update_batch.py runs both and takes each part of the rule away in turn.
#pragma once
#include <cstddef>
#include <cstdint>
#include <utility>
#include <vector>
namespace ub_batch {
/// What has to happen before a piece joins the run: nothing; the regions so far copied (their source buffer is another,
/// no barrier); the run ended by an overlap; the run ended by another destination. The last two: the regions so far
/// copied, then a barrier, then a new run
enum Before : int { JOIN = 0, COPY = 1, RUN_OVERLAP = 2, RUN_IMPORT = 3 };
template <class Buffer>
struct Run {
Buffer src{}, dst{}; // (the stream buffer the pending regions' bytes are in; the buffer the run writes)
bool any = false; // (a run is open: dst is set)
size_t pending = 0; // (regions not copied yet)
std::vector<std::pair<uint64_t, uint64_t>> written; // ([at, end) in dst written since the run began, copied or not)
bool overlaps(uint64_t at, uint64_t end) const {
for (const auto& w : written)
if (at < w.second && w.first < end) return true;
return false;
}
/// Before a piece from stream buffer s into [at, end) of d joins
Before before(Buffer s, Buffer d, uint64_t at, uint64_t end) const {
if (!any) return JOIN;
if (d != dst) return RUN_IMPORT;
if (overlaps(at, end)) return RUN_OVERLAP;
if (pending && s != src) return COPY;
return JOIN;
}
void clear() { any = false, pending = 0, written.clear(); }
void begin(Buffer d) { dst = d, any = true, pending = 0, written.clear(); } // (a new run, into d)
void add(Buffer s, uint64_t at, uint64_t end) { src = s, pending++, written.push_back({at, end}); } // (a region joins)
void wrote(uint64_t at, uint64_t end) { written.push_back({at, end}); } // (bytes of dst written some other way in this run)
void copied() { pending = 0; } // (the pending regions went into a copy)
};
} // namespace ub_batch

34
gpu/vk/upscale_off.h Normal file
View File

@@ -0,0 +1,34 @@
// Which of the game's network dispatches the plain stretch stands in for, and how it draws (no game, no GPU: a rule).
//
// (Triforce, 11-22, the performance audit's GPUWORK-20, after Source A's.) The game upscales its 3D with a small neural
// network every frame: fd029a317a9a4b94 from 3/4 size, 9dc98871eb7fc101 at full size, and one network of its own for each
// other share of the Render resolution row. vk_draw.cpp taau_stand_in already draws the first one's output with a
// stretch of its input (shaders/taau_upscale.comp, which undoes the network's square-root encoding) where the network
// runs on software matrix multiplies and costs most of a frame; 9dc98871 still ran every frame there (2.6 ms of GPU
// time at 2560x1440 on a mid-range AMD card, as Source A measured its like).
//
// "Off" leaves every gameplay upscale network out: its input stretched linearly into its output, no FXAA of the
// stand-in's and no history bound (the game's own SMAA has already run on the input and stays). It is opt-in
// (RX_VK_UPSCALE_OFF=1, or the lab's "upoff" phases): the picture is softer below 100% and has none of the network's
// sharpening. The networks are still compiled as before, so leaving Off again needs nothing made.
#pragma once
namespace upscale_off {
struct Choice {
bool stand_in; // the stretch draws the dispatch's output; the network is not dispatched
bool temporal; // as the stand-in was before: its history bound and FXAA. false: the plain linear stretch
bool by_shape; // taken for its shape alone: the caller must still see the scene's formats at both ends
};
/// listed: the program is one RX_VK_TAAU_KEYS names (by default fd029a317a9a4b94 on software multiplies);
/// off: the upscaler is Off;
/// by_key: the program is one of the two gameplay networks known by key;
/// shaped: it has a gameplay upscale's shape (a network sampling one texture into one image).
constexpr Choice choose(bool listed, bool off, bool by_key, bool shaped) {
if (off && by_key) return {true, false, false};
if (off && shaped && !listed) return {true, false, true};
return {listed, true, false};
}
} // namespace upscale_off

View File

@@ -2,6 +2,8 @@
Upscaler, and the names it shows for DLSS's (and FSR 3.1's and FSR 4's) modes. No Vulkan, no menu: tests/test_upscaler_rows.cpp checks them. */
#pragma once
#include <cstdint>
namespace mkwvk::upscaler_rows {
/// The Upscaler row (opts.dlss, RX_VK_DLSS): the game's own network, NVIDIA DLSS, NVIDIA DLAA, AMD FSR 3.1 (built from AMD's
@@ -189,7 +191,18 @@ inline int window_filter(unsigned picture_w, unsigned picture_h, unsigned dest_w
/// generation transformer, preset M (NVIDIA's own pick for Performance); 2 the same generation as tuned for Ultra
/// Performance, preset L; 3 NVIDIA's default, preset 0 (the runtime picks per mode, and a runtime update can change it)
enum { MODEL_K = 0, MODEL_M = 1, MODEL_L = 2, MODEL_DEFAULT = 3, kModels = 4 };
inline unsigned model_preset(int model) {
/// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The K row on a card with less
/// than 7 GB of video memory asks for preset E, NVIDIA's last CNN model: less GPU time and less of NGX's own memory than
/// the transformer there (Source A's, a 6 GB laptop RTX 3050 at 1080p DLSS Performance: 15.75 to 14.20 ms of GPU a frame,
/// 85 to 58 MB), with more ghosting on grass and thin objects at the smallest render sizes, and NVIDIA marks it deprecated.
/// vram: the card's video memory in bytes; 0 when the rule is off (vk_dlss.cpp dlss_cnn_vram: RX_VK_DLSS_CNN_VRAM is not
/// 1, the default) or the size is not known: K as chosen. M, L and NVIDIA's default stay as chosen, and RX_VK_DLSS_PRESET
/// goes over all of it
constexpr unsigned kPresetE = 5;
constexpr uint64_t kCnnVramBytes = 7ull << 30;
inline bool cnn_for_vram(int model, uint64_t vram) { return model == MODEL_K && vram && vram < kCnnVramBytes; }
inline unsigned model_preset(int model, uint64_t vram = 0) {
if (cnn_for_vram(model, vram)) return kPresetE;
switch (model) {
case MODEL_M: return 13;
case MODEL_L: return 12;
@@ -202,8 +215,8 @@ inline unsigned model_preset(int model) {
inline bool preset_env(const char* env) {
return env && ((*env >= 'A' && *env <= 'Z') || (*env >= 'a' && *env <= 'z') || (env[0] == '0' && !env[1]));
}
inline unsigned dlss_preset(int model, const char* env) {
if (!preset_env(env)) return model_preset(model);
inline unsigned dlss_preset(int model, const char* env, uint64_t vram = 0) {
if (!preset_env(env)) return model_preset(model, vram);
if (*env == '0') return 0;
return (unsigned)(*env >= 'a' ? *env - 'a' + 1 : *env - 'A' + 1);
}

View File

@@ -150,6 +150,27 @@ enum { HOS_MODS_PRESS_UP = 1, HOS_MODS_PRESS_DOWN = 2, HOS_MODS_PRESS_LEFT = 4,
text. HOS_OPT_TEXT_CHAR, option_set (the window's thread, host/display.c): a code point typed, or 8 Backspace, 127
Ctrl+Backspace (a pasted text comes as its characters) */
enum { HOS_OPT_TEXT_CHAR = 159, HOS_OPT_TEXT_WANTED };
/* ((Triforce) 11-20) The Mods screen's music (host/mods_music.c). HOS_OPT_MODS_MUSIC, option: 0 none; 1 the title's list
is there with its Mods entry (the tune is read from the game's files ahead of its use); 2 the Mods screen is open from
the title's list: the game's shop theme plays quietly and the game's own sound, the title's music, is faded out
under it, and back when the screen shuts */
enum { HOS_OPT_MODS_MUSIC = 161 };
/* ((Triforce) 11-20) The Mods entry with the game's own buttons, not by a clock of its own. HOS_OPT_MODS_ALPHA,
option_set: how much of the list's last button the game shows this frame, in thousandths (0 to 1000: the button's own
alpha as the game animates it, host/oot_menu.c), told before that frame's HOS_OPT_MODS_LIST. A HOS_OPT_MODS_LIST without
HOS_MODS_LIST_SHOWN says that the list is there no longer for the entry (another screen has the game's cursor): the
entry goes at once, as the game's buttons did */
enum { HOS_OPT_MODS_ALPHA = 162 };
/* (Triforce) Achievements (host/achievements.c keeps them, on unless HOS_ACHIEVEMENTS=0; gpu/vk/vk_menu.cpp and
gpu/vk/achievement_rows.h show them: the System tab's Achievements page, and a notice when one is earned; 165 to 169 are
theirs). HOS_OPT_ACH_ASK, option: 1 from a renderer that shows them. Once it answers the host names its list, in the list's
order: HOS_OPT_ACH_TEXT, option_set: the next byte of an achievement's name, UTF-8, then a line feed and the line about it,
then a line feed and the day it was earned when it was; HOS_OPT_ACH_ADD, option_set: the bytes sent so far are the next
achievement's, the value its icon (0 to 255, host/achievements.h ach_icon) and HOS_ACH_EARNED when it is earned;
HOS_OPT_ACH_COUNT, option_set: the list is whole (0: none). HOS_OPT_ACH_EARNED, option_set: the achievement at that place
in the list was earned now, the bytes sent so far the day: the notice shows it. One at a time, the host waits between two */
enum { HOS_OPT_ACH_TEXT = 165, HOS_OPT_ACH_ADD, HOS_OPT_ACH_COUNT, HOS_OPT_ACH_EARNED, HOS_OPT_ACH_ASK };
enum { HOS_ACH_EARNED = 256 };
/* (hookshot) the mouse over the window while it is free (not captured): the Insert menu's (host/display.c,
host/posix/display_sdl.c). x and y: the cursor in the window's pixels, of a window width x height pixels (the

View File

@@ -29,6 +29,8 @@
#endif
#include "dlss_checks.h"
#include "dlss_feature.h" // ((Triforce, the upscaler audit's UPS-01) the render size the feature is made for)
#include "lab_ab.h" // ((Triforce, 11-22, PRESENT-38) the lab's "dlssstrict" phases)
#include "upscaler_rows.h"
#include "vk_menu.h"
#include "vk_paths.h"
@@ -76,7 +78,13 @@ struct State {
NVSDK_NGX_Handle* feature = nullptr;
uint32_t feature_rw = 0, feature_rh = 0, feature_ow = 0, feature_oh = 0; // (its largest render size, its output)
uint32_t min_rw = 0, min_rh = 0;
// ((Triforce, 11-21, the upscaler audit's UPS-01) made for the one render size it is given (dlss_feature.h: a fixed share),
// not for the largest of its mode's range; and whether the render size was seen changing under one output in this run
// (the game's dynamic resolution), after which every feature is made for the range, as all were before)
bool feature_exact = false;
bool range_seen = false;
unsigned feature_preset = ~0u; // (the preset it was made with: another one from the menu makes it again)
dlss_checks::PresetSettle settle; // ((Triforce, 11-21, PRESENT-37) the row's preset, taken once it has held: dlss_checks.h)
bool feature_hdr = false; // (made for linear HDR colour, NGX's IsHDR: the other kind of colour makes it again)
uint64_t features_made = 0; // (how many times: a feature made again in play costs a stall, so each says why)
// (a feature not made for want of video memory: tried again from retry_frame on, kMemoryRetries times before it is given
@@ -287,7 +295,17 @@ bool ngx_init() {
/// DLSS's preset: the DLSS model row's, RX_VK_DLSS_PRESET over it (upscaler_rows.h dlss_preset)
unsigned wanted_preset() {
static const char* const env = getenv("RX_VK_DLSS_PRESET");
return upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr);
// ((Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) with RX_VK_DLSS_CNN_VRAM=1 the
// K row on a card under 7 GB asks for preset E, NVIDIA's CNN model (upscaler_rows.h cnn_for_vram); said once)
const uint64_t vram = dlss_cnn_vram();
const unsigned preset = upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr, vram);
static bool told;
if (!told && vram && preset != upscaler_rows::dlss_preset(opts.dlss_model, env && *env ? env : nullptr)) {
told = true;
logf("vk: NVIDIA DLSS: %llu MB of video memory: the K row draws with preset E (RX_VK_DLSS_CNN_VRAM=1)",
(unsigned long long)(vram >> 20));
}
return preset;
}
/// DLSS's feature for this output size and preset, made when either changes (cmd: its creation's work): the engine's
@@ -295,10 +313,17 @@ unsigned wanted_preset() {
int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t oh, bool hdr, uint32_t* in_w,
uint32_t* in_h, const char** fail) {
State& s = st();
const unsigned preset = wanted_preset();
if (s.feature && s.feature_ow == ow && s.feature_oh == oh && rw <= s.feature_rw && rh <= s.feature_rh &&
rw >= s.min_rw && rh >= s.min_rh && s.feature_preset == preset && s.feature_hdr == hdr)
// ((Triforce, 11-21, the performance audit's PRESENT-37, after Source A's) the DLSS model row's preset once the row has
// held it for RX_VK_DLSS_MODEL_SETTLE frames (30; 0: at once, as before): no feature for a preset stepped past)
static const uint64_t settle_frames = dlss_checks::settle_frames(getenv("RX_VK_DLSS_MODEL_SETTLE"));
const unsigned preset = s.settle.pick(wanted_preset(), s.feature_preset, s.feature != nullptr, texture_frame_no(), settle_frames);
// ((Triforce, UPS-01) the feature there is takes this frame: its one render size, or any of its range (dlss_feature.h takes)
const bool same_feature = s.feature && s.feature_ow == ow && s.feature_oh == oh && s.feature_preset == preset && s.feature_hdr == hdr;
if (same_feature && dlss_feature::takes(s.feature_exact, s.feature_rw, s.feature_rh, s.min_rw, s.min_rh, rw, rh))
return *in_w = s.feature_rw, *in_h = s.feature_rh, 1;
// (a feature of one render size, and a frame of another under the same output: the game changes the render size, so
// from here on the features of this run are made for their mode's whole range)
const bool found_range = dlss_feature::found_range(same_feature, s.feature_exact, s.feature_rw, s.feature_rh, rw, rh);
static const char* const kNoMemory = "no video memory for its feature: tried again in a while";
if (s.retry_frame && texture_frame_no() < s.retry_frame) return *fail = kNoMemory, 0;
// (why it is made: in the log with its time, so a stall in play can be matched with one)
@@ -311,50 +336,90 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t
snprintf(why, sizeof why, "preset %c, was %c (the DLSS model row)", upscaler_rows::preset_letter(preset),
upscaler_rows::preset_letter(s.feature_preset));
else if (s.feature_hdr != hdr) snprintf(why, sizeof why, "the colour is %s now", hdr ? "HDR" : "LDR");
else if (found_range)
snprintf(why, sizeof why, "the render size went from %ux%u to %ux%u: for its mode's whole range from now on", s.feature_rw,
s.feature_rh, rw, rh);
else
snprintf(why, sizeof why, "the render size %ux%u left its range, %ux%u to %ux%u", rw, rh, s.min_rw, s.min_rh,
s.feature_rw, s.feature_rh);
const uint64_t key = (uint64_t)ow << 44 ^ (uint64_t)oh << 24 ^ (uint64_t)preset << 1 ^ (hdr ? 1 : 0) ^ 1ull << 63;
if (s.refused == key) return *fail = "its feature was not made", 0; // (until a DLSS setting changes: upstream #2550)
if (s.feature) {
NVSDK_NGX_Handle* old = s.feature;
temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });
s.feature = nullptr;
}
if (found_range) s.range_seen = true;
// (the quality mode whose range of render sizes holds this one: the network's 3/4 is DLSS's MaxQuality range)
// (DLAA's when the render size is the output's)
static const NVSDK_NGX_PerfQuality_Value scaled[] = {NVSDK_NGX_PerfQuality_Value_MaxQuality, NVSDK_NGX_PerfQuality_Value_DLAA,
NVSDK_NGX_PerfQuality_Value_Balanced, NVSDK_NGX_PerfQuality_Value_MaxPerf,
NVSDK_NGX_PerfQuality_Value_UltraPerformance};
static const NVSDK_NGX_PerfQuality_Value same[] = {NVSDK_NGX_PerfQuality_Value_DLAA, NVSDK_NGX_PerfQuality_Value_MaxQuality};
const bool aa = rw == ow && rh == oh;
// ((Triforce, 11-22, the performance audit's PRESENT-32, after Source A's) with RX_VK_DLSS_PREWARM=1 the one-size modes
// only for NVIDIA DLAA: DLSS at the output's size (the Render resolution row's Auto, before the game lowers it) takes
// the scaled mode whose range holds it, and the smaller sizes after it draw with the same feature instead of making a
// second one in play (dlss_checks.h dlaa_feature). Off by default: as before, one size whenever the two are equal
static const bool dlaa_rule = dlss_checks::prewarm_rule(getenv("RX_VK_DLSS_PREWARM"));
const bool aa = dlss_checks::dlaa_feature(dlaa_rule, temporal_mode() == upscaler_rows::DLAA, rw, rh, ow, oh);
const NVSDK_NGX_PerfQuality_Value* modes = aa ? same : scaled;
const size_t nmodes = aa ? 2 : 5;
NVSDK_NGX_PerfQuality_Value mode = modes[0];
unsigned max_w = 0, max_h = 0, min_w = 0, min_h = 0;
bool found = aa;
if (aa) max_w = min_w = ow, max_h = min_h = oh; // (DLAA: the render size is the output's, one size)
for (size_t i = 0; i < nmodes && !found; i++) {
const NVSDK_NGX_PerfQuality_Value m = modes[i];
unsigned w = 0, h = 0, xw = 0, xh = 0, nw = 0, nh = 0;
float sharp = 0;
if (NVSDK_NGX_FAILED(NGX_DLSS_GET_OPTIMAL_SETTINGS(s.caps, ow, oh, m, &w, &h, &xw, &xh, &nw, &nh, &sharp)) || !w) continue;
if (rw >= nw && rh >= nh && rw <= xw && rh <= xh) {
mode = m, max_w = xw, max_h = xh, min_w = nw, min_h = nh, found = true;
break;
// ((Triforce, 11-22, the performance audit's PRESENT-38, after Source A's) every mode's range first, then the one that
// takes this size (dlss_checks.h mode_for): the first that holds it, else the first it is outside of by a rounding. The
// Render resolution row's shares are rounded up and NGX's sizes down, so at 2560x1440 the row's 33% (854x480) was in
// no range (Ultra Performance is 853x480), DLSS was refused every frame and the game's own upscale ran with DLSS
// chosen. RX_VK_DLSS_MODE_SLACK=0, or the lab's "dlssstrict" phases: only a range that holds the size, as before
if (!found) {
dlss_checks::ModeRange ranges[5];
for (size_t i = 0; i < nmodes; i++) {
unsigned w = 0, h = 0, xw = 0, xh = 0, nw = 0, nh = 0;
float sharp = 0;
if (NVSDK_NGX_FAILED(NGX_DLSS_GET_OPTIMAL_SETTINGS(s.caps, ow, oh, modes[i], &w, &h, &xw, &xh, &nw, &nh, &sharp)) || !w) continue;
ranges[i] = {nw, nh, xw, xh, true};
}
static const bool slack_env = !(getenv("RX_VK_DLSS_MODE_SLACK") && atoi(getenv("RX_VK_DLSS_MODE_SLACK")) == 0);
bool widened = false;
const int at = dlss_checks::mode_for(ranges, nmodes, rw, rh, ow, oh, slack_env && !lab_ab::on("dlssstrict"), &widened);
if (at >= 0) {
const dlss_checks::ModeRange& r = ranges[at];
// (a size taken by the slack is in the range from here on: the feature's reuse test takes it next frame)
mode = modes[at], found = true;
min_w = std::min<unsigned>(r.min_w, rw), min_h = std::min<unsigned>(r.min_h, rh);
max_w = std::max<unsigned>(r.max_w, rw), max_h = std::max<unsigned>(r.max_h, rh);
static bool told;
if (widened && !told) {
told = true;
logf("vk: NVIDIA DLSS: %ux%u is a rounding outside mode %d's sizes for %ux%u (%ux%u to %ux%u): taken as that "
"mode's (RX_VK_DLSS_MODE_SLACK=0: refused, the game's upscale runs)", rw, rh, (int)mode, ow, oh, r.min_w,
r.min_h, r.max_w, r.max_h);
}
}
}
// (a size no mode takes: the feature there is stays, so the next size one takes does not make it again)
if (!found) return *fail = "no DLSS mode takes this render size", 0;
if (s.feature) { // (only now is the old one let go: a mode takes this size, and a feature is made for it)
NVSDK_NGX_Handle* old = s.feature;
temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });
s.feature = nullptr;
}
// (the DLSS model row's preset for every mode, K by default: NVIDIA's transformer model. 0: NVIDIA's own per mode)
static_assert(NVSDK_NGX_DLSS_Hint_Render_Preset_K == 11 && NVSDK_NGX_DLSS_Hint_Render_Preset_L == 12 &&
NVSDK_NGX_DLSS_Hint_Render_Preset_M == 13 && NVSDK_NGX_DLSS_Hint_Render_Preset_Default == 0);
NVSDK_NGX_DLSS_Hint_Render_Preset_M == 13 && NVSDK_NGX_DLSS_Hint_Render_Preset_Default == 0 &&
NVSDK_NGX_DLSS_Hint_Render_Preset_E == 5); // ((Triforce, PRESENT-36) E: upscaler_rows.h kPresetE)
for (const char* k : {NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_DLAA, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Quality,
NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Balanced, NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_Performance,
NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_UltraPerformance,
NVSDK_NGX_Parameter_DLSS_Hint_Render_Preset_UltraQuality})
NVSDK_NGX_Parameter_SetUI(s.params, k, preset);
NVSDK_NGX_DLSS_Create_Params cp{};
cp.Feature.InWidth = max_w, cp.Feature.InHeight = max_h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh;
// ((Triforce, 11-21, the upscaler audit's UPS-01) a fixed share's feature for its own render size, not the largest its
// mode allows (NGX answers the output's own size for that: 2.25 times the pixels at Quality, 9 at Ultra Performance, for
// a feature whose cost and memory follow the size it is made for; NVIDIA's guide has the render size passed to the
// creation). For the mode's whole range where the game changes the render size: the Render resolution row's Auto, or a
// size seen changing under one output in this run. RX_VK_DLSS_FEATURE_MAX=1: always the largest, as before
static const bool whole_range = dlss_feature::whole_range(getenv("RX_VK_DLSS_FEATURE_MAX"));
const dlss_feature::Size made =
dlss_feature::size(aa, whole_range || s.range_seen || opts.resolution == 0, rw, rh, max_w, max_h);
cp.Feature.InWidth = made.w, cp.Feature.InHeight = made.h, cp.Feature.InTargetWidth = ow, cp.Feature.InTargetHeight = oh;
cp.Feature.InPerfQualityValue = mode;
// (LDR colour, already tonemapped: no exposure; motion at the render size, not jittered; depth not reversed)
cp.InFeatureCreateFlags = NVSDK_NGX_DLSS_Feature_Flags_MVLowRes;
@@ -367,7 +432,7 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t
const double made_ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
note_work(cmd); // (NGX's own commands, which vk_work.h does not see: the next full_barrier is not left out)
if (NVSDK_NGX_FAILED(r) || !s.feature) {
logf("vk: NVIDIA DLSS: its feature was not made (%#x) for %ux%u -> %ux%u", (unsigned)r, max_w, max_h, ow, oh);
logf("vk: NVIDIA DLSS: its feature was not made (%#x) for %ux%u -> %ux%u", (unsigned)r, made.w, made.h, ow, oh);
s.feature = nullptr;
// (out of video memory passes, when the game lets an area's textures go: not the end of DLSS for the run, as any
// failure was before. Anything else is)
@@ -387,13 +452,15 @@ int prepare(VkCommandBuffer cmd, uint32_t rw, uint32_t rh, uint32_t ow, uint32_t
return *fail = "its feature was not made", 0;
}
s.retries = 0, s.retry_frame = 0, s.refused = 0;
s.feature_rw = max_w, s.feature_rh = max_h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h;
s.feature_rw = made.w, s.feature_rh = made.h, s.feature_ow = ow, s.feature_oh = oh, s.min_rw = min_w, s.min_rh = min_h;
s.feature_exact = made.exact;
s.feature_preset = preset, s.feature_hdr = hdr;
s.features_made++;
logf("vk: NVIDIA DLSS: feature %llu made (%s; frame %llu, %.1f ms of CPU): %ux%u (%ux%u to %ux%u) -> %ux%u, mode %d, "
"preset %c%s", (unsigned long long)s.features_made, why, (unsigned long long)texture_frame_no(), made_ms, rw, rh,
min_w, min_h, max_w, max_h, ow, oh, (int)mode, upscaler_rows::preset_letter(preset),
preset ? "" : " (NVIDIA's default)");
"preset %c%s; made for %ux%u%s", (unsigned long long)s.features_made, why, (unsigned long long)texture_frame_no(),
made_ms, rw, rh, min_w, min_h, max_w, max_h, ow, oh, (int)mode, upscaler_rows::preset_letter(preset),
preset ? "" : " (NVIDIA's default)", made.w, made.h,
made.exact ? ", this render size alone (RX_VK_DLSS_FEATURE_MAX=1: its mode's largest)" : ", the largest of its range");
*in_w = s.feature_rw, *in_h = s.feature_rh;
return 2; // (a new history)
}
@@ -428,6 +495,7 @@ void release() {
temporal_later([old] { NVSDK_NGX_VULKAN_ReleaseFeature(old); });
s.feature = nullptr;
s.feature_rw = s.feature_rh = s.feature_ow = s.feature_oh = s.min_rw = s.min_rh = 0;
s.feature_exact = false;
}
uint64_t memory() {
@@ -485,4 +553,13 @@ void dlss_device_extensions(std::vector<const char*>&, const std::vector<VkExten
#endif
// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The card's video memory for the
// DLSS model row's CNN rule, 0 while the rule is off: RX_VK_DLSS_CNN_VRAM=1 turns it on (Source A has it on for every card
// under 7 GB; Source B has the same rule behind a switch that is off, for preset E's ghosting and because NVIDIA marks it
// deprecated: here off, as there, until the owner has looked at the picture). RX_VK_TEST_CARD_MB plays a smaller card
uint64_t dlss_cnn_vram() {
static const bool on = env_switch("RX_VK_DLSS_CNN_VRAM") == 1;
return on ? device_vram_bytes() : 0;
}
} // namespace mkwvk

View File

@@ -26,5 +26,10 @@ void dlss_instance_extensions(std::vector<const char*>& want, const std::vector<
void dlss_device_extensions(std::vector<const char*>& want, const std::vector<VkExtensionProperties>& all);
/// DLSS as the temporal upscaler's engine (DLSS and DLAA on the Upscaler row); nullptr in a build without it
const TemporalEngine* dlss_engine();
/// (Triforce, 11-22, the performance audit's PRESENT-36, after Source A's and Source B's) The card's video memory for the
/// DLSS model row's CNN rule (upscaler_rows.h cnn_for_vram: the K row draws with preset E under 7 GB), in bytes; 0 unless
/// RX_VK_DLSS_CNN_VRAM=1 (off by default: the picture is the owner's call). DLSS's preset and the menu's row both ask
/// this, so they agree. In every build
uint64_t dlss_cnn_vram();
} // namespace mkwvk

Some files were not shown because too many files have changed in this diff Show More