Merge branch 'claude/pr-intake-4-947f9e' into test-all-fixes

# Conflicts:
#	runtime/vk_texture.c
This commit is contained in:
chrg6
2026-11-10 14:18:21 -05:00
30 changed files with 1218 additions and 81 deletions

View File

@@ -4,7 +4,8 @@
// PlayerCameraParam.bgyml inside Pack/Actor/PlayerCamera.pack.zs. romfs.c serves a copy of that pack with every entry
// multiplied (cam_pack_build), so the setting raises the camera's top speed as well as its speed at a partial tilt, at
// any of the five sensitivities. The mouse camera reaches the game as a right-stick deflection (input.c, mouse_camera),
// which divides it by the same factor (cam_speed_applied), so NX2NAVI_MOUSE_SENS alone sets the mouse's speed.
// divided by the same factor (cam_speed_applied) before it is limited to full tilt (input_mouse_stick), so
// NX2NAVI_MOUSE_SENS alone sets the mouse's turn per movement, and its top speed is the game's with the factor in it.
//
// The pack is a SARC archive compressed with the dictionary pack.zsdic of Pack/ZsDic.pack.zs. A float in a BYML array
// is stored in the array itself, so the multiplied values take the bytes of the old ones and no offset moves.

View File

@@ -33,6 +33,7 @@ static int save_ok = 1; // the last cfg_save_to wrote (o
static const char *const known[] = {
"CORES", "TM_DISPATCH", "NVN2_CS_TABLE", "SKIP_SHADERS", "MOVIE_DEBAND", "MOVIE_FPS", "MOVIE_SKIP", "MOVIE_OVERLAY", "MOVIES", "NPLN", "NVN2_PROGRAMS", "NVN_TRAP", "OCARINA_REALTIME", "ONESHOT_BUDGET", "ONESHOT_WAIT_MS", "OCARINA_STEP", "PHYSICS_STEPS", "PICKUP_COUNT", "STEP_STEADY", "VI_HZ", "VI_VRR", "TIME_DUMP", "SHADER_DUMP", "SHOT_AFTER", "SHOT_DIR", "SHOT_EVERY", // Switch 2 port: movie skip (romfs.c), NVN2 proc traps (nvn_procs.c)
"AB", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_DEVICE", "AUDIO_GAME_PRIO", "AUDIO_OFF", "AUDIO_PREFILL_MS", "AUDIO_STATS",
"BARRIER_GAME",
"BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_LSTICK_DOWN", "BIND_LSTICK_LEFT",
"BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS",
"BIND_PLUS", "BIND_R", "BIND_RCLICK", "BIND_RIGHT", "BIND_RSTICK_DOWN", "BIND_RSTICK_LEFT", "BIND_RSTICK_RIGHT", "BIND_RSTICK_UP",
@@ -53,6 +54,7 @@ static const char *const known[] = {
"MV_SURVEY",
"ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "PRESENT_HELD", "HELD_SYNC",
"PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW",
"POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP",
"PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC",
"REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "RTSS", "RTSS_ENUM", "SCENE_SHARE",
"PRESNAP_PLACE",
@@ -68,7 +70,7 @@ static const char *const known[] = {
"ZSTD_CHECK", "ZSTD_NATIVE",
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL",
"RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS",
"RECORDER_CORE", "RECORDER_PRIO", "REPLAY_TIME", "LOCK_TIME", "POOL_CREATE_BUDGET", "POOL_CREATE_MS", "POOL_LOAD_BACKLOG", "POOL_LOAD_MS", "VA_REPORT",
"VMA_BLOCK_MB", "ASTC_BC_PROBE", "SYNC_WAKE_ALL", "LOG_ASYNC",
};
@@ -124,8 +126,8 @@ static const struct { const char *key, *section; } sections[] = {
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"VK_NAN_PRESERVE", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
};
@@ -268,6 +270,7 @@ static const CfgMeta metas[] = {
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha
{"PASS_CRUMBS", "", "0|1", RESTART}, // vk_crumbs.c pass marks; unset: VK_AMD_buffer_marker only
{"VA_REPORT", "0", "0|1|2", RESTART}, // vk_vaddr.c GPU address history for a lost device's report; 2: call stacks
{"INPUT_RECORD", "", NULL, RESTART},
};

View File

@@ -330,18 +330,22 @@ int probe_cfg(void) {
CHECK(st[0] == -32767 && st[1] == 0, "keys: left arrow is right stick left");
input_map_stick(dn, 0, st);
CHECK(st[0] == 0 && st[1] == 0, "keys: arrows leave the left stick alone");
input_mouse_stick(30, 0, 1.0 / 60, 1, st);
input_mouse_stick(30, 0, 1.0 / 60, 1, 1, st);
CHECK(st[0] == 32767 * 1800 / 3000 && st[1] == 0, "mouse: 30 counts in 1/60 s is 60% right");
input_mouse_stick(0, -10, 1.0 / 60, 1, st);
input_mouse_stick(0, -10, 1.0 / 60, 1, 1, st);
CHECK(st[0] == 0 && st[1] > 0, "mouse: moving up is stick up");
input_mouse_stick(30, 0, 1.0 / 30, 1, st);
input_mouse_stick(30, 0, 1.0 / 30, 1, 1, st);
CHECK(st[0] == 32767 * 900 / 3000, "mouse: the same counts over twice the time are half the deflection");
input_mouse_stick(15, 0, 1.0 / 60, 2, st);
input_mouse_stick(15, 0, 1.0 / 60, 2, 1, st);
CHECK(st[0] == 32767 * 1800 / 3000, "mouse: doubling sensitivity halves the counts needed (15 counts at 2 = 30 at 1)");
input_mouse_stick(1000, 1000, 1.0 / 60, 1, st);
input_mouse_stick(1000, 1000, 1.0 / 60, 1, 1, st);
CHECK((int64_t)st[0] * st[0] + (int64_t)st[1] * st[1] <= 32768ll * 32768, "mouse: clamped to the unit circle");
input_mouse_stick(30, 0, 1.0 / 60, 0, st);
input_mouse_stick(30, 0, 1.0 / 60, 0, 1, st);
CHECK(st[0] == 0 && st[1] == 0, "mouse: sensitivity 0 is off");
input_mouse_stick(30, 0, 1.0 / 60, 1, 1.5f, st);
CHECK(st[0] == 13107 && st[1] == 0, "mouse: a camera speed of 1.5 divides the deflection, so the turn per count stays");
input_mouse_stick(1000, 0, 1.0 / 60, 1, 1.5f, st);
CHECK(st[0] == 32767 && st[1] == 0, "mouse: a fast flick reaches full tilt at a camera speed of 1.5 too");
input_pad_stick(3000, -2000, 0.15f, st);
CHECK(st[0] == 0 && st[1] == 0, "pad stick: an 11% drift inside a 15% dead zone reads centred");
input_pad_stick(4000, -3000, 0, st);

View File

@@ -161,6 +161,7 @@ uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n);
uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n); // 0x05
uint32_t mem_query(uint64_t a, RtMemoryInfo *out); // 0x06
int mem_readable(uint64_t a, uint64_t n); // every page readable
uint32_t mem_map_gen(void); // changes with everything mem_readable answers from (mapping, heap, permissions)
int mem_read_hold(uint64_t a, uint64_t n); // mem_readable that keeps the range mapped until mem_read_release
void mem_read_release(void);
uint32_t mem_map_physical(uint64_t a, uint64_t n); // 0x2C

View File

@@ -368,17 +368,15 @@ static void mouse_camera(int32_t out[2], int selector) {
if (fr != frame) {
double dt = age < 0.004 ? 0.004 : age > 0.1 ? 0.1 : age;
ikb_mouse_take(&dx, &dy);
input_mouse_stick(dx, dy, dt, mouse_sens_get(), held);
input_mouse_stick(dx, dy, dt, mouse_sens_get(), cam_speed_applied(), held);
frame = fr;
t0 = now;
} else if (age > 0.1) {
held[0] = held[1] = 0; // frames stopped: do not hold a deflection
}
}
double k = cam_speed_applied(), x = held[0] / k, y = held[1] / k, len = sqrt(x * x + y * y);
if (len > STICK_MAX) x *= STICK_MAX / len, y *= STICK_MAX / len;
out[0] = (int32_t)lround(x);
out[1] = (int32_t)lround(y);
out[0] = held[0];
out[1] = held[1];
}
static float stick_dead[2] = {-1, -1}; // input_pad_deadzone of the left and right stick; live (cfg_version)

View File

@@ -349,10 +349,10 @@ void input_map_stick(const uint8_t down[KEY_COUNT], int stick, int32_t out[2]) {
out[1] = dy * m;
}
void input_mouse_stick(int32_t dx, int32_t dy, double dt_s, float sens, int32_t out[2]) {
void input_mouse_stick(int32_t dx, int32_t dy, double dt_s, float sens, float cam, int32_t out[2]) {
out[0] = out[1] = 0;
if (!(dt_s > 0) || !(sens > 0)) return;
double k = STICK_MAX * (double)sens / (MOUSE_FULL_CPS * dt_s);
if (!(dt_s > 0) || !(sens > 0) || !(cam > 0)) return;
double k = STICK_MAX * (double)sens / (MOUSE_FULL_CPS * dt_s * cam);
double x = dx * k, y = -dy * k, len = sqrt(x * x + y * y);
if (len > STICK_MAX) { x *= STICK_MAX / len; y *= STICK_MAX / len; }
out[0] = (int32_t)lrint(x);

View File

@@ -117,8 +117,11 @@ int input_map_held(const uint8_t down[KEY_COUNT], int action);
// unit circle.
void input_map_stick(const uint8_t down[KEY_COUNT], int stick, int32_t out[2]);
// Mouse motion in raw counts over dt_s seconds to right-stick deflection: full deflection at 3000 / sens counts/s.
// `cam` is the factor the game multiplies the right stick's turn rates by (cam_speed_applied). The deflection is
// divided by it, so the turn per count stays what sens says, and only then clamped: a clamp taken first would hold the
// mouse's fastest turn at the game's unmultiplied top speed, however high the factor.
// Clamped to the unit circle, up (negative dy) is +y.
void input_mouse_stick(int32_t dx, int32_t dy, double dt_s, float sens, int32_t out[2]);
void input_mouse_stick(int32_t dx, int32_t dy, double dt_s, float sens, float cam, int32_t out[2]);
// A gamepad stick (x, y in -32768..32767, up is +y) through a round dead zone of `dead` (fraction of full deflection,
// 0..0.5): inside the circle the stick reads 0, and the rest of the travel is scaled to start at 0 at the edge, so a
// drifting stick rests at 0 and full deflection still reaches full. The result stays inside the unit square.

View File

@@ -683,6 +683,11 @@ static uint64_t image_bytes(void) {
static int overlaps(uint64_t a, uint64_t n, uint64_t b, uint64_t m) { return a < b + m && b < a + n; }
static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n <= r.size && a - r.base <= r.size - n; }
// Bumped (lock held exclusively) by every change of what is mapped or readable: vk_exec_draw.c's texture pool looks the
// slots over unreadable storage up again only then (and now and then).
static uint32_t map_gen;
uint32_t mem_map_gen(void) { return __atomic_load_n(&map_gen, __ATOMIC_ACQUIRE); }
// ---- 0x01 SetHeapSize
uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) {
plat_mutex_lock(&lock);
@@ -693,6 +698,7 @@ uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) {
if (size > heap_size) commit(L.heap.base + heap_size, size - heap_size, "heap");
if (size < heap_size) decommit(L.heap.base + size, heap_size - size, "heap");
heap_size = size;
__atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE);
plat_mutex_unlock(&lock);
if (addr) *addr = L.heap.base;
return R_OK;
@@ -749,6 +755,7 @@ uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm) {
}
memset(t, (int)perm, n / PAGE);
protect(a, n, perm == PERM_RW ? PLAT_PROT_RW : perm == PERM_R ? PLAT_PROT_READ : PLAT_PROT_NONE, "module data");
__atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE);
plat_mutex_unlock(&lock);
rt_log("%s data permission +%#llx size %#llx perm %u", m->name, (unsigned long long)(a - m->base),
(unsigned long long)n, perm);
@@ -820,6 +827,7 @@ static uint32_t stack_move(int unmap, uint64_t dst, uint64_t src, uint64_t n) {
stack_maps[nstack_maps++] = (StackMap){dst, src, n};
}
out:
__atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE);
plat_mutex_unlock(&lock);
return r;
}
@@ -872,6 +880,7 @@ static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) {
alias_bytes += fresh * PAGE;
}
out:
__atomic_store_n(&map_gen, map_gen + 1, __ATOMIC_RELEASE);
plat_mutex_unlock(&lock);
if (r) rt_log("%sMapPhysicalMemory %#llx+%#llx failed %#x (alias %#llx used, heap %#llx)", unmap ? "Un" : "",
(unsigned long long)a, (unsigned long long)n, r, (unsigned long long)alias_bytes,

View File

@@ -1567,6 +1567,22 @@ int plat_module_of(uintptr_t addr, char *name, size_t cap, uintptr_t *base) {
return snprintf(name, cap, "%s", n ? n + 1 : d.dli_fname) < (int)cap;
}
typedef struct { uintptr_t *pc; int n, max; } StackWalk;
static _Unwind_Reason_Code stack_step(struct _Unwind_Context *ctx, void *p) {
StackWalk *w = p;
uintptr_t ip = _Unwind_GetIP(ctx);
if (!ip || w->n == w->max) return _URC_END_OF_STACK;
w->pc[w->n++] = ip;
return _URC_NO_REASON;
}
int plat_stack(uintptr_t *pc, int max) {
StackWalk w = {pc, 0, max};
_Unwind_Backtrace(stack_step, &w);
return w.n;
}
int plat_exe_path(char *out, size_t cap) {
ssize_t n = readlink("/proc/self/exe", out, cap - 1);
if (n <= 0 || (size_t)n >= cap - 1) return 0;
@@ -2373,6 +2389,22 @@ static void spawn_close_from(posix_spawn_file_actions_t *fa, int from) {
if (posix_spawn_file_actions_addclosefrom_np) posix_spawn_file_actions_addclosefrom_np(fa, from);
}
// The calling thread's mask with every processor this process uses, for the time of a posix_spawn: a child starts
// with the mask of the thread that spawned it, and after plat_cpu_reserve no thread has them all (the recording thread
// has its core alone, every other thread lacks that core). The game plat_restart starts would begin without the core,
// take the next fast one for its own recording thread, and so have one core fewer after every Restart.
// 1 when the mask was widened: *saved is then the mask to put back.
static int spawn_mask_whole(cpu_set_t *saved) {
cpu_set_t whole;
plat_mutex_lock(&cpu_lock);
int taken = cpu_taker != 0;
CPU_OR(&whole, &cpu_taken, &cpu_others);
plat_mutex_unlock(&cpu_lock);
if (!taken || sched_getaffinity(0, sizeof *saved, saved)) return 0;
CPU_OR(&whole, &whole, saved);
return !CPU_EQUAL(&whole, saved) && !sched_setaffinity(0, sizeof whole, &whole);
}
int plat_restart(void) {
// the arguments, NUL-separated, from /proc/self/cmdline
static char args[16384];
@@ -2399,8 +2431,11 @@ int plat_restart(void) {
posix_spawnattr_setsigdefault(&at, &all);
posix_spawnattr_setflags(&at, POSIX_SPAWN_SETSIGMASK | POSIX_SPAWN_SETSIGDEF);
setenv(RESTART_WAIT, wait, 1);
cpu_set_t mask;
int whole = spawn_mask_whole(&mask);
pid_t pid;
int e = posix_spawn(&pid, exe, &fa, &at, argv, environ);
if (whole) sched_setaffinity(0, sizeof mask, &mask);
unsetenv(RESTART_WAIT);
posix_spawnattr_destroy(&at);
posix_spawn_file_actions_destroy(&fa);

View File

@@ -416,6 +416,8 @@ uintptr_t plat_exe_base(void); // where the executabl
// The module (executable or library) holding code address addr: its file name without directories into name, its load
// address into *base. 0 when none.
int plat_module_of(uintptr_t addr, char *name, size_t cap, uintptr_t *base);
// Return addresses of the calling thread's stack into pc, innermost first, at most max: their count.
int plat_stack(uintptr_t *pc, int max);
#define PLAT_PATH_MAX 1024 // path buffers of the runtime
#ifdef _WIN32
#define PLAT_EXE_SUFFIX ".exe" // of the programs the runtime starts (nx2navi_xlate)

View File

@@ -1562,6 +1562,13 @@ int plat_module_of(uintptr_t addr, char *name, size_t cap, uintptr_t *base) {
return WideCharToMultiByte(CP_UTF8, 0, n, -1, name, (int)cap, NULL, NULL) != 0;
}
int plat_stack(uintptr_t *pc, int max) {
void *f[62];
USHORT n = RtlCaptureStackBackTrace(1, max < 62 ? (DWORD)max : 62, f, NULL);
for (USHORT i = 0; i < n; i++) pc[i] = (uintptr_t)f[i];
return n;
}
int plat_exe_path(char *out, size_t cap) {
wchar_t w[WPATH];
DWORD n = GetModuleFileNameW(NULL, w, WPATH);

View File

@@ -82,6 +82,15 @@ int vk_selftest(void); // 0 on success
// process, after the GPU breadcrumbs (vk_crumbs_report). Any other result returns at once.
void vk_device_fault_report(VkResult r);
// ---- GPU address history (vk_vaddr.c, NX2NAVI_VA_REPORT=1, 2 with call stacks): VK_EXT_device_address_binding_report's
// reports, kept so vk_device_fault_report can name the objects that held a faulting address.
int vk_vaddr_wanted(void);
void vk_vaddr_start(VkInstance instance); // create_instance, after vkCreateInstance
void vk_vaddr_device(int supported); // create_device: the driver reports address bindings
void vk_vaddr_fault(uint64_t addr, uint64_t precision); // logs the reports whose range covered addr's page
void vk_vaddr_dump(void); // the whole history to vaddr-history.txt beside the log
void vk_vaddr_stop(VkInstance instance); // before vkDestroyInstance
// ---- GPU breadcrumbs (vk_crumbs.c): with VK_AMD_buffer_marker, each crumb has the GPU write its code into host memory
// twice, when the command processor reaches it and when all work before it has finished. After a lost device the two
// codes bracket the work that hung. Without the extension every call returns at once.
@@ -556,8 +565,42 @@ uint32_t vk_texture_entry_gen(void); // vk_texture.c: bumped by
// Slots whose token may have changed since *pos (0 at first; then the value these calls leave): fn is called for each
// (some more than once) and *pos advanced. 0 when too many changes came since: the caller looks at every slot. *pos
// from vk_texture_dirty_pos_locked after such a walk. Texture lock held.
int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg);
// gfn, when given, gets each slot group of a bucket in place of fn for its members.
int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg);
// Slot groups (vk_texture.c): the GPU-written slots of one address bucket with one descriptor hash kh, which resolve
// alike. head the first member, vk_texture_group_next_locked the next (-1: the end), n how many; stamp and fmask the
// texture pool's (vk_exec_draw.c pool_dirty_group: 1 when every member holds the same views, of the kinds fmask; a slot
// that joins goes to the tail, refilled on its own), set 0 by vk_texture_groups_unstamp_locked. Texture lock held.
typedef struct { uint64_t kh; int32_t head, n; uint32_t stamp, fmask; } VkSlotGroup;
VkSlotGroup *vk_texture_group_locked(int g);
int vk_texture_group_next_locked(int id);
void vk_texture_groups_unstamp_locked(void);
uint64_t vk_texture_dirty_pos_locked(void);
// The bindless pool's entries (vk_texture.c): after a pool fill of slot id, notes the entry it resolved to (1; 0: none)
// and gives its number for vk_texture_pool_hold_locked in *ent (0: none); a submission's freshness check of every
// noted entry (returns how many). Texture lock held.
int vk_texture_pool_note_locked(int id, uint32_t *ent);
int vk_texture_pool_fresh_locked(void);
// A pool slot that held the entry old (0: none) now holds ent: the entries slots hold get the use stamps a lookup
// leaves (vk_texture.c touch) from the check by list. Texture lock held.
void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent);
// A slot whose last lookup found no entry, looked up again: 1 it resolves now; 0 still none, its storage may become
// readable; -1 none until its token changes (a CPU slot whose texture was released after its registration, no storage).
int vk_texture_slot_retry_locked(int id);
// vk_memory.c, for the pool's freshness check: the generation table's lock, shared; with it held, 1 when every 2 MiB
// region of [addr, addr+size) has this epoch's write-watch snapshot and no page stamped after since (nothing in it can
// have changed), 0 when a closer look is due (vk_mem_changed_snap).
void vk_mem_gen_lock_shared(void);
void vk_mem_gen_unlock_shared(void);
int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch);
// The same check's reason when it fails: 1 a region without this epoch's snapshot yet, 2 one untracked, 3 one written
// since, 4 nothing to check (no size, beyond the regions); 0 unchanged.
int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch);
// The lock exclusively, for the snapshot of the regions of [addr, addr+size) that have none in epoch yet (the number of
// write-watch queries it took).
void vk_mem_gen_lock(void);
void vk_mem_gen_unlock(void);
uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch);
void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers)
void vk_sampler_aniso_live(void); // each present: NX2NAVI_ANISO after a settings change (tiled samplers)
uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered

View File

@@ -219,6 +219,7 @@ static void present_image(const tex_t *t, const int crop[4], uint64_t at_ns, uin
if (!vram_probe) vram_probe = 1, perfmon_set_vram_game(vk_mem_device_used_mb); // the overlay's "game" beside VRAM
vk_exec_frame_log(t, frame);
dynres_frame(); // before vk_dlaa_frame reads the DLSS mode it may override
vk_exec_barrier_log(frame);
vk_dlaa_frame();
vk_sampler_aniso_live(); // NX2NAVI_ANISO changed in the menu: the surface textures' samplers take the new level
vk_mvsurvey_frame();

View File

@@ -141,8 +141,9 @@ static int create_instance(void) {
rt_log("vk: NX2NAVI_VK_VALIDATION set but %s is not installed; continuing without", layers[0]);
validate = 0;
}
// DLSS Frame Generation: Streamline's plugins call its naming functions even when they say it is off
if (validate || (vk_dlssg_wanted() && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME)))
// DLSS Frame Generation: Streamline's plugins call its naming functions even when they say it is off; the GPU
// address history (vk_vaddr.c) gets the driver's binding reports through a messenger
if (validate || ((vk_dlssg_wanted() || vk_vaddr_wanted()) && instance_ext_present(VK_EXT_DEBUG_UTILS_EXTENSION_NAME)))
ext[n_ext++] = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
// armed, not wanted: the menu switches HDR on during the run (vk_hdr.c), and the extensions are fixed here
if (vk_hdr_armed() && instance_ext_present(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME)) // HDR10 surface formats
@@ -168,6 +169,7 @@ static int create_instance(void) {
.pfnUserCallback = debug_cb};
if (f) f(g_vk.instance, &mi, NULL, &g_vk.messenger);
}
vk_vaddr_start(g_vk.instance);
return 1;
}
@@ -299,6 +301,7 @@ typedef struct {
VkPhysicalDeviceShaderMaximalReconvergenceFeaturesKHR mrc;
VkPhysicalDeviceMaintenance5FeaturesKHR m5;
VkPhysicalDevicePipelineBinaryFeaturesKHR pbin;
VkPhysicalDeviceAddressBindingReportFeaturesEXT abr;
} FeatureChain;
static int create_device(void) {
@@ -362,6 +365,12 @@ static int create_device(void) {
CHAIN(fbl, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_FEATURES_EXT);
CHAIN(lr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_LOCAL_READ_FEATURES_KHR);
}
// The GPU address history for a lost device's report (vk_vaddr.c, NX2NAVI_VA_REPORT=1)
int want_va = vk_vaddr_wanted() && has_ext(ep, ne, VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME);
if (want_va) {
ext[n_ext++] = VK_EXT_DEVICE_ADDRESS_BINDING_REPORT_EXTENSION_NAME;
CHAIN(abr, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ADDRESS_BINDING_REPORT_FEATURES_EXT);
}
vk_dlaa_device_exts(ep, ne, ext, &n_ext, 40); // NGX's; the eight slots after it stay for the eight below
// FSR 3.1 (vk_ffx.cpp): its Vulkan back end looks vkGetBufferMemoryRequirements2KHR up by that name, which resolves
// only with the extension enabled (the core function has no suffix); without it the back end called address 0.
@@ -431,6 +440,7 @@ static int create_device(void) {
else if (!h->textureCompressionBC) miss = "textureCompressionBC";
if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; }
if (!c->fault.deviceFault) g_vk.has_device_fault = 0;
if (vk_vaddr_wanted()) vk_vaddr_device(want_va && c->abr.reportAddressBinding);
if (g_vk.has_vrs) { // the rate is set per draw; no rate attachment and no per-primitive rates
c->vrs.attachmentFragmentShadingRate = c->vrs.primitiveFragmentShadingRate = VK_FALSE;
if (!c->vrs.pipelineFragmentShadingRate) g_vk.has_vrs = 0;
@@ -537,6 +547,7 @@ static void destroy_all(void) {
(PFN_vkDestroyDebugUtilsMessengerEXT)vkGetInstanceProcAddr(g_vk.instance, "vkDestroyDebugUtilsMessengerEXT");
if (f) f(g_vk.instance, g_vk.messenger, NULL);
}
vk_vaddr_stop(g_vk.instance);
if (g_vk.instance) vkDestroyInstance(g_vk.instance, NULL);
g_errors_total += g_vk.validation_errors;
memset(&g_vk, 0, sizeof g_vk);
@@ -596,6 +607,7 @@ void vk_device_fault_report(VkResult r) {
if (!get) {
rt_log("vk: device lost; no fault report (VK_EXT_device_fault %s)", c && c->has_device_fault ? "has no entry point"
: "not enabled");
vk_vaddr_dump(); // the history still names what held a page another report (the kernel's log) gives
plat_mutex_unlock(&lock);
return;
}
@@ -613,13 +625,16 @@ void vk_device_fault_report(VkResult r) {
rt_log("vk: device fault: \"%s\" (%u address infos, %u vendor infos, vendor binary %llu bytes)", info.description,
n.addressInfoCount, n.vendorInfoCount, (unsigned long long)binary);
// The faulting access lies in the precision-aligned range around the reported address.
for (uint32_t i = 0; i < n.addressInfoCount; i++)
for (uint32_t i = 0; i < n.addressInfoCount; i++) {
rt_log("vk: device fault address %u: %s at %#llx, precision %#llx", i, fault_address_type(ai[i].addressType),
(unsigned long long)ai[i].reportedAddress, (unsigned long long)ai[i].addressPrecision);
vk_vaddr_fault(ai[i].reportedAddress, ai[i].addressPrecision);
}
for (uint32_t i = 0; i < n.vendorInfoCount; i++)
rt_log("vk: device fault vendor info %u: \"%s\" code %#llx data %#llx", i, vi[i].description,
(unsigned long long)vi[i].vendorFaultCode, (unsigned long long)vi[i].vendorFaultData);
}
vk_vaddr_dump();
free(ai);
free(vi);
plat_mutex_unlock(&lock);

View File

@@ -585,8 +585,71 @@ static VkImageMemoryBarrier2 layout_barrier(VkImage img, VkImageAspectFlags aspe
// Full barrier. The images in the feedback layout become want[] (pass begin) or none (every other barrier): the others
// go back to GENERAL and the new ones leave it, in the same command.
// NX2NAVI_BARRIER_GAME (1; 0: every pass begin and dispatch a full barrier; ab: in the B blocks): the game's own
// barriers order its work, as on the Switch. A pass that
// samples what an earlier one drew, a dispatch that reads what another dispatch wrote, an indirect draw whose arguments
// a dispatch wrote: on the Switch each needs nvnCommandBufferBarrier (ORDER_* and INVALIDATE_* bits) in the game's
// command stream, which the replay keeps as a full barrier (OP_BARRIER). What the GPU orders by itself there is kept: a
// switch between graphics and compute work drains the one before (the 3D and compute classes share the engine). With
// every pass begin and every dispatch a full barrier, 2.3 ms of a 9.7 ms GPU frame in Kokiri Forest was barriers.
// With the switch:
// - a pass begin after an earlier pass and no dispatch since the last full barrier gets an attachment barrier
// (colour and depth/stencil output before the next pass's loads and writes: Vulkan orders attachment access only
// inside one pass instance), after neither nothing;
// - a dispatch after other dispatches only gets none;
// - a pass after a dispatch, and a dispatch after a pass, get a full barrier, the drain of the switch;
// - either gets a full barrier as before after one the replay itself asked for (vk_exec_barrier: its copies,
// clears, fills, uniform updates and stand-in copies call it before their commands, which sets bar_dirty), and
// when feedback-loop targets change layout.
// Copies, clears and the game's barriers keep their full barriers.
//
// The default is 1 on NVIDIA GPUs and 0 on every other. The game placed its barriers for an NVIDIA GPU, and what it
// left out is what such a GPU orders by itself; Vulkan asks for a barrier wherever a pass samples what an earlier pass
// drew or a dispatch reads what an earlier one wrote, whether the game recorded one or not. On an RTX 5090 the picture
// was the same with the switch on and off (34 screenshots across A/B blocks in Kokiri Forest) and the GPU time fell by
// 0.5 to 0.6 ms of 5.4 a frame; no other vendor's driver has run it, so there the setting has to be turned on by hand.
static int barrier_game(void) {
static int mode = -1;
if (mode < 0) {
const VkCtx *c = vk_ctx();
int nvidia = c && c->props.vendorID == 0x10de;
mode = rt_knob("BARRIER_GAME", nvidia ? "1" : "0");
if (mode)
rt_log("vk_exec: NX2NAVI_BARRIER_GAME=%s: passes and dispatches are ordered by the game's barriers",
mode == 2 ? "ab" : "1");
}
return RT_KNOB_ON(mode);
}
// NX2NAVI_TRACE=perf: barriers per frame by kind (vk_exec_barrier_log)
enum { BS_FULL, BS_GAME, BS_PASS_FULL, BS_PASS_ATT, BS_PASS_NONE, BS_DISP_FULL, BS_DISP_NONE, BS_N };
static uint64_t bstat[BS_N];
static const char *const bs_name[BS_N] = {"replay full", "game", "pass full", "pass attachment", "pass none",
"dispatch full", "dispatch none"};
enum { BR_OTHER, BR_GAME, BR_PASS, BR_DISPATCH };
static void barrier_to_why(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant, int why);
static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant) {
barrier_to_why(ex, want, wasp, nwant, BR_OTHER);
}
static void attachment_barrier(VkExec *ex) {
const VkPipelineStageFlags2 st = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT |
VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT;
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = st,
.srcAccessMask = VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
.dstStageMask = st,
.dstAccessMask = VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT |
VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT |
VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(ex->cb, &di);
ex->recorded++;
}
static void barrier_to_why(VkExec *ex, const VkImage *want, const VkImageAspectFlags *wasp, int nwant, int why) {
vk_exec_pass_end(ex);
if (why == BR_OTHER) ex->bar_dirty = 1; // also when it falls together with the last one: its commands follow
VkImageMemoryBarrier2 ib[18];
uint32_t n = 0;
for (int i = 0; i < ex->nfbl; i++)
@@ -599,6 +662,21 @@ static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags
for (int i = 0; i < nwant; i++) ex->fbl_img[i] = want[i], ex->fbl_aspect[i] = wasp[i];
ex->nfbl = nwant;
if (ex->barrier_at == ex->recorded && !n) return;
if (!n && !ex->bar_dirty && ((why == BR_PASS && !ex->bar_dispatches) || (why == BR_DISPATCH && !ex->bar_passes)) &&
barrier_game()) {
if (why == BR_DISPATCH) {
bstat[BS_DISP_NONE]++;
} else if (ex->bar_passes) {
attachment_barrier(ex);
bstat[BS_PASS_ATT]++;
} else {
bstat[BS_PASS_NONE]++;
}
return;
}
bstat[why == BR_GAME ? BS_GAME : why == BR_PASS ? BS_PASS_FULL : why == BR_DISPATCH ? BS_DISP_FULL : BS_FULL]++;
if (why != BR_OTHER) ex->bar_dirty = 0; // after BR_OTHER the replay's own commands follow: the next pass or
ex->bar_passes = ex->bar_dispatches = 0; // dispatch orders after them
vk_exec_frame.fb_layouts += n;
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
@@ -613,6 +691,25 @@ static void barrier_to(VkExec *ex, const VkImage *want, const VkImageAspectFlags
void vk_exec_barrier(VkExec *ex) { barrier_to(ex, NULL, NULL, 0); }
void vk_exec_barrier_dispatch(VkExec *ex) {
barrier_to_why(ex, NULL, NULL, 0, BR_DISPATCH);
ex->bar_dispatches++;
}
void vk_exec_barrier_log(uint64_t frame) {
static uint64_t last;
if (!(rt_trace & RT_TRACE_PERF) || frame % 600) return;
double f = frame > last ? (double)(frame - last) : 1.0;
last = frame;
char s[320];
int o = 0;
for (int k = 0; k < BS_N; k++) {
o += snprintf(s + o, sizeof s - o, "%s%s %.1f", k ? ", " : "", bs_name[k], bstat[k] / f);
bstat[k] = 0;
}
rt_log("vk_exec: frame %llu: barriers per frame: %s", (unsigned long long)frame, s);
}
void vk_exec_pass_begin(VkExec *ex) {
if (ex->in_pass) return;
resolve_rt(ex);
@@ -630,7 +727,8 @@ void vk_exec_pass_begin(VkExec *ex) {
if (has_image(want, nwant, t->image)) continue;
want[nwant] = t->image, wasp[nwant++] = i < 8 ? VK_IMAGE_ASPECT_COLOR_BIT : t->img->aspect;
}
barrier_to(ex, want, wasp, nwant); // passes, clears, copies and dispatches see each other's results
barrier_to_why(ex, want, wasp, nwant, BR_PASS); // passes, clears, copies and dispatches see each other's results
ex->bar_passes++;
ex->fbl_mask = fbl;
ex->fb_dirty = 0;
ex->fb_read = 0;
@@ -1390,7 +1488,7 @@ static const op_hdr *walk(VkExec *ex, const op_hdr *h, int depth) {
case OP_COPY_STRIDE: ex->copy_stride = ((const p_copy_stride *)a)->stride; break;
case OP_SET_TEX_POOL: ex->tex_pool = ((const p_set_pool *)a)->pool, ex->tex_serial++; break;
case OP_SET_SAMP_POOL: ex->samp_pool = ((const p_set_pool *)a)->pool; break;
case OP_BARRIER: vk_exec_barrier(ex); break;
case OP_BARRIER: barrier_to_why(ex, NULL, NULL, 0, BR_GAME); break;
case OP_DRAW: case OP_DRAW_IDX: case OP_DISPATCH: case OP_UPDATE_UB: case OP_DRAW_IDX_IND: case OP_DISPATCH_IND:
if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) vk_exec_frame.draws_seen++, split_draws++;
if (h->op == OP_DISPATCH || h->op == OP_DISPATCH_IND) vk_exec_frame.dispatches++, split_draws++;

View File

@@ -103,6 +103,10 @@ typedef struct VkExec {
void *views; // image views created for this command buffer (freed at its fence)
int nviews, cap_views;
int barrier_at; // recorded right after the last full barrier (-1: none yet)
// NX2NAVI_BARRIER_GAME (vk_exec.c barrier_to_why): the last full barrier was one the replay itself needed (a copy,
// clear or fill of its own follows it), so the next pass or dispatch gets a full one too; passes and dispatches
// begun since the last full barrier
int bar_dirty, bar_passes, bar_dispatches;
// Feedback loops (vk.h VkCtx.feedback, vk_exec_draw.c feedback_prepare). Learned targets are attached in
// ATTACHMENT_FEEDBACK_LOOP_OPTIMAL_EXT; they stay in it across consecutive passes and go back to GENERAL at the next
@@ -128,6 +132,11 @@ void vk_exec_pass_end(VkExec *ex);
// Full memory barrier (all commands, all memory); ends the open pass first. Copies and dispatches order themselves with
// it. Skipped when nothing was recorded since the last one.
void vk_exec_barrier(VkExec *ex);
// The barrier before a game dispatch: a full one, or with NX2NAVI_BARRIER_GAME none unless the replay's own commands
// came since the last full barrier (vk_exec.c barrier_to_why). Ends the open pass.
void vk_exec_barrier_dispatch(VkExec *ex);
// NX2NAVI_TRACE=perf, every 600 frames: the barriers per frame by kind.
void vk_exec_barrier_log(uint64_t frame);
// vk_texture.c (NX2NAVI_TEX_ALIAS): commands that must run at the replay's current position. vk_exec_inline_begin returns
// the command buffer being recorded after a full barrier (an open pass is ended; *reopen says so), or VK_NULL_HANDLE
// when this thread is not recording a replay. vk_exec_inline_end begins the pass again when it was open.

View File

@@ -1743,6 +1743,9 @@ static int tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int stor
// absent from that submit's page tables, a GPU page fault on the 128x128 probe pass of Kokiri Forest's first frame
// (RADV's radv_wait_for_vm_map_updates hid it, as did NX2NAVI_SPLIT_DRAWS=0).
volatile uint32_t vk_pool_kinds_used;
// Per texture slot, what pool_fill keeps between refills: the descriptor hash of the registration it was read under,
// and the lookup memo stamp its entries were last written from (0: none) with the kinds that write covered.
typedef struct { uint64_t kh; uint32_t kreg, stamp, fmask; } PoolSlot;
#define POOL_SETS VK_POOL_SETS
#define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write
static struct {
@@ -1758,7 +1761,24 @@ static struct {
uint64_t logn; // writes so far
uint64_t *last; // per entry: the log position of its newest write (catch-up writes it once, there)
uint64_t switches, waits, catchup, written, rewrites, catch_ns, sync_ns, n_slots, n_refill, n_fresh;
uint64_t n_walk[2], walk_visit[2], walk_refill[2], walk_ns[2]; // dirty walks: at a submission's start, within one
uint64_t n_visit; // slots the dirty walks looked at
uint64_t n_gwalk, n_gskip, n_gskip_slots, n_grep; // POOL_GROUP: groups walked, left alone and their slots, first
// members looked up for the memo
uint64_t n_gchk_writes, n_gchk_groups; // POOL_GROUP_CHECK: writes the groups left alone got all the same, and in how many
uint64_t n_gchk_pending, n_gchk_bad; // of members registered since their last fill (not counted above); this group's
uint64_t n_gnouni, n_gchanged; // groups walked member by member: not known to hold one resolution, first member changed
int gmode; // pool_group() as of the last sync
uint64_t n_by_reg, n_by_addr; // refills whose token changed by a registration, by an address generation only
uint64_t full_ns, n_full; // the walks over every slot (a submission's first sync) and their time
uint64_t ep_ns, n_ep, n_ep_ent; // POOL_FAST: a submission's freshness check of the noted entries instead
uint64_t unres_ns, n_unres_walks; // of it: the walks over the slots without an entry
uint32_t unres_mg; // mem_map_gen of the last such walk
int unres_more; // it left a lookup for the creation budget: the next submission walks again
PoolSlot *ps; // [id]
uint8_t *unres_on; // [id]: its last fill resolved to no entry (POOL_FAST: retried each submission)
int *unres, nunres, cunres; // those slots (some may have left: unres_on clear)
uint32_t *ent; // [id]: the entry its last fill resolved to, held there (pool_hold); 0: none
uint64_t dirty_pos; // vk_texture_dirty_slots_locked position (0: the next sync walks every slot)
int ready, uab, sealed; // sealed: a part of the current submission already reads sets[cur]
uint32_t kinds, gen, epoch, sgen;
@@ -1836,7 +1856,10 @@ static VkDescriptorSetLayout pool_layout_make(void) {
PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok);
PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view);
PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp);
if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory");
PL.ps = calloc(VK_POOL_TEX, sizeof *PL.ps);
PL.unres_on = calloc(VK_POOL_TEX, 1);
PL.ent = calloc(VK_POOL_TEX, sizeof *PL.ent);
if (!PL.tok || !PL.view || !PL.samp || !PL.ps || !PL.unres_on || !PL.ent) rt_fatal("vk: out of memory");
rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)%s",
VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP,
PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets,
@@ -2071,6 +2094,7 @@ uint64_t vk_pool_takes(void) { return __atomic_load_n(&pool_takes, __ATOMIC_ACQU
// The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now.
static uint64_t pool_memo_hits; // pool_fill: slots that shared another slot's lookup (defined with pool_memo below)
static uint64_t pool_memo_same; // of those, slots whose entries held the shared views already
static void pool_catch_up(uint64_t from, uint64_t to) {
VkDescriptorSet set = PL.set;
@@ -2089,23 +2113,47 @@ static void pool_catch_up(uint64_t from, uint64_t to) {
pool_flush();
PL.switches++;
if (PL.switches % 3600 == 0) { // diagnosis: what invalidates the slots (vk_texture.c vk_texture_bucket_stats)
char bs[320];
char bs[768];
vk_texture_bucket_stats(bs, sizeof bs);
rt_log("vk: texture pool buckets over 3600 switches: %s", bs);
}
if (PL.switches % 3600 == 0)
rt_log("vk: texture pool sets: %llu switches, %llu waited for the GPU, %.1f log entries and %.1f writes per "
"catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.0f slots, %.0f "
"refilled: %.0f registered, %.0f address changes, %.0f shared a lookup; %.0f checked); %llu identical registrations kept; "
"catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.3f ms of it in "
"%.2f walks over every slot, %.3f ms in %.2f checks of %.0f listed entries and %d slots without one "
"(%.3f ms in %.2f walks over those); "
"%.0f slots, %.0f refilled: %.0f registered, %.0f address changes, %.0f "
"shared a lookup, %.0f of them holding its views already; %.0f checked); %llu identical registrations kept; "
"%llu refreshes created nothing for their frame's time (pool_create_ms), %llu lookups of slots left in the frame skipped",
(unsigned long long)PL.switches, (unsigned long long)PL.waits, (double)PL.catchup / (double)PL.switches,
(double)PL.written / (double)PL.switches, (unsigned long long)PL.rewrites,
PL.catch_ns / 1e6 / (double)PL.switches, PL.sync_ns / 1e6 / (double)PL.switches,
PL.full_ns / 1e6 / (double)PL.switches, (double)PL.n_full / (double)PL.switches,
PL.ep_ns / 1e6 / (double)PL.switches, (double)PL.n_ep / (double)PL.switches,
PL.n_ep ? (double)PL.n_ep_ent / (double)PL.n_ep : 0.0, PL.nunres,
PL.unres_ns / 1e6 / (double)PL.switches, (double)PL.n_unres_walks / (double)PL.switches,
(double)PL.n_slots / (double)PL.switches, (double)PL.n_refill / (double)PL.switches,
(double)PL.n_by_reg / (double)PL.switches, (double)PL.n_by_addr / (double)PL.switches,
(double)pool_memo_hits / (double)PL.switches,
(double)pool_memo_hits / (double)PL.switches, (double)pool_memo_same / (double)PL.switches,
(double)PL.n_fresh / (double)PL.switches, (unsigned long long)vk_texture_same_regs(),
(unsigned long long)PL.n_frame_stops, (unsigned long long)PL.n_defer_skips);
if (PL.switches % 3600 == 0)
rt_log("vk: texture pool dirty walks per switch: %.2f at a submission's start (%.0f slots looked at, %.0f "
"refilled, %.3f ms), %.2f within one (%.0f looked at, %.0f refilled, %.3f ms)",
(double)PL.n_walk[0] / (double)PL.switches, (double)PL.walk_visit[0] / (double)PL.switches,
(double)PL.walk_refill[0] / (double)PL.switches, PL.walk_ns[0] / 1e6 / (double)PL.switches,
(double)PL.n_walk[1] / (double)PL.switches, (double)PL.walk_visit[1] / (double)PL.switches,
(double)PL.walk_refill[1] / (double)PL.switches, PL.walk_ns[1] / 1e6 / (double)PL.switches);
if (PL.switches % 3600 == 0)
rt_log("vk: texture pool slot groups per switch (NX2NAVI_POOL_GROUP): %.0f walked, %.0f left alone with %.0f "
"slots, %.0f first members refilled (%.0f changed), %.0f not known to hold one resolution; check "
"(NX2NAVI_POOL_GROUP_CHECK): %llu writes in %llu groups left alone so far, %llu more to members registered "
"since their last fill",
(double)PL.n_gwalk / (double)PL.switches, (double)PL.n_gskip / (double)PL.switches,
(double)PL.n_gskip_slots / (double)PL.switches, (double)PL.n_grep / (double)PL.switches,
(double)PL.n_gchanged / (double)PL.switches, (double)PL.n_gnouni / (double)PL.switches,
(unsigned long long)PL.n_gchk_writes, (unsigned long long)PL.n_gchk_groups,
(unsigned long long)PL.n_gchk_pending);
}
// Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the
@@ -2151,22 +2199,75 @@ static int pool_on(void) {
// transient render target registered over its storage, and every finalize and initialize of that target changes their
// tokens, so a refill looked up the same texture ~400 times, ~15,000 lookups a frame in Kokiri Forest. An entry holds the
// views one slot got, for the slots with its hash, address generation and entry generation.
// Its stamp changes only when one of its views does (NX2NAVI_POOL_FAST, default on): a slot whose entries were written
// from the entry at the stamp it has now (PoolSlot) holds those views already, so a refill that finds the entry so
// compares and writes nothing. That is most refills: a transient render target released and taken back moves the
// address generation of its storage, ~1,300 slots in Kokiri Forest, and every slot resolves as before (18,500 of 18,800
// refills a submission shared a lookup, each comparing up to 16 views a quarter megabyte apart: 1.9 ms).
#define POOL_MEMO 4096
static struct { uint64_t kh; uint32_t agen, egen, mask; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO];
static uint64_t pool_memo_hits;
static struct { uint64_t kh; uint32_t agen, egen, mask, stamp, got, ent; VkImageView view[VK_POOL_KINDS]; } pool_memo[POOL_MEMO];
static uint32_t pool_stamp; // the last stamp given out
static uint64_t pool_memo_hits, pool_memo_same;
static int pool_fast(void) { // ab: in the B blocks
static int on = -1;
if (on < 0) on = rt_knob("POOL_FAST", "1");
return RT_KNOB_ON(on);
}
// Slot id's last fill resolved to the entry ent (vk_texture_pool_note_locked; 0: none). vk_texture.c counts the slots
// that hold each entry: the entries held are the ones the walk over every slot reached with a lookup in each
// submission, and the check by list leaves their use stamps from that count (vk_texture_pool_hold_locked).
static void pool_hold(uint32_t id, uint32_t ent) {
if (PL.ent[id] == ent) return;
vk_texture_pool_hold_locked(PL.ent[id], ent);
PL.ent[id] = ent;
}
// Slot id's last fill resolved to no entry (got 0) or to one: the slots without one are looked up again each
// submission (pool_sync_entries), as the walk over every slot did: an entry their storage could not have then may be
// made now, and its address generation then refills them.
static void pool_unres(uint32_t id, int got) {
if (got || PL.unres_on[id]) {
if (got) PL.unres_on[id] = 0;
return;
}
PL.unres_on[id] = 1;
if (PL.nunres == PL.cunres) {
int n = PL.cunres ? PL.cunres * 2 : 1024;
int *p = realloc(PL.unres, sizeof *p * (size_t)n);
if (!p) rt_fatal("vk: out of memory");
PL.unres = p, PL.cunres = n;
}
PL.unres[PL.nunres++] = (int)id;
}
static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) {
vk_texture_create_deferred_locked();
uint64_t kh = tok ? vk_texture_slot_khash_locked((int)id) : 0;
PoolSlot *ps = &PL.ps[id];
uint64_t kh = 0;
if (tok) { // the descriptor hash changes only with a registration (the token's upper half)
uint32_t reg = (uint32_t)(tok >> 32);
if (ps->kreg != reg || !ps->kh) ps->kh = vk_texture_slot_khash_locked((int)id), ps->kreg = reg;
kh = ps->kh;
}
uint32_t ag = (uint32_t)tok;
typeof(pool_memo[0]) *m = kh ? &pool_memo[(kh >> 20) % POOL_MEMO] : NULL;
if (m && m->kh == kh && m->agen == ag && m->egen == vk_texture_entry_gen() && (m->mask & mask) == mask) {
pool_memo_hits++;
pool_unres(id, (int)m->got);
pool_hold(id, m->ent);
if (ps->stamp && ps->stamp == m->stamp && (ps->fmask & mask) == mask && pool_fast()) {
pool_memo_same++;
return 1;
}
for (int k = 0; k < VK_POOL_KINDS; k++) {
if (!(mask >> k & 1)) continue;
VkImageView *e = &PL.view[(size_t)k * VK_POOL_TEX + id];
if (*e != m->view[k]) *e = m->view[k], pool_write((uint32_t)k, id, m->view[k], VK_NULL_HANDLE);
}
pool_memo_hits++;
ps->fmask = ps->stamp == m->stamp ? ps->fmask | mask : mask;
ps->stamp = m->stamp;
return 1;
}
for (int k = 0; k < VK_POOL_KINDS; k++) {
@@ -2180,12 +2281,29 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) {
if (*e != ii.imageView) *e = ii.imageView, pool_write((uint32_t)k, id, ii.imageView, VK_NULL_HANDLE);
}
int ok = !vk_texture_create_deferred_locked();
uint32_t ent = 0;
int got = vk_texture_pool_note_locked((int)id, &ent);
if (tok) pool_unres(id, got);
pool_hold(id, tok ? ent : 0);
ps->stamp = 0; // its entries match no memo entry, unless the one below
if (m && ok) { // the generation after the lookups: an entry they created is part of what they resolved
uint32_t eg = vk_texture_entry_gen();
if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0;
// a view that changes gives the entry a new stamp; one kept outside the mask keeps its value, so a kind that
// comes back with another view is a change too
int changed = !m->stamp;
for (int k = 0; k < VK_POOL_KINDS; k++)
if (mask >> k & 1) m->view[k] = PL.view[(size_t)k * VK_POOL_TEX + id];
if (mask >> k & 1) {
VkImageView v = PL.view[(size_t)k * VK_POOL_TEX + id];
if (m->view[k] != v) m->view[k] = v, changed = 1;
}
if (m->kh != kh || m->agen != ag || m->egen != eg) m->kh = kh, m->agen = ag, m->egen = eg, m->mask = 0;
m->mask |= mask;
m->got = (uint32_t)got, m->ent = ent;
if (changed) {
if (!++pool_stamp) pool_stamp = 1;
m->stamp = pool_stamp;
}
ps->stamp = m->stamp, ps->fmask = mask;
}
return ok;
}
@@ -2237,6 +2355,7 @@ static int pool_refill(VkExec *ex, uint32_t id, uint32_t used, uint64_t t, int s
static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_locked: a slot whose token may have changed
PoolDirty *d = arg;
PL.n_visit++;
if (id >= d->n) return;
uint64_t t = vk_texture_slot_token(id);
if (t == PL.tok[id]) return;
@@ -2253,6 +2372,76 @@ static int pool_current(void) {
vk_texture_slots_gen() == PL.gen && vk_texture_epoch() == PL.epoch && vk_sampler_gen() == PL.sgen;
}
// NX2NAVI_POOL_GROUP=1 (default), 0, ab: a dirty bucket's slot groups (vk_texture.c) are walked whole. The members of a
// group resolve alike, so once a walk of the group brought every member to the current resolution (every fill done, an
// entry for each) they all hold the same views, and later the group is left alone when its first member still holds the
// current resolution: its token did not change (its views are what that token resolves to), or its refill wrote nothing.
// A slot that joined since is at the tail and refilled on its own (its registration's entry on the dirty ring).
// The members keep their older tokens: their views are what those tokens resolved to, and what the current ones do.
// (Comparing the memo's stamps instead left half the groups to their members: the memo's entries are shared between
// hashes, so the same views came back under a new stamp.)
static int pool_group(void) {
static int mode = -1;
if (mode < 0) mode = rt_knob("POOL_GROUP", "1");
return RT_KNOB_ON(mode);
}
// NX2NAVI_POOL_GROUP_CHECK=1: a group left alone is walked member by member all the same, and the writes that walk
// makes are counted (none when the rule holds; the pool log's slot groups line), but for the members registered since
// their last fill: their registration's entry on the dirty ring refills them in the same walk (counted apart).
static int pool_group_check(void) {
static int on = -1;
if (on < 0) on = rt_knob("POOL_GROUP_CHECK", "0") == 1; // a diagnosis: no settings row
return on;
}
static void pool_dirty_group(int g, void *arg) {
PoolDirty *d = arg;
uint64_t chk = 0;
VkSlotGroup *G = vk_texture_group_locked(g);
int rep = G->head;
PL.n_gwalk++;
if (!G->stamp) PL.n_gnouni++;
if (G->stamp && (G->fmask & d->used) == d->used && rep >= 0 && rep < d->n && !PL.unres_on[rep]) {
uint64_t t = vk_texture_slot_token(rep);
int same = t == PL.tok[rep];
if (!same) { // the first member's refill: nothing written, an entry found, its token taken
uint64_t l0 = PL.logn;
PL.n_grep++;
pool_dirty_slot(rep, arg);
same = PL.tok[rep] == t && PL.logn == l0 && !PL.unres_on[rep];
if (!same) PL.n_gchanged++;
}
if (same) {
PL.n_gskip++, PL.n_gskip_slots += (uint64_t)G->n;
if (!pool_group_check()) return;
chk = PL.logn + 1;
}
}
int r0 = d->retry, uni = 1; // member by member: then all hold the current resolution, unless a fill was left
d->retry = 0;
for (int id = G->head; id >= 0; id = vk_texture_group_next_locked(id)) {
if (id >= d->n) continue; // outside the pool
if (chk) { // the check: a member registered since its last fill is refilled by its own entry anyway
uint64_t l0 = PL.logn;
int pending = vk_texture_slot_token(id) >> 32 != PL.tok[id] >> 32;
pool_dirty_slot(id, arg);
if (PL.logn != l0) {
if (pending) PL.n_gchk_pending += PL.logn - l0;
else PL.n_gchk_writes += PL.logn - l0, PL.n_gchk_bad++;
}
} else {
pool_dirty_slot(id, arg);
}
if (PL.unres_on[id]) uni = 0;
}
if (d->retry) uni = 0;
d->retry |= r0;
G = vk_texture_group_locked(g);
G->stamp = (uint32_t)uni, G->fmask = uni ? d->used : 0;
if (chk && PL.n_gchk_bad) PL.n_gchk_groups++, PL.n_gchk_bad = 0;
}
static void pool_sync_entries(VkExec *ex) {
uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED);
uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen();
@@ -2277,12 +2466,51 @@ static void pool_sync_entries(VkExec *ex) {
vk_texture_create_budget_locked(spent ? 0 : pool_create_budget());
PL.n_frame_stops += (uint64_t)spent;
// Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that
// changed (vk_texture_dirty_slots_locked), not the whole table
// changed (vk_texture_dirty_slots_locked), not the whole table. With NX2NAVI_POOL_FAST (default on) a new
// submission too: its freshness check goes through the entries the slots resolved to and the slots that
// resolved to none (vk_texture_pool_fresh_locked, pool_unres), not every slot.
PoolDirty pd = {ex, used, n, 0, spent};
int walked = !fresh && ep == PL.epoch && !PL.retry && PL.dirty_pos &&
vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, &pd);
int newep = ep != PL.epoch;
uint64_t v0 = PL.n_visit, f0 = PL.n_refill, w0 = kern_now_ns();
int grp = pool_group();
if (grp && !PL.gmode) vk_texture_groups_unstamp_locked(); // members were refilled one by one meanwhile
PL.gmode = grp;
int walked = !fresh && (!newep || pool_fast()) && !PL.retry && PL.dirty_pos &&
vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd);
if (walked) {
int at = newep ? 0 : 1;
PL.n_walk[at]++, PL.walk_visit[at] += PL.n_visit - v0, PL.walk_refill[at] += PL.n_refill - f0;
PL.walk_ns[at] += kern_now_ns() - w0;
}
retry = pd.retry;
if (walked && newep) {
uint64_t te = kern_now_ns();
PL.n_ep_ent += (uint64_t)vk_texture_pool_fresh_locked();
// The slots without an entry are over storage the guest has not mapped (stale descriptors, 1,160 in
// Kokiri Forest): looked up again when the mapping changed, while a lookup is left for the creation budget,
// and every 8th submission for the rest (an entry whose creation failed, an image at their address that
// took GPU content since: slot_resolve's alias rule)
uint32_t mg = mem_map_gen();
uint64_t tu = kern_now_ns();
int walk_unres = mg != PL.unres_mg || PL.unres_more || !(PL.n_ep & 7);
if (walk_unres) PL.unres_mg = mg, PL.n_unres_walks++, vk_texture_create_deferred_locked();
for (int i = 0; walk_unres && i < PL.nunres;) {
int id = PL.unres[i];
uint64_t t = PL.unres_on[id] ? vk_texture_slot_token(id) : 0;
int r = t && t == PL.tok[id] ? vk_texture_slot_retry_locked(id) : -1;
if (r) { // resolves now (an entry it made changes its token: refilled), was refilled since, or
PL.unres_on[id] = 0; // resolves to none until its token changes (the refill lists it again)
PL.unres[i] = PL.unres[--PL.nunres];
continue;
}
i++;
}
if (walk_unres) PL.unres_more = vk_texture_create_deferred_locked();
PL.unres_ns += kern_now_ns() - tu;
PL.n_ep++, PL.ep_ns += kern_now_ns() - te;
}
if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked();
uint64_t tw = walked ? 0 : kern_now_ns();
for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync
uint64_t t = vk_texture_slot_token(id);
if (t != PL.tok[id]) {
@@ -2298,7 +2526,10 @@ static void pool_sync_entries(VkExec *ex) {
vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check
}
}
if (!walked) PL.n_slots += (uint64_t)n;
if (!walked) {
PL.n_slots += (uint64_t)n, PL.n_full++, PL.full_ns += kern_now_ns() - tw;
vk_texture_groups_unstamp_locked(); // the walk over every slot refilled members one by one
}
vk_texture_create_budget_locked(-1);
}
PL.retry = retry != 0;
@@ -5182,7 +5413,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) {
VkPipeline p = vk_pipeline_compute(cs, &info);
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS
vk_exec_barrier(ex); // ends the pass; earlier writes are visible to the shader
vk_exec_barrier_dispatch(ex); // ends the pass; earlier writes are visible to the shader
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
c0_dim[0] = (uint32_t)d->x, c0_dim[1] = (uint32_t)d->y, c0_dim[2] = (uint32_t)d->z;
twin_dispatch = 1;
@@ -5217,7 +5448,7 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) {
VkPipeline p = vk_pipeline_compute(cs, &info);
if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped");
if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS
vk_exec_barrier(ex);
vk_exec_barrier_dispatch(ex);
VK_MEM_USE(ip);
bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p);
c0_dim[0] = c0_dim[1] = c0_dim[2] = 0;

View File

@@ -157,6 +157,7 @@ static void top_up(const VkCtx *c, uint32_t type) {
static void headroom_main(void *arg) {
(void)arg;
__atomic_store_n(&H.tid, plat_thread_id(), __ATOMIC_RELEASE);
if (vk_vaddr_wanted()) rt_log("vk: headroom: thread %u", H.tid); // names this thread in the address history (vk_vaddr.c)
plat_thread_priority(PLAT_PRIO_BELOW_NORMAL);
uint64_t watch = 0; // memory types seen so far
for (;;) {

View File

@@ -914,6 +914,43 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e
return untracked ? -1 : changed;
}
void vk_mem_gen_lock_shared(void) { plat_mutex_lock_shared(&gen_lock); }
void vk_mem_gen_unlock_shared(void) { plat_mutex_unlock_shared(&gen_lock); }
void vk_mem_gen_lock(void) { plat_mutex_lock(&gen_lock); }
void vk_mem_gen_unlock(void) { plat_mutex_unlock(&gen_lock); }
int vk_mem_unchanged_why_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch) {
if (!size || !epoch) return 4;
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
if (r1 >= SNAP_REGIONS) return 4;
for (uint64_t r = r0; r <= r1; r++) { // REG_SHIFT is SNAP_SHIFT: reg_max is the newest stamp in the region
if (snap_at[r] != epoch) return 1;
if (snap_untracked[r]) return 2;
if (reg_max[r] > since) return 3;
}
return 0;
}
int vk_mem_unchanged_locked(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch) {
return !vk_mem_unchanged_why_locked(addr, size, since, epoch);
}
uint64_t vk_mem_snap_range_locked(uint64_t addr, uint64_t size, uint32_t epoch) {
if (!size || !epoch) return 0;
uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull;
uint64_t r0 = lo >> SNAP_SHIFT, r1 = (hi - 1) >> SNAP_SHIFT;
if (r1 >= SNAP_REGIONS) return 0;
for (uint64_t r = r0; r <= r1; r++)
if (snap_at[r] != epoch) {
uint64_t q = snap_regions(r0, r1, epoch);
n_snap_calls += q;
return q;
}
return 0;
}
void vk_mem_snap_stats(uint64_t *checks, uint64_t *calls, uint64_t *untracked) {
*checks = __atomic_exchange_n(&n_snap_checks, 0, __ATOMIC_RELAXED);
*calls = __atomic_exchange_n(&n_snap_calls, 0, __ATOMIC_RELAXED);

View File

@@ -90,6 +90,8 @@ typedef struct Tex {
uint64_t ow_frame; // gfx_counters.frames when overwritten was set
uint8_t overwritten; // released, and a texture at another address was drawn over its storage (overwrite_others)
uint8_t in_gwin; // in gwin[]
int pidx; // 1 + its index in pent[] (vk_texture_pool_note_locked); 0: not listed
int ph; // 1 + its index in pref[] (pool_ref); 0: none
} Tex;
// A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the
@@ -297,32 +299,135 @@ static void tex_gen_flip(Tex *e, int plain) {
dirty_push(DIRTY_BUCKET | b);
}
// Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register).
static void slot_rebucket(int id, uint32_t b) {
// Slot groups: the GPU-written slots of one address bucket that share a descriptor hash (TexSlot.khash) resolve alike
// under the same address and entry generations, as the texture pool's memo has it (vk_exec_draw.c pool_fill). Ocarina
// keeps up to ~400 descriptors of each transient render target registered over its storage, and the fullest bucket,
// 1,281 slots, moves 8 times a submission in Kokiri Forest: 12,800 of the 17,600 slots a submission's dirty walks
// refilled held their views already. The dirty walk can hand each group to the pool whole (vk_texture_dirty_slots_locked's
// gfn), which leaves its members alone when their resolution did not change (NX2NAVI_POOL_GROUP). A slot is in its
// bucket's own list (khash 0: a CPU-written texture) or in its group's member list, through the same links. A slot that
// joins a group goes to its tail and leaves the group's stamp as it was: its registration put it on the dirty ring, so
// the walk that follows refills it on its own, and the head, whose state the pool tests, is a member from before. (The
// game's 118 registrations a submission cleared the stamps of the groups they joined, and the walk at a submission's
// start went member by member as before: 11,300 of 11,550 slots.)
typedef struct { VkSlotGroup pub; int32_t gnext, gprev, tail; uint32_t bucket; } SlotGroup;
static SlotGroup *grp;
static int ngrp, cgrp, grp_free = -1, ngrp_live;
static uint64_t n_grouped; // slots in groups
static int32_t bucket_ghead[AGEN_N]; // per bucket: its first group (-1: none; set with the lists)
static int32_t *slot_grp; // per slot id: its group, -1 none
typedef struct { uint64_t kh, b; } GrpKey;
static nvn_hmap grp_by = {.klen = sizeof(GrpKey)}; // -> group + 1
static int grp_get(uint64_t kh, uint32_t b) {
GrpKey k = {kh, b};
void *v = nvn_hm_get(&grp_by, &k);
if (v) return (int)((intptr_t)v - 1);
int g;
if (grp_free >= 0) {
g = grp_free, grp_free = grp[g].gnext;
} else {
if (ngrp == cgrp) {
int n = cgrp ? cgrp * 2 : 1024;
SlotGroup *p = realloc(grp, sizeof *p * (size_t)n);
if (!p) rt_fatal("vk: out of memory for texture slot groups");
grp = p, cgrp = n;
}
g = ngrp++;
}
SlotGroup *G = &grp[g];
memset(G, 0, sizeof *G);
G->pub.kh = kh, G->pub.head = G->tail = -1, G->bucket = b;
G->gprev = -1, G->gnext = bucket_ghead[b];
if (G->gnext >= 0) grp[G->gnext].gprev = g;
bucket_ghead[b] = g;
nvn_hm_put(&grp_by, &k, (void *)(intptr_t)(g + 1));
ngrp_live++;
return g;
}
static void grp_drop(int g) {
SlotGroup *G = &grp[g];
GrpKey k = {G->pub.kh, G->bucket};
nvn_hm_del(&grp_by, &k);
if (G->gprev >= 0) grp[G->gprev].gnext = G->gnext;
else bucket_ghead[G->bucket] = G->gnext;
if (G->gnext >= 0) grp[G->gnext].gprev = G->gprev;
G->pub.kh = 0, G->pub.head = G->tail = -1, G->pub.n = 0, G->pub.stamp = 0;
G->gnext = grp_free, grp_free = g;
ngrp_live--;
}
// Places slot id in bucket b: at the tail of its group when it has a descriptor hash, else at the head of the bucket's
// own list (the lists exist once a slot was registered: vk_texture_register).
static void slot_place(int id, uint32_t b, uint64_t kh) {
if (slot_prev[id] != -2) { // out of its old list
int32_t g = slot_grp[id];
int32_t *head = g >= 0 ? &grp[g].pub.head : &bucket_head[slot_bucket[id]];
if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id];
else bucket_head[slot_bucket[id]] = slot_next[id];
else *head = slot_next[id];
if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id];
else if (g >= 0) grp[g].tail = slot_prev[id];
slot_grp[id] = -1;
if (g >= 0) {
n_grouped--;
if (!--grp[g].pub.n) grp_drop(g);
}
}
slot_bucket[id] = (uint16_t)b;
if (kh) {
int g = grp_get(kh, b); // may move grp
SlotGroup *G = &grp[g];
slot_grp[id] = g;
G->pub.n++, n_grouped++;
slot_next[id] = -1;
slot_prev[id] = G->tail;
if (G->tail >= 0) slot_next[G->tail] = id;
else G->pub.head = id;
G->tail = id;
return;
}
slot_prev[id] = -1;
slot_next[id] = bucket_head[b];
if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id;
bucket_head[b] = id;
}
VkSlotGroup *vk_texture_group_locked(int g) { return &grp[g].pub; }
int vk_texture_group_next_locked(int id) { return slot_next[id]; }
void vk_texture_groups_unstamp_locked(void) {
for (int g = 0; g < ngrp; g++) grp[g].pub.stamp = 0;
}
uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; }
int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) {
// A bucket listed several times since *pos is walked once a round: the walk reads its slots' tokens after every change
// listed before the round began (the lock is held). Changes the walk itself makes (an entry a lookup created) are the
// next round's. A bucket's slot groups go to gfn when given, else their members to fn.
int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void (*gfn)(int g, void *arg), void *arg) {
if (!slot_next || ndirty - *pos > DIRTY_N) {
*pos = ndirty;
return 0;
}
for (uint64_t q = *pos; q < ndirty; q++) {
uint32_t x = dirty[q % DIRTY_N];
if (x & DIRTY_BUCKET)
for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg);
else fn((int)x, arg);
static uint32_t seen[AGEN_N], round;
for (uint64_t q = *pos; q < ndirty;) {
uint64_t end = ndirty;
if (!++round) memset(seen, 0, sizeof seen), round = 1;
for (; q < end; q++) {
uint32_t x = dirty[q % DIRTY_N];
if (x & DIRTY_BUCKET) {
uint32_t b = x & ~DIRTY_BUCKET;
if (seen[b] == round) continue;
seen[b] = round;
for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) fn(id, arg);
for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) { // the walk registers nothing
if (gfn) gfn(g, arg);
else for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) fn(id, arg);
}
} else {
fn((int)x, arg);
}
}
}
*pos = ndirty;
return 1;
@@ -1131,10 +1236,189 @@ static void rel_unlink(Tex *e) {
e->rel_prev = e->rel_next = NULL;
}
// The entries the bindless texture pool's slots resolved to, for a submission's freshness check (make_fresh_snap once
// per entry), which looked at every slot instead: 33,000 slots, 1.5 ms of the replay thread a submission in Kokiri
// Forest. vk_exec_draw.c pool_fill notes a slot's entry after it resolved the slot; only entries whose content the CPU
// provides are listed (the check does nothing for the others; one the GPU took over, gpu_owned, never uploads again
// and leaves), and an entry stays listed until it is destroyed: the list holds every entry a slot resolves to, and
// some that no slot does any more. Each keeps its storage range and the page generation of its last check here, so
// that the check of an entry whose 2 MiB regions took no write since (vk_mem_unchanged_locked, one shared lock for the
// whole list) touches neither the entry nor the lock: 5,600 entries took 0.77 ms a submission one by one. Texture lock
// held.
typedef struct { Tex *e; uint64_t addr, size; uint32_t since; int ok; } PEnt; // ok: uploaded as of since
static PEnt *pent;
static int npent, cpent, *pq, cpq;
static uint64_t n_pool_checks, n_pool_skips;
// The region check's misses (vk_mem_unchanged_why_locked): not uploaded, a region without this epoch's snapshot yet, one
// untracked, one written since, nothing to check; and the snapshot queries its own pass made (NX2NAVI_POOL_PRESNAP).
static uint64_t n_pool_miss[5], n_presnap_q;
static void make_fresh_snap(Tex *e);
static void pool_unlist(Tex *e) {
if (!e->pidx) return;
int i = e->pidx - 1;
pent[i] = pent[--npent];
pent[i].e->pidx = i + 1;
e->pidx = 0;
}
// The entries the pool's slots hold: for each, how many slots' last fill resolved to it (vk_exec_draw.c pool_hold; a fill
// that shared another slot's lookup counts for the entry that lookup found). The walk over every slot looked each slot
// up in each submission, which left its entry's use stamp (touch): the idle clock reads it, and start_at's alias rule
// reads the frame of an entry with GPU content. The check by list is no lookup, so it leaves the stamps from these
// counts (pool_touch): in each submission for the GPU-owned entries, in turn for the others, each once in
// POOL_TOUCH_DIV submissions (IDLE_ROUNDS is 240 of them). An entry keeps its place until it is destroyed and no slot
// holds it any more: a destroyed one is refilled out of its slots by the bump of its bucket.
#define POOL_TOUCH_DIV 64
typedef struct { Tex *e; uint32_t refs; int next; uint8_t gown; } PRef; // e NULL: destroyed, or free (refs 0, next)
static PRef *pref;
static int npref, cpref, pref_free; // pref_free: 1 + the first free place
static void touch(Tex *e);
static uint32_t pool_ref(Tex *e) {
if (e->ph) return (uint32_t)e->ph;
int i;
if (pref_free) {
i = pref_free - 1, pref_free = pref[i].next;
} else {
if (npref == cpref) {
int n = cpref ? cpref * 2 : 1024;
PRef *q = realloc(pref, sizeof *q * (size_t)n);
if (!q) rt_fatal("vk: out of memory for the texture pool's entry counts");
pref = q, cpref = n;
}
i = npref++;
}
pref[i] = (PRef){e, 0, 0, e->pub.gpu_owned};
return (uint32_t)(e->ph = i + 1);
}
void vk_texture_pool_hold_locked(uint32_t old, uint32_t ent) {
if (ent) pref[ent - 1].refs++;
if (old && !--pref[old - 1].refs && !pref[old - 1].e) pref[old - 1].next = pref_free, pref_free = (int)old;
}
static void pool_forget(Tex *e) { // destroy_tex
if (!e->ph) return;
PRef *q = &pref[e->ph - 1];
q->e = NULL;
if (!q->refs) q->next = pref_free, pref_free = e->ph;
e->ph = 0;
}
static void pool_touch(void) {
static int at;
for (int i = 0; i < npref; i++)
if (pref[i].gown && pref[i].refs && pref[i].e) touch(pref[i].e);
for (int k = (npref + POOL_TOUCH_DIV - 1) / POOL_TOUCH_DIV; k-- > 0;) {
if (at >= npref) at = 0;
const PRef *q = &pref[at++];
if (!q->gown && q->refs && q->e) touch(q->e);
}
}
// e becomes GPU-owned: the list's check does nothing for it any more, and its use stamp is left in each submission.
static void pool_gpu_owned(Tex *e) {
pool_unlist(e);
if (e->ph) pref[e->ph - 1].gown = 1;
}
int vk_texture_pool_note_locked(int id, uint32_t *ent) {
*ent = 0;
if (id < 0 || id >= ntslots || !tslots[id].used) return 0;
// The caller's lookups left the slot resolved under its bucket's generation, unless it made none (a fill without a
// token or a kind): an entry cached under another generation may be destroyed, so the slot is looked up first.
if (tslots[id].ce && tslots[id].cgen != agen[sinfo[id] & 0xffff]) vk_texture_by_id_locked(id, NULL, NULL);
Tex *e = tslots[id].ce;
if (!e) return 0;
if (!e->pidx && e->pub.mirrored && !e->pub.gpu_owned) {
if (npent == cpent) {
int n = cpent ? cpent * 2 : 1024;
PEnt *p = realloc(pent, sizeof *p * (size_t)n);
int *q = realloc(pq, sizeof *q * (size_t)n);
if (!p || !q) rt_fatal("vk: out of memory for the texture pool's entry list");
pent = p, pq = q, cpent = cpq = n;
}
pent[npent++] = (PEnt){e, e->pub.addr, e->t.L.size, 0, 0};
e->pidx = npent;
}
*ent = pool_ref(e);
return 1;
}
// Released entries are left out: a slot resolves to none of those whose content the CPU provides (slot_resolve's stale
// rule), and the game may be unmapping their storage.
// NX2NAVI_POOL_PRESNAP=1 (default), 0, ab: the check takes the write-watch snapshot of an entry's regions that have
// none in this submission yet before it looks at them (one exclusive hold of the generation table's lock for the list),
// so an unchanged entry in such a region passes there. Without it 97 % of the 5,600 entries missed the region check for
// want of a snapshot and went the long way, one entry and one lock at a time. With it 97 % pass, 3.5 snapshot queries
// a submission: the check 0.50 to 0.19 ms a submission, the replay thread -0.37 ms against the control.
static int pool_presnap(void) {
static int mode = -1;
if (mode < 0) mode = rt_knob("POOL_PRESNAP", "1");
return RT_KNOB_ON(mode);
}
// The submission being recorded checked the pool's entries by this list: for the idle clock it walked the pool
// (vk_texture_new_submit).
static int pool_by_list;
int vk_texture_pool_fresh_locked(void) {
int nq = 0, ps = pool_presnap();
pool_by_list = 1;
if (ps) vk_mem_gen_lock();
else vk_mem_gen_lock_shared();
for (int i = 0; i < npent; i++) {
const PEnt *p = &pent[i];
if (p->ok) {
if (ps) n_presnap_q += vk_mem_snap_range_locked(p->addr, p->size, submit_epoch);
int why = vk_mem_unchanged_why_locked(p->addr, p->size, p->since, submit_epoch);
if (!why) continue;
n_pool_miss[why]++;
} else {
n_pool_miss[0]++;
}
pq[nq++] = i;
}
if (ps) vk_mem_gen_unlock();
else vk_mem_gen_unlock_shared();
n_pool_skips += (uint64_t)(npent - nq);
for (int k = nq; k-- > 0;) { // from the end: an entry leaving swaps the last one into its place
int i = pq[k];
PEnt *p = &pent[i];
Tex *e = p->e;
if (e->pub.gpu_owned) {
pool_unlist(e);
continue;
}
if (e->released) continue;
if (!e->uploaded || e->checked != submit_epoch) {
make_fresh_snap(e);
e->checked = submit_epoch;
n_pool_checks++;
}
p->since = e->wgen, p->ok = e->uploaded;
}
pool_touch();
return npent;
}
// slot_resolve's last answer was none because of its stale rule (vk_texture_slot_retry_locked)
static int resolve_stale;
int vk_texture_slot_retry_locked(int id) {
if (id < 0 || id >= ntslots || !tslots[id].used || !tslots[id].t.mem) return -1;
resolve_stale = 0;
if (vk_texture_by_id_locked(id, NULL, NULL)) return 1;
return resolve_stale ? -1 : 0;
}
static void retire_remove(Tex *e);
static void destroy_tex(Tex *e) {
if (e->retiring) retire_remove(e);
pool_unlist(e);
pool_forget(e);
own_remove(e);
gwin_remove(e);
n_site[0]++;
@@ -1196,6 +1480,10 @@ static int comes_back_intact(const Tex *e) { return e->pub.mirrored && !e->pub.g
// texture object, nothing samples it. The clock is look_round, which only a submission whose slot checks reached half
// of the registered pool slots moves on (vk_texture_new_submit): counted in submissions, a pause screen or a fade that
// draws nothing through the pool made the whole world's textures idle within seconds.
// With NX2NAVI_POOL_FAST the refresh checks the entries by list instead (vk_texture_pool_fresh_locked), which is no
// lookup. Such a submission counts for the clock all the same, and the entries the pool's slots hold get their stamp
// from the count of those slots (pool_touch), each at least once in POOL_TOUCH_DIV submissions: an entry a slot names
// never comes near IDLE_ROUNDS.
#define IDLE_ROUNDS 240
static uint32_t look_round = 1; // submissions whose lookups walked the texture pool
static uint32_t round_checks; // slot checks of the submission being recorded (vk_texture_by_id_locked)
@@ -1208,6 +1496,7 @@ static void touch(Tex *e) {
}
void vk_texture_pool_ctx(int on) { pool_ctx = on; }
static int idle(const Tex *e, uint64_t now);
// NX2NAVI_TEX_RELEASED_MB (default a thirty-second of the installed memory, 512 MiB to 2 GiB; 0 = no limit) bounds the
@@ -1584,6 +1873,7 @@ static uint64_t evict_disused(uint64_t want) {
}
retiring[nretiring++] = e;
e->retiring = 1, e->retire_takes = takes, e->retire_due = 0, e->retire_frame = now;
pool_unlist(e); // no slot resolves to it in the pool refresh; taken back, the refill of its slots lists it again
retiring_bytes += e->bytes;
freed += e->bytes;
tex_gen_bump(e->key.addr);
@@ -2424,6 +2714,7 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes
vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen);
e->uploaded = 1;
pool_gpu_owned(e);
e->pub.gpu_owned = 1;
}
e->chash_ok = 0;
@@ -2579,6 +2870,7 @@ void vk_texture_gpu_written(const tex_t *t) {
e->scan_frame = now + 1;
overwrite_others(e, now);
}
pool_gpu_owned(e);
e->pub.gpu_owned = 1;
e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged"
if (e->pub.skip) { // texture detail leaves out what the GPU can write (detail_skip); said if it ever does
@@ -2666,10 +2958,11 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
if (!sinfo) {
uint64_t *p = calloc(1 << 20, sizeof *p);
slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20);
slot_grp = malloc(sizeof *slot_grp << 20);
slot_bucket = calloc(1 << 20, sizeof *slot_bucket);
if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens");
for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1;
for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1;
if (!p || !slot_next || !slot_prev || !slot_grp || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens");
for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1, slot_grp[i] = -1;
for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1, bucket_ghead[i] = -1;
__atomic_store_n(&sinfo, p, __ATOMIC_RELEASE);
}
TexSlot *s = &tslots[id];
@@ -2695,7 +2988,6 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
slots_gen++;
if (!++reg_serial) reg_serial = 1;
__atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE);
slot_rebucket(id, agen_idx((uintptr_t)t->mem));
dirty_push((uint32_t)id);
s->has_view = v != NULL;
if (v) s->v = *v;
@@ -2711,6 +3003,7 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) {
for (size_t i = 0; i < sizeof s->v; i++) h = (h ^ b[i]) * 0x100000001b3ull;
s->khash = (h ^ s->has_view) | 1;
}
slot_place(id, agen_idx((uintptr_t)t->mem), s->khash);
unlock();
}
@@ -2777,6 +3070,7 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) {
else if (!e && !cpu_slot) e = own && own->superseded ? NULL : own; // a superseded one: entry_of takes it back
else if (!e && cpu_slot && s->finalized) stale = 1;
}
resolve_stale = stale;
if (!e && s->t.mem && !stale) {
// evicted by disuse: the pool refresh keeps the stand-in, without spending the creation budget on it
if (pool_ctx && holds.n && nvn_hm_get(&holds, &k)) resolve_held = 1;
@@ -2845,7 +3139,27 @@ void vk_texture_bucket_stats(char *out, size_t n) {
(unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone,
(unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]],
bump_count[top[0]], bump_count[top[1]], bump_count[top[2]], bump_count[top[3]]);
n_bumps = n_undone = 0;
size_t used = strlen(out);
if (used < n)
snprintf(out + used, n - used, "; pool freshness check: %llu entries checked, %llu passed by their regions (missed: "
"%llu not uploaded, %llu a region without a snapshot yet, %llu untracked, %llu written since, %llu "
"nothing to check; %llu snapshot queries of NX2NAVI_POOL_PRESNAP)",
(unsigned long long)n_pool_checks, (unsigned long long)n_pool_skips,
(unsigned long long)n_pool_miss[0], (unsigned long long)n_pool_miss[1],
(unsigned long long)n_pool_miss[2], (unsigned long long)n_pool_miss[3],
(unsigned long long)n_pool_miss[4], (unsigned long long)n_presnap_q);
used = strlen(out);
if (used < n)
snprintf(out + used, n - used, "; %d slot groups holding %llu slots", ngrp_live, (unsigned long long)n_grouped);
int held = 0, gown = 0;
for (int i = 0; i < npref; i++)
if (pref[i].refs && pref[i].e) held++, gown += pref[i].gown;
used = strlen(out);
if (used < n)
snprintf(out + used, n - used, "; pool slots hold %d entries, %d of them GPU-owned (stamped in each submission "
"checked by list)", held, gown);
n_bumps = n_undone = n_pool_checks = n_pool_skips = n_presnap_q = 0;
memset(n_pool_miss, 0, sizeof n_pool_miss);
memset(n_site, 0, sizeof n_site);
memset(bump_count, 0, sizeof bump_count);
unlock();
@@ -2888,8 +3202,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
s->cheld = (uint8_t)(!s->ce && resolve_held);
}
Tex *e = s->ce;
// a draw binding: the texture is in use, whether or not the pool walk checked it first in this submission
if (e && !pool_ctx) e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED);
// a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission
// (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too)
if (e && !pool_ctx) touch(e);
// Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a
// replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not
// mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A
@@ -2946,8 +3261,9 @@ uint32_t vk_texture_epoch(void) { return __atomic_load_n(&submit_epoch, __ATOMIC
void vk_texture_new_submit(void) {
lock();
if (round_checks && (uint64_t)round_checks * 2 >= pool_slots) look_round++; // the last one walked the pool
round_checks = 0;
// the last one walked the pool, or checked its entries by list
if (pool_by_list || (round_checks && (uint64_t)round_checks * 2 >= pool_slots)) look_round++;
round_checks = 0, pool_by_list = 0;
if (!++submit_epoch) submit_epoch = 1;
rate_tick();
sweep_pressure();