Merge the performance pass's texture generation and write-back changes into the test line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-09 23:43:10 -05:00
11 changed files with 186 additions and 43 deletions

View File

@@ -162,9 +162,9 @@
# made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay.
# 2 (diagnosis): the same marks, and such textures give way to stand-ins in the texture pool, but none is freed.
# tex_disuse = 1
# 1: a texture the game releases and takes straight back, with nothing else changed at its address, leaves the texture
# table's slots there as they were, instead of having them all looked up again (large towns release and take back
# thousands a second). 0: every release and take-back counts as a change, as before.
# 1: a texture the game releases or takes back, with nothing else changed at its address, puts the texture table's
# slots there back in a state they were in before, and they are not all looked up again (large towns release and take
# back thousands a second). 0: every release and take-back counts as a new state.
# tex_gen_undo = 1
# Host zstd decoder in place of the game's (0 = the game's), and a check that decodes every call both ways and logs
# the differences (1 = on; slow). Ocarina of Time 1.1.1 keeps the game's decoder while zstd_native is unset: its host
@@ -598,6 +598,10 @@
# are made inaccessible until their first read or write, which is logged with its instruction ("mem: write-back
# probe"). The bytes are there all along; the probe only watches, at the cost of the faults it takes.
# wb_probe = 0
# 1: what the GPU wrote goes back to the game's memory in a GPU submission of its own, and the wait after each frame's
# first submission ends when that submission's passes are through, without waiting for the copy back (3 to 4 ms of
# every frame). 0: the copy back is part of the submission, and the wait covers it.
# wb_split = 1
# 1: the game's flushes of mirrored memory are copied into the mirrors even where the write watch already sees every
# page the game wrote; 0 (default) copies only the written pages there, which saved 47-57 MiB of copies a frame in
# Kokiri Forest.

View File

@@ -62,7 +62,7 @@ static const char *const known[] = {
"SWKBD_ANSWER",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP",
@@ -119,7 +119,7 @@ static const struct { const char *key, *section; } sections[] = {
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"VK_NAN_PRESERVE", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
};
@@ -232,7 +232,7 @@ static const CfgMeta metas[] = {
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
// diagnostics
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha

View File

@@ -145,6 +145,9 @@ typedef struct {
uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value
// (gfx_submit_queue names the game's queue)
int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up
// As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest
// memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split.
int (*exec_wait)(uint64_t value, uint64_t timeout_ns);
uint64_t (*fence_done)(void); // highest value whose side effects are written
// Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that
// has passed or at_ns is 0.

View File

@@ -146,6 +146,7 @@ const GfxBackend gfx_null_backend = {
.program_release = null_program_release,
.submit = null_submit,
.fence_wait = null_fence_wait,
.exec_wait = null_fence_wait,
.fence_done = null_fence_done,
.present = null_present,
.finish = null_finish,

View File

@@ -660,6 +660,9 @@ NVN_API(nvnQueueSubmitCommands) {
// point). Copies of the data at submit made it worse, a mix of two frames' versions.
// NX2NAVI_SUBMIT_FINISH=<mask>: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits
// after every submission, 0 never.
// The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the
// write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on. The fences
// the game itself waits on cover them as before.
static long long finish_mask = -2;
if (finish_mask == -2) {
const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH");
@@ -667,7 +670,7 @@ NVN_API(nvnQueueSubmitCommands) {
if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all
}
int idx = nvn_submits_in_frame++;
if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX);
if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->exec_wait(f, UINT64_MAX);
}
// The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when

View File

@@ -570,6 +570,11 @@ void vk_sync_start(void); // vk_init (fatal without a device) a
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet.
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns);
int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own
VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs
int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record
uint64_t vk_sync_fence_done(void); // highest done value
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
void vk_sync_finish(void); // every submitted value done

View File

@@ -183,6 +183,11 @@ int vk_replay_async(void) {
return on;
}
// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's.
static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) {
return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns);
}
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
@@ -606,6 +611,7 @@ const GfxBackend gfx_vulkan_backend = {
.program_release = vk_be_program_release,
.submit = vk_be_submit,
.fence_wait = vk_sync_fence_wait,
.exec_wait = vk_be_exec_wait,
.fence_done = vk_sync_fence_done,
.present = vk_be_present,
.frame_start = vk_dlssg_frame_start,

View File

@@ -1325,6 +1325,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
vk_exec_pass_end(ex);
x_here = 0;
if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes
// The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the
// wait after a frame's first submission then ends when the submission's passes are through, not its write-backs.
if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) {
ex->cb = vk_sync_split_exec();
ex->dirty = VK_EXEC_DIRTY_ALL;
vk_exec_draw_new_cb();
}
if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
if (ex->nviews) {
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);

View File

@@ -2331,6 +2331,8 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) {
return j->n + 2;
}
int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); }
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
if (!vk_mirror_on) return 0;
int rec = 0;

View File

@@ -62,7 +62,7 @@ static struct {
int n;
VkSubmitInfo2 si[HOLD_CAP];
VkCommandBufferSubmitInfo cbi[HOLD_CAP][2];
VkSemaphoreSubmitInfo sig[HOLD_CAP];
VkSemaphoreSubmitInfo sig[HOLD_CAP][2];
uint64_t last; // the newest timeline value a held submission signals, 0 none
uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried
} H;
@@ -71,6 +71,26 @@ static volatile uint64_t held_t0; // when the oldest held submission was hel
static volatile uint64_t queued_v; // the newest timeline value handed to the queue
static int hold_max = -1;
// ---- the executed timeline
// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest
// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's
// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes
// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every
// frame with the render thread idle.
// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the
// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the
// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split
// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among
// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's.
static VkSemaphore exec_tl;
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
static int wb_split = -1;
int vk_sync_wb_split(void) {
if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT");
return wb_split && exec_tl != VK_NULL_HANDLE;
}
static void hold_flush_locked(VkCtx *c) {
if (!H.n) return;
VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
@@ -111,7 +131,10 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) {
if (si->commandBufferInfoCount)
memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount);
H.si[i].pCommandBufferInfos = H.cbi[i];
if (si->signalSemaphoreInfoCount) H.sig[i] = si->pSignalSemaphoreInfos[0], H.si[i].pSignalSemaphoreInfos = &H.sig[i];
if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl)
memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount);
H.si[i].pSignalSemaphoreInfos = H.sig[i];
}
if (value) H.last = value;
uint64_t now = kern_now_ns();
if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE);
@@ -430,6 +453,12 @@ void vk_sync_start(void) {
if (reserved < now) reserved = now;
stop = 0;
plat_mutex_unlock(&st_lock);
// the executed timeline starts where the timeline is; without it every exec wait is a fence wait
VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now};
VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et};
if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE;
exec_sig = now;
vk_mem_note_submit(now);
completion_thread = plat_thread_new(completion_main, c, "vk completion");
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
@@ -484,7 +513,12 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
return cb;
}
VkCommandBuffer vk_sync_split(void) {
static VkCommandBuffer split_part(int exec);
VkCommandBuffer vk_sync_split(void) { return split_part(0); }
// The part that ends here is the last with the submission's own commands: it signals the executed timeline.
VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); }
static VkCommandBuffer split_part(int exec) {
VkCtx *c = vk_ctx();
VkCommandBuffer cb = slots[cur_slot].cb;
vk_gpu_passes_part(cb, 1);
@@ -496,6 +530,10 @@ VkCommandBuffer vk_sync_split(void) {
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
.pCommandBufferInfos = pro ? cbi : cbi + 1};
VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
.value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
if (exec && exec_tl && exec_sig < cur_fence)
si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence;
ts_submitted();
queue_submit(c, &si, 0); // held or handed to the queue (submit batching)
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
@@ -556,12 +594,17 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
s = &slots[cur_slot];
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
// the timeline, and the executed timeline unless a part of this submission signalled it already (split_part)
VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT},
{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}};
int both = exec_tl && exec_sig < fence;
if (both) exec_sig = fence;
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
.pSignalSemaphoreInfos = &sig};
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u,
.pSignalSemaphoreInfos = sig};
ts_submitted();
queue_submit(c, &si, fence); // held or handed to the queue (submit batching)
s->fence = recorded ? fence : 0;
@@ -635,6 +678,27 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
return ok;
}
// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the
// CPU side effects of its fence may still be to come. 1 when reached.
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
vk_sync_start();
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
uint64_t p0 = perf_begin();
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
VkCtx *c = vk_ctx();
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
.pSemaphores = &exec_tl, .pValues = &value};
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
vk_device_fault_report(r);
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
}
perf_end(PERF_FENCE, p0);
perf_gpu_wait(p0);
return r == VK_SUCCESS;
}
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
@@ -661,6 +725,8 @@ void vk_sync_shutdown(void) {
plat_mutex_lock(&sub_lock);
VkCtx *c = vk_ctx();
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL);
exec_tl = VK_NULL_HANDLE;
perf_gpu_stats = NULL;
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
ts_pool = VK_NULL_HANDLE;

View File

@@ -57,7 +57,7 @@ typedef struct Tex {
uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it
int uploaded, released;
uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve)
uint32_t rel_agen, rel_pre; // its address generation after and before the release's bump (tex_gen_unrelease)
uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release
uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale)
uint32_t back_gap; // most presents between a release and its taking back so far (stale_after)
uint32_t n_up; // uploads of this entry (vk_texture_debug)
@@ -203,13 +203,35 @@ static uint16_t *slot_bucket;
static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; }
// tex_gen_bump calls, by site: destroy, create, take-back, release; and the take-backs that undid their release's bump
// instead (tex_gen_unrelease), for vk_texture_bucket_stats
// tex_gen_bump calls, by site: destroy, create, take-back, release; and the releases and take-backs that went back to a
// generation their bucket's state had before (tex_gen_flip), for vk_texture_bucket_stats
static uint64_t n_bumps, n_site[4], n_undone;
static uint32_t bump_count[AGEN_N];
// The highest generation each bucket ever had. A bump takes the next one after it, not the one after the current
// value, so a value a bucket went back from (tex_gen_unrelease) is never given to another state.
// The highest generation each bucket ever had. A new state takes the next one after it, not the one after the current
// value, so a value a bucket went back from (tex_gen_flip) is never given to another state.
static uint32_t agen_top[AGEN_N];
// What a slot at an address resolves to depends on the entries there and on which of them are released
// (slot_resolve). Per bucket: epoch counts every change other than a release or a take-back (an entry made, destroyed,
// retired, held or taken back with more than its flag changing), sig is the released entries' uids XORed, which a
// release and a take-back each flip. (bucket, epoch, sig) names the state; gs remembers the generation each state
// got, so a bucket that comes back to a state gets that state's generation back.
static uint32_t bepoch[AGEN_N];
static uint64_t bsig[AGEN_N];
#define GS_N 8192
static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N];
// A slot lookup is resolving (vk_texture_by_id_locked around slot_resolve). The lookup's callers stamp what they keep
// with the generation they read before it: the slot's own cache, the pool's tokens, the descriptor, type and feedback
// memos (vk_exec_draw.c). A lookup that itself changes the state (it takes a released entry back for a slot registered
// since the release) leaves such a stamp naming the state it left, with a result that state would not give: the next
// release of that entry is later than the slot's registration, the slot is stale then (slot_resolve), and the kept
// result would hand out the released entry, whose storage the game may be unmapping. So the state such a lookup
// leaves loses its remembered generation (tex_gen_flip): it gets a new one when the bucket is next in it, and the
// stamp never matches again.
static int in_lookup;
static uint32_t gs_at(uint32_t b) {
return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51);
}
static void tex_gen_bump(uint64_t addr) {
n_bumps++;
@@ -217,35 +239,59 @@ static void tex_gen_bump(uint64_t addr) {
if (!++tex_gen) tex_gen = 1;
slots_gen++;
uint32_t b = agen_idx(addr);
bepoch[b]++;
__atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE);
gs[gs_at(b)] = (typeof(gs[0])){bsig[b], b, bepoch[b], agen[b], 1};
dirty_push(DIRTY_BUCKET | b);
}
// NX2NAVI_TEX_GEN_UNDO (default 1; 0: a take-back bumps like any change). Ocarina finalizes and initializes transient
// textures all through a frame: in a large town about 40,000 releases and as many take-backs per 3,600 submissions,
// each a bump of its address bucket, and the buckets with the most slots (1,300 registered over one address) were the
// ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it and spent its 6 ms of a
// frame in half the frames there, which is also when new textures arrive late.
// A release followed by the take-back of the same entry, with no other change at its address between them, leaves every
// slot there resolving as before the release: slot_resolve reads the entries at the address and their released flags,
// and those are as they were. So the take-back puts the bucket's generation back to its value from before the
// release, and whatever was resolved then counts as current again: the pool's tokens, the descriptor memo and the
// slots' own cached entries. What was resolved while the entry was released carries the generation in between, which
// no state gets again (agen_top). The global generations still move, so their readers look, and find nothing changed.
// Not for an entry that was superseded or overwritten while released: its take-back changes more than the flag.
static void tex_gen_unrelease(Tex *e) {
// NX2NAVI_TEX_GEN_UNDO (default 1; 0: every release and take-back is a new generation). Ocarina finalizes and
// initializes transient textures all through a frame: in a large town about 40,000 releases and as many take-backs
// per 3,600 submissions, each a bump of its address bucket, and the buckets with the most slots (1,300 registered over
// one address) were the ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it.
// A release or a take-back changes one entry's released flag and nothing else, so it moves the bucket between states
// it has usually been in before: the textures the game keeps released and takes back for a moment, and those it
// releases and takes straight back. The bucket then gets the generation that state had, and whatever was resolved in
// that state counts as current again: the pool's tokens, the descriptor memo and the slots' own cached entries. A
// state seen for the first time gets a new generation (agen_top), and any other change starts a new epoch, so a
// generation never stands for two states. The global generations still move, so their readers look, and find the
// tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than
// the flag, and is a change like any other.
// One input of slot_resolve is not in the state: the stale rule compares a slot's registration with the entry's
// release (rel_serial), which is later at every release. A slot is stale at every later release once it is at one, so
// what it resolves to in a released state stays right; the one that is not stale takes the entry back, which is the
// lookup-made change in_lookup deals with.
static void tex_gen_flip(Tex *e, int plain) {
static int on = -1;
static uint64_t uids;
if (on < 0) on = rt_knob("TEX_GEN_UNDO", "1") == 1;
uint32_t b = agen_idx(e->key.addr), at = e->rel_agen;
e->rel_agen = 0;
if (!on || !at || agen[b] != at || e->superseded || e->overwritten) {
if (!e->uid) {
uint64_t z = ++uids * 0x9E3779B97F4A7C15ull;
z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull, z = (z ^ z >> 27) * 0x94D049BB133111EBull;
e->uid = (z ^ z >> 31) | 1;
}
uint32_t b = agen_idx(e->key.addr);
if (in_lookup) { // see in_lookup: the state this lookup leaves is not to get its generation again
typeof(gs[0]) *m0 = &gs[gs_at(b)];
if (m0->valid && m0->b == b && m0->epoch == bepoch[b] && m0->sig == bsig[b]) m0->valid = 0;
}
bsig[b] ^= e->uid;
if (!on || plain) {
tex_gen_bump(e->key.addr);
return;
}
n_undone++;
if (!++tex_gen) tex_gen = 1;
slots_gen++;
__atomic_store_n(&agen[b], e->rel_pre, __ATOMIC_RELEASE);
typeof(gs[0]) *m = &gs[gs_at(b)];
if (m->valid && m->b == b && m->epoch == bepoch[b] && m->sig == bsig[b]) {
n_undone++;
__atomic_store_n(&agen[b], m->gen, __ATOMIC_RELEASE);
} else {
n_bumps++;
bump_count[b]++;
__atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE);
*m = (typeof(*m)){bsig[b], b, bepoch[b], agen[b], 1};
}
dirty_push(DIRTY_BUCKET | b);
}
@@ -2134,7 +2180,7 @@ static Tex *entry_of(const tex_t *t) {
e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one
uint64_t gap = e->samp_frame - e->rel_frame;
if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap;
e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_unrelease(e);
e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten);
e->overwritten = 0;
if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work
e->superseded = 0, e->reused = 1;
@@ -2201,9 +2247,7 @@ static void mark_released(Tex *e) { // tex_lock held; e is not released
nreleased++;
rel_append(e);
n_site[3]++;
e->rel_pre = agen[agen_idx(e->key.addr)];
tex_gen_bump(e->key.addr);
e->rel_agen = agen[agen_idx(e->key.addr)];
tex_gen_flip(e, 0);
}
// e, in the GPU-only address window, is written by the GPU in frame fr: the entries its storage runs over are
@@ -2473,8 +2517,8 @@ void vk_texture_bucket_stats(char *out, size_t n) {
}
uint64_t hot = 0; // bumps weighted by the slots they invalidate
for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b];
snprintf(out, n, "%llu bumps (destroy %llu, create %llu, take-back %llu, release %llu; %llu take-backs put their "
"release's generation back instead), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u",
snprintf(out, n, "%llu new generations (destroy %llu, create %llu, take-back %llu, release %llu; %llu releases and "
"take-backs went back to a generation their state had), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u",
(unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1],
(unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone,
(unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]],
@@ -2513,7 +2557,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
view_t v;
int hv;
resolve_held = 0;
in_lookup = 1;
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
in_lookup = 0;
s->cview = v, s->chas_view = (uint8_t)hv;
s->cgen = g;
s->cepoch = 0;