mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 07:31:06 -04:00
Merge the performance pass's texture generation and write-back changes into the test line.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -62,7 +62,7 @@ static const char *const known[] = {
|
||||
"SWKBD_ANSWER",
|
||||
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
|
||||
"TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP",
|
||||
@@ -119,7 +119,7 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
|
||||
{"VK_NAN_PRESERVE", "performance"},
|
||||
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
|
||||
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
|
||||
{"VMA_BLOCK_MB", "diagnostics"}, {"ASTC_BC_PROBE", "diagnostics"},
|
||||
};
|
||||
@@ -232,7 +232,7 @@ static const CfgMeta metas[] = {
|
||||
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
|
||||
// diagnostics
|
||||
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
|
||||
{"WB_FENCE_LIMIT", "65536", "0..1073741824", RESTART}, {"WB_MERGE", "1", "0|1", RESTART}, {"WB_EXACT", "1", "0|1", RESTART}, {"MEM_CENSUS", "0", "0|1", RESTART}, {"WB_NO_REUP", "1", "0|1", RESTART}, {"WB_PROBE", "0", "0|1", RESTART}, {"WB_SPLIT", "1", "0|1", RESTART}, {"FLUSH_UPLOAD", "0", "0|1", RESTART}, {"GPA_TWIN_MAX_KB", "0", "0..1048576", RESTART},
|
||||
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
|
||||
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
|
||||
{"ASTC_BC_PROBE", "0", "0|1", RESTART}, // vk_texture.c astc_probe_alpha
|
||||
|
||||
@@ -145,6 +145,9 @@ typedef struct {
|
||||
uint64_t (*submit)(const uint64_t *handles, int n); // replays; returns this submission's fence value
|
||||
// (gfx_submit_queue names the game's queue)
|
||||
int (*fence_wait)(uint64_t value, uint64_t timeout_ns); // 1 reached; caller has given its core up
|
||||
// As fence_wait, but only until the GPU has executed the submission's own commands: its write-backs to guest
|
||||
// memory and its fence's side effects may follow (vk_sync.c exec_tl). The same as fence_wait where there is no split.
|
||||
int (*exec_wait)(uint64_t value, uint64_t timeout_ns);
|
||||
uint64_t (*fence_done)(void); // highest value whose side effects are written
|
||||
// Queues the present and returns at once; the host present goes out at at_ns (kern_now_ns clock), at once when that
|
||||
// has passed or at_ns is 0.
|
||||
|
||||
@@ -146,6 +146,7 @@ const GfxBackend gfx_null_backend = {
|
||||
.program_release = null_program_release,
|
||||
.submit = null_submit,
|
||||
.fence_wait = null_fence_wait,
|
||||
.exec_wait = null_fence_wait,
|
||||
.fence_done = null_fence_done,
|
||||
.present = null_present,
|
||||
.finish = null_finish,
|
||||
|
||||
@@ -660,6 +660,9 @@ NVN_API(nvnQueueSubmitCommands) {
|
||||
// point). Copies of the data at submit made it worse, a mix of two frames' versions.
|
||||
// NX2NAVI_SUBMIT_FINISH=<mask>: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits
|
||||
// after every submission, 0 never.
|
||||
// The wait is for the submission's passes, which is what the rewrite races with: exec_wait leaves out the
|
||||
// write-backs of what they wrote (vk_sync.c exec_tl), which nothing here needs before the game goes on. The fences
|
||||
// the game itself waits on cover them as before.
|
||||
static long long finish_mask = -2;
|
||||
if (finish_mask == -2) {
|
||||
const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH");
|
||||
@@ -667,7 +670,7 @@ NVN_API(nvnQueueSubmitCommands) {
|
||||
if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all
|
||||
}
|
||||
int idx = nvn_submits_in_frame++;
|
||||
if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX);
|
||||
if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->exec_wait(f, UINT64_MAX);
|
||||
}
|
||||
|
||||
// The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when
|
||||
|
||||
@@ -570,6 +570,11 @@ void vk_sync_start(void); // vk_init (fatal without a device) a
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
|
||||
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
|
||||
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
|
||||
// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet.
|
||||
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns);
|
||||
int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own
|
||||
VkCommandBuffer vk_sync_split_exec(void); // vk_sync_split for the part before a submission's write-backs
|
||||
int vk_mem_mirror_wb_pending(void); // vk_memory.c: vk_mem_mirror_end has write-backs to record
|
||||
uint64_t vk_sync_fence_done(void); // highest done value
|
||||
uint64_t vk_sync_issued(void); // newest submitted value (the one being recorded is this + 1)
|
||||
void vk_sync_finish(void); // every submitted value done
|
||||
|
||||
@@ -183,6 +183,11 @@ int vk_replay_async(void) {
|
||||
return on;
|
||||
}
|
||||
|
||||
// With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's.
|
||||
static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) {
|
||||
return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns);
|
||||
}
|
||||
|
||||
// The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end.
|
||||
int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); }
|
||||
|
||||
@@ -606,6 +611,7 @@ const GfxBackend gfx_vulkan_backend = {
|
||||
.program_release = vk_be_program_release,
|
||||
.submit = vk_be_submit,
|
||||
.fence_wait = vk_sync_fence_wait,
|
||||
.exec_wait = vk_be_exec_wait,
|
||||
.fence_done = vk_sync_fence_done,
|
||||
.present = vk_be_present,
|
||||
.frame_start = vk_dlssg_frame_start,
|
||||
|
||||
@@ -1325,6 +1325,13 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t
|
||||
vk_exec_pass_end(ex);
|
||||
x_here = 0;
|
||||
if (ex->recorded) vk_exec_barrier(ex), vk_exec_pool_marks(ex); // later submissions see this one's writes
|
||||
// The write-backs in a part of their own, after one that signals the executed timeline (vk_sync.c exec_tl): the
|
||||
// wait after a frame's first submission then ends when the submission's passes are through, not its write-backs.
|
||||
if (synced && ex->recorded && vk_sync_wb_split() && vk_mem_mirror_wb_pending() && vk_exec_pool_split(ex->fence)) {
|
||||
ex->cb = vk_sync_split_exec();
|
||||
ex->dirty = VK_EXEC_DIRTY_ALL;
|
||||
vk_exec_draw_new_cb();
|
||||
}
|
||||
if (synced) ex->recorded += vk_mem_mirror_end(ex->cb, fence), vk_usnap_end(fence); // VRAM mirror bytes the GPU wrote go back to guest memory
|
||||
if (ex->nviews) {
|
||||
ViewList *l = malloc(sizeof *l + sizeof(VkImageView) * (size_t)ex->nviews);
|
||||
|
||||
@@ -2331,6 +2331,8 @@ static int verify_record(VkCommandBuffer cb, uint64_t fence) {
|
||||
return j->n + 2;
|
||||
}
|
||||
|
||||
int vk_mem_mirror_wb_pending(void) { return vk_mirror_on && (nsnl || nwbl); }
|
||||
|
||||
int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) {
|
||||
if (!vk_mirror_on) return 0;
|
||||
int rec = 0;
|
||||
|
||||
@@ -62,7 +62,7 @@ static struct {
|
||||
int n;
|
||||
VkSubmitInfo2 si[HOLD_CAP];
|
||||
VkCommandBufferSubmitInfo cbi[HOLD_CAP][2];
|
||||
VkSemaphoreSubmitInfo sig[HOLD_CAP];
|
||||
VkSemaphoreSubmitInfo sig[HOLD_CAP][2];
|
||||
uint64_t last; // the newest timeline value a held submission signals, 0 none
|
||||
uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried
|
||||
} H;
|
||||
@@ -71,6 +71,26 @@ static volatile uint64_t held_t0; // when the oldest held submission was hel
|
||||
static volatile uint64_t queued_v; // the newest timeline value handed to the queue
|
||||
static int hold_max = -1;
|
||||
|
||||
// ---- the executed timeline
|
||||
// A submission's write-backs (vk_memory.c vk_mem_mirror_end: what its shaders wrote into VRAM mirrors goes back to guest
|
||||
// memory) are the last thing in it, 3 to 4 ms of GPU time a frame in Ocarina, 60 MiB merged. The wait after a frame's
|
||||
// first submission (nvn_api_core.c nvnQueueSubmitCommands) is there so that the GPU has run that submission's passes
|
||||
// before the game rewrites what they read; it does not need the write-backs, and waited for them too: 5 ms of every
|
||||
// frame with the render thread idle.
|
||||
// exec_tl is a second timeline that reaches a submission's value when the GPU has executed its own commands: the
|
||||
// replay ends the part before the write-backs (vk_exec.c, vk_sync_split_exec), that part signals exec_tl, and the
|
||||
// write-backs follow in a part of their own, which signals the timeline as before. A submission without that split
|
||||
// signals both at its end. vk_sync_exec_wait waits on exec_tl; every other wait, the fences the game asks for among
|
||||
// them, still covers the write-backs. NX2NAVI_WB_SPLIT=0: no split, and the wait is the fence's.
|
||||
static VkSemaphore exec_tl;
|
||||
static uint64_t exec_sig; // sub_lock: the newest value a submitted part signals on exec_tl
|
||||
static int wb_split = -1;
|
||||
|
||||
int vk_sync_wb_split(void) {
|
||||
if (wb_split < 0) wb_split = cfg_bool("NX2NAVI_WB_SPLIT");
|
||||
return wb_split && exec_tl != VK_NULL_HANDLE;
|
||||
}
|
||||
|
||||
static void hold_flush_locked(VkCtx *c) {
|
||||
if (!H.n) return;
|
||||
VkResult r = vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE);
|
||||
@@ -111,7 +131,10 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) {
|
||||
if (si->commandBufferInfoCount)
|
||||
memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount);
|
||||
H.si[i].pCommandBufferInfos = H.cbi[i];
|
||||
if (si->signalSemaphoreInfoCount) H.sig[i] = si->pSignalSemaphoreInfos[0], H.si[i].pSignalSemaphoreInfos = &H.sig[i];
|
||||
if (si->signalSemaphoreInfoCount) { // at most two: the timeline and the executed timeline (exec_tl)
|
||||
memcpy(H.sig[i], si->pSignalSemaphoreInfos, sizeof H.sig[i][0] * si->signalSemaphoreInfoCount);
|
||||
H.si[i].pSignalSemaphoreInfos = H.sig[i];
|
||||
}
|
||||
if (value) H.last = value;
|
||||
uint64_t now = kern_now_ns();
|
||||
if (i == 0) __atomic_store_n(&held_t0, now, __ATOMIC_RELEASE);
|
||||
@@ -430,6 +453,12 @@ void vk_sync_start(void) {
|
||||
if (reserved < now) reserved = now;
|
||||
stop = 0;
|
||||
plat_mutex_unlock(&st_lock);
|
||||
// the executed timeline starts where the timeline is; without it every exec wait is a fence wait
|
||||
VkSemaphoreTypeCreateInfo et = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
|
||||
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE, .initialValue = now};
|
||||
VkSemaphoreCreateInfo ei = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, .pNext = &et};
|
||||
if (vkCreateSemaphore(c->device, &ei, NULL, &exec_tl) != VK_SUCCESS) exec_tl = VK_NULL_HANDLE;
|
||||
exec_sig = now;
|
||||
vk_mem_note_submit(now);
|
||||
completion_thread = plat_thread_new(completion_main, c, "vk completion");
|
||||
if (!completion_thread) rt_fatal("vk: cannot start the completion thread");
|
||||
@@ -484,7 +513,12 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
|
||||
return cb;
|
||||
}
|
||||
|
||||
VkCommandBuffer vk_sync_split(void) {
|
||||
static VkCommandBuffer split_part(int exec);
|
||||
VkCommandBuffer vk_sync_split(void) { return split_part(0); }
|
||||
// The part that ends here is the last with the submission's own commands: it signals the executed timeline.
|
||||
VkCommandBuffer vk_sync_split_exec(void) { return split_part(1); }
|
||||
|
||||
static VkCommandBuffer split_part(int exec) {
|
||||
VkCtx *c = vk_ctx();
|
||||
VkCommandBuffer cb = slots[cur_slot].cb;
|
||||
vk_gpu_passes_part(cb, 1);
|
||||
@@ -496,6 +530,10 @@ VkCommandBuffer vk_sync_split(void) {
|
||||
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = cb}};
|
||||
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = pro ? 2 : 1,
|
||||
.pCommandBufferInfos = pro ? cbi : cbi + 1};
|
||||
VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
|
||||
.value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
|
||||
if (exec && exec_tl && exec_sig < cur_fence)
|
||||
si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence;
|
||||
ts_submitted();
|
||||
queue_submit(c, &si, 0); // held or handed to the queue (submit batching)
|
||||
vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted
|
||||
@@ -556,12 +594,17 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) {
|
||||
s = &slots[cur_slot];
|
||||
VkCommandBufferSubmitInfo cbi[2] = {{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = pro},
|
||||
{.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = s->cb}};
|
||||
VkSemaphoreSubmitInfo sig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
|
||||
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT};
|
||||
// the timeline, and the executed timeline unless a part of this submission signalled it already (split_part)
|
||||
VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline,
|
||||
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT},
|
||||
{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl,
|
||||
.value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}};
|
||||
int both = exec_tl && exec_sig < fence;
|
||||
if (both) exec_sig = fence;
|
||||
VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
|
||||
.commandBufferInfoCount = (pro ? 1u : 0u) + (recorded ? 1u : 0u),
|
||||
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = 1,
|
||||
.pSignalSemaphoreInfos = &sig};
|
||||
.pCommandBufferInfos = pro ? cbi : cbi + 1, .signalSemaphoreInfoCount = both ? 2u : 1u,
|
||||
.pSignalSemaphoreInfos = sig};
|
||||
ts_submitted();
|
||||
queue_submit(c, &si, fence); // held or handed to the queue (submit batching)
|
||||
s->fence = recorded ? fence : 0;
|
||||
@@ -635,6 +678,27 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns) {
|
||||
return ok;
|
||||
}
|
||||
|
||||
// Waits until the GPU has executed submission value's own commands (the executed timeline); its write-backs and the
|
||||
// CPU side effects of its fence may still be to come. 1 when reached.
|
||||
int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns) {
|
||||
vk_sync_start();
|
||||
if (!vk_sync_wb_split()) return vk_sync_fence_wait(value, timeout_ns);
|
||||
if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1;
|
||||
uint64_t p0 = perf_begin();
|
||||
hold_release(value, 0, 0); // a held submission the waiter needs goes to the queue now
|
||||
VkCtx *c = vk_ctx();
|
||||
VkSemaphoreWaitInfo wi = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, .semaphoreCount = 1,
|
||||
.pSemaphores = &exec_tl, .pValues = &value};
|
||||
VkResult r = vkWaitSemaphores(c->device, &wi, timeout_ns);
|
||||
if (r != VK_SUCCESS && r != VK_TIMEOUT) {
|
||||
vk_device_fault_report(r);
|
||||
rt_fatal("vk: the wait for a submission's commands failed (%d; device lost?)", (int)r);
|
||||
}
|
||||
perf_end(PERF_FENCE, p0);
|
||||
perf_gpu_wait(p0);
|
||||
return r == VK_SUCCESS;
|
||||
}
|
||||
|
||||
uint64_t vk_sync_fence_done(void) { return __atomic_load_n(&done, __ATOMIC_ACQUIRE); }
|
||||
|
||||
uint64_t vk_sync_issued(void) { return __atomic_load_n(&issued, __ATOMIC_ACQUIRE); }
|
||||
@@ -661,6 +725,8 @@ void vk_sync_shutdown(void) {
|
||||
plat_mutex_lock(&sub_lock);
|
||||
VkCtx *c = vk_ctx();
|
||||
if (c && cmd_pool) vkDestroyCommandPool(c->device, cmd_pool, NULL);
|
||||
if (c && exec_tl) vkDestroySemaphore(c->device, exec_tl, NULL);
|
||||
exec_tl = VK_NULL_HANDLE;
|
||||
perf_gpu_stats = NULL;
|
||||
if (c && ts_pool) vkDestroyQueryPool(c->device, ts_pool, NULL);
|
||||
ts_pool = VK_NULL_HANDLE;
|
||||
|
||||
@@ -57,7 +57,7 @@ typedef struct Tex {
|
||||
uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it
|
||||
int uploaded, released;
|
||||
uint32_t rel_serial; // reg_serial when it was released: slots registered before then are stale (slot_resolve)
|
||||
uint32_t rel_agen, rel_pre; // its address generation after and before the release's bump (tex_gen_unrelease)
|
||||
uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release
|
||||
uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale)
|
||||
uint32_t back_gap; // most presents between a release and its taking back so far (stale_after)
|
||||
uint32_t n_up; // uploads of this entry (vk_texture_debug)
|
||||
@@ -203,13 +203,35 @@ static uint16_t *slot_bucket;
|
||||
|
||||
static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; }
|
||||
|
||||
// tex_gen_bump calls, by site: destroy, create, take-back, release; and the take-backs that undid their release's bump
|
||||
// instead (tex_gen_unrelease), for vk_texture_bucket_stats
|
||||
// tex_gen_bump calls, by site: destroy, create, take-back, release; and the releases and take-backs that went back to a
|
||||
// generation their bucket's state had before (tex_gen_flip), for vk_texture_bucket_stats
|
||||
static uint64_t n_bumps, n_site[4], n_undone;
|
||||
static uint32_t bump_count[AGEN_N];
|
||||
// The highest generation each bucket ever had. A bump takes the next one after it, not the one after the current
|
||||
// value, so a value a bucket went back from (tex_gen_unrelease) is never given to another state.
|
||||
// The highest generation each bucket ever had. A new state takes the next one after it, not the one after the current
|
||||
// value, so a value a bucket went back from (tex_gen_flip) is never given to another state.
|
||||
static uint32_t agen_top[AGEN_N];
|
||||
// What a slot at an address resolves to depends on the entries there and on which of them are released
|
||||
// (slot_resolve). Per bucket: epoch counts every change other than a release or a take-back (an entry made, destroyed,
|
||||
// retired, held or taken back with more than its flag changing), sig is the released entries' uids XORed, which a
|
||||
// release and a take-back each flip. (bucket, epoch, sig) names the state; gs remembers the generation each state
|
||||
// got, so a bucket that comes back to a state gets that state's generation back.
|
||||
static uint32_t bepoch[AGEN_N];
|
||||
static uint64_t bsig[AGEN_N];
|
||||
#define GS_N 8192
|
||||
static struct { uint64_t sig; uint32_t b, epoch, gen, valid; } gs[GS_N];
|
||||
// A slot lookup is resolving (vk_texture_by_id_locked around slot_resolve). The lookup's callers stamp what they keep
|
||||
// with the generation they read before it: the slot's own cache, the pool's tokens, the descriptor, type and feedback
|
||||
// memos (vk_exec_draw.c). A lookup that itself changes the state (it takes a released entry back for a slot registered
|
||||
// since the release) leaves such a stamp naming the state it left, with a result that state would not give: the next
|
||||
// release of that entry is later than the slot's registration, the slot is stale then (slot_resolve), and the kept
|
||||
// result would hand out the released entry, whose storage the game may be unmapping. So the state such a lookup
|
||||
// leaves loses its remembered generation (tex_gen_flip): it gets a new one when the bucket is next in it, and the
|
||||
// stamp never matches again.
|
||||
static int in_lookup;
|
||||
|
||||
static uint32_t gs_at(uint32_t b) {
|
||||
return (uint32_t)(((bsig[b] ^ ((uint64_t)b << 32 | bepoch[b])) * 0x9E3779B97F4A7C15ull) >> 51);
|
||||
}
|
||||
|
||||
static void tex_gen_bump(uint64_t addr) {
|
||||
n_bumps++;
|
||||
@@ -217,35 +239,59 @@ static void tex_gen_bump(uint64_t addr) {
|
||||
if (!++tex_gen) tex_gen = 1;
|
||||
slots_gen++;
|
||||
uint32_t b = agen_idx(addr);
|
||||
bepoch[b]++;
|
||||
__atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE);
|
||||
gs[gs_at(b)] = (typeof(gs[0])){bsig[b], b, bepoch[b], agen[b], 1};
|
||||
dirty_push(DIRTY_BUCKET | b);
|
||||
}
|
||||
|
||||
// NX2NAVI_TEX_GEN_UNDO (default 1; 0: a take-back bumps like any change). Ocarina finalizes and initializes transient
|
||||
// textures all through a frame: in a large town about 40,000 releases and as many take-backs per 3,600 submissions,
|
||||
// each a bump of its address bucket, and the buckets with the most slots (1,300 registered over one address) were the
|
||||
// ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it and spent its 6 ms of a
|
||||
// frame in half the frames there, which is also when new textures arrive late.
|
||||
// A release followed by the take-back of the same entry, with no other change at its address between them, leaves every
|
||||
// slot there resolving as before the release: slot_resolve reads the entries at the address and their released flags,
|
||||
// and those are as they were. So the take-back puts the bucket's generation back to its value from before the
|
||||
// release, and whatever was resolved then counts as current again: the pool's tokens, the descriptor memo and the
|
||||
// slots' own cached entries. What was resolved while the entry was released carries the generation in between, which
|
||||
// no state gets again (agen_top). The global generations still move, so their readers look, and find nothing changed.
|
||||
// Not for an entry that was superseded or overwritten while released: its take-back changes more than the flag.
|
||||
static void tex_gen_unrelease(Tex *e) {
|
||||
// NX2NAVI_TEX_GEN_UNDO (default 1; 0: every release and take-back is a new generation). Ocarina finalizes and
|
||||
// initializes transient textures all through a frame: in a large town about 40,000 releases and as many take-backs
|
||||
// per 3,600 submissions, each a bump of its address bucket, and the buckets with the most slots (1,300 registered over
|
||||
// one address) were the ones bumped most. The texture pool refresh refilled about 6,000 slots per submission for it.
|
||||
// A release or a take-back changes one entry's released flag and nothing else, so it moves the bucket between states
|
||||
// it has usually been in before: the textures the game keeps released and takes back for a moment, and those it
|
||||
// releases and takes straight back. The bucket then gets the generation that state had, and whatever was resolved in
|
||||
// that state counts as current again: the pool's tokens, the descriptor memo and the slots' own cached entries. A
|
||||
// state seen for the first time gets a new generation (agen_top), and any other change starts a new epoch, so a
|
||||
// generation never stands for two states. The global generations still move, so their readers look, and find the
|
||||
// tokens equal. plain: the take-back of an entry that was superseded or overwritten while released changes more than
|
||||
// the flag, and is a change like any other.
|
||||
// One input of slot_resolve is not in the state: the stale rule compares a slot's registration with the entry's
|
||||
// release (rel_serial), which is later at every release. A slot is stale at every later release once it is at one, so
|
||||
// what it resolves to in a released state stays right; the one that is not stale takes the entry back, which is the
|
||||
// lookup-made change in_lookup deals with.
|
||||
static void tex_gen_flip(Tex *e, int plain) {
|
||||
static int on = -1;
|
||||
static uint64_t uids;
|
||||
if (on < 0) on = rt_knob("TEX_GEN_UNDO", "1") == 1;
|
||||
uint32_t b = agen_idx(e->key.addr), at = e->rel_agen;
|
||||
e->rel_agen = 0;
|
||||
if (!on || !at || agen[b] != at || e->superseded || e->overwritten) {
|
||||
if (!e->uid) {
|
||||
uint64_t z = ++uids * 0x9E3779B97F4A7C15ull;
|
||||
z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull, z = (z ^ z >> 27) * 0x94D049BB133111EBull;
|
||||
e->uid = (z ^ z >> 31) | 1;
|
||||
}
|
||||
uint32_t b = agen_idx(e->key.addr);
|
||||
if (in_lookup) { // see in_lookup: the state this lookup leaves is not to get its generation again
|
||||
typeof(gs[0]) *m0 = &gs[gs_at(b)];
|
||||
if (m0->valid && m0->b == b && m0->epoch == bepoch[b] && m0->sig == bsig[b]) m0->valid = 0;
|
||||
}
|
||||
bsig[b] ^= e->uid;
|
||||
if (!on || plain) {
|
||||
tex_gen_bump(e->key.addr);
|
||||
return;
|
||||
}
|
||||
n_undone++;
|
||||
if (!++tex_gen) tex_gen = 1;
|
||||
slots_gen++;
|
||||
__atomic_store_n(&agen[b], e->rel_pre, __ATOMIC_RELEASE);
|
||||
typeof(gs[0]) *m = &gs[gs_at(b)];
|
||||
if (m->valid && m->b == b && m->epoch == bepoch[b] && m->sig == bsig[b]) {
|
||||
n_undone++;
|
||||
__atomic_store_n(&agen[b], m->gen, __ATOMIC_RELEASE);
|
||||
} else {
|
||||
n_bumps++;
|
||||
bump_count[b]++;
|
||||
__atomic_store_n(&agen[b], ++agen_top[b], __ATOMIC_RELEASE);
|
||||
*m = (typeof(*m)){bsig[b], b, bepoch[b], agen[b], 1};
|
||||
}
|
||||
dirty_push(DIRTY_BUCKET | b);
|
||||
}
|
||||
|
||||
@@ -2134,7 +2180,7 @@ static Tex *entry_of(const tex_t *t) {
|
||||
e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // in use again, as a new one
|
||||
uint64_t gap = e->samp_frame - e->rel_frame;
|
||||
if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap;
|
||||
e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_unrelease(e);
|
||||
e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten);
|
||||
e->overwritten = 0;
|
||||
if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work
|
||||
e->superseded = 0, e->reused = 1;
|
||||
@@ -2201,9 +2247,7 @@ static void mark_released(Tex *e) { // tex_lock held; e is not released
|
||||
nreleased++;
|
||||
rel_append(e);
|
||||
n_site[3]++;
|
||||
e->rel_pre = agen[agen_idx(e->key.addr)];
|
||||
tex_gen_bump(e->key.addr);
|
||||
e->rel_agen = agen[agen_idx(e->key.addr)];
|
||||
tex_gen_flip(e, 0);
|
||||
}
|
||||
|
||||
// e, in the GPU-only address window, is written by the GPU in frame fr: the entries its storage runs over are
|
||||
@@ -2473,8 +2517,8 @@ void vk_texture_bucket_stats(char *out, size_t n) {
|
||||
}
|
||||
uint64_t hot = 0; // bumps weighted by the slots they invalidate
|
||||
for (int b = 0; b < AGEN_N; b++) hot += (uint64_t)bump_count[b] * occ[b];
|
||||
snprintf(out, n, "%llu bumps (destroy %llu, create %llu, take-back %llu, release %llu; %llu take-backs put their "
|
||||
"release's generation back instead), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u",
|
||||
snprintf(out, n, "%llu new generations (destroy %llu, create %llu, take-back %llu, release %llu; %llu releases and "
|
||||
"take-backs went back to a generation their state had), %llu slot invalidations; fullest buckets %u/%u/%u/%u slots, bumped %u/%u/%u/%u",
|
||||
(unsigned long long)n_bumps, (unsigned long long)n_site[0], (unsigned long long)n_site[1],
|
||||
(unsigned long long)n_site[2], (unsigned long long)n_site[3], (unsigned long long)n_undone,
|
||||
(unsigned long long)hot, occ[top[0]], occ[top[1]], occ[top[2]], occ[top[3]],
|
||||
@@ -2513,7 +2557,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) {
|
||||
view_t v;
|
||||
int hv;
|
||||
resolve_held = 0;
|
||||
in_lookup = 1;
|
||||
s->ce = slot_resolve(s, &v, &hv); // entry_of may create an entry and bump g: g read before it
|
||||
in_lookup = 0;
|
||||
s->cview = v, s->chas_view = (uint8_t)hv;
|
||||
s->cgen = g;
|
||||
s->cepoch = 0;
|
||||
|
||||
Reference in New Issue
Block a user