Merge branch 'main' of https://github.com/tangerineturtle1/NX2navi into test-all-fixes

# Conflicts:
#	runtime/cfg.c
This commit is contained in:
chrg6
2026-11-10 14:27:47 -05:00
14 changed files with 1255 additions and 27 deletions

View File

@@ -48,7 +48,7 @@ static const char *const known[] = {
"HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE",
"HDR_LIVE",
"HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
"MV_SURVEY",
@@ -65,7 +65,7 @@ static const char *const known[] = {
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
"SWKBD_ANSWER",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_ALIAS", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
@@ -118,14 +118,14 @@ static const struct { const char *key, *section; } sections[] = {
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
{"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"},
{"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"},
{"COMPUTE_KEEP", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"DEPTH_ALIAS", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
{"VK_NAN_PRESERVE", "performance"},
{"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"},
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"},
@@ -239,6 +239,8 @@ static const CfgMeta metas[] = {
{"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take
{"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold
{"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth
{"OQ_BATCH", "1", "0|1", RESTART}, // vk_exec_xfer.c oq_take
{"GATE_WAIT_MS", "10", "0..100", RESTART}, // vk_exec.c vk_exec_gate_wait
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},
{"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART},
@@ -265,6 +267,7 @@ static const CfgMeta metas[] = {
{"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART},
{"VK_RECONVERGE", "1", "0|1", RESTART}, // vk_pipeline.c spirv_reconverge
{"VK_NAN_PRESERVE", "", "0|1", RESTART}, // vk_pipeline.c spirv_nan_preserve; unset: every driver but NVIDIA's
{"VK_NAN_BITS", "", "0|1", RESTART}, // vk_pipeline.c spirv_nan_bits; unset: AMD's driver for RDNA 1 and 2
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
// diagnostics
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},

View File

@@ -387,6 +387,7 @@ static uint64_t submit_sync(const uint64_t *handles, int n) {
if (!vk_exec_held()) ph_flush(&PH.after);
}
vk_present_drain();
vk_exec_gate_wait(handles, n); // a submission that begins with a wait for the CPU's signal: before its copies
vk_texture_new_submit();
uint64_t f = vk_sync_submit(handles, n);
// The submission stopped at a wait whose signal the CPU gave while it was being replayed (vk_exec_xfer.c "events

View File

@@ -1192,6 +1192,72 @@ static int cont_ready(const Cont *c) {
return !c->at || vk_exec_event_met(nvn_op_payload(c->at)) || vk_exec_submit_seq - c->since > CONT_MAX;
}
// ---- the gate: a submission that begins with a wait for the CPU's signal (NX2NAVI_GATE_WAIT_MS, default 10; 0 off)
// Ocarina's main queue begins with a wait for an event the game signals from the CPU once it has written what the
// frame reads (vk_exec_xfer.c "events the CPU signals"). The render thread submits that queue without waiting for the
// signal, and polls the event afterwards, a millisecond at a time, before it presents. In Hyrule Field the signal
// came 1 to 2 ms after the submit call in about every other frame: the main thread first waits for the previous
// frame's fence and reads back what it needs. The submission then took its copies (the mirror sync and the uniform
// snapshot, 1.1 to 1.7 ms), met the wait unmet and was held; the render thread polled, presented, and the present
// replayed the frame (vk_backend.c held_resume) under a value the game's fence does not stand for, so the game's next
// wait for that fence returned with the GPU still at work and the frame after was replayed inside its submit call
// again: frames of 23 and 15 ms in turn where each takes 17.
// So when a submission would stop at such a wait before it has recorded anything, the submit call waits for the
// signal here, before the copies are taken, as the console's GPU waits at the wait itself: the copies are then
// taken after the game has finished writing, and the frame is replayed in the call the game's fence stands for.
// Only state may precede the wait in the command buffer, and the event must be one the CPU has signalled before;
// the wait ends after NX2NAVI_GATE_WAIT_MS at the latest, and the submission is then held as before.
static struct { uint64_t n, ns, late; } GW; // since the last perf line: waits, their time, waits that ran out
static const void *gate_first_wait(const op_hdr *h) {
for (int k = 0; k < 256; k++) {
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) return NULL;
const void *a = nvn_op_payload(h);
switch (h->op) {
case OP_WAIT_EVENT: return a;
case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue;
// state the walk only stores: nothing is recorded and nothing reaches the CPU before the wait
case OP_BEGIN: case OP_SET_RT: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_BIND_PROGRAM:
case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO:
case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_STENCIL: case OP_POLY_OFFSET:
case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_COPY_STRIDE: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP:
case OP_RASTER_DISCARD: case OP_PRIM_RESTART: case OP_PUSH_CC:
break;
default: return NULL; // a draw, a copy, a signal, a call: the walk decides, as before
}
h = nvn_op_next(h);
}
return NULL;
}
void vk_exec_gate_wait(const uint64_t *handles, int n) {
static int64_t cap_ns = -1;
if (cap_ns < 0) {
int ms = cfg_int("NX2NAVI_GATE_WAIT_MS");
cap_ns = ms > 0 ? (int64_t)ms * 1000000 : 0;
}
if (!cap_ns || n < 1 || !handles[0] || !gfx_submit_queue || !evo_enabled()) return;
const void *w = gate_first_wait(nvn_op_first(handles[0]));
if (!w || vk_exec_event_holds_now(w) || !vk_exec_event_cpu_known((const void *)((const p_wait_event *)w)->val)) return;
uint64_t t0 = kern_now_ns(), now = t0;
for (unsigned i = 0; !vk_exec_event_holds_now(w); i++) {
if ((i & 15) == 15 && (now = kern_now_ns()) - t0 >= (uint64_t)cap_ns) {
GW.late++;
break;
}
if (i < 20000) plat_pause();
else plat_yield();
}
GW.n++, GW.ns += kern_now_ns() - t0;
if (GW.n >= 600) {
if (rt_trace & RT_TRACE_PERF)
rt_log("perf: gate: %llu submissions waited for the CPU's signal before their copies, %.2f ms on average; "
"%llu waited %lld ms without it and were held", (unsigned long long)GW.n, (double)GW.ns / (double)GW.n / 1e6,
(unsigned long long)GW.late, (long long)(cap_ns / 1000000));
GW.n = GW.ns = GW.late = 0;
}
}
int vk_exec_held_ready(void) {
for (int s = 0; evo_pending && s < EVO_QUEUES; s++)
if (evq[s].head && cont_ready(evq[s].head)) return 1;

View File

@@ -252,6 +252,11 @@ extern int vk_exec_signalled; // set by every replayed SignalEve
// Events the CPU signals (vk_exec_xfer.c "events the CPU signals").
void vk_exec_event_cpu(const void *event); // any thread: nvnEventSignal is about to write the event's value
void vk_exec_event_sync(int begin); // vk_exec_record: its mirror sync and uniform copy are taken now / it ends
int vk_exec_event_cpu_known(const void *event); // the CPU has signalled this event before
int vk_exec_event_holds_now(const void *payload); // an OP_WAIT_EVENT's condition on the value in memory now
// vk_backend.c submit_sync, before a submission is recorded: waits for the CPU's signal when the submission begins
// with a wait for it (vk_exec.c "the gate").
void vk_exec_gate_wait(const uint64_t *handles, int n);
int vk_exec_event_at_sync(void); // NX2NAVI_EVENT_AT_SYNC
uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after the copies would have passed, since the last call
// Since the last call, which resets them: waits, waits reached before their signal, those of them a later replayed

View File

@@ -265,38 +265,103 @@ void vk_exec_oq_draw(VkExec *ex) {
void vk_exec_oq_pass_end(VkExec *ex) { oq_close(ex); }
static void oq_batch_drop(void); // the results kept by the completion thread (oq_take)
void vk_exec_xfer_shutdown(void) {
if (oq_state > 0) vkDestroyQueryPool(vk_ctx()->device, oq_pool, NULL);
oq_pool = VK_NULL_HANDLE, oq_state = 0, oq_next = 0, oq_counting = oq_reported = oq_keep = 0, oq_open = -1;
oq_nseg = 0, oq_lost = 0, oq_count = 0;
memset((void *)oq_busy, 0, sizeof oq_busy);
oq_batch_drop();
}
// The completion thread reads the results in batches (NX2NAVI_OQ_BATCH, default 1; 0: every segment by itself, as
// before). The game tests one proxy per report, so a segment holds one query, and a frame of Hyrule Field has about a
// hundred of them: one vkGetQueryPoolResults and one vkResetQueryPool each. On an RX 7900 XT (26.10.1) each of those
// calls goes into the kernel, and the two hundred took 3.9 ms of every frame on the completion thread, which runs a
// submission's callbacks before it lets the submission's waiters go: the game's main thread waits for the previous
// frame's fence before it lets the next frame's GPU work start, so the frame after one replayed inside its submit
// call began 4 to 6 ms late with every thread idle. Slots come from the ring in order, so the slots of the segments
// that follow lie behind the one asked for: one call reads OQ_BATCH slots from there on with their availability, the
// available ones are kept (the results stay in the pool until the reset), and the segments after it take theirs from
// what is kept. A slot not available in the batch (its part had not run yet) is read again when its segment comes.
// The slots taken are reset in runs, one vkResetQueryPool for each: a run ends when a slot that does not continue it
// is taken or a new batch is read. Until then its slots stay busy for the replay, at most OQ_BATCH of the ring's
// OQ_N, also while the game reports nothing.
#define OQ_BATCH 256
static struct {
uint32_t first, n; // the slots kept: [first, first + n)
uint64_t v[OQ_BATCH * 2]; // per slot: samples, available
uint8_t taken[OQ_BATCH];
uint32_t r0, rn; // the run of taken slots still to be reset
uint64_t calls, slots, segs; // since the last perf line: result calls, slots they read, segments served
} OQB;
static int oq_batch_mode = -1;
static void oq_batch_drop(void) { OQB.n = OQB.rn = 0; }
static void oq_reset_flush(VkDevice dev) {
if (!OQB.rn) return;
vkResetQueryPool(dev, oq_pool, OQB.r0, OQB.rn);
for (uint32_t k = 0; k < OQB.rn; k++) __atomic_store_n(&oq_busy[OQB.r0 + k], 0, __ATOMIC_RELEASE);
OQB.rn = 0;
}
static void oq_read(VkDevice dev, uint32_t first, uint32_t n) {
VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0],
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
if (r != VK_SUCCESS && r != VK_NOT_READY) {
vk_device_fault_report(r);
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
}
OQB.first = first, OQB.n = n;
memset(OQB.taken, 0, n);
OQB.calls++, OQB.slots += n;
}
// The samples of slot q, whose submission has completed; *avail 0 when the query has no result.
static uint64_t oq_take(VkDevice dev, uint32_t q, int *avail) {
uint32_t k = q - OQB.first;
if (!(q >= OQB.first && k < OQB.n && !OQB.taken[k] && OQB.v[2 * k + 1])) {
// not kept, or kept as unavailable (read before its part ran): from this slot on, up to the replay's next slot
oq_reset_flush(dev); // before the kept results go: the run's slots are among them
uint32_t next = __atomic_load_n(&oq_next, __ATOMIC_RELAXED) % OQ_N, n = OQ_N - q;
if (next > q && next - q < n) n = next - q;
if (n > OQ_BATCH) n = OQ_BATCH;
if (oq_batch_mode == 0) n = 1;
oq_read(dev, q, n);
k = 0;
}
OQB.taken[k] = 1;
if (OQB.rn && q != OQB.r0 + OQB.rn) oq_reset_flush(dev);
if (!OQB.rn) OQB.r0 = q;
OQB.rn++;
*avail = OQB.v[2 * k + 1] != 0;
return OQB.v[2 * k];
}
static void oq_cb(void *arg) {
OqSegment *s = arg;
VkDevice dev = vk_ctx()->device;
uint64_t sum = 0, v[64 * 2];
for (uint32_t i = 0; i < s->n;) {
uint32_t j = i + 1;
while (j < s->n && j - i < 64 && s->q[j] == s->q[j - 1] + 1) j++;
if (oq_batch_mode < 0) oq_batch_mode = cfg_bool("NX2NAVI_OQ_BATCH");
uint64_t sum = 0;
for (uint32_t i = 0; i < s->n; i++) {
// The fence has passed, so a query that is still unavailable will not become available: waiting would block the
// completion thread, and with it every later callback, for good. It counts as visible (one sample).
VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0],
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
if (r != VK_SUCCESS && r != VK_NOT_READY) {
vk_device_fault_report(r);
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
int avail;
uint64_t v = oq_take(dev, s->q[i], &avail);
if (avail) sum += v;
else {
sum += 1;
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]);
}
vkResetQueryPool(dev, oq_pool, s->q[i], j - i);
for (uint32_t k = i; k < j; k++) {
if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)];
else {
sum += 1;
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[k]);
}
__atomic_store_n(&oq_busy[s->q[k]], 0, __ATOMIC_RELEASE);
}
i = j;
}
if (s->n && ++OQB.segs >= 12000) {
if (rt_trace & RT_TRACE_PERF)
rt_log("perf: occlusion queries: %llu segments read with %llu result calls of %.1f slots on average",
(unsigned long long)OQB.segs, (unsigned long long)OQB.calls,
OQB.calls ? (double)OQB.slots / (double)OQB.calls : 0.0);
OQB.calls = OQB.slots = OQB.segs = 0;
}
if (s->reset) oq_count = 0;
else {
@@ -412,6 +477,22 @@ void vk_exec_event_cpu(const void *event) {
plat_mutex_unlock(&cpu_ev_lock);
}
// 1 when the CPU has signalled this event before (nvnEventSignal): its waits are gates the game opens from the CPU.
int vk_exec_event_cpu_known(const void *event) {
plat_mutex_lock(&cpu_ev_lock);
int i = 0;
while (i < cpu_ev_n && cpu_ev[i].val != event) i++;
int known = i < cpu_ev_n;
plat_mutex_unlock(&cpu_ev_lock);
return known;
}
// The wait's condition on the value in memory now, whatever the replay has seen. Any thread.
int vk_exec_event_holds_now(const void *payload) {
const p_wait_event *p = payload;
return ev_holds(__atomic_load_n(p->val, __ATOMIC_ACQUIRE), p->mode, p->value);
}
void vk_exec_event_sync(int begin) {
plat_mutex_lock(&cpu_ev_lock);
ev_sync_seq = begin ? cpu_ev_seq : UINT64_MAX;

View File

@@ -87,6 +87,8 @@ static struct {
VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available)
int reconverge; // modules with subgroup ops get MaximallyReconvergesKHR (spirv_reconverge)
uint32_t nan_preserve; // float widths that get SignedZeroInfNanPreserve (spirv_nan_preserve): NP_16|NP_32|NP_64
int nan_bits; // NaN tests as integer bit tests (spirv_nan_bits)
volatile uint32_t nan_bits_tests, nan_bits_selects, nan_bits_modules; // rewritten so far (the first log line)
VkDynamicState dyn[32];
uint32_t n_dyn;
PFN_vkCmdPushDescriptorSetKHR push;
@@ -658,6 +660,12 @@ static void nan_preserve_init(VkCtx *c) {
(fc.shaderSignedZeroInfNanPreserveFloat32 ? NP_32 : 0) |
(fc.shaderSignedZeroInfNanPreserveFloat64 ? NP_64 : 0);
P.nan_preserve = want && (can & NP_32) ? can : 0;
// NX2NAVI_VK_NAN_BITS: 1 every driver, 0 none; unset: AMD's driver for RDNA 1 and 2 (spirv_nan_bits)
const char *nb = cfg_get("NX2NAVI_VK_NAN_BITS");
P.nan_bits = nb && *nb ? strcmp(nb, "0") != 0 : vk_caps.amd_sc;
if (P.nan_bits)
rt_log("vk_pipeline: game shaders test for NaN on the bits of the number, not as a float (%s)",
nb && *nb ? "NX2NAVI_VK_NAN_BITS=1" : "AMD's driver for RDNA 1 and 2");
if (P.nan_preserve)
rt_log("vk_pipeline: game shaders keep NaN, infinity and signed zero (SignedZeroInfNanPreserve for %s32%s-bit "
"floats%s)", can & NP_16 ? "16, " : "", can & NP_64 ? ", 64" : "", set ? ", NX2NAVI_VK_NAN_PRESERVE=1" : "");
@@ -1478,6 +1486,125 @@ static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widt
return o;
}
// NaN tests that no compiler can drop (P.nan_bits). The game's shaders and the translator's own guards test for NaN with
// OpIsNan: 48,717 of an install's 54,959 translated modules, 26 tests each on average (the game's unordered
// comparisons, and the guard that keeps NaN out of every float colour output, patch 0027, so that one NaN texel does not
// spread through bloom and blur into blocks of black and white). SignedZeroInfNanPreserve asks the driver to keep NaN,
// but whether a compiler that reasons about floats keeps an isnan through every optimisation is up to it. The reports
// of black and white blocks, on Navi, on Link next to her and on bright surfaces seen from inside a house, come from
// RX 5000 and 6000 cards (#41, #110, #117), whose Windows driver compiles with "AMD proprietary shader compiler"
// (vk_caps.amd_sc); RDNA 3 and 4 (LLPC) draw the same scenes clean. It is not known whether that compiler keeps the
// mode: on it each float32 test becomes integer arithmetic on the bits, which means the same under any float
// assumption:
// isnan(x) -> (bitcast<uint>(x) & 0x7fffffff) > 0x7f800000
// for scalars and vectors of 32-bit floats; tests of 16- and 64-bit floats stay. A float32 select with a constant
// operand whose bits are a NaN (an integer mask such as 0xffffffff that the translator types as float) becomes a
// select of the bits (issue #156: on an RX 9070 XT without the mode such a select between 1 and -1 gave 2^32).
// NULL: nothing to change (or no memory); *tests and *selects count what changed.
static uint32_t *spirv_nan_bits(const uint32_t *w, uint64_t n, uint64_t *out_n, uint32_t *tests, uint32_t *selects) {
enum { OP_UNDEF = 1, OP_TYPE_BOOL = 20, OP_SPEC_CONSTANT = 50, OP_SPEC_CONSTANT_COMPOSITE = 51, OP_STORE = 62,
OP_BITCAST = 124, OP_IS_NAN = 156, OP_SELECT = 169, OP_UGREATER = 172, OP_BITWISE_AND = 199 };
*tests = *selects = 0;
if (n < 5 || w[3] >= (1u << 22)) return NULL;
uint32_t bound = w[3];
uint32_t *rtype = calloc(bound, sizeof *rtype), *vec_comp = calloc(bound, sizeof *vec_comp);
uint8_t *f32 = calloc(bound, 1), *vec_n = calloc(bound, 1), *nan_const = calloc(bound, 1);
uint32_t *o = NULL;
if (!rtype || !vec_comp || !f32 || !vec_n || !nan_const) goto out;
uint32_t tu = 0, uvec[5] = {0}; // uint32 and its vectors by component count
uint64_t fn_at = 0;
int vec_used[5] = {0};
// Ids that are not results of the instruction's second word, inside functions: first assignment wins anyway, as
// a definition comes before its uses (OpPhi operands aside, which this never reads).
for (uint64_t i = 5; i < n;) {
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
if (!l || i + l > n) goto out;
const uint32_t *a = w + i + 1;
if (!fn_at) {
if (op == SPV_OP_FUNCTION) fn_at = i;
else if (op == SPV_OP_TYPE_FLOAT && l >= 3 && a[0] < bound && a[1] == 32) f32[a[0]] = 1;
else if (op == SPV_OP_TYPE_INT && l >= 4 && a[1] == 32 && a[2] == 0 && !tu) tu = a[0];
else if (op == SPV_OP_TYPE_VECTOR && l >= 4 && a[0] < bound && a[2] >= 2 && a[2] <= 4) {
vec_comp[a[0]] = a[1], vec_n[a[0]] = (uint8_t)a[2];
if (tu && a[1] == tu && !uvec[a[2]]) uvec[a[2]] = a[0];
} else if ((op == SPV_OP_CONSTANT || op == SPV_OP_CONSTANT_COMPOSITE || op == SPV_OP_CONSTANT_NULL ||
op == OP_UNDEF || op == SPV_OP_VARIABLE || op == OP_SPEC_CONSTANT ||
op == OP_SPEC_CONSTANT_COMPOSITE) && l >= 3 && a[1] < bound) {
rtype[a[1]] = a[0];
if (op == SPV_OP_CONSTANT && l == 4 && a[0] < bound && f32[a[0]] && (a[2] & 0x7fffffffu) > 0x7f800000u)
nan_const[a[1]] = 1;
}
}
if (fn_at) {
int no_result = op == OP_STORE || op == 63 || op == 64 || op == 8 || op == 317 || (op >= 246 && op <= 255) ||
op == 99 || op == 224 || op == 225 || op == 228 || op == 56 || op == 218 || op == 219 ||
op == 5294 || op == 5295 || op == 4416 || op == 5380 || op == 5364 || op == 5365;
if (!no_result && op != SPV_OP_FUNCTION && l >= 3 && a[1] < bound && !rtype[a[1]]) rtype[a[1]] = a[0];
if (op == OP_IS_NAN && l == 4 && a[2] < bound) {
uint32_t t = rtype[a[2]];
if (t < bound && f32[t]) ++*tests;
else if (t < bound && vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]]) ++*tests, vec_used[vec_n[t]] = 1;
} else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound &&
(nan_const[a[3]] || nan_const[a[4]])) {
++*selects;
}
}
i += l;
}
if (!fn_at || (!*tests && !*selects)) goto out;
// the declarations go right before the first function: uint32 and its vectors where missing, then the constants
uint32_t next = bound, decl[64], nd = 0;
if (!tu) tu = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_INT, decl[nd++] = tu, decl[nd++] = 32, decl[nd++] = 0;
uint32_t c_mag = next++, c_inf = next++, cv_mag[5] = {0}, cv_inf[5] = {0};
for (int k = 2; k <= 4; k++)
if (vec_used[k] && !uvec[k])
uvec[k] = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_VECTOR, decl[nd++] = uvec[k], decl[nd++] = tu, decl[nd++] = (uint32_t)k;
decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_mag, decl[nd++] = 0x7fffffffu;
decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_inf, decl[nd++] = 0x7f800000u;
for (int k = 2; k <= 4; k++) {
if (!vec_used[k]) continue;
cv_mag[k] = next++, cv_inf[k] = next++;
decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_mag[k];
for (int j = 0; j < k; j++) decl[nd++] = c_mag;
decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_inf[k];
for (int j = 0; j < k; j++) decl[nd++] = c_inf;
}
uint64_t cap = n + nd + 10ull * *tests + 12ull * *selects;
o = malloc(cap * 4);
if (!o) goto out;
memcpy(o, w, fn_at * 4);
uint64_t p = fn_at;
memcpy(o + p, decl, nd * 4), p += nd;
for (uint64_t i = fn_at; i < n;) {
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
const uint32_t *a = w + i + 1;
uint32_t t = op == OP_IS_NAN && l == 4 && a[2] < bound ? rtype[a[2]] : 0;
int k = t && t < bound ? (f32[t] ? 1 : vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]] ? vec_n[t] : 0) : 0;
if (k) {
uint32_t ut = k == 1 ? tu : uvec[k], mag = k == 1 ? c_mag : cv_mag[k], inf = k == 1 ? c_inf : cv_inf[k];
uint32_t bits = next++, low = next++;
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = ut, o[p++] = bits, o[p++] = a[2];
o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = ut, o[p++] = low, o[p++] = bits, o[p++] = mag;
o[p++] = 5u << 16 | OP_UGREATER, o[p++] = a[0], o[p++] = a[1], o[p++] = low, o[p++] = inf;
} else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound &&
(nan_const[a[3]] || nan_const[a[4]])) {
uint32_t x = next++, y = next++, s = next++;
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = x, o[p++] = a[3];
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = y, o[p++] = a[4];
o[p++] = 6u << 16 | OP_SELECT, o[p++] = tu, o[p++] = s, o[p++] = a[2], o[p++] = x, o[p++] = y;
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = a[0], o[p++] = a[1], o[p++] = s;
} else {
memcpy(o + p, w + i, l * 4), p += l;
}
i += l;
}
o[3] = next;
*out_n = p;
out:
free(rtype), free(vec_comp), free(f32), free(vec_n), free(nan_const);
return o;
}
static int spirv_mark_list_test(const char *list); // texture pool marks, below
// --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures.
@@ -1523,6 +1650,42 @@ int vk_pipeline_spirv_selftest(void) {
failures++;
}
printf("spirv: SignedZeroInfNanPreserve rewrite: %s\n", failures ? "FAIL" : "ok");
{ // spirv_nan_bits: a scalar and a vec2 isnan, a select on a NaN constant; no uint32 type in the module
static const uint32_t m[] = {
0x07230203, 0x00010300, 0, 20, 0,
2u << 16 | 17, 1, // OpCapability Shader
3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450
5u << 16 | 15, 5, 1, 0x6e69616d, 0, // OpEntryPoint GLCompute %1 "main"
2u << 16 | 19, 2, // %2 void
3u << 16 | 33, 3, 2, // %3 fn
3u << 16 | 22, 4, 32, // %4 float
2u << 16 | 20, 5, // %5 bool
4u << 16 | 23, 6, 4, 2, // %6 vec2
4u << 16 | 23, 7, 5, 2, // %7 bvec2
4u << 16 | 43, 4, 8, 0xffffffffu, // %8 = NaN bits
4u << 16 | 43, 4, 9, 0x3f800000u, // %9 = 1.0
5u << 16 | 44, 6, 10, 8, 9, // %10 = vec2(%8, %9)
3u << 16 | 41, 5, 11, // %11 = OpConstantTrue
5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction
2u << 16 | 248, 12, // OpLabel
4u << 16 | 156, 5, 13, 9, // %13 = OpIsNan bool %9
4u << 16 | 156, 7, 14, 10, // %14 = OpIsNan bvec2 %10
6u << 16 | 169, 4, 15, 11, 9, 8, // %15 = OpSelect float %11 %9 %8
1u << 16 | 253, 1u << 16 | 56}; // OpReturn, OpFunctionEnd
uint64_t on = 0;
uint32_t tests = 0, sels = 0, isnan_left = 0, ugt = 0, int32 = 0, bitcasts = 0;
uint32_t *o = spirv_nan_bits(m, sizeof m / 4, &on, &tests, &sels);
for (uint64_t i = 5; o && i < on;) {
uint32_t op = o[i] & 0xffff, l = o[i] >> 16;
if (!l) break;
isnan_left += op == 156, ugt += op == 172, int32 += op == 21 && o[i + 2] == 32, bitcasts += op == 124;
i += l;
}
int bad = !o || tests != 2 || sels != 1 || isnan_left || ugt != 2 || int32 != 1 || bitcasts != 5 || o[3] <= 20;
printf("spirv: integer NaN tests rewrite: %s\n", bad ? "FAIL" : "ok");
failures += bad;
free(o);
}
const char *list = cfg_get("NX2NAVI_SPV_MARK_TEST");
if (list && *list) failures += spirv_mark_list_test(list);
return failures;
@@ -1867,6 +2030,18 @@ static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_o
free(fixed);
w = fixed = np, words = np_n;
}
uint64_t nb_n = 0;
uint32_t nb_tests = 0, nb_sels = 0;
uint32_t *nb = P.nan_bits ? spirv_nan_bits(w, words, &nb_n, &nb_tests, &nb_sels) : NULL;
if (nb) {
free(fixed);
w = fixed = nb, words = nb_n;
__atomic_fetch_add(&P.nan_bits_tests, nb_tests, __ATOMIC_RELAXED);
__atomic_fetch_add(&P.nan_bits_selects, nb_sels, __ATOMIC_RELAXED);
if (__atomic_add_fetch(&P.nan_bits_modules, 1, __ATOMIC_RELAXED) == 1)
rt_log("vk_pipeline: first module with NaN tests made integer: %u tests, %u selects of NaN bits", nb_tests,
nb_sels);
}
if (P.feedback >= 2) mm_hash = fnv(FNV0, w, (size_t)words * 4);
VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w};
VkShaderModule m = VK_NULL_HANDLE;