mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-19 19:29:57 -04:00
Merge branch 'main' of https://github.com/tangerineturtle1/NX2navi into test-all-fixes
# Conflicts: # runtime/cfg.c
This commit is contained in:
@@ -48,7 +48,7 @@ static const char *const known[] = {
|
||||
"HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE",
|
||||
"HDR_LIVE",
|
||||
"HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
|
||||
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
|
||||
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "OQ_BATCH", "GATE_WAIT_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
|
||||
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
|
||||
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
|
||||
"MV_SURVEY",
|
||||
@@ -65,7 +65,7 @@ static const char *const known[] = {
|
||||
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
|
||||
"SWKBD_ANSWER",
|
||||
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_ALIAS", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
|
||||
"VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
|
||||
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
|
||||
"ZSTD_CHECK", "ZSTD_NATIVE",
|
||||
"BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA",
|
||||
@@ -118,14 +118,14 @@ static const struct { const char *key, *section; } sections[] = {
|
||||
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
|
||||
{"PIPE_FRAME_WAIT_MS", "performance"}, {"PIPE_SKIP_STEADY", "performance"},
|
||||
{"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"},
|
||||
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"},
|
||||
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"},
|
||||
{"COMPUTE_KEEP", "performance"},
|
||||
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
|
||||
{"TEX_ALIAS", "performance"}, {"DEPTH_ALIAS", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
|
||||
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
|
||||
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
|
||||
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
|
||||
{"VK_NAN_PRESERVE", "performance"},
|
||||
{"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"},
|
||||
{"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"},
|
||||
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"},
|
||||
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"}, {"VA_REPORT", "diagnostics"},
|
||||
@@ -239,6 +239,8 @@ static const CfgMeta metas[] = {
|
||||
{"PIPE_FAST_WORKERS", "1", "0|1", RESTART}, // vk_pipeline.c lib_worker, fast_take
|
||||
{"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold
|
||||
{"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth
|
||||
{"OQ_BATCH", "1", "0|1", RESTART}, // vk_exec_xfer.c oq_take
|
||||
{"GATE_WAIT_MS", "10", "0..100", RESTART}, // vk_exec.c vk_exec_gate_wait
|
||||
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},
|
||||
{"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold
|
||||
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART},
|
||||
@@ -265,6 +267,7 @@ static const CfgMeta metas[] = {
|
||||
{"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART},
|
||||
{"VK_RECONVERGE", "1", "0|1", RESTART}, // vk_pipeline.c spirv_reconverge
|
||||
{"VK_NAN_PRESERVE", "", "0|1", RESTART}, // vk_pipeline.c spirv_nan_preserve; unset: every driver but NVIDIA's
|
||||
{"VK_NAN_BITS", "", "0|1", RESTART}, // vk_pipeline.c spirv_nan_bits; unset: AMD's driver for RDNA 1 and 2
|
||||
{"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag
|
||||
// diagnostics
|
||||
{"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART},
|
||||
|
||||
@@ -387,6 +387,7 @@ static uint64_t submit_sync(const uint64_t *handles, int n) {
|
||||
if (!vk_exec_held()) ph_flush(&PH.after);
|
||||
}
|
||||
vk_present_drain();
|
||||
vk_exec_gate_wait(handles, n); // a submission that begins with a wait for the CPU's signal: before its copies
|
||||
vk_texture_new_submit();
|
||||
uint64_t f = vk_sync_submit(handles, n);
|
||||
// The submission stopped at a wait whose signal the CPU gave while it was being replayed (vk_exec_xfer.c "events
|
||||
|
||||
@@ -1192,6 +1192,72 @@ static int cont_ready(const Cont *c) {
|
||||
return !c->at || vk_exec_event_met(nvn_op_payload(c->at)) || vk_exec_submit_seq - c->since > CONT_MAX;
|
||||
}
|
||||
|
||||
// ---- the gate: a submission that begins with a wait for the CPU's signal (NX2NAVI_GATE_WAIT_MS, default 10; 0 off)
|
||||
// Ocarina's main queue begins with a wait for an event the game signals from the CPU once it has written what the
|
||||
// frame reads (vk_exec_xfer.c "events the CPU signals"). The render thread submits that queue without waiting for the
|
||||
// signal, and polls the event afterwards, a millisecond at a time, before it presents. In Hyrule Field the signal
|
||||
// came 1 to 2 ms after the submit call in about every other frame: the main thread first waits for the previous
|
||||
// frame's fence and reads back what it needs. The submission then took its copies (the mirror sync and the uniform
|
||||
// snapshot, 1.1 to 1.7 ms), met the wait unmet and was held; the render thread polled, presented, and the present
|
||||
// replayed the frame (vk_backend.c held_resume) under a value the game's fence does not stand for, so the game's next
|
||||
// wait for that fence returned with the GPU still at work and the frame after was replayed inside its submit call
|
||||
// again: frames of 23 and 15 ms in turn where each takes 17.
|
||||
// So when a submission would stop at such a wait before it has recorded anything, the submit call waits for the
|
||||
// signal here, before the copies are taken, as the console's GPU waits at the wait itself: the copies are then
|
||||
// taken after the game has finished writing, and the frame is replayed in the call the game's fence stands for.
|
||||
// Only state may precede the wait in the command buffer, and the event must be one the CPU has signalled before;
|
||||
// the wait ends after NX2NAVI_GATE_WAIT_MS at the latest, and the submission is then held as before.
|
||||
static struct { uint64_t n, ns, late; } GW; // since the last perf line: waits, their time, waits that ran out
|
||||
|
||||
static const void *gate_first_wait(const op_hdr *h) {
|
||||
for (int k = 0; k < 256; k++) {
|
||||
if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) return NULL;
|
||||
const void *a = nvn_op_payload(h);
|
||||
switch (h->op) {
|
||||
case OP_WAIT_EVENT: return a;
|
||||
case OP_JUMP: h = (const op_hdr *)(uintptr_t)((const p_jump *)a)->target; continue;
|
||||
// state the walk only stores: nothing is recorded and nothing reaches the CPU before the wait
|
||||
case OP_BEGIN: case OP_SET_RT: case OP_VIEWPORTS: case OP_SCISSOR: case OP_DEPTH_RANGE: case OP_BIND_PROGRAM:
|
||||
case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: case OP_BIND_UBO:
|
||||
case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_STENCIL: case OP_POLY_OFFSET:
|
||||
case OP_SET_TEX_POOL: case OP_SET_SAMP_POOL: case OP_COPY_STRIDE: case OP_BLEND_COLOR: case OP_DEPTH_CLAMP:
|
||||
case OP_RASTER_DISCARD: case OP_PRIM_RESTART: case OP_PUSH_CC:
|
||||
break;
|
||||
default: return NULL; // a draw, a copy, a signal, a call: the walk decides, as before
|
||||
}
|
||||
h = nvn_op_next(h);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void vk_exec_gate_wait(const uint64_t *handles, int n) {
|
||||
static int64_t cap_ns = -1;
|
||||
if (cap_ns < 0) {
|
||||
int ms = cfg_int("NX2NAVI_GATE_WAIT_MS");
|
||||
cap_ns = ms > 0 ? (int64_t)ms * 1000000 : 0;
|
||||
}
|
||||
if (!cap_ns || n < 1 || !handles[0] || !gfx_submit_queue || !evo_enabled()) return;
|
||||
const void *w = gate_first_wait(nvn_op_first(handles[0]));
|
||||
if (!w || vk_exec_event_holds_now(w) || !vk_exec_event_cpu_known((const void *)((const p_wait_event *)w)->val)) return;
|
||||
uint64_t t0 = kern_now_ns(), now = t0;
|
||||
for (unsigned i = 0; !vk_exec_event_holds_now(w); i++) {
|
||||
if ((i & 15) == 15 && (now = kern_now_ns()) - t0 >= (uint64_t)cap_ns) {
|
||||
GW.late++;
|
||||
break;
|
||||
}
|
||||
if (i < 20000) plat_pause();
|
||||
else plat_yield();
|
||||
}
|
||||
GW.n++, GW.ns += kern_now_ns() - t0;
|
||||
if (GW.n >= 600) {
|
||||
if (rt_trace & RT_TRACE_PERF)
|
||||
rt_log("perf: gate: %llu submissions waited for the CPU's signal before their copies, %.2f ms on average; "
|
||||
"%llu waited %lld ms without it and were held", (unsigned long long)GW.n, (double)GW.ns / (double)GW.n / 1e6,
|
||||
(unsigned long long)GW.late, (long long)(cap_ns / 1000000));
|
||||
GW.n = GW.ns = GW.late = 0;
|
||||
}
|
||||
}
|
||||
|
||||
int vk_exec_held_ready(void) {
|
||||
for (int s = 0; evo_pending && s < EVO_QUEUES; s++)
|
||||
if (evq[s].head && cont_ready(evq[s].head)) return 1;
|
||||
|
||||
@@ -252,6 +252,11 @@ extern int vk_exec_signalled; // set by every replayed SignalEve
|
||||
// Events the CPU signals (vk_exec_xfer.c "events the CPU signals").
|
||||
void vk_exec_event_cpu(const void *event); // any thread: nvnEventSignal is about to write the event's value
|
||||
void vk_exec_event_sync(int begin); // vk_exec_record: its mirror sync and uniform copy are taken now / it ends
|
||||
int vk_exec_event_cpu_known(const void *event); // the CPU has signalled this event before
|
||||
int vk_exec_event_holds_now(const void *payload); // an OP_WAIT_EVENT's condition on the value in memory now
|
||||
// vk_backend.c submit_sync, before a submission is recorded: waits for the CPU's signal when the submission begins
|
||||
// with a wait for it (vk_exec.c "the gate").
|
||||
void vk_exec_gate_wait(const uint64_t *handles, int n);
|
||||
int vk_exec_event_at_sync(void); // NX2NAVI_EVENT_AT_SYNC
|
||||
uint64_t vk_exec_event_late_take(void); // waits only a CPU signal after the copies would have passed, since the last call
|
||||
// Since the last call, which resets them: waits, waits reached before their signal, those of them a later replayed
|
||||
|
||||
@@ -265,38 +265,103 @@ void vk_exec_oq_draw(VkExec *ex) {
|
||||
|
||||
void vk_exec_oq_pass_end(VkExec *ex) { oq_close(ex); }
|
||||
|
||||
static void oq_batch_drop(void); // the results kept by the completion thread (oq_take)
|
||||
|
||||
void vk_exec_xfer_shutdown(void) {
|
||||
if (oq_state > 0) vkDestroyQueryPool(vk_ctx()->device, oq_pool, NULL);
|
||||
oq_pool = VK_NULL_HANDLE, oq_state = 0, oq_next = 0, oq_counting = oq_reported = oq_keep = 0, oq_open = -1;
|
||||
oq_nseg = 0, oq_lost = 0, oq_count = 0;
|
||||
memset((void *)oq_busy, 0, sizeof oq_busy);
|
||||
oq_batch_drop();
|
||||
}
|
||||
|
||||
// The completion thread reads the results in batches (NX2NAVI_OQ_BATCH, default 1; 0: every segment by itself, as
|
||||
// before). The game tests one proxy per report, so a segment holds one query, and a frame of Hyrule Field has about a
|
||||
// hundred of them: one vkGetQueryPoolResults and one vkResetQueryPool each. On an RX 7900 XT (26.10.1) each of those
|
||||
// calls goes into the kernel, and the two hundred took 3.9 ms of every frame on the completion thread, which runs a
|
||||
// submission's callbacks before it lets the submission's waiters go: the game's main thread waits for the previous
|
||||
// frame's fence before it lets the next frame's GPU work start, so the frame after one replayed inside its submit
|
||||
// call began 4 to 6 ms late with every thread idle. Slots come from the ring in order, so the slots of the segments
|
||||
// that follow lie behind the one asked for: one call reads OQ_BATCH slots from there on with their availability, the
|
||||
// available ones are kept (the results stay in the pool until the reset), and the segments after it take theirs from
|
||||
// what is kept. A slot not available in the batch (its part had not run yet) is read again when its segment comes.
|
||||
// The slots taken are reset in runs, one vkResetQueryPool for each: a run ends when a slot that does not continue it
|
||||
// is taken or a new batch is read. Until then its slots stay busy for the replay, at most OQ_BATCH of the ring's
|
||||
// OQ_N, also while the game reports nothing.
|
||||
#define OQ_BATCH 256
|
||||
static struct {
|
||||
uint32_t first, n; // the slots kept: [first, first + n)
|
||||
uint64_t v[OQ_BATCH * 2]; // per slot: samples, available
|
||||
uint8_t taken[OQ_BATCH];
|
||||
uint32_t r0, rn; // the run of taken slots still to be reset
|
||||
uint64_t calls, slots, segs; // since the last perf line: result calls, slots they read, segments served
|
||||
} OQB;
|
||||
static int oq_batch_mode = -1;
|
||||
|
||||
static void oq_batch_drop(void) { OQB.n = OQB.rn = 0; }
|
||||
|
||||
static void oq_reset_flush(VkDevice dev) {
|
||||
if (!OQB.rn) return;
|
||||
vkResetQueryPool(dev, oq_pool, OQB.r0, OQB.rn);
|
||||
for (uint32_t k = 0; k < OQB.rn; k++) __atomic_store_n(&oq_busy[OQB.r0 + k], 0, __ATOMIC_RELEASE);
|
||||
OQB.rn = 0;
|
||||
}
|
||||
|
||||
static void oq_read(VkDevice dev, uint32_t first, uint32_t n) {
|
||||
VkResult r = vkGetQueryPoolResults(dev, oq_pool, first, n, sizeof OQB.v[0] * 2 * n, OQB.v, 2 * sizeof OQB.v[0],
|
||||
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
|
||||
if (r != VK_SUCCESS && r != VK_NOT_READY) {
|
||||
vk_device_fault_report(r);
|
||||
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
|
||||
}
|
||||
OQB.first = first, OQB.n = n;
|
||||
memset(OQB.taken, 0, n);
|
||||
OQB.calls++, OQB.slots += n;
|
||||
}
|
||||
|
||||
// The samples of slot q, whose submission has completed; *avail 0 when the query has no result.
|
||||
static uint64_t oq_take(VkDevice dev, uint32_t q, int *avail) {
|
||||
uint32_t k = q - OQB.first;
|
||||
if (!(q >= OQB.first && k < OQB.n && !OQB.taken[k] && OQB.v[2 * k + 1])) {
|
||||
// not kept, or kept as unavailable (read before its part ran): from this slot on, up to the replay's next slot
|
||||
oq_reset_flush(dev); // before the kept results go: the run's slots are among them
|
||||
uint32_t next = __atomic_load_n(&oq_next, __ATOMIC_RELAXED) % OQ_N, n = OQ_N - q;
|
||||
if (next > q && next - q < n) n = next - q;
|
||||
if (n > OQ_BATCH) n = OQ_BATCH;
|
||||
if (oq_batch_mode == 0) n = 1;
|
||||
oq_read(dev, q, n);
|
||||
k = 0;
|
||||
}
|
||||
OQB.taken[k] = 1;
|
||||
if (OQB.rn && q != OQB.r0 + OQB.rn) oq_reset_flush(dev);
|
||||
if (!OQB.rn) OQB.r0 = q;
|
||||
OQB.rn++;
|
||||
*avail = OQB.v[2 * k + 1] != 0;
|
||||
return OQB.v[2 * k];
|
||||
}
|
||||
|
||||
static void oq_cb(void *arg) {
|
||||
OqSegment *s = arg;
|
||||
VkDevice dev = vk_ctx()->device;
|
||||
uint64_t sum = 0, v[64 * 2];
|
||||
for (uint32_t i = 0; i < s->n;) {
|
||||
uint32_t j = i + 1;
|
||||
while (j < s->n && j - i < 64 && s->q[j] == s->q[j - 1] + 1) j++;
|
||||
if (oq_batch_mode < 0) oq_batch_mode = cfg_bool("NX2NAVI_OQ_BATCH");
|
||||
uint64_t sum = 0;
|
||||
for (uint32_t i = 0; i < s->n; i++) {
|
||||
// The fence has passed, so a query that is still unavailable will not become available: waiting would block the
|
||||
// completion thread, and with it every later callback, for good. It counts as visible (one sample).
|
||||
VkResult r = vkGetQueryPoolResults(dev, oq_pool, s->q[i], j - i, sizeof v, v, 2 * sizeof v[0],
|
||||
VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
|
||||
if (r != VK_SUCCESS && r != VK_NOT_READY) {
|
||||
vk_device_fault_report(r);
|
||||
rt_fatal("vk: occlusion query results failed (%d)", (int)r);
|
||||
int avail;
|
||||
uint64_t v = oq_take(dev, s->q[i], &avail);
|
||||
if (avail) sum += v;
|
||||
else {
|
||||
sum += 1;
|
||||
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[i]);
|
||||
}
|
||||
vkResetQueryPool(dev, oq_pool, s->q[i], j - i);
|
||||
for (uint32_t k = i; k < j; k++) {
|
||||
if (v[2 * (k - i) + 1]) sum += v[2 * (k - i)];
|
||||
else {
|
||||
sum += 1;
|
||||
LOG_ONCE("vk: occlusion query slot %u not available after its fence, counted as visible", s->q[k]);
|
||||
}
|
||||
__atomic_store_n(&oq_busy[s->q[k]], 0, __ATOMIC_RELEASE);
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
if (s->n && ++OQB.segs >= 12000) {
|
||||
if (rt_trace & RT_TRACE_PERF)
|
||||
rt_log("perf: occlusion queries: %llu segments read with %llu result calls of %.1f slots on average",
|
||||
(unsigned long long)OQB.segs, (unsigned long long)OQB.calls,
|
||||
OQB.calls ? (double)OQB.slots / (double)OQB.calls : 0.0);
|
||||
OQB.calls = OQB.slots = OQB.segs = 0;
|
||||
}
|
||||
if (s->reset) oq_count = 0;
|
||||
else {
|
||||
@@ -412,6 +477,22 @@ void vk_exec_event_cpu(const void *event) {
|
||||
plat_mutex_unlock(&cpu_ev_lock);
|
||||
}
|
||||
|
||||
// 1 when the CPU has signalled this event before (nvnEventSignal): its waits are gates the game opens from the CPU.
|
||||
int vk_exec_event_cpu_known(const void *event) {
|
||||
plat_mutex_lock(&cpu_ev_lock);
|
||||
int i = 0;
|
||||
while (i < cpu_ev_n && cpu_ev[i].val != event) i++;
|
||||
int known = i < cpu_ev_n;
|
||||
plat_mutex_unlock(&cpu_ev_lock);
|
||||
return known;
|
||||
}
|
||||
|
||||
// The wait's condition on the value in memory now, whatever the replay has seen. Any thread.
|
||||
int vk_exec_event_holds_now(const void *payload) {
|
||||
const p_wait_event *p = payload;
|
||||
return ev_holds(__atomic_load_n(p->val, __ATOMIC_ACQUIRE), p->mode, p->value);
|
||||
}
|
||||
|
||||
void vk_exec_event_sync(int begin) {
|
||||
plat_mutex_lock(&cpu_ev_lock);
|
||||
ev_sync_seq = begin ? cpu_ev_seq : UINT64_MAX;
|
||||
|
||||
@@ -87,6 +87,8 @@ static struct {
|
||||
VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available)
|
||||
int reconverge; // modules with subgroup ops get MaximallyReconvergesKHR (spirv_reconverge)
|
||||
uint32_t nan_preserve; // float widths that get SignedZeroInfNanPreserve (spirv_nan_preserve): NP_16|NP_32|NP_64
|
||||
int nan_bits; // NaN tests as integer bit tests (spirv_nan_bits)
|
||||
volatile uint32_t nan_bits_tests, nan_bits_selects, nan_bits_modules; // rewritten so far (the first log line)
|
||||
VkDynamicState dyn[32];
|
||||
uint32_t n_dyn;
|
||||
PFN_vkCmdPushDescriptorSetKHR push;
|
||||
@@ -658,6 +660,12 @@ static void nan_preserve_init(VkCtx *c) {
|
||||
(fc.shaderSignedZeroInfNanPreserveFloat32 ? NP_32 : 0) |
|
||||
(fc.shaderSignedZeroInfNanPreserveFloat64 ? NP_64 : 0);
|
||||
P.nan_preserve = want && (can & NP_32) ? can : 0;
|
||||
// NX2NAVI_VK_NAN_BITS: 1 every driver, 0 none; unset: AMD's driver for RDNA 1 and 2 (spirv_nan_bits)
|
||||
const char *nb = cfg_get("NX2NAVI_VK_NAN_BITS");
|
||||
P.nan_bits = nb && *nb ? strcmp(nb, "0") != 0 : vk_caps.amd_sc;
|
||||
if (P.nan_bits)
|
||||
rt_log("vk_pipeline: game shaders test for NaN on the bits of the number, not as a float (%s)",
|
||||
nb && *nb ? "NX2NAVI_VK_NAN_BITS=1" : "AMD's driver for RDNA 1 and 2");
|
||||
if (P.nan_preserve)
|
||||
rt_log("vk_pipeline: game shaders keep NaN, infinity and signed zero (SignedZeroInfNanPreserve for %s32%s-bit "
|
||||
"floats%s)", can & NP_16 ? "16, " : "", can & NP_64 ? ", 64" : "", set ? ", NX2NAVI_VK_NAN_PRESERVE=1" : "");
|
||||
@@ -1478,6 +1486,125 @@ static uint32_t *spirv_nan_preserve(const uint32_t *w, uint64_t n, uint32_t widt
|
||||
return o;
|
||||
}
|
||||
|
||||
// NaN tests that no compiler can drop (P.nan_bits). The game's shaders and the translator's own guards test for NaN with
|
||||
// OpIsNan: 48,717 of an install's 54,959 translated modules, 26 tests each on average (the game's unordered
|
||||
// comparisons, and the guard that keeps NaN out of every float colour output, patch 0027, so that one NaN texel does not
|
||||
// spread through bloom and blur into blocks of black and white). SignedZeroInfNanPreserve asks the driver to keep NaN,
|
||||
// but whether a compiler that reasons about floats keeps an isnan through every optimisation is up to it. The reports
|
||||
// of black and white blocks, on Navi, on Link next to her and on bright surfaces seen from inside a house, come from
|
||||
// RX 5000 and 6000 cards (#41, #110, #117), whose Windows driver compiles with "AMD proprietary shader compiler"
|
||||
// (vk_caps.amd_sc); RDNA 3 and 4 (LLPC) draw the same scenes clean. It is not known whether that compiler keeps the
|
||||
// mode: on it each float32 test becomes integer arithmetic on the bits, which means the same under any float
|
||||
// assumption:
|
||||
// isnan(x) -> (bitcast<uint>(x) & 0x7fffffff) > 0x7f800000
|
||||
// for scalars and vectors of 32-bit floats; tests of 16- and 64-bit floats stay. A float32 select with a constant
|
||||
// operand whose bits are a NaN (an integer mask such as 0xffffffff that the translator types as float) becomes a
|
||||
// select of the bits (issue #156: on an RX 9070 XT without the mode such a select between 1 and -1 gave 2^32).
|
||||
// NULL: nothing to change (or no memory); *tests and *selects count what changed.
|
||||
static uint32_t *spirv_nan_bits(const uint32_t *w, uint64_t n, uint64_t *out_n, uint32_t *tests, uint32_t *selects) {
|
||||
enum { OP_UNDEF = 1, OP_TYPE_BOOL = 20, OP_SPEC_CONSTANT = 50, OP_SPEC_CONSTANT_COMPOSITE = 51, OP_STORE = 62,
|
||||
OP_BITCAST = 124, OP_IS_NAN = 156, OP_SELECT = 169, OP_UGREATER = 172, OP_BITWISE_AND = 199 };
|
||||
*tests = *selects = 0;
|
||||
if (n < 5 || w[3] >= (1u << 22)) return NULL;
|
||||
uint32_t bound = w[3];
|
||||
uint32_t *rtype = calloc(bound, sizeof *rtype), *vec_comp = calloc(bound, sizeof *vec_comp);
|
||||
uint8_t *f32 = calloc(bound, 1), *vec_n = calloc(bound, 1), *nan_const = calloc(bound, 1);
|
||||
uint32_t *o = NULL;
|
||||
if (!rtype || !vec_comp || !f32 || !vec_n || !nan_const) goto out;
|
||||
uint32_t tu = 0, uvec[5] = {0}; // uint32 and its vectors by component count
|
||||
uint64_t fn_at = 0;
|
||||
int vec_used[5] = {0};
|
||||
// Ids that are not results of the instruction's second word, inside functions: first assignment wins anyway, as
|
||||
// a definition comes before its uses (OpPhi operands aside, which this never reads).
|
||||
for (uint64_t i = 5; i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
if (!l || i + l > n) goto out;
|
||||
const uint32_t *a = w + i + 1;
|
||||
if (!fn_at) {
|
||||
if (op == SPV_OP_FUNCTION) fn_at = i;
|
||||
else if (op == SPV_OP_TYPE_FLOAT && l >= 3 && a[0] < bound && a[1] == 32) f32[a[0]] = 1;
|
||||
else if (op == SPV_OP_TYPE_INT && l >= 4 && a[1] == 32 && a[2] == 0 && !tu) tu = a[0];
|
||||
else if (op == SPV_OP_TYPE_VECTOR && l >= 4 && a[0] < bound && a[2] >= 2 && a[2] <= 4) {
|
||||
vec_comp[a[0]] = a[1], vec_n[a[0]] = (uint8_t)a[2];
|
||||
if (tu && a[1] == tu && !uvec[a[2]]) uvec[a[2]] = a[0];
|
||||
} else if ((op == SPV_OP_CONSTANT || op == SPV_OP_CONSTANT_COMPOSITE || op == SPV_OP_CONSTANT_NULL ||
|
||||
op == OP_UNDEF || op == SPV_OP_VARIABLE || op == OP_SPEC_CONSTANT ||
|
||||
op == OP_SPEC_CONSTANT_COMPOSITE) && l >= 3 && a[1] < bound) {
|
||||
rtype[a[1]] = a[0];
|
||||
if (op == SPV_OP_CONSTANT && l == 4 && a[0] < bound && f32[a[0]] && (a[2] & 0x7fffffffu) > 0x7f800000u)
|
||||
nan_const[a[1]] = 1;
|
||||
}
|
||||
}
|
||||
if (fn_at) {
|
||||
int no_result = op == OP_STORE || op == 63 || op == 64 || op == 8 || op == 317 || (op >= 246 && op <= 255) ||
|
||||
op == 99 || op == 224 || op == 225 || op == 228 || op == 56 || op == 218 || op == 219 ||
|
||||
op == 5294 || op == 5295 || op == 4416 || op == 5380 || op == 5364 || op == 5365;
|
||||
if (!no_result && op != SPV_OP_FUNCTION && l >= 3 && a[1] < bound && !rtype[a[1]]) rtype[a[1]] = a[0];
|
||||
if (op == OP_IS_NAN && l == 4 && a[2] < bound) {
|
||||
uint32_t t = rtype[a[2]];
|
||||
if (t < bound && f32[t]) ++*tests;
|
||||
else if (t < bound && vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]]) ++*tests, vec_used[vec_n[t]] = 1;
|
||||
} else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound &&
|
||||
(nan_const[a[3]] || nan_const[a[4]])) {
|
||||
++*selects;
|
||||
}
|
||||
}
|
||||
i += l;
|
||||
}
|
||||
if (!fn_at || (!*tests && !*selects)) goto out;
|
||||
// the declarations go right before the first function: uint32 and its vectors where missing, then the constants
|
||||
uint32_t next = bound, decl[64], nd = 0;
|
||||
if (!tu) tu = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_INT, decl[nd++] = tu, decl[nd++] = 32, decl[nd++] = 0;
|
||||
uint32_t c_mag = next++, c_inf = next++, cv_mag[5] = {0}, cv_inf[5] = {0};
|
||||
for (int k = 2; k <= 4; k++)
|
||||
if (vec_used[k] && !uvec[k])
|
||||
uvec[k] = next++, decl[nd++] = 4u << 16 | SPV_OP_TYPE_VECTOR, decl[nd++] = uvec[k], decl[nd++] = tu, decl[nd++] = (uint32_t)k;
|
||||
decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_mag, decl[nd++] = 0x7fffffffu;
|
||||
decl[nd++] = 4u << 16 | SPV_OP_CONSTANT, decl[nd++] = tu, decl[nd++] = c_inf, decl[nd++] = 0x7f800000u;
|
||||
for (int k = 2; k <= 4; k++) {
|
||||
if (!vec_used[k]) continue;
|
||||
cv_mag[k] = next++, cv_inf[k] = next++;
|
||||
decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_mag[k];
|
||||
for (int j = 0; j < k; j++) decl[nd++] = c_mag;
|
||||
decl[nd++] = (uint32_t)(3 + k) << 16 | SPV_OP_CONSTANT_COMPOSITE, decl[nd++] = uvec[k], decl[nd++] = cv_inf[k];
|
||||
for (int j = 0; j < k; j++) decl[nd++] = c_inf;
|
||||
}
|
||||
uint64_t cap = n + nd + 10ull * *tests + 12ull * *selects;
|
||||
o = malloc(cap * 4);
|
||||
if (!o) goto out;
|
||||
memcpy(o, w, fn_at * 4);
|
||||
uint64_t p = fn_at;
|
||||
memcpy(o + p, decl, nd * 4), p += nd;
|
||||
for (uint64_t i = fn_at; i < n;) {
|
||||
uint32_t op = w[i] & 0xffff, l = w[i] >> 16;
|
||||
const uint32_t *a = w + i + 1;
|
||||
uint32_t t = op == OP_IS_NAN && l == 4 && a[2] < bound ? rtype[a[2]] : 0;
|
||||
int k = t && t < bound ? (f32[t] ? 1 : vec_n[t] && vec_comp[t] < bound && f32[vec_comp[t]] ? vec_n[t] : 0) : 0;
|
||||
if (k) {
|
||||
uint32_t ut = k == 1 ? tu : uvec[k], mag = k == 1 ? c_mag : cv_mag[k], inf = k == 1 ? c_inf : cv_inf[k];
|
||||
uint32_t bits = next++, low = next++;
|
||||
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = ut, o[p++] = bits, o[p++] = a[2];
|
||||
o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = ut, o[p++] = low, o[p++] = bits, o[p++] = mag;
|
||||
o[p++] = 5u << 16 | OP_UGREATER, o[p++] = a[0], o[p++] = a[1], o[p++] = low, o[p++] = inf;
|
||||
} else if (op == OP_SELECT && l == 6 && a[0] < bound && f32[a[0]] && a[3] < bound && a[4] < bound &&
|
||||
(nan_const[a[3]] || nan_const[a[4]])) {
|
||||
uint32_t x = next++, y = next++, s = next++;
|
||||
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = x, o[p++] = a[3];
|
||||
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = tu, o[p++] = y, o[p++] = a[4];
|
||||
o[p++] = 6u << 16 | OP_SELECT, o[p++] = tu, o[p++] = s, o[p++] = a[2], o[p++] = x, o[p++] = y;
|
||||
o[p++] = 4u << 16 | OP_BITCAST, o[p++] = a[0], o[p++] = a[1], o[p++] = s;
|
||||
} else {
|
||||
memcpy(o + p, w + i, l * 4), p += l;
|
||||
}
|
||||
i += l;
|
||||
}
|
||||
o[3] = next;
|
||||
*out_n = p;
|
||||
out:
|
||||
free(rtype), free(vec_comp), free(f32), free(vec_n), free(nan_const);
|
||||
return o;
|
||||
}
|
||||
|
||||
static int spirv_mark_list_test(const char *list); // texture pool marks, below
|
||||
|
||||
// --selftest (main.c): spirv_nan_preserve on a small fragment module, without a device. Returns the number of failures.
|
||||
@@ -1523,6 +1650,42 @@ int vk_pipeline_spirv_selftest(void) {
|
||||
failures++;
|
||||
}
|
||||
printf("spirv: SignedZeroInfNanPreserve rewrite: %s\n", failures ? "FAIL" : "ok");
|
||||
{ // spirv_nan_bits: a scalar and a vec2 isnan, a select on a NaN constant; no uint32 type in the module
|
||||
static const uint32_t m[] = {
|
||||
0x07230203, 0x00010300, 0, 20, 0,
|
||||
2u << 16 | 17, 1, // OpCapability Shader
|
||||
3u << 16 | 14, 0, 1, // OpMemoryModel Logical GLSL450
|
||||
5u << 16 | 15, 5, 1, 0x6e69616d, 0, // OpEntryPoint GLCompute %1 "main"
|
||||
2u << 16 | 19, 2, // %2 void
|
||||
3u << 16 | 33, 3, 2, // %3 fn
|
||||
3u << 16 | 22, 4, 32, // %4 float
|
||||
2u << 16 | 20, 5, // %5 bool
|
||||
4u << 16 | 23, 6, 4, 2, // %6 vec2
|
||||
4u << 16 | 23, 7, 5, 2, // %7 bvec2
|
||||
4u << 16 | 43, 4, 8, 0xffffffffu, // %8 = NaN bits
|
||||
4u << 16 | 43, 4, 9, 0x3f800000u, // %9 = 1.0
|
||||
5u << 16 | 44, 6, 10, 8, 9, // %10 = vec2(%8, %9)
|
||||
3u << 16 | 41, 5, 11, // %11 = OpConstantTrue
|
||||
5u << 16 | 54, 2, 1, 0, 3, // %1 = OpFunction
|
||||
2u << 16 | 248, 12, // OpLabel
|
||||
4u << 16 | 156, 5, 13, 9, // %13 = OpIsNan bool %9
|
||||
4u << 16 | 156, 7, 14, 10, // %14 = OpIsNan bvec2 %10
|
||||
6u << 16 | 169, 4, 15, 11, 9, 8, // %15 = OpSelect float %11 %9 %8
|
||||
1u << 16 | 253, 1u << 16 | 56}; // OpReturn, OpFunctionEnd
|
||||
uint64_t on = 0;
|
||||
uint32_t tests = 0, sels = 0, isnan_left = 0, ugt = 0, int32 = 0, bitcasts = 0;
|
||||
uint32_t *o = spirv_nan_bits(m, sizeof m / 4, &on, &tests, &sels);
|
||||
for (uint64_t i = 5; o && i < on;) {
|
||||
uint32_t op = o[i] & 0xffff, l = o[i] >> 16;
|
||||
if (!l) break;
|
||||
isnan_left += op == 156, ugt += op == 172, int32 += op == 21 && o[i + 2] == 32, bitcasts += op == 124;
|
||||
i += l;
|
||||
}
|
||||
int bad = !o || tests != 2 || sels != 1 || isnan_left || ugt != 2 || int32 != 1 || bitcasts != 5 || o[3] <= 20;
|
||||
printf("spirv: integer NaN tests rewrite: %s\n", bad ? "FAIL" : "ok");
|
||||
failures += bad;
|
||||
free(o);
|
||||
}
|
||||
const char *list = cfg_get("NX2NAVI_SPV_MARK_TEST");
|
||||
if (list && *list) failures += spirv_mark_list_test(list);
|
||||
return failures;
|
||||
@@ -1867,6 +2030,18 @@ static VkShaderModule make_module(const uint32_t *w, uint64_t words, int early_o
|
||||
free(fixed);
|
||||
w = fixed = np, words = np_n;
|
||||
}
|
||||
uint64_t nb_n = 0;
|
||||
uint32_t nb_tests = 0, nb_sels = 0;
|
||||
uint32_t *nb = P.nan_bits ? spirv_nan_bits(w, words, &nb_n, &nb_tests, &nb_sels) : NULL;
|
||||
if (nb) {
|
||||
free(fixed);
|
||||
w = fixed = nb, words = nb_n;
|
||||
__atomic_fetch_add(&P.nan_bits_tests, nb_tests, __ATOMIC_RELAXED);
|
||||
__atomic_fetch_add(&P.nan_bits_selects, nb_sels, __ATOMIC_RELAXED);
|
||||
if (__atomic_add_fetch(&P.nan_bits_modules, 1, __ATOMIC_RELAXED) == 1)
|
||||
rt_log("vk_pipeline: first module with NaN tests made integer: %u tests, %u selects of NaN bits", nb_tests,
|
||||
nb_sels);
|
||||
}
|
||||
if (P.feedback >= 2) mm_hash = fnv(FNV0, w, (size_t)words * 4);
|
||||
VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w};
|
||||
VkShaderModule m = VK_NULL_HANDLE;
|
||||
|
||||
Reference in New Issue
Block a user