From 00f06e669e8266986a7da5c0f0eaa7631182efd0 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Mon, 9 Nov 2026 21:54:32 -0500 Subject: [PATCH] Fix nine runtime faults a tester's report named: a stack overrun on mesh draws, lost YMM halves on preemption, and seven more. - The mesh stage's storage table starts 0x90 bytes before the end of the 2 KiB constant bank, and all sixteen slots were written: 0x6C bytes past a stack buffer on every mesh draw. It holds the nine slots that fit. - The prebuilt runtime is compiled without AVX, so its preemption stub saved only XMM0-15 while the lifted code of an AVX2 processor keeps data in the upper YMM halves, which the stub's callees may clear. Both bodies are built now and the one for the processor is chosen at the start. - After an FSR 3.1 frame generation error the window's swapchain was still AMD's object but went to the plain Vulkan calls. Swapchain calls follow whose swapchain it is, and a failure takes it down as a switch in the menu does. - A movie is paced at its own 60 fps under a lower limit too: at 40 fps it ended 25 s behind its sound. - A lost device left the wait for an upload spinning for good; it ends the run with the device's report. - The compute pipeline key had four bytes of padding that were hashed and compared, so a prebuilt pipeline could be missed and built again. - A thread's TLS pointer was published before the TLS was allocated, so the slot was always 0. - A fatal exit puts RivaTuner Statistics Server's profile back, and an ini saved as UTF-16 is read. Co-Authored-By: Claude Fable 5.2 --- runtime/cfg.c | 33 ++++++ runtime/cfg_probe.c | 14 +++ runtime/dispatch.c | 3 + runtime/frame_rate.c | 9 +- runtime/kernel.c | 215 ++++++--------------------------------- runtime/kernel_ap_stub.h | 172 +++++++++++++++++++++++++++++++ runtime/vk_exec_draw.c | 14 +-- runtime/vk_fg.cpp | 16 ++- runtime/vk_memory.c | 8 +- runtime/vk_pipeline.c | 6 +- 10 files changed, 290 insertions(+), 200 deletions(-) create mode 100644 runtime/kernel_ap_stub.h diff --git a/runtime/cfg.c b/runtime/cfg.c index d766cf3e..31e0cdb2 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -298,12 +298,45 @@ static void migrate(void) { prompts ? prompts->value : "xbox"); } +// A file saved as UTF-16 (Notepad's "Unicode": the byte order mark FF FE or FE FF, then two bytes per character) as +// UTF-8 in a new buffer; NULL for any other file. Read as bytes such a file ends at the zero after its first character: +// no line was taken, and every setting kept its default without a word in the log. +static uint8_t *utf16_text(const uint8_t *d, uint64_t size) { + if (size < 2 || !((d[0] == 0xFF && d[1] == 0xFE) || (d[0] == 0xFE && d[1] == 0xFF))) return NULL; + int be = d[0] == 0xFE; + uint8_t *out = malloc((size_t)(size / 2) * 3 + 1), *o = out; + if (!out) return NULL; + for (uint64_t i = 2; i + 1 < size; i += 2) { + uint32_t c = be ? (uint32_t)d[i] << 8 | d[i + 1] : (uint32_t)d[i + 1] << 8 | d[i]; + if (!c) break; + if (c >= 0xD800 && c < 0xDC00 && i + 3 < size) { // a pair: one character above U+FFFF + uint32_t lo = be ? (uint32_t)d[i + 2] << 8 | d[i + 3] : (uint32_t)d[i + 3] << 8 | d[i + 2]; + if (lo >= 0xDC00 && lo < 0xE000) c = 0x10000 + ((c - 0xD800) << 10) + (lo - 0xDC00), i += 2; + } + if (c < 0x80) *o++ = (uint8_t)c; + else if (c < 0x800) *o++ = (uint8_t)(0xC0 | c >> 6), *o++ = (uint8_t)(0x80 | (c & 0x3F)); + else if (c < 0x10000) + *o++ = (uint8_t)(0xE0 | c >> 12), *o++ = (uint8_t)(0x80 | (c >> 6 & 0x3F)), *o++ = (uint8_t)(0x80 | (c & 0x3F)); + else + *o++ = (uint8_t)(0xF0 | c >> 18), *o++ = (uint8_t)(0x80 | (c >> 12 & 0x3F)), + *o++ = (uint8_t)(0x80 | (c >> 6 & 0x3F)), *o++ = (uint8_t)(0x80 | (c & 0x3F)); + } + *o = 0; + return out; +} + int cfg_load(const char *path) { clear(); __atomic_add_fetch(&version, 1, __ATOMIC_RELEASE); // the lookup memo (cfg_memo) forgets the freed strings uint64_t size = 0; uint8_t *data = plat_file_read(path, &size); if (!data) return 0; + uint8_t *wide = utf16_text(data, size); + if (wide) { + free(data); + data = wide, size = strlen((const char *)wide); + rt_log("cfg: %s is saved as UTF-16; read as that (the menu saves it as UTF-8)", path); + } char *p = (char *)data; if (size >= 3 && !memcmp(p, "\xEF\xBB\xBF", 3)) p += 3; // editors add a BOM for (int line = 1; *p; line++) { diff --git a/runtime/cfg_probe.c b/runtime/cfg_probe.c index 5c1ce5f4..585c2f89 100644 --- a/runtime/cfg_probe.c +++ b/runtime/cfg_probe.c @@ -221,6 +221,20 @@ int probe_cfg(void) { cfg_set("NX2NAVI_VOLUME", "60"); CHECK(cfg_save_to(path) == 1, "reset: volume 60 again for the environment case"); + // a file saved as UTF-16 (little and big endian, with a line ending of two characters) reads as the same lines + static const char wtext[] = "[audio]\r\nvolume = 35\r\n[input]\r\nmotion = 0\r\n"; + for (int be = 0; be < 2; be++) { + uint8_t wbuf[2 + 2 * sizeof wtext]; + size_t wn = 0; + wbuf[wn++] = be ? 0xFE : 0xFF, wbuf[wn++] = be ? 0xFF : 0xFE; + for (size_t i = 0; i + 1 < sizeof wtext; i++) + wbuf[wn++] = be ? 0 : (uint8_t)wtext[i], wbuf[wn++] = be ? (uint8_t)wtext[i] : 0; + CHECK(plat_file_write("cfg-probe-utf16.ini", wbuf, wn) && cfg_load("cfg-probe-utf16.ini") == 1 && + EQ(cfg_get("NX2NAVI_VOLUME"), "35") && EQ(cfg_get("NX2NAVI_MOTION"), "0"), + be ? "an ini saved as UTF-16, big endian, is not read" : "an ini saved as UTF-16, little endian, is not read"); + } + plat_file_delete("cfg-probe-utf16.ini"); + // a key the environment sets is skipped and keeps its file line CHECK(cfg_load(path) == 1 && EQ(cfg_get("NX2NAVI_VOLUME"), "60"), "reset: reload"); cfg_setenv("NX2NAVI_VOLUME", "70"); diff --git a/runtime/dispatch.c b/runtime/dispatch.c index 6f7be646..1e651f61 100644 --- a/runtime/dispatch.c +++ b/runtime/dispatch.c @@ -55,6 +55,9 @@ void rt_fatal(const char *fmt, ...) { rt_log("fatal: memory commit: this process %.0f MB, the system %.0f of %.0f MB", proc, used, limit); kern_free_cores_note(); prof_sampler_write(); // NX2NAVI_PROFILE: what was sampled until now, else a crash loses the whole profile + // RivaTuner Statistics Server's profile as it was: left with the game's limit in it, the next run would take that + // limit for the player's own (rtss_limit.c; it gives up after a second if the failing thread holds its lock) + rtss_limit_restore(); fflush(stdout); fflush(stderr); _Exit(70); diff --git a/runtime/frame_rate.c b/runtime/frame_rate.c index 4a5343f7..2f7228c7 100644 --- a/runtime/frame_rate.c +++ b/runtime/frame_rate.c @@ -197,7 +197,10 @@ static void frame_limit_live(void) { // NX2NAVI_MOVIE_FPS (default 60; 0 = off): the game's frame rate while a movie plays (ov_movie2.c ov_movie_live). A // movie's event counts the game's updates as 60 Hz frames whatever the step: at 144 fps the intro's narration lines // started 2.4 times too early, each over the one before, and its closing fade to black came at 65 s of the movie's -// 154, hiding the whole dream. The movies are 60 fps, so the cap shows the same pictures. +// 154, hiding the whole dream. The movies are 60 fps, so the cap shows the same pictures. It is the rate in both +// directions: under a lower limit (40 fps, or 60 shown with frame generation, which is 30 of the game's) the movie ran +// at that fraction of its speed and ended many seconds behind its sound, so the game is paced at the movie's rate +// while one plays, above the limit too. static int movie_cap_hz(void) { static int hz = -1; if (hz < 0) { @@ -244,7 +247,7 @@ uint64_t frame_step_cap_ns(void) { if (!!movie_hz != movie_was) { movie_was = !!movie_hz; if (movie_logs++ < 40) { // the first 20 movies - if (movie_hz) rt_log("frame rate: a movie plays: at most %d fps until it ends (NX2NAVI_MOVIE_FPS)", movie_hz); + if (movie_hz) rt_log("frame rate: a movie plays: %d fps until it ends (NX2NAVI_MOVIE_FPS)", movie_hz); else rt_log("frame rate: the movie ended: the frame limit again"); } } @@ -272,7 +275,7 @@ uint64_t frame_step_cap_ns(void) { // limit over that factor (half, a third, a quarter), so the screen shows the limit. With VSync and the limit at the // display's rate, the generated frames then fill the display's refreshes and none is dropped. if (gen > 1) hz /= gen; - if (movie_hz && hz > movie_hz) hz = movie_hz; + if (movie_hz) hz = movie_hz; rtss_limit_want(rtss ? (int64_t)(hz * 1000 + 0.5) : -1); if (rtss_limit_active()) return 0; return (uint64_t)(1e9 / hz + 0.5); diff --git a/runtime/kernel.c b/runtime/kernel.c index f5aad9af..c97103c0 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -376,7 +376,18 @@ static int g_ap_log; // redirects logged so far (AP_LOG_N) #define AP_LOG_N 40 static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok; -void kern_ap_stub(void); +void kern_ap_stub_ymm(void), kern_ap_stub_xmm(void); +static void (*ap_stub)(void) = kern_ap_stub_xmm; // the stub for this processor (ap_init) + +// The processor has AVX and the system saves YMM state across thread switches (OSXSAVE, XCR0 bits 1 and 2). +static int cpu_keeps_ymm(void) { + unsigned a = 1, b, c = 0, d; + __asm__("cpuid" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + if ((c & 0x18000000u) != 0x18000000u) return 0; + unsigned lo, hi; + __asm__("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + return (lo & 6) == 6; +} static void ap_init(void) { if (g_ap >= 0) return; @@ -393,8 +404,9 @@ static void ap_init(void) { if (f < lo) lo = f; if (f > hi) hi = f; } - const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc, - (uintptr_t)rt_dispatch, (uintptr_t)rt_call}; + ap_stub = cpu_keeps_ymm() ? kern_ap_stub_ymm : kern_ap_stub_xmm; + const uintptr_t host[] = {(uintptr_t)kern_ap_stub_ymm, (uintptr_t)kern_ap_stub_xmm, (uintptr_t)kern_checkpoint, + (uintptr_t)rt_svc, (uintptr_t)rt_dispatch, (uintptr_t)rt_call}; int bad = !n || lo >= hi; for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi; if (bad) { @@ -408,9 +420,10 @@ static void ap_init(void) { } } rt_log("kernel: asynchronous preemption of virtual cores %s, priority inheritance and preemption disable %s; lifted " - "code exe+%#llx-exe+%#llx", g_ap_int ? "on" : "off", g_ap ? "on (NX2NAVI_ASYNC_PREEMPT=0 turns all off)" : "off", + "code exe+%#llx-exe+%#llx; an interrupted core's %s registers are kept", g_ap_int ? "on" : "off", + g_ap ? "on (NX2NAVI_ASYNC_PREEMPT=0 turns all off)" : "off", (unsigned long long)(g_gen_lo ? g_gen_lo - plat_exe_base() : 0), - (unsigned long long)(g_gen_hi ? g_gen_hi - plat_exe_base() : 0)); + (unsigned long long)(g_gen_hi ? g_gen_hi - plat_exe_base() : 0), ap_stub == kern_ap_stub_ymm ? "YMM" : "XMM"); } // Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the @@ -437,187 +450,25 @@ static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; } // in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at // entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs // balanced. The waiter never writes the owner's stack. -// An x86-64-v2 build (no __AVX__) saves XMM0-15 with SSE moves into the same 32-byte slots instead: its game code -// keeps nothing in the upper YMM halves, and the CPUs it is for have no VEX encoding, so a vmovdqa there faults with -// an illegal instruction the first time a core is interrupted. -#ifdef __AVX__ +// Two bodies of the stub (kernel_ap_stub.h): kern_ap_stub_ymm saves YMM0-15 and is the one used wherever the processor +// and the system keep YMM state (ap_stub, chosen by ap_init). The lifted code of such a PC is compiled with AVX2 and +// keeps data in the upper halves, which every function the stub calls is free to clear, whatever the runtime itself was +// compiled for. kern_ap_stub_xmm saves XMM0-15 with SSE moves: code on a processor without AVX has nothing in the upper +// halves, and a VEX move there is an illegal instruction. +#define AP_NAME "kern_ap_stub_ymm" #define AP_VMOV "vmovdqa" #define AP_V "%ymm" #define AP_VZEROUPPER " vzeroupper\n" -#else +#include "kernel_ap_stub.h" +#undef AP_NAME +#undef AP_VMOV +#undef AP_V +#undef AP_VZEROUPPER +#define AP_NAME "kern_ap_stub_xmm" #define AP_VMOV "movdqa" #define AP_V "%xmm" #define AP_VZEROUPPER "" -#endif -#ifdef _WIN32 -__asm__(".text\n" - ".globl kern_ap_stub\n" - ".def kern_ap_stub; .scl 2; .type 32; .endef\n" - ".p2align 4\n" - "kern_ap_stub:\n" - ".seh_proc kern_ap_stub\n" - " leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq - ".seh_stackalloc 8\n" - " pushfq\n" - ".seh_stackalloc 8\n" - " pushq %rax\n" - ".seh_pushreg %rax\n" - " pushq %rcx\n" - ".seh_pushreg %rcx\n" - " pushq %rdx\n" - ".seh_pushreg %rdx\n" - " pushq %r8\n" - ".seh_pushreg %r8\n" - " pushq %r9\n" - ".seh_pushreg %r9\n" - " pushq %r10\n" - ".seh_pushreg %r10\n" - " pushq %r11\n" - ".seh_pushreg %r11\n" - " pushq %rbp\n" - ".seh_pushreg %rbp\n" - " movq %rsp, %rbp\n" - ".seh_setframe %rbp, 0\n" - ".seh_endprologue\n" - " andq $-32, %rsp\n" - " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) - " " AP_VMOV " " AP_V "0, 64(%rsp)\n" - " " AP_VMOV " " AP_V "1, 96(%rsp)\n" - " " AP_VMOV " " AP_V "2, 128(%rsp)\n" - " " AP_VMOV " " AP_V "3, 160(%rsp)\n" - " " AP_VMOV " " AP_V "4, 192(%rsp)\n" - " " AP_VMOV " " AP_V "5, 224(%rsp)\n" - " " AP_VMOV " " AP_V "6, 256(%rsp)\n" - " " AP_VMOV " " AP_V "7, 288(%rsp)\n" - " " AP_VMOV " " AP_V "8, 320(%rsp)\n" - " " AP_VMOV " " AP_V "9, 352(%rsp)\n" - " " AP_VMOV " " AP_V "10, 384(%rsp)\n" - " " AP_VMOV " " AP_V "11, 416(%rsp)\n" - " " AP_VMOV " " AP_V "12, 448(%rsp)\n" - " " AP_VMOV " " AP_V "13, 480(%rsp)\n" - " " AP_VMOV " " AP_V "14, 512(%rsp)\n" - " " AP_VMOV " " AP_V "15, 544(%rsp)\n" - " stmxcsr 32(%rsp)\n" - " movl $0x1f80, 36(%rsp)\n" - " ldmxcsr 36(%rsp)\n" - " cld\n" - AP_VZEROUPPER - " call kern_ap_rip\n" - " movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq - " call kern_ap_yield\n" - " ldmxcsr 32(%rsp)\n" - " " AP_VMOV " 64(%rsp), " AP_V "0\n" - " " AP_VMOV " 96(%rsp), " AP_V "1\n" - " " AP_VMOV " 128(%rsp), " AP_V "2\n" - " " AP_VMOV " 160(%rsp), " AP_V "3\n" - " " AP_VMOV " 192(%rsp), " AP_V "4\n" - " " AP_VMOV " 224(%rsp), " AP_V "5\n" - " " AP_VMOV " 256(%rsp), " AP_V "6\n" - " " AP_VMOV " 288(%rsp), " AP_V "7\n" - " " AP_VMOV " 320(%rsp), " AP_V "8\n" - " " AP_VMOV " 352(%rsp), " AP_V "9\n" - " " AP_VMOV " 384(%rsp), " AP_V "10\n" - " " AP_VMOV " 416(%rsp), " AP_V "11\n" - " " AP_VMOV " 448(%rsp), " AP_V "12\n" - " " AP_VMOV " 480(%rsp), " AP_V "13\n" - " " AP_VMOV " 512(%rsp), " AP_V "14\n" - " " AP_VMOV " 544(%rsp), " AP_V "15\n" - " movq %rbp, %rsp\n" - " popq %rbp\n" - " popq %r11\n" - " popq %r10\n" - " popq %r9\n" - " popq %r8\n" - " popq %rdx\n" - " popq %rcx\n" - " popq %rax\n" - " popfq\n" - " leaq 8(%rsp), %rsp\n" - " jmpq *-8(%rsp)\n" - ".seh_endproc\n"); -#else -// The System V version: rsi and rdi are volatile there too, and the interrupted code may keep data in the 128 bytes -// below its stack pointer (the red zone), so the return slot and the saves go below that. Without a red zone in the -// way the slot is at entry - 136; the exit restores rsp to the entry value and jumps through the slot. -__asm__(".text\n" - ".globl kern_ap_stub\n" - ".type kern_ap_stub, @function\n" - ".p2align 4\n" - "kern_ap_stub:\n" - " .cfi_startproc\n" - " leaq -136(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq - " pushfq\n" - " pushq %rax\n" - " pushq %rcx\n" - " pushq %rdx\n" - " pushq %rsi\n" - " pushq %rdi\n" - " pushq %r8\n" - " pushq %r9\n" - " pushq %r10\n" - " pushq %r11\n" - " pushq %rbp\n" - " movq %rsp, %rbp\n" - " andq $-32, %rsp\n" - " subq $576, %rsp\n" // 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) - " " AP_VMOV " " AP_V "0, 64(%rsp)\n" - " " AP_VMOV " " AP_V "1, 96(%rsp)\n" - " " AP_VMOV " " AP_V "2, 128(%rsp)\n" - " " AP_VMOV " " AP_V "3, 160(%rsp)\n" - " " AP_VMOV " " AP_V "4, 192(%rsp)\n" - " " AP_VMOV " " AP_V "5, 224(%rsp)\n" - " " AP_VMOV " " AP_V "6, 256(%rsp)\n" - " " AP_VMOV " " AP_V "7, 288(%rsp)\n" - " " AP_VMOV " " AP_V "8, 320(%rsp)\n" - " " AP_VMOV " " AP_V "9, 352(%rsp)\n" - " " AP_VMOV " " AP_V "10, 384(%rsp)\n" - " " AP_VMOV " " AP_V "11, 416(%rsp)\n" - " " AP_VMOV " " AP_V "12, 448(%rsp)\n" - " " AP_VMOV " " AP_V "13, 480(%rsp)\n" - " " AP_VMOV " " AP_V "14, 512(%rsp)\n" - " " AP_VMOV " " AP_V "15, 544(%rsp)\n" - " stmxcsr 32(%rsp)\n" - " movl $0x1f80, 36(%rsp)\n" - " ldmxcsr 36(%rsp)\n" - " cld\n" - AP_VZEROUPPER - " call kern_ap_rip\n" - " movq %rax, 88(%rbp)\n" // the return slot: rbp + 10 pushes + pushfq - " call kern_ap_yield\n" - " ldmxcsr 32(%rsp)\n" - " " AP_VMOV " 64(%rsp), " AP_V "0\n" - " " AP_VMOV " 96(%rsp), " AP_V "1\n" - " " AP_VMOV " 128(%rsp), " AP_V "2\n" - " " AP_VMOV " 160(%rsp), " AP_V "3\n" - " " AP_VMOV " 192(%rsp), " AP_V "4\n" - " " AP_VMOV " 224(%rsp), " AP_V "5\n" - " " AP_VMOV " 256(%rsp), " AP_V "6\n" - " " AP_VMOV " 288(%rsp), " AP_V "7\n" - " " AP_VMOV " 320(%rsp), " AP_V "8\n" - " " AP_VMOV " 352(%rsp), " AP_V "9\n" - " " AP_VMOV " 384(%rsp), " AP_V "10\n" - " " AP_VMOV " 416(%rsp), " AP_V "11\n" - " " AP_VMOV " 448(%rsp), " AP_V "12\n" - " " AP_VMOV " 480(%rsp), " AP_V "13\n" - " " AP_VMOV " 512(%rsp), " AP_V "14\n" - " " AP_VMOV " 544(%rsp), " AP_V "15\n" - " movq %rbp, %rsp\n" - " popq %rbp\n" - " popq %r11\n" - " popq %r10\n" - " popq %r9\n" - " popq %r8\n" - " popq %rdi\n" - " popq %rsi\n" - " popq %rdx\n" - " popq %rcx\n" - " popq %rax\n" - " popfq\n" - " leaq 136(%rsp), %rsp\n" - " jmpq *-136(%rsp)\n" - " .cfi_endproc\n" - ".size kern_ap_stub, .-kern_ap_stub\n"); -#endif +#include "kernel_ap_stub.h" enum { AP_NONE, AP_RETRY, AP_SENT }; @@ -657,7 +508,7 @@ static int ap_try(RtThread *w) { } else { o->ap_rip = rip; o->ap_pending = 1; - to = (uint64_t)(uintptr_t)kern_ap_stub; + to = (uint64_t)(uintptr_t)ap_stub; } if (!plat_intr_resume(o->hthr, to)) { o->ap_pending = 0; @@ -1440,8 +1291,8 @@ static RtThread *thr_new(int prio, int core, uint64_t mask) { // TLR (0x200) and the "current thread object" the SDK reads through TLR+0x1F8, in one guest-visible block. t->tlr = mem_kernel_alloc(TLR_SIZE + THREAD_OBJ_SIZE, TLR_SIZE); *(uint64_t *)(t->tlr + 0x1F8) = (uint64_t)(uintptr_t)(t->tlr + TLR_SIZE); - *(uint64_t *)(t->tlr + 0x1F0) = (uint64_t)(uintptr_t)t->tls; // the SDK reads the TLS pointer here and makes it TPIDR_EL0 t->tls = mem_kernel_alloc(TLS_SIZE, 16); + *(uint64_t *)(t->tlr + 0x1F0) = (uint64_t)(uintptr_t)t->tls; // the SDK reads the TLS pointer here and makes it TPIDR_EL0 c->tpidrro = (uint64_t)(uintptr_t)t->tlr; c->tpidr = (uint64_t)(uintptr_t)t->tls; c->excl_addr = ~0ull; diff --git a/runtime/kernel_ap_stub.h b/runtime/kernel_ap_stub.h new file mode 100644 index 00000000..de8aba66 --- /dev/null +++ b/runtime/kernel_ap_stub.h @@ -0,0 +1,172 @@ +// The body of kern_ap_stub (kernel.c, "asynchronous preemption"), included once per register width with AP_NAME the +// function's name, AP_VMOV and AP_V the move and the register prefix that save the vector registers, and AP_VZEROUPPER +// the instruction that ends the use of the upper halves (or nothing). No include guard: kernel.c includes it twice. +#ifdef _WIN32 +__asm__(".text\n" + ".globl " AP_NAME "\n" + ".def " AP_NAME "; .scl 2; .type 32; .endef\n" + ".p2align 4\n" + AP_NAME ":\n" + ".seh_proc " AP_NAME "\n" + " leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq + ".seh_stackalloc 8\n" + " pushfq\n" + ".seh_stackalloc 8\n" + " pushq %rax\n" + ".seh_pushreg %rax\n" + " pushq %rcx\n" + ".seh_pushreg %rcx\n" + " pushq %rdx\n" + ".seh_pushreg %rdx\n" + " pushq %r8\n" + ".seh_pushreg %r8\n" + " pushq %r9\n" + ".seh_pushreg %r9\n" + " pushq %r10\n" + ".seh_pushreg %r10\n" + " pushq %r11\n" + ".seh_pushreg %r11\n" + " pushq %rbp\n" + ".seh_pushreg %rbp\n" + " movq %rsp, %rbp\n" + ".seh_setframe %rbp, 0\n" + ".seh_endprologue\n" + " andq $-32, %rsp\n" + " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) + " " AP_VMOV " " AP_V "0, 64(%rsp)\n" + " " AP_VMOV " " AP_V "1, 96(%rsp)\n" + " " AP_VMOV " " AP_V "2, 128(%rsp)\n" + " " AP_VMOV " " AP_V "3, 160(%rsp)\n" + " " AP_VMOV " " AP_V "4, 192(%rsp)\n" + " " AP_VMOV " " AP_V "5, 224(%rsp)\n" + " " AP_VMOV " " AP_V "6, 256(%rsp)\n" + " " AP_VMOV " " AP_V "7, 288(%rsp)\n" + " " AP_VMOV " " AP_V "8, 320(%rsp)\n" + " " AP_VMOV " " AP_V "9, 352(%rsp)\n" + " " AP_VMOV " " AP_V "10, 384(%rsp)\n" + " " AP_VMOV " " AP_V "11, 416(%rsp)\n" + " " AP_VMOV " " AP_V "12, 448(%rsp)\n" + " " AP_VMOV " " AP_V "13, 480(%rsp)\n" + " " AP_VMOV " " AP_V "14, 512(%rsp)\n" + " " AP_VMOV " " AP_V "15, 544(%rsp)\n" + " stmxcsr 32(%rsp)\n" + " movl $0x1f80, 36(%rsp)\n" + " ldmxcsr 36(%rsp)\n" + " cld\n" + AP_VZEROUPPER + " call kern_ap_rip\n" + " movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq + " call kern_ap_yield\n" + " ldmxcsr 32(%rsp)\n" + " " AP_VMOV " 64(%rsp), " AP_V "0\n" + " " AP_VMOV " 96(%rsp), " AP_V "1\n" + " " AP_VMOV " 128(%rsp), " AP_V "2\n" + " " AP_VMOV " 160(%rsp), " AP_V "3\n" + " " AP_VMOV " 192(%rsp), " AP_V "4\n" + " " AP_VMOV " 224(%rsp), " AP_V "5\n" + " " AP_VMOV " 256(%rsp), " AP_V "6\n" + " " AP_VMOV " 288(%rsp), " AP_V "7\n" + " " AP_VMOV " 320(%rsp), " AP_V "8\n" + " " AP_VMOV " 352(%rsp), " AP_V "9\n" + " " AP_VMOV " 384(%rsp), " AP_V "10\n" + " " AP_VMOV " 416(%rsp), " AP_V "11\n" + " " AP_VMOV " 448(%rsp), " AP_V "12\n" + " " AP_VMOV " 480(%rsp), " AP_V "13\n" + " " AP_VMOV " 512(%rsp), " AP_V "14\n" + " " AP_VMOV " 544(%rsp), " AP_V "15\n" + " movq %rbp, %rsp\n" + " popq %rbp\n" + " popq %r11\n" + " popq %r10\n" + " popq %r9\n" + " popq %r8\n" + " popq %rdx\n" + " popq %rcx\n" + " popq %rax\n" + " popfq\n" + " leaq 8(%rsp), %rsp\n" + " jmpq *-8(%rsp)\n" + ".seh_endproc\n"); +#else +// The System V version: rsi and rdi are volatile there too, and the interrupted code may keep data in the 128 bytes +// below its stack pointer (the red zone), so the return slot and the saves go below that. Without a red zone in the +// way the slot is at entry - 136; the exit restores rsp to the entry value and jumps through the slot. +__asm__(".text\n" + ".globl " AP_NAME "\n" + ".type " AP_NAME ", @function\n" + ".p2align 4\n" + AP_NAME ":\n" + " .cfi_startproc\n" + " leaq -136(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq + " pushfq\n" + " pushq %rax\n" + " pushq %rcx\n" + " pushq %rdx\n" + " pushq %rsi\n" + " pushq %rdi\n" + " pushq %r8\n" + " pushq %r9\n" + " pushq %r10\n" + " pushq %r11\n" + " pushq %rbp\n" + " movq %rsp, %rbp\n" + " andq $-32, %rsp\n" + " subq $576, %rsp\n" // 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) + " " AP_VMOV " " AP_V "0, 64(%rsp)\n" + " " AP_VMOV " " AP_V "1, 96(%rsp)\n" + " " AP_VMOV " " AP_V "2, 128(%rsp)\n" + " " AP_VMOV " " AP_V "3, 160(%rsp)\n" + " " AP_VMOV " " AP_V "4, 192(%rsp)\n" + " " AP_VMOV " " AP_V "5, 224(%rsp)\n" + " " AP_VMOV " " AP_V "6, 256(%rsp)\n" + " " AP_VMOV " " AP_V "7, 288(%rsp)\n" + " " AP_VMOV " " AP_V "8, 320(%rsp)\n" + " " AP_VMOV " " AP_V "9, 352(%rsp)\n" + " " AP_VMOV " " AP_V "10, 384(%rsp)\n" + " " AP_VMOV " " AP_V "11, 416(%rsp)\n" + " " AP_VMOV " " AP_V "12, 448(%rsp)\n" + " " AP_VMOV " " AP_V "13, 480(%rsp)\n" + " " AP_VMOV " " AP_V "14, 512(%rsp)\n" + " " AP_VMOV " " AP_V "15, 544(%rsp)\n" + " stmxcsr 32(%rsp)\n" + " movl $0x1f80, 36(%rsp)\n" + " ldmxcsr 36(%rsp)\n" + " cld\n" + AP_VZEROUPPER + " call kern_ap_rip\n" + " movq %rax, 88(%rbp)\n" // the return slot: rbp + 10 pushes + pushfq + " call kern_ap_yield\n" + " ldmxcsr 32(%rsp)\n" + " " AP_VMOV " 64(%rsp), " AP_V "0\n" + " " AP_VMOV " 96(%rsp), " AP_V "1\n" + " " AP_VMOV " 128(%rsp), " AP_V "2\n" + " " AP_VMOV " 160(%rsp), " AP_V "3\n" + " " AP_VMOV " 192(%rsp), " AP_V "4\n" + " " AP_VMOV " 224(%rsp), " AP_V "5\n" + " " AP_VMOV " 256(%rsp), " AP_V "6\n" + " " AP_VMOV " 288(%rsp), " AP_V "7\n" + " " AP_VMOV " 320(%rsp), " AP_V "8\n" + " " AP_VMOV " 352(%rsp), " AP_V "9\n" + " " AP_VMOV " 384(%rsp), " AP_V "10\n" + " " AP_VMOV " 416(%rsp), " AP_V "11\n" + " " AP_VMOV " 448(%rsp), " AP_V "12\n" + " " AP_VMOV " 480(%rsp), " AP_V "13\n" + " " AP_VMOV " 512(%rsp), " AP_V "14\n" + " " AP_VMOV " 544(%rsp), " AP_V "15\n" + " movq %rbp, %rsp\n" + " popq %rbp\n" + " popq %r11\n" + " popq %r10\n" + " popq %r9\n" + " popq %r8\n" + " popq %rdi\n" + " popq %rsi\n" + " popq %rdx\n" + " popq %rcx\n" + " popq %rax\n" + " popfq\n" + " leaq 136(%rsp), %rsp\n" + " jmpq *-136(%rsp)\n" + " .cfi_endproc\n" + ".size " AP_NAME ", .-" AP_NAME "\n"); +#endif diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 6d00dccc..46b8730e 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1042,8 +1042,8 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, return 1; } -static void c0_table(uint8_t *at, const p_bind_buf b[16]) { - for (int r = 0; r < 16; r++) { +static void c0_table(uint8_t *at, const p_bind_buf b[16], int rows) { + for (int r = 0; r < rows; r++) { uint64_t a = b[r].addr; uint32_t n = (uint32_t)b[r].size; memcpy(at + 16 * r, &a, 8); @@ -1056,8 +1056,9 @@ static void c0_table(uint8_t *at, const p_bind_buf b[16]) { // candidate base from here and compares the global pointer against it. Compute: dispatch size at 0x200, the uniform // buffer table at 0x210 + 16 r. Graphics: the largest point size (Maxwell's limit, GLSLC clamps gl_PointSize to it) at // 0x610, the base instance at 0x644. NVN2 task and mesh stages: storage tables at 0x670 and 0x770 (the game binds -// slots 0-2 there, and their programs read 0x670.. and 0x770..). Other stages: zeros. Unbound slots read zero, since the -// replay keeps them zero. +// slots 0-2 there, and their programs read 0x670.. and 0x770..). The mesh table starts 0x90 bytes before the bank's +// end: it holds the 9 slots that fit, and sixteen would write 0x6C bytes past the bank. Other stages: zeros. Unbound +// slots read zero, since the replay keeps them zero. // Byte offset of a stage's storage buffer table in c0, -1 for none. static int c0_ssbo_base(int stage) { return stage == NVN_STAGE_VS ? 0x110 : stage == NVN_STAGE_FS ? 0x510 : stage == NVN_STAGE_CS ? 0x310 @@ -1069,11 +1070,12 @@ void vk_c0_fill(uint8_t bank[VK_C0_BYTES], int stage, const p_bind_buf ssbo[16], memset(bank, 0, VK_C0_BYTES); int base = c0_ssbo_base(stage); if (base < 0) return; - c0_table(bank + base, ssbo); + int rows = (int)(VK_C0_BYTES - (uint32_t)base) / 16; + c0_table(bank + base, ssbo, rows < 16 ? rows : 16); if (stage == NVN_STAGE_TASK || stage == NVN_STAGE_MESH) return; if (stage == NVN_STAGE_CS) { memcpy(bank + 0x200, dispatch, 12); - c0_table(bank + 0x210, ubo); + c0_table(bank + 0x210, ubo, 16); return; } const float max_point = 2047.0f; diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 1135796a..5673d1bc 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -82,9 +82,11 @@ void fail(const char *what, unsigned r) { // It needs DLSS's or FSR 3.1's motion vectors, so it also needs one of them on. Not on a device made through // Streamline's proxies (NVIDIA's frame generation set at the start, vk_dlssg_dpa): AMD's swapchain there crashed in // NVIDIA's driver when the menu switched from NVIDIA's frame generation to AMD's. That switch waits for a restart, -// as the menu says (menu.cpp row_framegen). +// as the menu says (menu.cpp row_framegen). Not after a failure (fail): the present code then takes AMD's swapchain +// down as it does for a switch in the menu, and a plain one follows. extern "C" int vk_fg_wanted(void) { - return !strcmp(cfg_value("NX2NAVI_FRAMEGEN"), "fsr3") && cfg_bool("NX2NAVI_DLAA") && vk_dlaa_wanted() && !vk_dlssg_dpa(); + return !F.failed && !strcmp(cfg_value("NX2NAVI_FRAMEGEN"), "fsr3") && cfg_bool("NX2NAVI_DLAA") && vk_dlaa_wanted() && + !vk_dlssg_dpa(); } // Always the three queues: the setting can be switched on during a run, and idle queues cost nothing. @@ -92,9 +94,12 @@ extern "C" uint32_t vk_fg_queue_count(void) { return 3; } extern "C" void vk_fg_queues(VkQueue present, VkQueue acquire) { F.present = present, F.acquire = acquire; } -extern "C" int vk_fg_active(void) { return F.sc != nullptr && !F.failed; } +// AMD's swapchain is in place: the window's swapchain handle is AMD's object, and every swapchain call has to go +// through AMD's functions. That holds after a failure too, until the present code has replaced the swapchain +// (vk_fg_wanted): the plain Vulkan calls must never get that handle. +extern "C" int vk_fg_active(void) { return F.sc != nullptr; } -extern "C" int vk_fg_generating(void) { return vk_fg_active() && __atomic_load_n(&F.on, __ATOMIC_RELAXED); } +extern "C" int vk_fg_generating(void) { return vk_fg_active() && !F.failed && __atomic_load_n(&F.on, __ATOMIC_RELAXED); } extern "C" VkResult vk_fg_create_swapchain(const VkSwapchainCreateInfoKHR *ci, VkSwapchainKHR *sc) { if (F.sc) return F.fn.pOutCreateSwapchainFFXAPI(vk_ctx()->device, ci, nullptr, sc, F.sc); @@ -120,6 +125,7 @@ extern "C" VkResult vk_fg_create_swapchain(const VkSwapchainCreateInfoKHR *ci, V } F.fn.header.type = FFX_API_QUERY_DESC_TYPE_FGSWAPCHAIN_FUNCTIONS_VK; if (vk_ffx_query(&F.sc, &F.fn.header) != FFX_API_RETURN_OK || !F.fn.pOutQueuePresentKHR) { + vk_ffx_destroy(&F.sc), F.sc = nullptr; // with it goes the swapchain it made: nothing was handed out yet fail("the swapchain functions", 0); return VK_ERROR_INITIALIZATION_FAILED; } @@ -162,7 +168,7 @@ static bool hdr_format(VkFormat f) { extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, VkExtent2D ext, VkImage hudless, VkImageView hudless_view, const int pic[4]) { (void)hudless_view; - if (!vk_fg_active()) return; + if (!vk_fg_active() || F.failed) return; vk_dlaa_hudless_wanted = 1; // the upscaling pass keeps a copy of the picture before the HUD (vk_dlaa.c hudless) const VkCtx *c = vk_ctx(); VkFfxImage depth, mv; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 3e1cd01a..ced07ef4 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -3023,7 +3023,13 @@ uint64_t vk_upload_flush(int wait) { } if (c && wait) { while (ninflight && done_seq < seq) { - vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); + // a lost device ends the wait at once and leaves the fence unsignalled: retire would take nothing and this + // loop would spin for good, holding the upload lock, instead of the run ending with the device's report + VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); + if (w != VK_SUCCESS) { + vk_device_fault_report(w); + rt_fatal("vk: the wait for an upload failed (%d)", (int)w); + } retire(c, 0); } } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 229887be..dcd39267 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -4267,7 +4267,7 @@ static struct { VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { if (info) *info = NULL; if (!cs || !ensure_init()) return VK_NULL_HANDLE; - struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1}; + struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); plat_mutex_lock(&P.lock); Pipe *p = pipe_find(&k, sizeof k, h); @@ -4487,7 +4487,7 @@ static void bg_main(void *arg) { plat_mutex_unlock(&LB.stat); if (!have) break; const uint8_t *iid = idbuf; - struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1}; + struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL; @@ -4567,7 +4567,7 @@ static void bg_start(void) { } static void lb_compute(const GfxShader *cs, const uint8_t iid[32]) { - struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1}; + struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL;