diff --git a/runtime/cfg.c b/runtime/cfg.c index d766cf3e..31e0cdb2 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -298,12 +298,45 @@ static void migrate(void) { prompts ? prompts->value : "xbox"); } +// A file saved as UTF-16 (Notepad's "Unicode": the byte order mark FF FE or FE FF, then two bytes per character) as +// UTF-8 in a new buffer; NULL for any other file. Read as bytes such a file ends at the zero after its first character: +// no line was taken, and every setting kept its default without a word in the log. +static uint8_t *utf16_text(const uint8_t *d, uint64_t size) { + if (size < 2 || !((d[0] == 0xFF && d[1] == 0xFE) || (d[0] == 0xFE && d[1] == 0xFF))) return NULL; + int be = d[0] == 0xFE; + uint8_t *out = malloc((size_t)(size / 2) * 3 + 1), *o = out; + if (!out) return NULL; + for (uint64_t i = 2; i + 1 < size; i += 2) { + uint32_t c = be ? (uint32_t)d[i] << 8 | d[i + 1] : (uint32_t)d[i + 1] << 8 | d[i]; + if (!c) break; + if (c >= 0xD800 && c < 0xDC00 && i + 3 < size) { // a pair: one character above U+FFFF + uint32_t lo = be ? (uint32_t)d[i + 2] << 8 | d[i + 3] : (uint32_t)d[i + 3] << 8 | d[i + 2]; + if (lo >= 0xDC00 && lo < 0xE000) c = 0x10000 + ((c - 0xD800) << 10) + (lo - 0xDC00), i += 2; + } + if (c < 0x80) *o++ = (uint8_t)c; + else if (c < 0x800) *o++ = (uint8_t)(0xC0 | c >> 6), *o++ = (uint8_t)(0x80 | (c & 0x3F)); + else if (c < 0x10000) + *o++ = (uint8_t)(0xE0 | c >> 12), *o++ = (uint8_t)(0x80 | (c >> 6 & 0x3F)), *o++ = (uint8_t)(0x80 | (c & 0x3F)); + else + *o++ = (uint8_t)(0xF0 | c >> 18), *o++ = (uint8_t)(0x80 | (c >> 12 & 0x3F)), + *o++ = (uint8_t)(0x80 | (c >> 6 & 0x3F)), *o++ = (uint8_t)(0x80 | (c & 0x3F)); + } + *o = 0; + return out; +} + int cfg_load(const char *path) { clear(); __atomic_add_fetch(&version, 1, __ATOMIC_RELEASE); // the lookup memo (cfg_memo) forgets the freed strings uint64_t size = 0; uint8_t *data = plat_file_read(path, &size); if (!data) return 0; + uint8_t *wide = utf16_text(data, size); + if (wide) { + free(data); + data = wide, size = strlen((const char *)wide); + rt_log("cfg: %s is saved as UTF-16; read as that (the menu saves it as UTF-8)", path); + } char *p = (char *)data; if (size >= 3 && !memcmp(p, "\xEF\xBB\xBF", 3)) p += 3; // editors add a BOM for (int line = 1; *p; line++) { diff --git a/runtime/cfg_probe.c b/runtime/cfg_probe.c index 5c1ce5f4..585c2f89 100644 --- a/runtime/cfg_probe.c +++ b/runtime/cfg_probe.c @@ -221,6 +221,20 @@ int probe_cfg(void) { cfg_set("NX2NAVI_VOLUME", "60"); CHECK(cfg_save_to(path) == 1, "reset: volume 60 again for the environment case"); + // a file saved as UTF-16 (little and big endian, with a line ending of two characters) reads as the same lines + static const char wtext[] = "[audio]\r\nvolume = 35\r\n[input]\r\nmotion = 0\r\n"; + for (int be = 0; be < 2; be++) { + uint8_t wbuf[2 + 2 * sizeof wtext]; + size_t wn = 0; + wbuf[wn++] = be ? 0xFE : 0xFF, wbuf[wn++] = be ? 0xFF : 0xFE; + for (size_t i = 0; i + 1 < sizeof wtext; i++) + wbuf[wn++] = be ? 0 : (uint8_t)wtext[i], wbuf[wn++] = be ? (uint8_t)wtext[i] : 0; + CHECK(plat_file_write("cfg-probe-utf16.ini", wbuf, wn) && cfg_load("cfg-probe-utf16.ini") == 1 && + EQ(cfg_get("NX2NAVI_VOLUME"), "35") && EQ(cfg_get("NX2NAVI_MOTION"), "0"), + be ? "an ini saved as UTF-16, big endian, is not read" : "an ini saved as UTF-16, little endian, is not read"); + } + plat_file_delete("cfg-probe-utf16.ini"); + // a key the environment sets is skipped and keeps its file line CHECK(cfg_load(path) == 1 && EQ(cfg_get("NX2NAVI_VOLUME"), "60"), "reset: reload"); cfg_setenv("NX2NAVI_VOLUME", "70"); diff --git a/runtime/dispatch.c b/runtime/dispatch.c index 6f7be646..1e651f61 100644 --- a/runtime/dispatch.c +++ b/runtime/dispatch.c @@ -55,6 +55,9 @@ void rt_fatal(const char *fmt, ...) { rt_log("fatal: memory commit: this process %.0f MB, the system %.0f of %.0f MB", proc, used, limit); kern_free_cores_note(); prof_sampler_write(); // NX2NAVI_PROFILE: what was sampled until now, else a crash loses the whole profile + // RivaTuner Statistics Server's profile as it was: left with the game's limit in it, the next run would take that + // limit for the player's own (rtss_limit.c; it gives up after a second if the failing thread holds its lock) + rtss_limit_restore(); fflush(stdout); fflush(stderr); _Exit(70); diff --git a/runtime/frame_rate.c b/runtime/frame_rate.c index 4a5343f7..2f7228c7 100644 --- a/runtime/frame_rate.c +++ b/runtime/frame_rate.c @@ -197,7 +197,10 @@ static void frame_limit_live(void) { // NX2NAVI_MOVIE_FPS (default 60; 0 = off): the game's frame rate while a movie plays (ov_movie2.c ov_movie_live). A // movie's event counts the game's updates as 60 Hz frames whatever the step: at 144 fps the intro's narration lines // started 2.4 times too early, each over the one before, and its closing fade to black came at 65 s of the movie's -// 154, hiding the whole dream. The movies are 60 fps, so the cap shows the same pictures. +// 154, hiding the whole dream. The movies are 60 fps, so the cap shows the same pictures. It is the rate in both +// directions: under a lower limit (40 fps, or 60 shown with frame generation, which is 30 of the game's) the movie ran +// at that fraction of its speed and ended many seconds behind its sound, so the game is paced at the movie's rate +// while one plays, above the limit too. static int movie_cap_hz(void) { static int hz = -1; if (hz < 0) { @@ -244,7 +247,7 @@ uint64_t frame_step_cap_ns(void) { if (!!movie_hz != movie_was) { movie_was = !!movie_hz; if (movie_logs++ < 40) { // the first 20 movies - if (movie_hz) rt_log("frame rate: a movie plays: at most %d fps until it ends (NX2NAVI_MOVIE_FPS)", movie_hz); + if (movie_hz) rt_log("frame rate: a movie plays: %d fps until it ends (NX2NAVI_MOVIE_FPS)", movie_hz); else rt_log("frame rate: the movie ended: the frame limit again"); } } @@ -272,7 +275,7 @@ uint64_t frame_step_cap_ns(void) { // limit over that factor (half, a third, a quarter), so the screen shows the limit. With VSync and the limit at the // display's rate, the generated frames then fill the display's refreshes and none is dropped. if (gen > 1) hz /= gen; - if (movie_hz && hz > movie_hz) hz = movie_hz; + if (movie_hz) hz = movie_hz; rtss_limit_want(rtss ? (int64_t)(hz * 1000 + 0.5) : -1); if (rtss_limit_active()) return 0; return (uint64_t)(1e9 / hz + 0.5); diff --git a/runtime/kernel.c b/runtime/kernel.c index f5aad9af..c97103c0 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -376,7 +376,18 @@ static int g_ap_log; // redirects logged so far (AP_LOG_N) #define AP_LOG_N 40 static uintptr_t g_gen_lo, g_gen_hi; // host code of the lifted modules: first and last function start static uint64_t g_ap_sent, g_ap_taken, g_ap_host, g_ap_pc, g_ap_deferred, g_ap_fail, g_ap_spin_ok; -void kern_ap_stub(void); +void kern_ap_stub_ymm(void), kern_ap_stub_xmm(void); +static void (*ap_stub)(void) = kern_ap_stub_xmm; // the stub for this processor (ap_init) + +// The processor has AVX and the system saves YMM state across thread switches (OSXSAVE, XCR0 bits 1 and 2). +static int cpu_keeps_ymm(void) { + unsigned a = 1, b, c = 0, d; + __asm__("cpuid" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + if ((c & 0x18000000u) != 0x18000000u) return 0; + unsigned lo, hi; + __asm__("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + return (lo & 6) == 6; +} static void ap_init(void) { if (g_ap >= 0) return; @@ -393,8 +404,9 @@ static void ap_init(void) { if (f < lo) lo = f; if (f > hi) hi = f; } - const uintptr_t host[] = {(uintptr_t)kern_ap_stub, (uintptr_t)kern_checkpoint, (uintptr_t)rt_svc, - (uintptr_t)rt_dispatch, (uintptr_t)rt_call}; + ap_stub = cpu_keeps_ymm() ? kern_ap_stub_ymm : kern_ap_stub_xmm; + const uintptr_t host[] = {(uintptr_t)kern_ap_stub_ymm, (uintptr_t)kern_ap_stub_xmm, (uintptr_t)kern_checkpoint, + (uintptr_t)rt_svc, (uintptr_t)rt_dispatch, (uintptr_t)rt_call}; int bad = !n || lo >= hi; for (unsigned i = 0; i < sizeof host / sizeof *host; i++) bad |= host[i] >= lo && host[i] <= hi; if (bad) { @@ -408,9 +420,10 @@ static void ap_init(void) { } } rt_log("kernel: asynchronous preemption of virtual cores %s, priority inheritance and preemption disable %s; lifted " - "code exe+%#llx-exe+%#llx", g_ap_int ? "on" : "off", g_ap ? "on (NX2NAVI_ASYNC_PREEMPT=0 turns all off)" : "off", + "code exe+%#llx-exe+%#llx; an interrupted core's %s registers are kept", g_ap_int ? "on" : "off", + g_ap ? "on (NX2NAVI_ASYNC_PREEMPT=0 turns all off)" : "off", (unsigned long long)(g_gen_lo ? g_gen_lo - plat_exe_base() : 0), - (unsigned long long)(g_gen_hi ? g_gen_hi - plat_exe_base() : 0)); + (unsigned long long)(g_gen_hi ? g_gen_hi - plat_exe_base() : 0), ap_stub == kern_ap_stub_ymm ? "YMM" : "XMM"); } // Horizon's priority inheritance: o counts at the priority of the best thread blocked on a guest mutex it owns (the @@ -437,187 +450,25 @@ static int ap_disabled(const RtThread *t) { return g_ap > 0 && ap_tlr(t)[0]; } // in kern_ap_yield and jumps back to kern_ap_rip(), whose value it first stores in the return slot it reserved at // entry, so a stack walk sees the interrupted function as its caller. No ret: the jump keeps call/return pairs // balanced. The waiter never writes the owner's stack. -// An x86-64-v2 build (no __AVX__) saves XMM0-15 with SSE moves into the same 32-byte slots instead: its game code -// keeps nothing in the upper YMM halves, and the CPUs it is for have no VEX encoding, so a vmovdqa there faults with -// an illegal instruction the first time a core is interrupted. -#ifdef __AVX__ +// Two bodies of the stub (kernel_ap_stub.h): kern_ap_stub_ymm saves YMM0-15 and is the one used wherever the processor +// and the system keep YMM state (ap_stub, chosen by ap_init). The lifted code of such a PC is compiled with AVX2 and +// keeps data in the upper halves, which every function the stub calls is free to clear, whatever the runtime itself was +// compiled for. kern_ap_stub_xmm saves XMM0-15 with SSE moves: code on a processor without AVX has nothing in the upper +// halves, and a VEX move there is an illegal instruction. +#define AP_NAME "kern_ap_stub_ymm" #define AP_VMOV "vmovdqa" #define AP_V "%ymm" #define AP_VZEROUPPER " vzeroupper\n" -#else +#include "kernel_ap_stub.h" +#undef AP_NAME +#undef AP_VMOV +#undef AP_V +#undef AP_VZEROUPPER +#define AP_NAME "kern_ap_stub_xmm" #define AP_VMOV "movdqa" #define AP_V "%xmm" #define AP_VZEROUPPER "" -#endif -#ifdef _WIN32 -__asm__(".text\n" - ".globl kern_ap_stub\n" - ".def kern_ap_stub; .scl 2; .type 32; .endef\n" - ".p2align 4\n" - "kern_ap_stub:\n" - ".seh_proc kern_ap_stub\n" - " leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq - ".seh_stackalloc 8\n" - " pushfq\n" - ".seh_stackalloc 8\n" - " pushq %rax\n" - ".seh_pushreg %rax\n" - " pushq %rcx\n" - ".seh_pushreg %rcx\n" - " pushq %rdx\n" - ".seh_pushreg %rdx\n" - " pushq %r8\n" - ".seh_pushreg %r8\n" - " pushq %r9\n" - ".seh_pushreg %r9\n" - " pushq %r10\n" - ".seh_pushreg %r10\n" - " pushq %r11\n" - ".seh_pushreg %r11\n" - " pushq %rbp\n" - ".seh_pushreg %rbp\n" - " movq %rsp, %rbp\n" - ".seh_setframe %rbp, 0\n" - ".seh_endprologue\n" - " andq $-32, %rsp\n" - " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) - " " AP_VMOV " " AP_V "0, 64(%rsp)\n" - " " AP_VMOV " " AP_V "1, 96(%rsp)\n" - " " AP_VMOV " " AP_V "2, 128(%rsp)\n" - " " AP_VMOV " " AP_V "3, 160(%rsp)\n" - " " AP_VMOV " " AP_V "4, 192(%rsp)\n" - " " AP_VMOV " " AP_V "5, 224(%rsp)\n" - " " AP_VMOV " " AP_V "6, 256(%rsp)\n" - " " AP_VMOV " " AP_V "7, 288(%rsp)\n" - " " AP_VMOV " " AP_V "8, 320(%rsp)\n" - " " AP_VMOV " " AP_V "9, 352(%rsp)\n" - " " AP_VMOV " " AP_V "10, 384(%rsp)\n" - " " AP_VMOV " " AP_V "11, 416(%rsp)\n" - " " AP_VMOV " " AP_V "12, 448(%rsp)\n" - " " AP_VMOV " " AP_V "13, 480(%rsp)\n" - " " AP_VMOV " " AP_V "14, 512(%rsp)\n" - " " AP_VMOV " " AP_V "15, 544(%rsp)\n" - " stmxcsr 32(%rsp)\n" - " movl $0x1f80, 36(%rsp)\n" - " ldmxcsr 36(%rsp)\n" - " cld\n" - AP_VZEROUPPER - " call kern_ap_rip\n" - " movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq - " call kern_ap_yield\n" - " ldmxcsr 32(%rsp)\n" - " " AP_VMOV " 64(%rsp), " AP_V "0\n" - " " AP_VMOV " 96(%rsp), " AP_V "1\n" - " " AP_VMOV " 128(%rsp), " AP_V "2\n" - " " AP_VMOV " 160(%rsp), " AP_V "3\n" - " " AP_VMOV " 192(%rsp), " AP_V "4\n" - " " AP_VMOV " 224(%rsp), " AP_V "5\n" - " " AP_VMOV " 256(%rsp), " AP_V "6\n" - " " AP_VMOV " 288(%rsp), " AP_V "7\n" - " " AP_VMOV " 320(%rsp), " AP_V "8\n" - " " AP_VMOV " 352(%rsp), " AP_V "9\n" - " " AP_VMOV " 384(%rsp), " AP_V "10\n" - " " AP_VMOV " 416(%rsp), " AP_V "11\n" - " " AP_VMOV " 448(%rsp), " AP_V "12\n" - " " AP_VMOV " 480(%rsp), " AP_V "13\n" - " " AP_VMOV " 512(%rsp), " AP_V "14\n" - " " AP_VMOV " 544(%rsp), " AP_V "15\n" - " movq %rbp, %rsp\n" - " popq %rbp\n" - " popq %r11\n" - " popq %r10\n" - " popq %r9\n" - " popq %r8\n" - " popq %rdx\n" - " popq %rcx\n" - " popq %rax\n" - " popfq\n" - " leaq 8(%rsp), %rsp\n" - " jmpq *-8(%rsp)\n" - ".seh_endproc\n"); -#else -// The System V version: rsi and rdi are volatile there too, and the interrupted code may keep data in the 128 bytes -// below its stack pointer (the red zone), so the return slot and the saves go below that. Without a red zone in the -// way the slot is at entry - 136; the exit restores rsp to the entry value and jumps through the slot. -__asm__(".text\n" - ".globl kern_ap_stub\n" - ".type kern_ap_stub, @function\n" - ".p2align 4\n" - "kern_ap_stub:\n" - " .cfi_startproc\n" - " leaq -136(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq - " pushfq\n" - " pushq %rax\n" - " pushq %rcx\n" - " pushq %rdx\n" - " pushq %rsi\n" - " pushq %rdi\n" - " pushq %r8\n" - " pushq %r9\n" - " pushq %r10\n" - " pushq %r11\n" - " pushq %rbp\n" - " movq %rsp, %rbp\n" - " andq $-32, %rsp\n" - " subq $576, %rsp\n" // 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) - " " AP_VMOV " " AP_V "0, 64(%rsp)\n" - " " AP_VMOV " " AP_V "1, 96(%rsp)\n" - " " AP_VMOV " " AP_V "2, 128(%rsp)\n" - " " AP_VMOV " " AP_V "3, 160(%rsp)\n" - " " AP_VMOV " " AP_V "4, 192(%rsp)\n" - " " AP_VMOV " " AP_V "5, 224(%rsp)\n" - " " AP_VMOV " " AP_V "6, 256(%rsp)\n" - " " AP_VMOV " " AP_V "7, 288(%rsp)\n" - " " AP_VMOV " " AP_V "8, 320(%rsp)\n" - " " AP_VMOV " " AP_V "9, 352(%rsp)\n" - " " AP_VMOV " " AP_V "10, 384(%rsp)\n" - " " AP_VMOV " " AP_V "11, 416(%rsp)\n" - " " AP_VMOV " " AP_V "12, 448(%rsp)\n" - " " AP_VMOV " " AP_V "13, 480(%rsp)\n" - " " AP_VMOV " " AP_V "14, 512(%rsp)\n" - " " AP_VMOV " " AP_V "15, 544(%rsp)\n" - " stmxcsr 32(%rsp)\n" - " movl $0x1f80, 36(%rsp)\n" - " ldmxcsr 36(%rsp)\n" - " cld\n" - AP_VZEROUPPER - " call kern_ap_rip\n" - " movq %rax, 88(%rbp)\n" // the return slot: rbp + 10 pushes + pushfq - " call kern_ap_yield\n" - " ldmxcsr 32(%rsp)\n" - " " AP_VMOV " 64(%rsp), " AP_V "0\n" - " " AP_VMOV " 96(%rsp), " AP_V "1\n" - " " AP_VMOV " 128(%rsp), " AP_V "2\n" - " " AP_VMOV " 160(%rsp), " AP_V "3\n" - " " AP_VMOV " 192(%rsp), " AP_V "4\n" - " " AP_VMOV " 224(%rsp), " AP_V "5\n" - " " AP_VMOV " 256(%rsp), " AP_V "6\n" - " " AP_VMOV " 288(%rsp), " AP_V "7\n" - " " AP_VMOV " 320(%rsp), " AP_V "8\n" - " " AP_VMOV " 352(%rsp), " AP_V "9\n" - " " AP_VMOV " 384(%rsp), " AP_V "10\n" - " " AP_VMOV " 416(%rsp), " AP_V "11\n" - " " AP_VMOV " 448(%rsp), " AP_V "12\n" - " " AP_VMOV " 480(%rsp), " AP_V "13\n" - " " AP_VMOV " 512(%rsp), " AP_V "14\n" - " " AP_VMOV " 544(%rsp), " AP_V "15\n" - " movq %rbp, %rsp\n" - " popq %rbp\n" - " popq %r11\n" - " popq %r10\n" - " popq %r9\n" - " popq %r8\n" - " popq %rdi\n" - " popq %rsi\n" - " popq %rdx\n" - " popq %rcx\n" - " popq %rax\n" - " popfq\n" - " leaq 136(%rsp), %rsp\n" - " jmpq *-136(%rsp)\n" - " .cfi_endproc\n" - ".size kern_ap_stub, .-kern_ap_stub\n"); -#endif +#include "kernel_ap_stub.h" enum { AP_NONE, AP_RETRY, AP_SENT }; @@ -657,7 +508,7 @@ static int ap_try(RtThread *w) { } else { o->ap_rip = rip; o->ap_pending = 1; - to = (uint64_t)(uintptr_t)kern_ap_stub; + to = (uint64_t)(uintptr_t)ap_stub; } if (!plat_intr_resume(o->hthr, to)) { o->ap_pending = 0; @@ -1440,8 +1291,8 @@ static RtThread *thr_new(int prio, int core, uint64_t mask) { // TLR (0x200) and the "current thread object" the SDK reads through TLR+0x1F8, in one guest-visible block. t->tlr = mem_kernel_alloc(TLR_SIZE + THREAD_OBJ_SIZE, TLR_SIZE); *(uint64_t *)(t->tlr + 0x1F8) = (uint64_t)(uintptr_t)(t->tlr + TLR_SIZE); - *(uint64_t *)(t->tlr + 0x1F0) = (uint64_t)(uintptr_t)t->tls; // the SDK reads the TLS pointer here and makes it TPIDR_EL0 t->tls = mem_kernel_alloc(TLS_SIZE, 16); + *(uint64_t *)(t->tlr + 0x1F0) = (uint64_t)(uintptr_t)t->tls; // the SDK reads the TLS pointer here and makes it TPIDR_EL0 c->tpidrro = (uint64_t)(uintptr_t)t->tlr; c->tpidr = (uint64_t)(uintptr_t)t->tls; c->excl_addr = ~0ull; diff --git a/runtime/kernel_ap_stub.h b/runtime/kernel_ap_stub.h new file mode 100644 index 00000000..de8aba66 --- /dev/null +++ b/runtime/kernel_ap_stub.h @@ -0,0 +1,172 @@ +// The body of kern_ap_stub (kernel.c, "asynchronous preemption"), included once per register width with AP_NAME the +// function's name, AP_VMOV and AP_V the move and the register prefix that save the vector registers, and AP_VZEROUPPER +// the instruction that ends the use of the upper halves (or nothing). No include guard: kernel.c includes it twice. +#ifdef _WIN32 +__asm__(".text\n" + ".globl " AP_NAME "\n" + ".def " AP_NAME "; .scl 2; .type 32; .endef\n" + ".p2align 4\n" + AP_NAME ":\n" + ".seh_proc " AP_NAME "\n" + " leaq -8(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq + ".seh_stackalloc 8\n" + " pushfq\n" + ".seh_stackalloc 8\n" + " pushq %rax\n" + ".seh_pushreg %rax\n" + " pushq %rcx\n" + ".seh_pushreg %rcx\n" + " pushq %rdx\n" + ".seh_pushreg %rdx\n" + " pushq %r8\n" + ".seh_pushreg %r8\n" + " pushq %r9\n" + ".seh_pushreg %r9\n" + " pushq %r10\n" + ".seh_pushreg %r10\n" + " pushq %r11\n" + ".seh_pushreg %r11\n" + " pushq %rbp\n" + ".seh_pushreg %rbp\n" + " movq %rsp, %rbp\n" + ".seh_setframe %rbp, 0\n" + ".seh_endprologue\n" + " andq $-32, %rsp\n" + " subq $576, %rsp\n" // 0: shadow space, 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) + " " AP_VMOV " " AP_V "0, 64(%rsp)\n" + " " AP_VMOV " " AP_V "1, 96(%rsp)\n" + " " AP_VMOV " " AP_V "2, 128(%rsp)\n" + " " AP_VMOV " " AP_V "3, 160(%rsp)\n" + " " AP_VMOV " " AP_V "4, 192(%rsp)\n" + " " AP_VMOV " " AP_V "5, 224(%rsp)\n" + " " AP_VMOV " " AP_V "6, 256(%rsp)\n" + " " AP_VMOV " " AP_V "7, 288(%rsp)\n" + " " AP_VMOV " " AP_V "8, 320(%rsp)\n" + " " AP_VMOV " " AP_V "9, 352(%rsp)\n" + " " AP_VMOV " " AP_V "10, 384(%rsp)\n" + " " AP_VMOV " " AP_V "11, 416(%rsp)\n" + " " AP_VMOV " " AP_V "12, 448(%rsp)\n" + " " AP_VMOV " " AP_V "13, 480(%rsp)\n" + " " AP_VMOV " " AP_V "14, 512(%rsp)\n" + " " AP_VMOV " " AP_V "15, 544(%rsp)\n" + " stmxcsr 32(%rsp)\n" + " movl $0x1f80, 36(%rsp)\n" + " ldmxcsr 36(%rsp)\n" + " cld\n" + AP_VZEROUPPER + " call kern_ap_rip\n" + " movq %rax, 72(%rbp)\n" // the return slot: rbp + 8 pushes + pushfq + " call kern_ap_yield\n" + " ldmxcsr 32(%rsp)\n" + " " AP_VMOV " 64(%rsp), " AP_V "0\n" + " " AP_VMOV " 96(%rsp), " AP_V "1\n" + " " AP_VMOV " 128(%rsp), " AP_V "2\n" + " " AP_VMOV " 160(%rsp), " AP_V "3\n" + " " AP_VMOV " 192(%rsp), " AP_V "4\n" + " " AP_VMOV " 224(%rsp), " AP_V "5\n" + " " AP_VMOV " 256(%rsp), " AP_V "6\n" + " " AP_VMOV " 288(%rsp), " AP_V "7\n" + " " AP_VMOV " 320(%rsp), " AP_V "8\n" + " " AP_VMOV " 352(%rsp), " AP_V "9\n" + " " AP_VMOV " 384(%rsp), " AP_V "10\n" + " " AP_VMOV " 416(%rsp), " AP_V "11\n" + " " AP_VMOV " 448(%rsp), " AP_V "12\n" + " " AP_VMOV " 480(%rsp), " AP_V "13\n" + " " AP_VMOV " 512(%rsp), " AP_V "14\n" + " " AP_VMOV " 544(%rsp), " AP_V "15\n" + " movq %rbp, %rsp\n" + " popq %rbp\n" + " popq %r11\n" + " popq %r10\n" + " popq %r9\n" + " popq %r8\n" + " popq %rdx\n" + " popq %rcx\n" + " popq %rax\n" + " popfq\n" + " leaq 8(%rsp), %rsp\n" + " jmpq *-8(%rsp)\n" + ".seh_endproc\n"); +#else +// The System V version: rsi and rdi are volatile there too, and the interrupted code may keep data in the 128 bytes +// below its stack pointer (the red zone), so the return slot and the saves go below that. Without a red zone in the +// way the slot is at entry - 136; the exit restores rsp to the entry value and jumps through the slot. +__asm__(".text\n" + ".globl " AP_NAME "\n" + ".type " AP_NAME ", @function\n" + ".p2align 4\n" + AP_NAME ":\n" + " .cfi_startproc\n" + " leaq -136(%rsp), %rsp\n" // not sub: the flags are still the interrupted code's until pushfq + " pushfq\n" + " pushq %rax\n" + " pushq %rcx\n" + " pushq %rdx\n" + " pushq %rsi\n" + " pushq %rdi\n" + " pushq %r8\n" + " pushq %r9\n" + " pushq %r10\n" + " pushq %r11\n" + " pushq %rbp\n" + " movq %rsp, %rbp\n" + " andq $-32, %rsp\n" + " subq $576, %rsp\n" // 32: MXCSR saved, 36: MXCSR default, 64: YMM0-15 (XMM0-15 without AVX) + " " AP_VMOV " " AP_V "0, 64(%rsp)\n" + " " AP_VMOV " " AP_V "1, 96(%rsp)\n" + " " AP_VMOV " " AP_V "2, 128(%rsp)\n" + " " AP_VMOV " " AP_V "3, 160(%rsp)\n" + " " AP_VMOV " " AP_V "4, 192(%rsp)\n" + " " AP_VMOV " " AP_V "5, 224(%rsp)\n" + " " AP_VMOV " " AP_V "6, 256(%rsp)\n" + " " AP_VMOV " " AP_V "7, 288(%rsp)\n" + " " AP_VMOV " " AP_V "8, 320(%rsp)\n" + " " AP_VMOV " " AP_V "9, 352(%rsp)\n" + " " AP_VMOV " " AP_V "10, 384(%rsp)\n" + " " AP_VMOV " " AP_V "11, 416(%rsp)\n" + " " AP_VMOV " " AP_V "12, 448(%rsp)\n" + " " AP_VMOV " " AP_V "13, 480(%rsp)\n" + " " AP_VMOV " " AP_V "14, 512(%rsp)\n" + " " AP_VMOV " " AP_V "15, 544(%rsp)\n" + " stmxcsr 32(%rsp)\n" + " movl $0x1f80, 36(%rsp)\n" + " ldmxcsr 36(%rsp)\n" + " cld\n" + AP_VZEROUPPER + " call kern_ap_rip\n" + " movq %rax, 88(%rbp)\n" // the return slot: rbp + 10 pushes + pushfq + " call kern_ap_yield\n" + " ldmxcsr 32(%rsp)\n" + " " AP_VMOV " 64(%rsp), " AP_V "0\n" + " " AP_VMOV " 96(%rsp), " AP_V "1\n" + " " AP_VMOV " 128(%rsp), " AP_V "2\n" + " " AP_VMOV " 160(%rsp), " AP_V "3\n" + " " AP_VMOV " 192(%rsp), " AP_V "4\n" + " " AP_VMOV " 224(%rsp), " AP_V "5\n" + " " AP_VMOV " 256(%rsp), " AP_V "6\n" + " " AP_VMOV " 288(%rsp), " AP_V "7\n" + " " AP_VMOV " 320(%rsp), " AP_V "8\n" + " " AP_VMOV " 352(%rsp), " AP_V "9\n" + " " AP_VMOV " 384(%rsp), " AP_V "10\n" + " " AP_VMOV " 416(%rsp), " AP_V "11\n" + " " AP_VMOV " 448(%rsp), " AP_V "12\n" + " " AP_VMOV " 480(%rsp), " AP_V "13\n" + " " AP_VMOV " 512(%rsp), " AP_V "14\n" + " " AP_VMOV " 544(%rsp), " AP_V "15\n" + " movq %rbp, %rsp\n" + " popq %rbp\n" + " popq %r11\n" + " popq %r10\n" + " popq %r9\n" + " popq %r8\n" + " popq %rdi\n" + " popq %rsi\n" + " popq %rdx\n" + " popq %rcx\n" + " popq %rax\n" + " popfq\n" + " leaq 136(%rsp), %rsp\n" + " jmpq *-136(%rsp)\n" + " .cfi_endproc\n" + ".size " AP_NAME ", .-" AP_NAME "\n"); +#endif diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 6d00dccc..46b8730e 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1042,8 +1042,8 @@ static int bind_range(VkExec *ex, uint64_t addr, uint64_t size, uint64_t align, return 1; } -static void c0_table(uint8_t *at, const p_bind_buf b[16]) { - for (int r = 0; r < 16; r++) { +static void c0_table(uint8_t *at, const p_bind_buf b[16], int rows) { + for (int r = 0; r < rows; r++) { uint64_t a = b[r].addr; uint32_t n = (uint32_t)b[r].size; memcpy(at + 16 * r, &a, 8); @@ -1056,8 +1056,9 @@ static void c0_table(uint8_t *at, const p_bind_buf b[16]) { // candidate base from here and compares the global pointer against it. Compute: dispatch size at 0x200, the uniform // buffer table at 0x210 + 16 r. Graphics: the largest point size (Maxwell's limit, GLSLC clamps gl_PointSize to it) at // 0x610, the base instance at 0x644. NVN2 task and mesh stages: storage tables at 0x670 and 0x770 (the game binds -// slots 0-2 there, and their programs read 0x670.. and 0x770..). Other stages: zeros. Unbound slots read zero, since the -// replay keeps them zero. +// slots 0-2 there, and their programs read 0x670.. and 0x770..). The mesh table starts 0x90 bytes before the bank's +// end: it holds the 9 slots that fit, and sixteen would write 0x6C bytes past the bank. Other stages: zeros. Unbound +// slots read zero, since the replay keeps them zero. // Byte offset of a stage's storage buffer table in c0, -1 for none. static int c0_ssbo_base(int stage) { return stage == NVN_STAGE_VS ? 0x110 : stage == NVN_STAGE_FS ? 0x510 : stage == NVN_STAGE_CS ? 0x310 @@ -1069,11 +1070,12 @@ void vk_c0_fill(uint8_t bank[VK_C0_BYTES], int stage, const p_bind_buf ssbo[16], memset(bank, 0, VK_C0_BYTES); int base = c0_ssbo_base(stage); if (base < 0) return; - c0_table(bank + base, ssbo); + int rows = (int)(VK_C0_BYTES - (uint32_t)base) / 16; + c0_table(bank + base, ssbo, rows < 16 ? rows : 16); if (stage == NVN_STAGE_TASK || stage == NVN_STAGE_MESH) return; if (stage == NVN_STAGE_CS) { memcpy(bank + 0x200, dispatch, 12); - c0_table(bank + 0x210, ubo); + c0_table(bank + 0x210, ubo, 16); return; } const float max_point = 2047.0f; diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 1135796a..5673d1bc 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -82,9 +82,11 @@ void fail(const char *what, unsigned r) { // It needs DLSS's or FSR 3.1's motion vectors, so it also needs one of them on. Not on a device made through // Streamline's proxies (NVIDIA's frame generation set at the start, vk_dlssg_dpa): AMD's swapchain there crashed in // NVIDIA's driver when the menu switched from NVIDIA's frame generation to AMD's. That switch waits for a restart, -// as the menu says (menu.cpp row_framegen). +// as the menu says (menu.cpp row_framegen). Not after a failure (fail): the present code then takes AMD's swapchain +// down as it does for a switch in the menu, and a plain one follows. extern "C" int vk_fg_wanted(void) { - return !strcmp(cfg_value("NX2NAVI_FRAMEGEN"), "fsr3") && cfg_bool("NX2NAVI_DLAA") && vk_dlaa_wanted() && !vk_dlssg_dpa(); + return !F.failed && !strcmp(cfg_value("NX2NAVI_FRAMEGEN"), "fsr3") && cfg_bool("NX2NAVI_DLAA") && vk_dlaa_wanted() && + !vk_dlssg_dpa(); } // Always the three queues: the setting can be switched on during a run, and idle queues cost nothing. @@ -92,9 +94,12 @@ extern "C" uint32_t vk_fg_queue_count(void) { return 3; } extern "C" void vk_fg_queues(VkQueue present, VkQueue acquire) { F.present = present, F.acquire = acquire; } -extern "C" int vk_fg_active(void) { return F.sc != nullptr && !F.failed; } +// AMD's swapchain is in place: the window's swapchain handle is AMD's object, and every swapchain call has to go +// through AMD's functions. That holds after a failure too, until the present code has replaced the swapchain +// (vk_fg_wanted): the plain Vulkan calls must never get that handle. +extern "C" int vk_fg_active(void) { return F.sc != nullptr; } -extern "C" int vk_fg_generating(void) { return vk_fg_active() && __atomic_load_n(&F.on, __ATOMIC_RELAXED); } +extern "C" int vk_fg_generating(void) { return vk_fg_active() && !F.failed && __atomic_load_n(&F.on, __ATOMIC_RELAXED); } extern "C" VkResult vk_fg_create_swapchain(const VkSwapchainCreateInfoKHR *ci, VkSwapchainKHR *sc) { if (F.sc) return F.fn.pOutCreateSwapchainFFXAPI(vk_ctx()->device, ci, nullptr, sc, F.sc); @@ -120,6 +125,7 @@ extern "C" VkResult vk_fg_create_swapchain(const VkSwapchainCreateInfoKHR *ci, V } F.fn.header.type = FFX_API_QUERY_DESC_TYPE_FGSWAPCHAIN_FUNCTIONS_VK; if (vk_ffx_query(&F.sc, &F.fn.header) != FFX_API_RETURN_OK || !F.fn.pOutQueuePresentKHR) { + vk_ffx_destroy(&F.sc), F.sc = nullptr; // with it goes the swapchain it made: nothing was handed out yet fail("the swapchain functions", 0); return VK_ERROR_INITIALIZATION_FAILED; } @@ -162,7 +168,7 @@ static bool hdr_format(VkFormat f) { extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, VkExtent2D ext, VkImage hudless, VkImageView hudless_view, const int pic[4]) { (void)hudless_view; - if (!vk_fg_active()) return; + if (!vk_fg_active() || F.failed) return; vk_dlaa_hudless_wanted = 1; // the upscaling pass keeps a copy of the picture before the HUD (vk_dlaa.c hudless) const VkCtx *c = vk_ctx(); VkFfxImage depth, mv; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 3e1cd01a..ced07ef4 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -3023,7 +3023,13 @@ uint64_t vk_upload_flush(int wait) { } if (c && wait) { while (ninflight && done_seq < seq) { - vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); + // a lost device ends the wait at once and leaves the fence unsignalled: retire would take nothing and this + // loop would spin for good, holding the upload lock, instead of the run ending with the device's report + VkResult w = vkWaitForFences(c->device, 1, &inflight[0].fence, VK_TRUE, UINT64_MAX); + if (w != VK_SUCCESS) { + vk_device_fault_report(w); + rt_fatal("vk: the wait for an upload failed (%d)", (int)w); + } retire(c, 0); } } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 229887be..dcd39267 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -4267,7 +4267,7 @@ static struct { VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { if (info) *info = NULL; if (!cs || !ensure_init()) return VK_NULL_HANDLE; - struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1}; + struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); plat_mutex_lock(&P.lock); Pipe *p = pipe_find(&k, sizeof k, h); @@ -4487,7 +4487,7 @@ static void bg_main(void *arg) { plat_mutex_unlock(&LB.stat); if (!have) break; const uint8_t *iid = idbuf; - struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1}; + struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL; @@ -4567,7 +4567,7 @@ static void bg_start(void) { } static void lb_compute(const GfxShader *cs, const uint8_t iid[32]) { - struct { const GfxShader *cs; uint32_t compute; } k = {cs, 1}; + struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; const char *why = NULL;