diff --git a/runtime/input.c b/runtime/input.c index 6e3fdcbf..3eac713a 100644 --- a/runtime/input.c +++ b/runtime/input.c @@ -786,8 +786,17 @@ static void poll_merged(uint64_t *buttons, int32_t *lx, int32_t *ly, int32_t *rx static int was_captured; static uint8_t mouse_ignore[KEY_COUNT]; int cap = ikb_captured(); + // A scan is one key state system call per key (about 95). The game reads several pads in a row: reads + // within a millisecond of a scan get its answers again. + static uint8_t raw[KEY_COUNT]; + static uint64_t raw_at; + uint64_t now = plat_now_ns(); + if (!raw_at || now - raw_at >= 1000000) { + for (int k = 1; k < KEY_COUNT; k++) raw[k] = (uint8_t)(ikb_down(k) != 0); + raw_at = now; + } for (int k = 1; k < KEY_COUNT; k++) { - down[k] = ikb_down(k); + down[k] = raw[k]; kbm_used |= down[k]; if (k < KEY_MOUSE_L || k > KEY_MOUSE_X2) continue; if (cap && !was_captured) mouse_ignore[k] = down[k]; diff --git a/runtime/kernel.c b/runtime/kernel.c index dd33700f..f5aad9af 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -267,7 +267,7 @@ static uint32_t block_inner(RtThread *t, int kind, int64_t ns) { t->wkind = W_NONE; return R_TIMEOUT; } - uint64_t now = kern_now_ns(); + uint64_t now = ns < 0 ? 0 : kern_now_ns(); // a wait without a timeout reads no clock (G is held here) int forever = ns < 0 || (uint64_t)ns > UINT64_MAX - now; uint64_t deadline = forever ? 0 : now + (uint64_t)ns; while (!t->woke) { @@ -1187,6 +1187,8 @@ void kern_core_stats(int *held_now, int *held_max) { unlock(); } +int kern_cores(void) { return g_active_cores; } + void kern_core_stats_reset(void) { lock(); g_held_max = 0; @@ -1747,12 +1749,21 @@ static int kern_svc_inner(Cpu *c, uint32_t imm) { unlock(); return 1; } - // Yield: hand the core to a waiting thread of equal or better priority. + // Yield: hand the core to a waiting thread of equal or better priority. Most yields find nobody waiting for a + // core: those take neither G nor the priority scan over every blocked thread (prio_of), which the game's yield + // loops paid thousands of times a second. The look is as racy as rt_spin_gate's: a thread that queues for a + // core right after it is found by the next yield, and kicks the owner itself in core_take meanwhile. + if (!__atomic_load_n(&g_core_wait, __ATOMIC_RELAXED)) { + plat_yield(); + return 1; + } lock(); - int give = 0; - int tp = prio_of(t); + int give = 0, tp = -1; for (RtThread *w = g_core_wait; w && !give; w = w->cnext) - give = t->cur_core >= 0 && core_allowed(w, t->cur_core) && prio_of(w) <= tp; + if (t->cur_core >= 0 && core_allowed(w, t->cur_core)) { + if (tp < 0) tp = prio_of(t); // only once a waiter may run on this core + give = prio_of(w) <= tp; + } if (give) { core_put(t); core_take(t); diff --git a/runtime/kernel.h b/runtime/kernel.h index 1cf1de62..3bde15c0 100644 --- a/runtime/kernel.h +++ b/runtime/kernel.h @@ -35,6 +35,7 @@ enum { KERN_WAIT_NONE, KERN_WAIT_LOCK, KERN_WAIT_CONDVAR, KERN_WAIT_ADDRESS, KER int kern_thread_state(uint32_t handle); // KERN_WAIT_*, KERN_THREAD_EXITED, or -1 for a bad handle void kern_core_stats(int *held_now, int *held_max); // virtual-core holders now and the most ever at once void kern_core_stats_reset(void); +int kern_cores(void); // the virtual cores that run threads: 6, or NX2NAVI_CORES extern volatile int kern_test_cv_old_order; // WaitProcessWideKeyAtomic: release the lock before setting // the key (the old order, the negative control) extern void (*volatile kern_test_cv_hook)(void); // called right after the lock release, holding the kernel lock diff --git a/runtime/kernel_probe.c b/runtime/kernel_probe.c index 62343235..0323177f 100644 --- a/runtime/kernel_probe.c +++ b/runtime/kernel_probe.c @@ -323,9 +323,19 @@ static void test_arbiter(Cpu *c) { uint32_t r = sv(c, 0x34, arb, 2, 5, 5000000); double ms = ms_since(t0); CHECK(r == KERN_R_TIMEOUT && ms >= 5.0, "wait type 2 with a 5 ms timeout: %#x after %.2f ms", r, ms); - CHECK(sv(c, 0x34, arb, 3, 0, 0) == KERN_R_INVALID_ENUM && sig(c, 3, 0, 0) == KERN_R_INVALID_ENUM, "type 3"); - printf("address arbiter: wait types 0-2 (less, decrement-less, equal), signal types 0-2 (wake, increment, modify by " - "waiters), 5 ms timeout after %.2f ms: %s\n", ms, failures == before ? "ok" : "FAIL"); + // wait type 3: the whole 64-bit word against x2 (kernel.c case 0x34); an address off 8 bytes is refused + static volatile uint64_t arb64 = 0x100000005ull; + uint64_t a64 = ga(&arb64); + t0 = kern_now_ns(); + r = sv(c, 0x34, a64, 3, 0x100000005ull, 2000000); + double ms3 = ms_since(t0); + CHECK(r == KERN_R_TIMEOUT && ms3 >= 2.0, "wait type 3 on an equal word with a 2 ms timeout: %#x after %.2f ms", r, ms3); + CHECK(sv(c, 0x34, a64, 3, 5, 0) == KERN_R_BAD_STATE, "wait type 3 on a word whose high half differs"); + CHECK(sv(c, 0x34, a64 + 4, 3, 0, 0) == 0xCC01u, "wait type 3 at an address off 8 bytes"); + CHECK(sv(c, 0x34, arb, 4, 0, 0) == KERN_R_INVALID_ENUM && sig(c, 3, 0, 0) == KERN_R_INVALID_ENUM, + "a wait type past 3 or a signal type past 2"); + printf("address arbiter: wait types 0-3 (less, decrement-less, equal, equal 64-bit), signal types 0-2 (wake, " + "increment, modify by waiters), 5 ms timeout after %.2f ms: %s\n", ms, failures == before ? "ok" : "FAIL"); } // ---- events @@ -423,18 +433,20 @@ static void test_events(Cpu *c) { } // ---- virtual cores: count the threads running guest code at once, and that no two of them share a core number -static volatile int32_t g_running, g_run_max, g_core_use[3], g_core_clash; +#define PROBE_CORES 8 // at least kernel.c NCORE +static volatile int32_t g_running, g_run_max, g_core_use[PROBE_CORES], g_core_clash; static void core_worker(Cpu *c) { for (int i = 0; i < CORE_RUNS; i++) { uint32_t core = sv(c, 0x10, 0, 0, 0, 0); - if (core > 2 || plat_inc32(&g_core_use[core > 2 ? 0 : core]) != 1) plat_inc32(&g_core_clash); + int known = core < (uint32_t)kern_cores() && core < PROBE_CORES; // a number past the cores is a clash too + if (!known || plat_inc32(&g_core_use[core]) != 1) plat_inc32(&g_core_clash); long n = plat_inc32(&g_running), m; while (n > (m = g_run_max) && plat_cas32(&g_run_max, n, m) != m) {} uint64_t until = kern_now_ns() + 20000; while (kern_now_ns() < until) {} plat_dec32(&g_running); - plat_dec32(&g_core_use[core > 2 ? 0 : core]); + if (known) plat_dec32(&g_core_use[core]); sv(c, 0x0B, i % 8 == 7 ? 50000 : 0, 0, 0, 0); } exit_thread(c); @@ -453,7 +465,9 @@ static void test_cores(Cpu *c) { kern_core_stats(&now, &held_max); uint32_t cpus = plat_cpu_usable(); int gate = !cfg_bool("NX2NAVI_FREE_CORES"); // the kernel's own switch - int ok = g_run_max <= 3 && held_max <= 3 && !g_core_clash && (cpus < 4 || g_run_max == 3); + int ncore = kern_cores(); // as many threads run guest code at once as there are virtual cores, where the host + // has more CPUs than that + int ok = g_run_max <= ncore && held_max <= ncore && !g_core_clash && ((int)cpus <= ncore || g_run_max == ncore); printf("virtual cores: %d threads x %d runs on %u host CPUs: at most %d running guest code at once, at most %d " "core holders (kernel count), %d core-number clashes: %s\n", CORE_WORKERS, CORE_RUNS, cpus, g_run_max, held_max, g_core_clash, !gate ? "not checked (NX2NAVI_FREE_CORES is set)" : ok ? "ok" : "FAIL"); diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index ce851b39..714cdeb0 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -300,6 +300,11 @@ static const tex_t *tex_pin(const NVNtexture *nt) { return &r->t; } +// The ops whose payload names pins (op_pins): five of the op kinds. The walks over every record of a recording +// (flatten_recording, pin_refs) test this mask before they call op_pins. +#define PIN_OPS (1ull << OP_SET_RT | 1ull << OP_COPY_B2T | 1ull << OP_COPY_T2B | 1ull << OP_CLEAR_TEX | 1ull << OP_COPY_T2T) +_Static_assert(OP_COUNT_ <= 64, "PIN_OPS holds one bit per op kind"); + // The pins an op's payload names, into out (at most 9); returns how many. static int op_pins(uint32_t op, const void *a, const tex_t *out[9]) { int k = 0; @@ -347,6 +352,7 @@ static uint32_t pin_refs(const uint8_t *copy, int delta) { uint64_t now = __atomic_load_n(&nsubmits, __ATOMIC_RELAXED); uint32_t n = 0; for (const op_hdr *h = nvn_op_first((uintptr_t)copy); h->op != OP_END; h = nvn_op_next(h)) { + if (h->op >= 64 || !(PIN_OPS >> h->op & 1)) continue; const tex_t *p[9]; int k = op_pins(h->op, nvn_op_payload(h), p); for (int i = 0; i < k; i++) pin_ref(p[i], delta, now); @@ -426,20 +432,38 @@ static void zombie_add(copy_prefix *pf) { // copy_lock held zombies[nzombies++] = pf; } -static void copy_free(copy_prefix *pf) { // copy_lock held +static void copy_free(copy_prefix *pf) { // copy_lock not held if (pf->pins) pin_refs((const uint8_t *)(pf + 1), -1); free(pf); } -static void zombie_sweep(uint64_t replayed) { // copy_lock held +// Released copies nothing reads any more are freed once copy_lock is dropped: uncounting a copy's pins walks its ops, +// and that walk and free() under the exclusive lock held up every thread that was recording or submitting (the +// main-loop thread waited there for 2.8% of its time). doom chains a copy for that, through made_at, which nothing +// reads after the release; doom_free frees the chain. +static copy_prefix *doom(copy_prefix *list, copy_prefix *pf) { // copy_lock held + pf->made_at = (uint64_t)(uintptr_t)list; + return pf; +} + +static void doom_free(copy_prefix *list) { // copy_lock not held + while (list) { + copy_prefix *next = (copy_prefix *)(uintptr_t)list->made_at; + copy_free(list); + list = next; + } +} + +static copy_prefix *zombie_sweep(uint64_t replayed, copy_prefix *doomed) { // copy_lock held; returns the doom chain for (uint32_t i = 0; i < nzombies;) { if (zombies[i]->used_at <= replayed) { - copy_free(zombies[i]); + doomed = doom(doomed, zombies[i]); zombies[i] = zombies[--nzombies]; } else { i++; } } + return doomed; } static copy_prefix *copy_find(uint64_t handle) { // copy_lock held @@ -488,7 +512,7 @@ static uint8_t *flatten_recording(const uint8_t *p, size_t n) { (!at && h.op != OP_BEGIN)) rt_fatal("nvn: corrupt recording at %p (op %u size %u)", (const void *)(p + at), h.op, h.size); const tex_t *pinned[9]; - int k = op_pins(h.op, out + sizeof(copy_prefix) + at + 8, pinned); + int k = PIN_OPS >> h.op & 1 ? op_pins(h.op, out + sizeof(copy_prefix) + at + 8, pinned) : 0; for (int i = 0; i < k; i++) pin_ref(pinned[i], 1, 0); pins_named += (uint64_t)k; at += h.size; @@ -532,8 +556,9 @@ static void copy_add(uintptr_t lo, uintptr_t hi, uint8_t *copy) { static void release_copies(uintptr_t lo, uintptr_t hi) { if (hi <= lo) return; uint64_t replayed = gfx_replayed_submits(); + copy_prefix *doomed = NULL; plat_mutex_lock(©_lock); - if (nzombies) zombie_sweep(replayed); + if (nzombies) doomed = zombie_sweep(replayed, doomed); for (uintptr_t pg = lo >> COPY_PAGE_SHIFT; pg <= (hi - 1) >> COPY_PAGE_SHIFT; pg++) { copy_bucket *b = nvn_hm_get(©_pages, &pg); if (!b) continue; @@ -547,7 +572,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { copy_bytes -= pf->bytes; pf->magic = 0; if (pf->used_at > replayed) zombie_add(pf); // a queued submission still reads it - else copy_free(pf); + else doomed = doom(doomed, pf); b->v[i] = b->v[--b->n]; } if (!b->n) { @@ -557,6 +582,7 @@ static void release_copies(uintptr_t lo, uintptr_t hi) { } } plat_mutex_unlock(©_lock); + doom_free(doomed); } // ---------- CommandBuffer: object and memory @@ -1237,8 +1263,9 @@ void nvn_submit_prepare(const uint64_t *handles, int n) { void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { uint64_t now = __atomic_add_fetch(&nsubmits, 1, __ATOMIC_RELAXED); uint64_t replayed = gfx_replayed_submits(); + copy_prefix *doomed = NULL; plat_mutex_lock(©_lock); - if (nzombies) zombie_sweep(replayed); + if (nzombies) doomed = zombie_sweep(replayed, doomed); for (int i = 0; i < n; i++) { copy_prefix *pf = copy_find(handles[i]); if (!pf) { @@ -1263,6 +1290,7 @@ void nvn_submit_prepare_async(const uint64_t *handles, uint64_t *out, int n) { } } plat_mutex_unlock(©_lock); + doom_free(doomed); for (int i = 0; i < n; i++) { int nd = resolve_deferred((const uint8_t *)(uintptr_t)out[i], 0); if (out[i] == handles[i]) ((copy_prefix *)(uintptr_t)out[i] - 1)->deferred = nd ? 2 : 1; diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 93ea7789..28aeeb1b 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -675,7 +675,7 @@ NVN_API(nvnQueueSubmitCommands) { static void obs_release_read(Cpu *c) { uint64_t lr = c->x[30], obj = c->x[19]; rt_original(c); - if (lr != 0x70008c70c8ull) return; + if (rt_game_version != 121 || lr != 0x70008c70c8ull) return; // the address is 1.2.1's: no other build has this producer there static int on = -1; if (on < 0) { on = cfg_bool("NX2NAVI_LOD_GUARD"); diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index 265f88b0..87f40703 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -770,7 +770,10 @@ static void sleep_until(uint64_t deadline) { uint64_t now = kern_now_ns(); if (now >= deadline) return; uint64_t d = deadline - now; - if (d >= 2000000) plat_sleep_ms((uint32_t)(d / 1000000 - 1)); + // The high-resolution wait never returns early and at most PLAT_WAIT_LATE_NS late, so it takes the wait up to + // that margin before the deadline and only the rest is spent yielding. The coarse sleep that stood here left + // 1 to 2 ms of every paced frame to the yield loop. + if (d > PLAT_WAIT_LATE_NS + 100000) plat_wait_ns(d - PLAT_WAIT_LATE_NS - 100000); else plat_yield(); } }