/* Guest threads: one host thread each, with its own Ctx, TLS block and thread-local region. TPIDR_EL0 (TLS): a 4 KB block per thread, TP in its middle. TOTK's main reads its one thread-local at TP+0x10 (local-exec, after AArch64's 16-byte TCB); the area below TP is zeroed room for whatever the SDK's musl keeps there. TPIDRRO_EL0 (thread-local region): 0x200 bytes; the SDK's nn::os keeps the current ThreadType* at +0x1F8, so lifted SDK code that reads it inline finds the same thread the HLE knows. */ #include "hos.h" #include "hos_x3d.h" #include "pace_wait.h" /* ((Triforce) hos_sleep_until_ns: the frame wait to its time) */ #include #include #define WIN32_LEAN_AND_MEAN #include #ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION /* (older Windows SDKs lack it: Windows 10 1803 and later take it) */ #define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002 #endif #include #ifdef __EMSCRIPTEN__ #include #include #include #endif struct HosThread { Ctx ctx; uint64_t guest_tt, entry, arg, stack, stack_size, id; uint64_t tls_block, tlr; int priority, ideal_core; uint64_t core_mask; uint64_t tls_slots[HOS_TLS_SLOTS]; char name[48]; HANDLE handle; int started; /* the kernel side (hos_svc.c): this thread's Horizon handle and the event it blocks on */ uint32_t kernel_handle; HANDLE wake; /* virtual cores (HOS_VCORES): the core held + 1 (0: none), the last one, a better waiter's request, queued; (Triforce) when a release granted it the core it waits for (vc_grant), until it runs (the vcores: line's grant to run) */ int vc_held, vc_last, vc_queued; volatile int vc_kick; int64_t vc_since, vc_grant_at; CONDITION_VARIABLE vc_cv; /* (hookshot) fibers: the context running now (NULL: ctx, the thread's own), the thread's host fiber once it switched to a guest fiber */ Ctx *volatile run_ctx; void *host_fiber; }; /* The SDK's own mutexes (InternalCriticalSection) tag a lock word with the owner's handle, read from ThreadType+0x1B0 through TLR+0x1F8 (sdk+0x13186c). Every thread gets a unique handle there. */ #define TT_HANDLE 0x1B0 static volatile LONG next_handle = 0x10000; uint32_t hos_thread_handle(HosThread *t) { return t->kernel_handle; } /* nnosInitialize gives the main thread the SDK's own ThreadType (it writes TLR+0x1F8): adopt it, so the HLE and the SDK agree on which object is the current thread. */ void hos_thread_adopt_guest(HosThread *t) { uint64_t tt = G64(t->tlr + 0x1F8); if (tt && tt != t->guest_tt) { t->guest_tt = tt; G32(tt + TT_HANDLE) = t->kernel_handle; } } int hos_thread_prio(HosThread *t) { return t->priority; } int hos_trace; extern const uint64_t hos_thread_start_hook, hos_thread_exit_hook; /* gen/hos_image.c */ static _Thread_local HosThread *cur; static SRWLOCK threads_lock = SRWLOCK_INIT; static HosThread *threads[1024]; static int nthreads; static volatile LONG64 next_id = 0x50; HosThread *hos_self(void) { return cur; } Ctx *hos_ctx(void) { return cur ? (cur->run_ctx ? cur->run_ctx : &cur->ctx) : NULL; } uint64_t hos_thread_guest(HosThread *t) { return t->guest_tt; } uint64_t hos_thread_id(HosThread *t) { return t->id; } const char *hos_thread_name(HosThread *t) { return t->name; } uint64_t *hos_tls_slots(HosThread *t) { return t->tls_slots; } /* The core a thread runs on. Most of TOTK's threads have no ideal core (-2, "use default") and a one-core affinity mask, so the mask decides; the engine keys per-core resources (its command memory arenas, for one) on GetCurrentCoreNumber, so threads pinned to different cores must report different ones. */ /* HOS_PIN_BY_CORE=1 (experiment): every game thread runs on the host's logical core 2 x its core number (one per physical core on SMT machines), so threads the game puts on the same core never run at the same time, as on the Switch, where TOTK keeps per-core data without locks (both MacNativeGames and totk-pc-recomp found it relies on that); they are still time-sliced at any instruction, not scheduled by priority as Horizon does. */ static void pin_by_core(HosThread *t) { #if defined(_WIN32) && !defined(__EMSCRIPTEN__) static int on = -1; if (on < 0) on = getenv("HOS_PIN_BY_CORE") != NULL; if (on && t->handle) SetThreadAffinityMask(t->handle, (DWORD_PTR)1 << (2 * hos_thread_core(t))); #else (void)t; #endif } void hos_thread_affinity(HosThread *t, int ideal, uint64_t mask) { if (ideal >= 0) t->ideal_core = ideal; if (mask) t->core_mask = mask; pin_by_core(t); } uint64_t hos_thread_mask(HosThread *t) { return t->core_mask; } void hos_thread_boost(int level) { #if defined(_WIN32) && !defined(__EMSCRIPTEN__) static int off = -1; if (off < 0) off = getenv("HOS_NO_PRIORITIES") != NULL; if (!off) SetThreadPriority(GetCurrentThread(), level >= 2 ? THREAD_PRIORITY_HIGHEST : THREAD_PRIORITY_ABOVE_NORMAL); #else (void)level; #endif } int hos_cores(void) { static int n; if (!n) { const char *e = getenv("HOS_CORES"); n = e && atoi(e) >= 1 && atoi(e) <= 8 ? atoi(e) : 6; /* (mkw-recomp: a Switch 2 application's six) */ if (!e && hos_game_cores()) n = hos_game_cores(); /* (totk_cores.c: the game set up for more cores) */ } return n; } int hos_thread_core(HosThread *t) { if (!t) return 0; if (t->vc_held) return t->vc_held - 1; /* (virtual cores: the one it holds) */ const int n = hos_cores(); if (t->ideal_core >= 0) return t->ideal_core % n; /* the mask's lowest core, wrapped (a thread pinned past the cores there are shares a core number: the extra BaseProc executors, totk_cores.c) */ return t->core_mask ? (int)(__builtin_ctzll(t->core_mask) % (unsigned)n) : 0; } void hos_thread_stack(HosThread *t, uint64_t *addr, uint64_t *size) { *addr = t->stack; *size = t->stack_size; } int hos_thread_priority(HosThread *t, int set, int value) { int old = t->priority; if (set) t->priority = value; return old; } #if defined(_WIN32) && !defined(__EMSCRIPTEN__) static SRWLOCK log_lock = SRWLOCK_INIT, log_write_lock = SRWLOCK_INIT; static CONDITION_VARIABLE log_cv = CONDITION_VARIABLE_INIT; static char *log_q[2048]; static int log_head, log_count; static volatile LONG log_async = -1; static volatile DWORD log_writer; /* log_main's thread */ static int log_take_locked(char **out) { int n = 0; while (log_count) { out[n++] = log_q[log_head]; log_head = (log_head + 1) % 2048; log_count--; } return n; } /* (hookshot) A batch goes out in one write. stderr is unbuffered (hos_main.c), so each line was a write of its own: 30-90 a second in play, every one extending the log file. With the game on a slow disk (a SATA SSD, 90% full) those writes kept the disk busy: its 4 KB reads took 10-200 ms where they take 0.4 ms, and what the game read from it meanwhile (a page of its own code, a file of the game's) waited as long: the long frames of a session. */ static void log_write(char **lines, int n) { size_t total = 0; for (int i = 0; i < n; i++) total += strlen(lines[i]); char *all = n > 1 ? (char *)malloc(total + 1) : NULL; if (all) { size_t at = 0; for (int i = 0; i < n; i++) { const size_t len = strlen(lines[i]); memcpy(all + at, lines[i], len); at += len; free(lines[i]); } fwrite(all, 1, total, stderr); free(all); } else { /* (one line, or no memory for the batch: line by line) */ for (int i = 0; i < n; i++) { fputs(lines[i], stderr); free(lines[i]); } } fflush(stderr); } /* (hookshot) and the writer waits a little after the line it is woken for, so the lines of that time go out together: a few writes a second however many lines there are. A queue half full goes out at once, before a burst can fill it (a full queue has the thread that logs write the line itself). HOS_LOG_BATCH_MS= (default 250, at most 2000; 0: the lines there are at each wake, as before). Nothing waiting is lost: every way out writes the queue first (log_flush, log_crash_filter), and the writer holds no lock while it waits */ enum { LOG_BATCH_FULL = 1024 }; static DWORD log_batch_ms(void) { static int ms = -1; if (ms < 0) { const char *e = getenv("HOS_LOG_BATCH_MS"); const int v = e && *e ? atoi(e) : 250; ms = v < 0 ? 0 : v > 2000 ? 2000 : v; } return (DWORD)ms; } static DWORD WINAPI log_main(void *p) { (void)p; static char *batch[2048]; log_writer = GetCurrentThreadId(); for (;;) { AcquireSRWLockExclusive(&log_lock); while (!log_count) SleepConditionVariableSRW(&log_cv, &log_lock, INFINITE, 0); /* (log_queue wakes it at each line: until the time is up or the queue half full) */ const ULONGLONG until = GetTickCount64() + log_batch_ms(); for (ULONGLONG now; log_count < LOG_BATCH_FULL && (now = GetTickCount64()) < until;) SleepConditionVariableSRW(&log_cv, &log_lock, (DWORD)(until - now), 0); const int n = log_take_locked(batch); AcquireSRWLockExclusive(&log_write_lock); ReleaseSRWLockExclusive(&log_lock); log_write(batch, n); ReleaseSRWLockExclusive(&log_write_lock); } } /* the queued lines straight to the handle (crash paths: no CRT stream, no write lock) */ static void log_drain_raw(void) { if (!TryAcquireSRWLockExclusive(&log_lock)) return; HANDLE err = GetStdHandle(STD_ERROR_HANDLE); while (log_count) { const char *line = log_q[log_head]; DWORD written; WriteFile(err, line, (DWORD)strlen(line), &written, NULL); log_head = (log_head + 1) % 2048; log_count--; } ReleaseSRWLockExclusive(&log_lock); } static void log_flush(void) { if (log_async <= 0) return; /* (a crash report on the writer itself, in its own write: it holds log_write_lock) */ if (GetCurrentThreadId() == log_writer) { log_drain_raw(); return; } static char *batch[2048]; AcquireSRWLockExclusive(&log_write_lock); AcquireSRWLockExclusive(&log_lock); const int n = log_take_locked(batch); ReleaseSRWLockExclusive(&log_lock); log_write(batch, n); ReleaseSRWLockExclusive(&log_write_lock); } static LPTOP_LEVEL_EXCEPTION_FILTER log_prev_filter; static LONG WINAPI log_crash_filter(EXCEPTION_POINTERS *e) { if (GetCurrentThreadId() == log_writer) log_drain_raw(); else { /* the writer may be in the middle of a batch: it finishes first (at most 2 s: a write stuck on the disk), then the rest of the queue goes out (before: the queue was left unwritten whenever the writer was busy) */ int got = 0; for (int i = 0; i < 2000 && !(got = TryAcquireSRWLockExclusive(&log_write_lock)); i++) Sleep(1); log_drain_raw(); if (got) ReleaseSRWLockExclusive(&log_write_lock); } return log_prev_filter ? log_prev_filter(e) : EXCEPTION_CONTINUE_SEARCH; } /* Lines are written by log_main, not by the thread that logs: a write that blocks (a console; the log file while the disk is busy: the render thread was seen held 140-310 ms in one line's write) holds no game thread. Nothing queued is lost when the game ends or crashes: hos_fatal, report_crash/report_fatal (mkw_glue.c), log_crash_filter, svcExitProcess, the window's close and the process's exit (atexit) write the queue first. HOS_LOG_SYNC=1: each line written by the thread that logs it, as files and pipes were before */ static int log_queue(const char *line) { if (log_async < 0) { AcquireSRWLockExclusive(&log_lock); if (log_async < 0) { int on = !getenv("HOS_LOG_SYNC"); if (on && !CreateThread(NULL, 0, log_main, NULL, 0, NULL)) on = 0; if (on) log_prev_filter = SetUnhandledExceptionFilter(log_crash_filter); if (on) atexit(log_flush); log_async = on; } ReleaseSRWLockExclusive(&log_lock); } if (!log_async) return 0; char *s = _strdup(line); if (!s) return 0; AcquireSRWLockExclusive(&log_lock); if (log_count < 2048) { log_q[(log_head + log_count) % 2048] = s; log_count++; ReleaseSRWLockExclusive(&log_lock); WakeConditionVariable(&log_cv); return 1; } ReleaseSRWLockExclusive(&log_lock); free(s); log_flush(); return 0; } #else static int log_queue(const char *line) { (void)line; return 0; } static void log_flush(void) {} #endif void hos_log_flush(void) { log_flush(); } void hos_log(const char *fmt, ...) { char buf[1024], line[1100]; va_list ap; va_start(ap, fmt); vsnprintf(buf, sizeof buf, fmt, ap); va_end(ap); snprintf(line, sizeof line, "[hos %s] %s\n", cur ? cur->name : "host", buf); if (log_queue(line)) return; fputs(line, stderr); fflush(stderr); } /* (Triforce, after NX2navi) 32 KiB of the calling thread's or fiber's stack kept back for its own stack overflow. Windows leaves a thread that ran out of stack about a guard page's worth, and the crash report (mkw_glue.c report_crash: its kilobyte buffers, hos_log's, the frame walk) needs more: it faulted again inside the report and the process ended with nothing in the log. The game's threads and fibers ask at their start (64 MB of stack reserved each, so a guest function that calls itself without end is the case), and each then commits that much more of its stack. The guarantee is the running fiber's own, so a fiber asks too. tests/test_crash_report.py */ void hos_stack_guarantee(void) { #ifdef _WIN32 ULONG bytes = 32768; SetThreadStackGuarantee(&bytes); #endif } /* (Triforce, after NX2navi) One end for a failed run. A crash is reported by every thread that faults, without waiting for the others, and a fatal error can come beside a crash from the same bad state (an allocation that failed here, a null pointer read there): the later one ended the process and cut the first report where it was. 1 for the first caller, which reports and ends the process; 0 for that thread again (a failure inside its own report: no second one); any other thread waits here, logging nothing (it may hold what the report needs), until the process ends, and ends it itself after 30 s if the first report never finishes. */ int hos_ending_enter(void) { static volatile LONG ending; const LONG me = (LONG)GetCurrentThreadId(); const LONG was = InterlockedCompareExchange(&ending, me, 0); if (!was) return 1; if (was == me) return 0; Sleep(30000); TerminateProcess(GetCurrentProcess(), 3); return 0; } void hos_fatal(Ctx *c, const char *fmt, ...) { char buf[1024]; va_list ap; va_start(ap, fmt); vsnprintf(buf, sizeof buf, fmt, ap); va_end(ap); if (!hos_ending_enter()) { /* (a fatal error inside this thread's own report: the report so far stands) */ TerminateProcess(GetCurrentProcess(), 3); ExitProcess(3); } if (!c) c = hos_ctx(); log_flush(); fprintf(stderr, "[hos %s] FATAL: %s", cur ? cur->name : "host", buf); if (c) fprintf(stderr, " (lr %#llx sp %#llx)", (unsigned long long)c->x[30], (unsigned long long)c->x[31]); fprintf(stderr, "\n"); fflush(stderr); void hos_backtrace(void), hos_hostcall_stats(void); hos_backtrace(); hos_hostcall_stats(); /* (Triforce, after NX2navi) TerminateProcess, not ExitProcess: ExitProcess stops the other threads wherever they are and then runs every library's detach on this thread, and a detach that faults there (an upscaler's, a driver's) put a crash report of its own after the lines above, or held the process open. Everything said here is written out already (log_flush, fflush). tests/test_device_lost_exit.py */ TerminateProcess(GetCurrentProcess(), 3); ExitProcess(3); /* (not reached) */ } /* Traps from the lifted code (rt/a64_rt.c): name the thread, then stop. */ void a64_on_trap(Ctx *c, uint64_t pc, uint32_t insn, const char *why) { hos_fatal(c, "%s at %#llx (insn %08x)", why, (unsigned long long)pc, insn); } static void setup_guest(HosThread *t) { t->tls_block = hos_sys_alloc(0x1000, 0x1000); t->tlr = hos_sys_alloc(0x200, 0x200); G64(t->tlr + 0x1F8) = t->guest_tt; t->kernel_handle = (uint32_t)InterlockedIncrement(&next_handle); G32(t->guest_tt + TT_HANDLE) = t->kernel_handle; /* (hookshot) SDK 22's InternalCriticalSection::IsLockedByCurrentThread reads the owner handle from TLR+0x110 directly (SDK 21: ThreadType+0x1B0 through TLR+0x1F8). Left 0, every free mutex looked held by the caller and the first heap lock aborted in pthread_mutex_lock. */ G32(t->tlr + 0x110) = t->kernel_handle; t->wake = CreateEventA(NULL, FALSE, FALSE, NULL); /* auto-reset: one signal, one wake-up */ Ctx *c = &t->ctx; memset(c, 0, sizeof *c); c->tpidr_el0 = t->tls_block + 0x800; c->tpidrro_el0 = t->tlr; c->thread = t; } static void track(HosThread *t) { AcquireSRWLockExclusive(&threads_lock); if (nthreads < 1024) threads[nthreads++] = t; ReleaseSRWLockExclusive(&threads_lock); } /* Host handles of every running guest thread, for the profiler (hos_prof.c). */ int hos_thread_handles(HANDLE *out, const char **names, int max) { int n = 0; AcquireSRWLockShared(&threads_lock); for (int i = 0; i < nthreads && n < max; i++) if (threads[i]->handle) { out[n] = threads[i]->handle; names[n] = threads[i]->name; n++; } ReleaseSRWLockShared(&threads_lock); return n; } /* (mkw-recomp) Every guest thread's name and guest registers as of its last call out of lifted code, with its frame chain from x29: what each thread of a stalled game waits in (mkw_glue.c, HOS_STATS_AFTER). */ void hos_thread_dump(void) { AcquireSRWLockShared(&threads_lock); for (int i = 0; i < nthreads; i++) { const Ctx *c = &threads[i]->ctx; char bt[400]; int n = 0; uint64_t fp = c->x[29]; for (int k = 0; k < 12 && fp && !(fp & 15) && fp < HOS_GUEST_SPACE && n < (int)sizeof bt - 24; k++) { MEMORY_BASIC_INFORMATION mbi; if (!VirtualQuery(GP(fp), &mbi, sizeof mbi) || mbi.State != MEM_COMMIT) break; n += snprintf(bt + n, sizeof bt - n, " %#llx", (unsigned long long)G64(fp + 8)); const uint64_t next = G64(fp); if (next <= fp) break; fp = next; } bt[n] = 0; hos_log("thread %-24s lr %#llx sp %#llx x0 %#llx; frames%s", threads[i]->name, (unsigned long long)c->x[30], (unsigned long long)c->x[31], (unsigned long long)c->x[0], bt); if (hos_game_is_mkw() && c->x[30] == 0x825c6844ULL) continue; /* (MKW's idle thread loop) */ /* registers (current at the call out), with the text a register (or the word it points to) leads to */ for (int r = 0; r < 29; r++) { uint64_t v = c->x[r], s = 0; MEMORY_BASIC_INFORMATION mbi; for (int hop = 0; hop < 2 && v > 0x100000 && v < HOS_GUEST_SPACE - 64; hop++) { if (!VirtualQuery(GP(v), &mbi, sizeof mbi) || mbi.State != MEM_COMMIT) break; const uint8_t *p = (const uint8_t *)GP(v); int k = 0; while (k < 48 && p[k] >= 0x20 && p[k] < 0x7f) k++; if (k >= 6 && (p[k] == 0 || k == 48)) { s = v; break; } v = G64(v); } if (s) hos_log("thread %s: x%d %#llx -> \"%.48s\"", threads[i]->name, r, (unsigned long long)c->x[r], (const char *)GP(s)); } } ReleaseSRWLockShared(&threads_lock); } /* HOS_FP_REPORT=1 (debugging): the FPCR / FPSR of the busiest game threads, with the thread stats */ int hos_thread_fp_report(char *out, int cap) { int len = 0; AcquireSRWLockShared(&threads_lock); for (int i = 0; i < nthreads && len < cap - 64; i++) { const char *nm = threads[i]->name; if (strstr(nm, "Main") || strstr(nm, "Worker") || strstr(nm, "Executor")) len += snprintf(out + len, cap - len, " %s fpcr %llx fpsr %llx;", nm, (unsigned long long)threads[i]->ctx.fpcr, (unsigned long long)threads[i]->ctx.fpsr); } ReleaseSRWLockShared(&threads_lock); return len; } HosThread *hos_thread_by_handle(uint32_t h) { HosThread *r = NULL; AcquireSRWLockShared(&threads_lock); for (int i = 0; i < nthreads; i++) if (threads[i]->kernel_handle == h) { r = threads[i]; break; } ReleaseSRWLockShared(&threads_lock); return r; } /* Block on the thread's wake event (the kernel layer's lock must not be held). */ void hos_thread_wait(HosThread *t, int64_t timeout_ns, int *timed_out) { DWORD ms = timeout_ns < 0 ? INFINITE : (DWORD)((timeout_ns + 999999) / 1000000); const int away = hos_vcore_away(); *timed_out = WaitForSingleObject(t->wake, ms) == WAIT_TIMEOUT; if (away) hos_vcore_back(); } void hos_thread_wake(HosThread *t) { SetEvent(t->wake); } HosThread *hos_thread_of(uint64_t guest_tt) { HosThread *r = NULL; AcquireSRWLockShared(&threads_lock); for (int i = 0; i < nthreads; i++) if (threads[i]->guest_tt == guest_tt) { r = threads[i]; break; } ReleaseSRWLockShared(&threads_lock); return r; } Ctx *hos_boot_main_thread(uint64_t stack_size) { HosThread *t = (HosThread *)calloc(1, sizeof *t); t->guest_tt = hos_sys_alloc(0x400, 0x10); /* a ThreadType for the main thread; the SDK never made one */ t->stack_size = stack_size; t->stack = hos_sys_alloc(stack_size, 0x1000); t->id = InterlockedIncrement64(&next_id); t->priority = 44; /* main thread priority in TOTK's NPDM range */ t->ideal_core = 0; t->started = 1; snprintf(t->name, sizeof t->name, "MainThread"); DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &t->handle, 0, FALSE, DUPLICATE_SAME_ACCESS); /* a real handle, for the profiler */ SetThreadDescription(GetCurrentThread(), L"MainThread"); /* (HOS_SAMPLE finds threads by it) */ hos_thread_pin(t->handle, t->name); setup_guest(t); t->ctx.x[31] = (t->stack + stack_size) & ~15ULL; cur = t; track(t); hos_vcore_back(); /* (virtual cores: before any guest code) */ return &t->ctx; } static DWORD WINAPI thread_main(void *p) { HosThread *t = (HosThread *)p; hos_stack_guarantee(); cur = t; Ctx *c = &t->ctx; c->x[0] = t->arg; c->x[30] = 0; c->x[31] = (t->stack + t->stack_size) & ~15ULL; A64Fn f = a64_lookup(t->entry); if (!f) hos_fatal(c, "thread %s: entry %#llx is not a function", t->name, (unsigned long long)t->entry); hos_vcore_back(); /* (virtual cores: before any guest code) */ if (hos_trace) hos_log("thread start: entry %#llx arg %#llx", (unsigned long long)t->entry, (unsigned long long)t->arg); /* What the SDK's own thread entry does before the thread function (see build.py thread_hooks): nnmusl's start hook gives the thread its TLS (thread pointer with the DTV at TP+8, every module's TLS block) and leaves the thread pointer at TLR+0x1F0. Without it every thread would share one DTV, and so every thread_local. */ uint64_t hook = hos_thread_start_hook ? G64(hos_thread_start_hook) : 0; if (hook) { c->x[0] = t->guest_tt; a64_icall(c, hook); if (G64(t->tlr + 0x1F0)) c->tpidr_el0 = G64(t->tlr + 0x1F0); if (hos_trace || getenv("HOS_TLS_TRACE")) hos_log("thread %s: TLS hook gave TP %#llx, DTV %#llx; ideal core %d mask %#llx priority %d", t->name, (unsigned long long)c->tpidr_el0, (unsigned long long)G64(c->tpidr_el0 + 8), t->ideal_core, (unsigned long long)t->core_mask, t->priority); c->x[31] = (t->stack + t->stack_size) & ~15ULL; } c->x[0] = t->arg; c->x[30] = 0; f(c); hook = hos_thread_exit_hook ? G64(hos_thread_exit_hook) : 0; if (hook) { c->x[0] = t->guest_tt; a64_icall(c, hook); } if (hos_trace) hos_log("thread exit"); hos_vcore_away(); return 0; } HosThread *hos_thread_create(uint64_t guest_tt, uint64_t entry, uint64_t arg, uint64_t stack, uint64_t stack_size, int priority, int core) { HosThread *t = (HosThread *)calloc(1, sizeof *t); t->guest_tt = guest_tt; t->entry = entry; t->arg = arg; t->stack = stack; t->stack_size = stack_size; t->priority = priority; t->ideal_core = core; t->id = InterlockedIncrement64(&next_id); snprintf(t->name, sizeof t->name, "Thread_%llx", (unsigned long long)entry); setup_guest(t); /* FPCR starts at 0, as the kernel does it; TOTK sets FZ itself (nnMain+0x474, thread entries) */ track(t); return t; } void hos_thread_start(HosThread *t) { if (t->started) return; t->started = 1; /* Host stack: lifted code nests a C call per guest call, so give it plenty. */ t->handle = CreateThread(NULL, 64 << 20, thread_main, t, STACK_SIZE_PARAM_IS_A_RESERVATION, NULL); if (!t->handle) hos_fatal(NULL, "CreateThread failed for %s", t->name); if (t->name[0]) { /* (named before it started: the OS name too, for profilers and HOS_SAMPLE) */ wchar_t w[48]; MultiByteToWideChar(CP_UTF8, 0, t->name, -1, w, 48); SetThreadDescription(t->handle, w); } hos_thread_pin(t->handle, t->name); pin_by_core(t); #ifdef __EMSCRIPTEN__ { void web_thread_named(HANDLE t, const char *name); web_thread_named(t->handle, t->name); } #endif } void hos_thread_join(HosThread *t) { if (!t->handle) return; const int away = hos_vcore_away(); /* (EvilOOT, 11-07: a freeze after Load, one thread joining another for good: a join of over 10 s says who joins whom, once) */ if (WaitForSingleObject(t->handle, 10000) == WAIT_TIMEOUT) { HosThread *self = hos_self(); hos_log("thread %s: joining %s for 10 s", self ? hos_thread_name(self) : "?", t->name); WaitForSingleObject(t->handle, INFINITE); } if (away) hos_vcore_back(); } void hos_thread_destroy(HosThread *t) { AcquireSRWLockExclusive(&threads_lock); for (int i = 0; i < nthreads; i++) if (threads[i] == t) { threads[i] = threads[--nthreads]; break; } ReleaseSRWLockExclusive(&threads_lock); if (t->handle) CloseHandle(t->handle); if (t->wake) CloseHandle(t->wake); free(t); } void hos_thread_set_name(HosThread *t, const char *name) { snprintf(t->name, sizeof t->name, "%s", name); if (t->handle) { wchar_t w[48]; MultiByteToWideChar(CP_UTF8, 0, t->name, -1, w, 48); SetThreadDescription(t->handle, w); hos_thread_pin(t->handle, t->name); } } /* HOS_PIN==,...[,*=] (experiment): host CPU affinity by thread name (game threads as they are named or started, host threads as they register); "*" for every other named thread. The first matching entry wins. (2026-10-20: the castle's MainThread and ModuleSystem workers each on a physical core of their own, the rest elsewhere, was slower than Windows' own scheduling: 63-68 against 69-71 fps.) */ /* HOS_PRIO==,... (experiment, same matching): host thread priority, -2 (lowest) to 2 (highest). (10-20: MainThread, ModuleSystem workers and BaseProc executors above normal: no change at the castle) */ typedef struct { char pat[40]; long long value; } NameSpec; static int parse_name_specs(const char *e, NameSpec *out, int max, int base) { int n = 0; while (e && *e && n < max) { const char *eq = strchr(e, '='), *end = strchr(e, ','); if (!end) end = e + strlen(e); if (!eq || eq > end) break; snprintf(out[n].pat, sizeof out[n].pat, "%.*s", (int)(eq - e), e); out[n++].value = strtoll(eq + 1, NULL, base); e = *end ? end + 1 : end; } return n; } static const NameSpec *match_name_spec(const NameSpec *s, int n, const char *name) { for (int i = 0; i < n; i++) if (strcmp(s[i].pat, "*") ? strstr(name, s[i].pat) != NULL : 1) return &s[i]; return NULL; } /* The last physical core (its logical processors' mask), kept for the Vulkan back end's replay thread (gpu/vk vk_replay.cpp pins it there): the game's threads run on the others. The replay is on the frame's critical path (MKW waits for the last frame's GPU work before it submits the next), and on a core shared with game threads it ran ~1.7x slower than alone. Only on machines with 8+ logical processors; HOS_RESERVE_CORE=0: none. (hookshot) On a CPU with cores of more than one kind, the last core of the fastest kind: the last core of all is an efficiency core on Intel's hybrid CPUs (an i7-12700K's logical processors 16-19), the slowest place for the replay (an i7-13700K's Kokiri Forest: 48.7 fps with it there, 60 on a performance core). EfficiencyClass is higher the faster the core, and the same for every core of a CPU of one kind (the last core, as before). Only with 4+ cores of the fastest kind (8+ logical processors always have them on a CPU of one kind): a laptop's 2 performance cores and 8 efficiency ones (i5-1235U) would keep one of the two from the game; then none is kept. The calling thread's processor group's cores: the masks the affinity calls take are its group's. Windows only: Linux keeps the last core, as before. */ /* (the choice: the cores' logical processor masks and EfficiencyClass, n of them; 0 for none. tests/test_reserved_core.py) */ static uint64_t pick_reserved_core(const uint64_t *masks, const unsigned char *classes, int n) { uint64_t last = 0; int best = -1, fast = 0, logical = 0; for (int i = 0; i < n; i++) { if (!masks[i]) continue; logical += __builtin_popcountll(masks[i]); if ((int)classes[i] > best) { best = classes[i], last = masks[i], fast = 1; } else if ((int)classes[i] == best) { fast++; if (masks[i] > last) last = masks[i]; } } if (logical < 8 || fast < 4) return 0; return last; } /* (Triforce, after NX2navi's cache placement) The die with the largest L3 cache, on a processor whose L3 caches differ: a two-die X3D part, 3D V-Cache on one die (host/hos_x3d.h has the rule and where nothing is done). The game's threads are kept on it (hos_thread_pin), and the core kept for the replay is its last one (last_fastest_core), not the processor's last, which is on the other die: the replay reads what the game's threads just wrote. Only where the rule finds such a die; everywhere else 0, and nothing changes. HOS_X3D=0: off there too. Other threads of the process (the driver's, the shader workers) stay where Windows puts them. Windows only: Linux as before. NX2navi keeps the whole process on the die and reserves no core there; here the replay's core stays, on the die, as the replay is pinned to it by default (gpu/vk/vk_replay.cpp ReplayPin). HOS_RESERVE_CORE=0 keeps none, as ever */ #if defined(_WIN32) /* (hos_x3d_pick over what Windows lists for the calling thread's processor group) */ static uint64_t x3d_detect(HosX3d *out) { static HosCache caches[1024]; static uint64_t cores[256]; int ncaches = 0, ncores = 0; HosX3d none = {HOS_X3D_NONE, 0, 0, 0, 0}; *out = none; DWORD_PTR proc = 0, sys = 0; if (!GetProcessAffinityMask(GetCurrentProcess(), &proc, &sys) || !proc) return 0; GROUP_AFFINITY ga; const WORD group = GetThreadGroupAffinity(GetCurrentThread(), &ga) ? ga.Group : 0; for (int pass = 0; pass < 2; pass++) { const LOGICAL_PROCESSOR_RELATIONSHIP rel = pass ? RelationProcessorCore : RelationCache; DWORD len = 0; if (GetLogicalProcessorInformationEx(rel, NULL, &len) || GetLastError() != ERROR_INSUFFICIENT_BUFFER) return 0; char *buf = (char *)malloc(len); if (!buf || !GetLogicalProcessorInformationEx(rel, (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)buf, &len)) { free(buf); return 0; } for (DWORD at = 0; at < len;) { const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *e = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)(buf + at); if (!e->Size) break; if (e->Relationship == RelationCache && e->Cache.GroupMask.Group == group && ncaches < 1024) { caches[ncaches].level = e->Cache.Level; caches[ncaches].mask = (uint64_t)e->Cache.GroupMask.Mask; caches[ncaches].bytes = e->Cache.CacheSize; ncaches++; } else if (e->Relationship == RelationProcessorCore && e->Processor.GroupCount && e->Processor.GroupMask[0].Group == group && ncores < 256) { cores[ncores++] = (uint64_t)e->Processor.GroupMask[0].Mask; } at += e->Size; } free(buf); } return hos_x3d_pick(caches, ncaches, cores, ncores, (uint64_t)proc, (int)GetActiveProcessorGroupCount(), out); } #endif static uint64_t hos_x3d_cpus(void) { static uint64_t mask = ~0ull; static SRWLOCK lk = SRWLOCK_INIT; AcquireSRWLockExclusive(&lk); if (mask == ~0ull) { mask = 0; #if defined(_WIN32) HosX3d x; const char *e = getenv("HOS_X3D"); const int off = e && *e && !atoi(e); const uint64_t die = x3d_detect(&x); if (x.why == HOS_X3D_USED && !off) mask = die; if (x.why == HOS_X3D_USED || x.why == HOS_X3D_FEW) /* (a processor with L3 caches of two sizes: said either way) */ hos_log("cpu: logical processors %#llx share the largest L3 cache (%llu MB, against %llu MB), %d cores: %s", (unsigned long long)x.die, (unsigned long long)(x.big >> 20), (unsigned long long)(x.small >> 20), x.cores, x.why == HOS_X3D_FEW ? "fewer than 8, so the game's threads are not kept there" : off ? "not used (HOS_X3D=0)" : "the game's threads and the replay's core are kept there (HOS_X3D=0: as before)"); #endif } const uint64_t m = mask; ReleaseSRWLockExclusive(&lk); return m; } #if defined(_WIN32) /* (pick_reserved_core over the calling thread's group's cores; *cores: how many, 0 when they could not be read) */ static uint64_t last_fastest_core(int *cores) { static uint64_t masks[256]; static unsigned char classes[256]; DWORD len = 0; int n = 0; *cores = 0; GROUP_AFFINITY ga; const WORD group = GetThreadGroupAffinity(GetCurrentThread(), &ga) ? ga.Group : 0; const uint64_t die = hos_x3d_cpus(); /* (Triforce: the cores of the die the game's threads are kept on, when there is one) */ if (GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &len) || GetLastError() != ERROR_INSUFFICIENT_BUFFER) return 0; char *buf = (char *)malloc(len); if (buf && GetLogicalProcessorInformationEx(RelationProcessorCore, (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)buf, &len)) for (DWORD at = 0; at < len && n < 256;) { const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *e = (const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)(buf + at); if (!e->Size) break; if (e->Relationship == RelationProcessorCore && e->Processor.GroupCount && e->Processor.GroupMask[0].Group == group && (!die || ((uint64_t)e->Processor.GroupMask[0].Mask & die))) { masks[n] = (uint64_t)e->Processor.GroupMask[0].Mask; classes[n] = e->Processor.EfficiencyClass; n++; } at += e->Size; } free(buf); *cores = n; return pick_reserved_core(masks, classes, n); } #endif uint64_t hos_reserved_cpus(void) { static uint64_t mask = ~0ull; if (mask != ~0ull) return mask; mask = 0; if (getenv("HOS_RESERVE_CORE") && !atoi(getenv("HOS_RESERVE_CORE"))) return mask; #if defined(_WIN32) { int cores = 0; const uint64_t fast = last_fastest_core(&cores); if (cores) return mask = fast; } #endif SYSTEM_LOGICAL_PROCESSOR_INFORMATION info[256]; DWORD len = sizeof info; if (!GetLogicalProcessorInformation(info, &len)) return mask; int logical = 0; uint64_t last = 0; for (DWORD i = 0; i < len / sizeof info[0]; i++) if (info[i].Relationship == RelationProcessorCore) { logical += __builtin_popcountll((uint64_t)info[i].ProcessorMask); if ((uint64_t)info[i].ProcessorMask > last) last = (uint64_t)info[i].ProcessorMask; } if (logical >= 8) mask = last; return mask; } void hos_thread_pin(void *h, const char *name) { const uint64_t reserved = hos_reserved_cpus(), die = hos_x3d_cpus(); if (reserved || die) { /* (Triforce: on the die with the largest L3 where there is one, hos_x3d.h; else as before) */ DWORD_PTR proc = 0, sys = 0; GetProcessAffinityMask(GetCurrentProcess(), &proc, &sys); const uint64_t want = hos_x3d_thread_mask((uint64_t)proc, die, reserved); if (want) SetThreadAffinityMask(h, (DWORD_PTR)want); } static NameSpec pins[24], prios[24]; static int npins = -1, nprios; static SRWLOCK lk = SRWLOCK_INIT; AcquireSRWLockExclusive(&lk); if (npins < 0) { npins = parse_name_specs(getenv("HOS_PIN"), pins, 24, 16); nprios = parse_name_specs(getenv("HOS_PRIO"), prios, 24, 10); } ReleaseSRWLockExclusive(&lk); const NameSpec *p = match_name_spec(pins, npins, name); if (p && p->value) SetThreadAffinityMask(h, (DWORD_PTR)p->value); static const int levels[5] = {THREAD_PRIORITY_LOWEST, THREAD_PRIORITY_BELOW_NORMAL, THREAD_PRIORITY_NORMAL, THREAD_PRIORITY_ABOVE_NORMAL, THREAD_PRIORITY_HIGHEST}; if ((p = match_name_spec(prios, nprios, name)) && p->value >= -2 && p->value <= 2) SetThreadPriority(h, levels[p->value + 2]); else if (strstr(name, "OptimizeChange") && !getenv("HOS_OCB_NORMAL")) { /* (OoT's OptimizeChangeBank (guest priority 27, a background thread) walks every placed object of the area by name, again and again (Kokiri: ~344 names, ~250k lookups a second). On the console it only gets the time nothing more important wants; at the host's normal priority it took the renderer's and the driver's cores and the frame waited (Kokiri, RX 6600 + 5600G: 17-20 fps; lowest: 21-28). HOS_OCB_NORMAL=1: as before) */ SetThreadPriority(h, THREAD_PRIORITY_LOWEST); } } /* ---- time ------------------------------------------------------------------------------------------------ */ uint64_t hos_ticks(void) { return recomp_cntpct(NULL); } int64_t hos_now_ns(void) { static LARGE_INTEGER freq; static int64_t ns_per_tick; LARGE_INTEGER now; if (!freq.QuadPart) { LARGE_INTEGER f; QueryPerformanceFrequency(&f); ns_per_tick = 1000000000LL % f.QuadPart == 0 ? 1000000000LL / f.QuadPart : 0; freq = f; } QueryPerformanceCounter(&now); if (ns_per_tick) return (int64_t)now.QuadPart * ns_per_tick; return (int64_t)((now.QuadPart / freq.QuadPart) * 1000000000LL + (now.QuadPart % freq.QuadPart) * 1000000000LL / freq.QuadPart); } static void sleep_ns(int64_t ns); /* (virtual cores: a sleep gives the core up, SleepThread(0) offers it to a waiter as good) */ void hos_sleep_ns(int64_t ns) { if (ns <= 0) { hos_vcore_yield(); SwitchToThread(); return; } const int away = hos_vcore_away(); sleep_ns(ns); if (away) hos_vcore_back(); } static void sleep_ns(int64_t ns) { /* A high-resolution waitable timer (one per thread) sleeps to within ~0.5 ms without spinning; Sleep() rounds to the timer resolution, so without one the last millisecond is spun (threads that sleep in short steps, the vibration thread for one, used 15% of a core spinning). */ static _Thread_local HANDLE timer; static _Thread_local int tried; int64_t until = hos_now_ns() + ns; #ifdef __EMSCRIPTEN__ /* The web has no high-resolution timer, and its clock is a call into JavaScript: spinning to the deadline below burned ~0.8 of a core across TOTK's sleeping threads (cores its game, render and WebGPU threads needed). A timed wait (a futex: Atomics.wait) to `slack` before the time, then the rest spun; slack follows how late the waits wake (HOS_WEB_SPIN_SLEEP=1: spun, as before). */ static int web_spin = -1; if (web_spin < 0) web_spin = getenv("HOS_WEB_SPIN_SLEEP") != NULL; static _Atomic int64_t slack = 300000; const int64_t s = atomic_load_explicit(&slack, memory_order_relaxed); if (!web_spin && ns > s + 50000) { const int64_t want = until - s; usleep((useconds_t)((want - hos_now_ns()) / 1000)); const int64_t late = hos_now_ns() - want; int64_t next = s + (late + 25000 - s) / 8; next = next < 50000 ? 50000 : next > 2000000 ? 2000000 : next; atomic_store_explicit(&slack, next, memory_order_relaxed); } while (hos_now_ns() < until) sched_yield(); return; #endif /* Under a millisecond: yielded away until the time (the timer's ~0.5 ms granularity made TOTK's 20 us waits for its job workers 25x longer, ~2000 a second on the frame's critical path). HOS_SHORT_SLEEP_TIMER=1: as before */ static int short_timer = -1; if (short_timer < 0) short_timer = getenv("HOS_SHORT_SLEEP_TIMER") != NULL; if (ns < 1000000 && !short_timer) { while (hos_now_ns() < until) if (!SwitchToThread()) YieldProcessor(); return; } if (!tried) { tried = 1; timer = CreateWaitableTimerExW(NULL, NULL, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); } if (timer) { LARGE_INTEGER due; due.QuadPart = -(ns / 100 > 0 ? ns / 100 : 1); /* relative, 100 ns units */ if (SetWaitableTimerEx(timer, &due, 0, NULL, NULL, NULL, 0)) { WaitForSingleObject(timer, INFINITE); return; } } if (ns > 2000000) Sleep((DWORD)((ns - 1000000) / 1000000)); while (hos_now_ns() < until) SwitchToThread(); } /* (Triforce) The game's frame wait to its time (pace_wait.h, host/nn_vi.c hos_vi_native_acquire): the timer up to the margin before it (sleep_ns, which takes the timer for a millisecond and more), the rest yielded away as sleep_ns yields a short wait; the core given up once for the whole of it, as hos_sleep_ns gives it up */ static void yield_for_wait(void) { if (!SwitchToThread()) YieldProcessor(); } void hos_sleep_until_ns(int64_t until, int64_t margin_ns) { if (until <= hos_now_ns()) return; const int away = hos_vcore_away(); #ifdef __EMSCRIPTEN__ (void)yield_for_wait, (void)margin_ns; /* (the web's own sleep_ns spins the end of a wait already) */ const int64_t left = until - hos_now_ns(); if (left > 0) sleep_ns(left); #else pace_wait_until(until, margin_ns, hos_now_ns, sleep_ns, yield_for_wait); #endif if (away) hos_vcore_back(); } /* ---- nn::os threads -------------------------------------------------------------------------------------- */ static HosThread *thread_arg(Ctx *c, uint64_t tt) { HosThread *t = hos_thread_of(tt); if (!t) hos_fatal(c, "unknown nn::os::ThreadType %#llx", (unsigned long long)tt); return t; } /* nn::os::CreateThread(ThreadType*, void (*)(void*), void* arg, void* stack, unsigned long size, int prio) */ HLE(_ZN2nn2os12CreateThreadEPNS0_10ThreadTypeEPFvPvES3_S3_mi) { HOS_TRACE_CALL("CreateThread"); hos_thread_create(ARG(0), ARG(1), ARG(2), ARG(3), ARG(4), (int)ARG(5), -2); RET(0); } /* nn::os::CreateThread(ThreadType*, void (*)(void*), void* arg, void* stack, unsigned long size, int prio, int idealCore) (hookshot: OoT's Porter thread pool uses this overload; left to the guest SDK it reached ThreadManagerImplByHorizon::MapAliasStack, which needs svcMapMemory, and aborted) */ HLE(_ZN2nn2os12CreateThreadEPNS0_10ThreadTypeEPFvPvES3_S3_mii) { HOS_TRACE_CALL("CreateThread"); hos_thread_create(ARG(0), ARG(1), ARG(2), ARG(3), ARG(4), (int)ARG(5), (int)ARG(6)); RET(0); } HLE(_ZN2nn2os11StartThreadEPNS0_10ThreadTypeE) { HOS_TRACE_CALL("StartThread"); hos_thread_start(thread_arg(c, ARG(0))); } HLE(_ZN2nn2os10WaitThreadEPNS0_10ThreadTypeE) { HOS_TRACE_CALL("WaitThread"); hos_thread_join(thread_arg(c, ARG(0))); } HLE(_ZN2nn2os13DestroyThreadEPNS0_10ThreadTypeE) { HOS_TRACE_CALL("DestroyThread"); HosThread *t = thread_arg(c, ARG(0)); hos_thread_join(t); hos_thread_destroy(t); } HLE(_ZN2nn2os13SetThreadNameEPNS0_10ThreadTypeEPKc) { HosThread *t = thread_arg(c, ARG(0)); hos_thread_set_name(t, ARG(1) ? (const char *)GP(ARG(1)) : ""); if (hos_trace) hos_log("SetThreadName %s", t->name); } HLE(_ZN2nn2os16GetCurrentThreadEv) { RET(cur->guest_tt); } HLE(_ZN2nn2os11GetThreadIdEPKNS0_10ThreadTypeE) { RET(thread_arg(c, ARG(0))->id); } HLE(_ZN2nn2os17GetThreadPriorityEPKNS0_10ThreadTypeE) { RET((uint32_t)hos_thread_priority(thread_arg(c, ARG(0)), 0, 0)); } HLE(_ZN2nn2os20ChangeThreadPriorityEPNS0_10ThreadTypeEi) { RET((uint32_t)hos_thread_priority(thread_arg(c, ARG(0)), 1, (int)ARG(1))); } /* nn::os::SetThreadCoreMask(ThreadType*, int idealCore, unsigned long mask) */ HLE(_ZN2nn2os17SetThreadCoreMaskEPNS0_10ThreadTypeEim) { HosThread *t = thread_arg(c, ARG(0)); hos_thread_affinity(t, (int)ARG(1), ARG(2)); } HLE(_ZN2nn2os20GetCurrentCoreNumberEv) { RET((uint32_t)hos_thread_core(cur)); } HLE(_ZN2nn2os26GetThreadAvailableCoreMaskEv) { RET((1u << hos_cores()) - 1); } /* cores 0-2: what an application gets */ HLE(_ZN2nn2os19GetCurrentStackInfoEPmS1_) { if (ARG(0)) G64(ARG(0)) = cur->stack; if (ARG(1)) G64(ARG(1)) = cur->stack_size; } HLE(_ZN2nn2os11YieldThreadEv) { hos_vcore_yield(); SwitchToThread(); } /* HOS_SLEEP_LOG=1 (diagnostic): guest sleeps by caller (LR) every 5 s: count, asked, slept */ void hos_sleep_logged(int64_t ns, uint64_t lr) { /* HOS_POLL_SLEEP_NS= (experiment): a guest sleep of up to 100 us lasts at most that long. TOTK's job system waits for its workers by sleeping 20 us and looking again (~90 times a frame at the castle, on MainThread and the workers): a shorter step notices the job's end sooner (the loop looks again; nothing else changes). Measured 2026-10-21 (castle, 2000 ns, 3 rounds): no change. */ static int64_t poll_cap = -1; if (poll_cap < 0) { const char *e = getenv("HOS_POLL_SLEEP_NS"); poll_cap = e ? atoll(e) : 0; } if (poll_cap && ns > poll_cap && ns <= 100000) ns = poll_cap; static int on = -1; if (on < 0) on = getenv("HOS_SLEEP_LOG") != NULL; if (!on) { hos_sleep_ns(ns); return; } const int64_t t0 = hos_now_ns(); hos_sleep_ns(ns); const int64_t took = hos_now_ns() - t0; static SRWLOCK lk = SRWLOCK_INIT; static struct { uint64_t lr; int64_t n, asked, slept; } tab[64]; static int64_t last; AcquireSRWLockExclusive(&lk); int i = 0; while (i < 64 && tab[i].lr && tab[i].lr != lr) i++; if (i < 64) { tab[i].lr = lr; tab[i].n++; tab[i].asked += ns > 0 ? ns : 0; tab[i].slept += took; } const int64_t now = hos_now_ns(); if (now - last > 5000000000LL) { last = now; for (int k = 0; k < 64 && tab[k].lr; k++) if (tab[k].n) hos_log("sleep: lr %#llx: %lld calls, asked %.2f ms, slept %.2f ms (%.1f us each)", (unsigned long long)tab[k].lr, (long long)tab[k].n, tab[k].asked / 1e6, tab[k].slept / 1e6, tab[k].slept / 1e3 / tab[k].n); memset(tab, 0, sizeof tab); } ReleaseSRWLockExclusive(&lk); } /* (hookshot) OoT's present waits for its frame's GPU work by polling an NVN event, sleeping ~1.6 ms between looks (0x80590500: `while (nvnEventGetValue(e) != 1) SleepThread(...)`, ~3.5 sleeps a frame, 2 ms each on Windows): the frame noticed the GPU's signal late, ~7 ms of the 25 ms frame asleep on its critical path. The GPU writes the event into guest memory itself, so the sleep looks at it between yields (its virtual core given up) and ends once it is 1, at the latest when the asked time is up. Only where the poll is nvnEventGetValue. HOS_PRESENT_POLL=1 (an experiment: no fps change measured, and it keeps a core busy while it looks) */ static int present_poll(Ctx *c, int64_t ns) { static int on = -1; static uint64_t fn_ok, fn_bad; if (on < 0) on = getenv("HOS_PRESENT_POLL") && atoi(getenv("HOS_PRESENT_POLL")) != 0; if (!on || c->x[30] != 0x80590520 || ns <= 0 || ns > 20000000) return 0; const uint64_t fp = G64(c->x[24]); if (fp != fn_ok) { if (fp == fn_bad) return 0; const char *hos_hostcall_name(uint64_t addr); const char *n = hos_hostcall_name(fp); if (!n || strcmp(n, "nvnEventGetValue")) { fn_bad = fp; return 0; } fn_ok = fp; hos_log("sleep: OoT's present poll looks at its event between yields (HOS_PRESENT_POLL)"); } int hos_nvn_event_value(uint64_t guest_event, uint32_t *v); const uint64_t ev = c->x[23] + (uint64_t)G32(c->x[22] + 0x164) * 64; uint32_t v = 0; if (!hos_nvn_event_value(ev, &v)) return 0; const int away = hos_vcore_away(); const int64_t end = hos_now_ns() + ns; while (v != 1 && hos_now_ns() < end) { SwitchToThread(); for (int i = 0; i < 64; i++) _mm_pause(); hos_nvn_event_value(ev, &v); } if (away) hos_vcore_back(); return 1; } HLE(_ZN2nn2os11SleepThreadENS_8TimeSpanE) { if (present_poll(c, (int64_t)ARG(0))) return; if (c->x[30] == 0x80590520 && getenv("HOS_SLEEP_LOG")) { /* (hookshot, diagnostic: OoT's present poll) */ static int logged; if (logged++ < 3) { const char *hos_hostcall_name(uint64_t addr); const uint64_t fp = G64(c->x[24]), sync = c->x[23] + (uint64_t)G32(c->x[22] + 0x164) * 64; const char *n = hos_hostcall_name(fp); hos_log("sleep: OoT's present poll: fn %#llx (%s), arg %#llx, sleep %lld ns", (unsigned long long)fp, n ? n : "guest code", (unsigned long long)sync, (long long)ARG(0)); } } hos_sleep_logged((int64_t)ARG(0), c->x[30]); } /* ---- TLS slots --------------------------------------------------------------------------------------------- */ static volatile LONG slot_used[HOS_TLS_SLOTS]; /* nn::os::AllocateTlsSlot(TlsSlot*, void (*)(unsigned long)) -> Result */ HLE(_ZN2nn2os15AllocateTlsSlotEPNS0_7TlsSlotEPFvmE) { for (int i = 0; i < HOS_TLS_SLOTS; i++) { if (!InterlockedCompareExchange(&slot_used[i], 1, 0)) { G32(ARG(0)) = (uint32_t)i; RET(0); return; } } RET(0x2C01); /* os::ResultOutOfResource */ } HLE(_ZN2nn2os11FreeTlsSlotENS0_7TlsSlotE) { if ((uint32_t)ARG(0) < HOS_TLS_SLOTS) slot_used[(uint32_t)ARG(0)] = 0; } HLE(_ZN2nn2os11GetTlsValueENS0_7TlsSlotE) { RET((uint32_t)ARG(0) < HOS_TLS_SLOTS ? cur->tls_slots[(uint32_t)ARG(0)] : 0); } HLE(_ZN2nn2os11SetTlsValueENS0_7TlsSlotEm) { if ((uint32_t)ARG(0) < HOS_TLS_SLOTS) cur->tls_slots[(uint32_t)ARG(0)] = ARG(1); } /* ---- ticks ------------------------------------------------------------------------------------------------- */ volatile uint64_t hos_gfx_module; /* HOS_TICK_LOG=1 (diagnostic): who reads the clock, by caller (LR), every 5 s */ static void tick_log(uint64_t lr) { static CRITICAL_SECTION cs; static volatile LONG once; static uint64_t lrs[512]; static uint32_t counts[512]; static int64_t next; if (!InterlockedExchange(&once, 1)) InitializeCriticalSection(&cs); EnterCriticalSection(&cs); for (int i = (int)((lr >> 2) * 2654435761u >> 8) & 511, n = 0; n < 512; n++, i = (i + 1) & 511) if (!lrs[i] || lrs[i] == lr) { lrs[i] = lr; counts[i]++; break; } const int64_t now = hos_now_ns(); if (now > next) { next = now + 5000000000LL; char line[1500]; int n = 0; for (int k = 0; k < 40 && n < 1400; k++) { int best = -1; for (int i = 0; i < 512; i++) if (counts[i] && (best < 0 || counts[i] > counts[best])) best = i; if (best < 0) break; n += snprintf(line + n, sizeof line - n, " %llx:%u", (unsigned long long)lrs[best], counts[best]); counts[best] = 0; } memset(counts, 0, sizeof counts); hos_log("tick readers in 5 s:%s", line); } LeaveCriticalSection(&cs); } HLE(_ZN2nn2os13GetSystemTickEv) { static int on = -1; if (on < 0) on = getenv("HOS_TICK_LOG") != NULL; if (on) tick_log(c->x[30]); /* (hookshot) The graphics module: x19 at its frame-end clock read (main+0x346c770 is after the bl), for nn_vi.c's real-time step */ if (c->x[30] == 0x8348c770) hos_gfx_module = c->x[19]; RET(hos_ticks()); } HLE(_ZN2nn2os22GetSystemTickFrequencyEv) { RET(HOS_TICK_HZ); } /* (hookshot) At the counter's own rate, HOS_TICK_HZ (31.25 MHz: 32 ns a tick). These converted at the Switch 1's 19.2 MHz (ticks = ns * 12 / 625) while GetSystemTick counts 31.25 MHz, so a tick span came out 1.63x too long: OoT's present slept 1.63 ms for its 1 ms poll step (0x80590500: SleepThread(ConvertToTimeSpan(frequency / 1000))), and any of its timing through these was off by as much. HOS_TICK_CONVERT_192=1: as before */ static int tick_convert_192(void) { static int on = -1; if (on < 0) on = getenv("HOS_TICK_CONVERT_192") != NULL; return on; } HLE(_ZN2nn2os13ConvertToTickENS_8TimeSpanE) { int64_t ns = (int64_t)ARG(0); if (tick_convert_192()) { RET((ns / 625) * 12 + (ns % 625) * 12 / 625); return; } RET(ns / (int64_t)(1000000000ULL / HOS_TICK_HZ)); } HLE(_ZN2nn2os17ConvertToTimeSpanENS0_4TickE) { int64_t t = (int64_t)ARG(0); if (tick_convert_192()) { RET((t / 12) * 625 + (t % 12) * 625 / 12); return; } RET(t * (int64_t)(1000000000ULL / HOS_TICK_HZ)); } /* ---- fibers (hookshot: nn::async's FiberTaskCoordinator, on OoT's nn.npln threads) ---------------------------- A guest fiber is a Win32 fiber with its own guest context: the lifted frames on its host stack keep using that context wherever it runs, so a fiber may resume on another thread. Switching in, it takes the running thread's identity (TLS registers, host thread object); the thread's own context stays in HosThread.ctx. The fiber function returns the FiberType to switch to next (nn::os semantics). */ typedef struct HosFiber { Ctx ctx; /* (first: a context's fiber is found from its address) */ uint64_t ft, fn, arg, stack, size; void *host; int started, done; struct HosFiber *next; } HosFiber; static SRWLOCK fibers_lock = SRWLOCK_INIT; static HosFiber *fibers; static HosFiber *fiber_of_ft(uint64_t ft) { AcquireSRWLockShared(&fibers_lock); HosFiber *f = fibers; while (f && f->ft != ft) f = f->next; ReleaseSRWLockShared(&fibers_lock); return f; } static HosFiber *fiber_of_ctx(Ctx *c) { AcquireSRWLockShared(&fibers_lock); HosFiber *f = fibers; while (f && &f->ctx != c) f = f->next; ReleaseSRWLockShared(&fibers_lock); return f; } static void fiber_switch(Ctx *from, HosFiber *to); static void WINAPI fiber_main(void *p) { HosFiber *f = (HosFiber *)p; hos_stack_guarantee(); /* (the guarantee is the fiber's own stack's) */ Ctx *c = &f->ctx; c->x[0] = f->arg; c->x[30] = 0; c->x[31] = (f->stack + f->size) & ~15ULL; a64_icall(c, f->fn); /* FiberType* (*)(void*) */ const uint64_t next = c->x[0]; f->done = 1; HosFiber *to = next ? fiber_of_ft(next) : NULL; /* (NULL: back to the thread's own context, as nn::os does) */ if (next && !to) hos_fatal(c, "fiber %#llx returned %#llx: not an initialized fiber", (unsigned long long)f->ft, (unsigned long long)next); fiber_switch(c, to); hos_fatal(c, "fiber %#llx: switched to after it returned", (unsigned long long)f->ft); } /* from: the context calling (the thread's own or a fiber's); to: NULL for the thread's own context */ static void fiber_switch(Ctx *from, HosFiber *to) { HosThread *t = cur; if (!t->host_fiber) { t->host_fiber = ConvertThreadToFiberEx(NULL, FIBER_FLAG_FLOAT_SWITCH); if (!t->host_fiber) hos_fatal(from, "ConvertThreadToFiber failed (%lu)", GetLastError()); } Ctx *dst = to ? &to->ctx : &t->ctx; if (dst == from) return; if (to) { if (to->done) hos_fatal(from, "SwitchToFiber(%#llx): that fiber has returned", (unsigned long long)to->ft); if (!to->started) { /* the thread's state it inherits: FPCR, then the identity below */ to->started = 1; to->ctx.fpcr = from->fpcr; to->ctx.fpsr = from->fpsr; } } /* the running thread's identity, whoever ran this context last */ dst->tpidr_el0 = t->ctx.tpidr_el0; dst->tpidrro_el0 = t->ctx.tpidrro_el0; dst->thread = t->ctx.thread; if (from->poll) dst->poll = 1; /* (a pending virtual-core request follows the thread) */ t->run_ctx = to ? &to->ctx : NULL; SwitchToFiber(to ? to->host : t->host_fiber); /* back in `from`, maybe on another thread: take that one's identity (fiber_switch's caller set it already) */ } /* InitializeFiber(FiberType*, FiberType* (*)(void*), void* arg, void* stack, unsigned long size, int flags) */ HLE(_ZN2nn2os15InitializeFiberEPNS0_9FiberTypeEPFS2_PvES3_S3_mi) { HosFiber *f = fiber_of_ft(ARG(0)); if (!f) { f = (HosFiber *)calloc(1, sizeof *f); f->ft = ARG(0); AcquireSRWLockExclusive(&fibers_lock); f->next = fibers, fibers = f; ReleaseSRWLockExclusive(&fibers_lock); } else if (f->host) { DeleteFiber(f->host); } memset(&f->ctx, 0, sizeof f->ctx); f->fn = ARG(1), f->arg = ARG(2), f->stack = ARG(3), f->size = ARG(4); f->started = f->done = 0; f->host = CreateFiberEx(64 << 10, 64 << 20, FIBER_FLAG_FLOAT_SWITCH, fiber_main, f); if (!f->host) hos_fatal(c, "CreateFiberEx failed (%lu)", GetLastError()); static volatile LONG logged; if (InterlockedIncrement(&logged) <= 8 || hos_trace) hos_log("fiber %#llx: function %#llx arg %#llx stack %#llx+%#llx flags %d", (unsigned long long)f->ft, (unsigned long long)f->fn, (unsigned long long)f->arg, (unsigned long long)f->stack, (unsigned long long)f->size, (int)ARG(5)); } HLE(_ZN2nn2os13FinalizeFiberEPNS0_9FiberTypeE) { HosFiber *f = fiber_of_ft(ARG(0)); if (f && f->host && &f->ctx != c) { DeleteFiber(f->host); f->host = NULL; f->done = 1; } } HLE(_ZN2nn2os15GetCurrentFiberEv) { HosFiber *f = c == &cur->ctx ? NULL : fiber_of_ctx(c); RET(f ? f->ft : 0); } /* SwitchToFiber(FiberType*): NULL goes back to the thread's own context */ HLE(_ZN2nn2os13SwitchToFiberEPNS0_9FiberTypeE) { HosFiber *to = NULL; if (ARG(0) && !(to = fiber_of_ft(ARG(0)))) hos_fatal(c, "SwitchToFiber(%#llx): not an initialized fiber", (unsigned long long)ARG(0)); fiber_switch(c, to); } /* Every thread's kernel handle (the tag its SDK mutexes hold), for hos_kernel_dump. */ void hos_thread_log_handles(void) { AcquireSRWLockShared(&threads_lock); /* (lr: the return address of the thread's last call, x0 its first argument: in a wait, which game code waits on what; the web has no stack walk) */ for (int i = 0; i < nthreads; i++) hos_log("kernel: handle %#x = %s: lr %#llx x0 %#llx x19 %#llx x20 %#llx x21 %#llx x22 %#llx", threads[i]->kernel_handle, threads[i]->name, (unsigned long long)threads[i]->ctx.x[30], (unsigned long long)threads[i]->ctx.x[0], (unsigned long long)threads[i]->ctx.x[19], (unsigned long long)threads[i]->ctx.x[20], (unsigned long long)threads[i]->ctx.x[21], (unsigned long long)threads[i]->ctx.x[22]); ReleaseSRWLockShared(&threads_lock); } /* ---- virtual cores (on unless HOS_VCORES=0) ------------------------------------------------------------------- Horizon runs at most one thread per core, and TOTK keys per-core data (allocators, job queues) by the current core number without locks; both MacNativeGames (threads.c) and totk-pc-recomp (runtime/kernel.c, MIT) schedule the game's threads on three virtual cores for it. Here a guest thread holds one of hos_cores() cores while it runs guest code and gives it up where Horizon would switch threads: in every block (hos_cv_sleep, hos_thread_wait, hos_sleep_ns, the NVN waits), at a yield, at exit. A free core goes to the best waiter allowed on it (lower priority value first, then the one waiting longest). A waiter better than a holder asks it to give the core up (vc_kick), which it does at its next checkpoint (every nn::os call and SVC: hos_vcore_checkpoint), as Horizon would run the better thread at once. Guest code with no checkpoint (a spin loop) keeps its core, so a waiter that has waited too long runs anyway without one (an "escape", logged with the holders): HOS_VCORE_ESCAPE_US for a better waiter (default 1000), HOS_VCORE_ESCAPE_EQ_US otherwise (20000). Every 5 s the log has a "vcores:" line. GetCurrentCoreNumber answers the core a thread holds. (Triforce) A waiter that asked a holder for its core spins for it before it sleeps (vc_spin). */ #if !defined(__EMSCRIPTEN__) static SRWLOCK vc_lock = SRWLOCK_INIT; static HosThread *vc_owner[8]; static HosThread *vc_queue[1024]; static int vc_nq; static struct { int64_t takes, waits, wait_ns, max_ns, escapes, kicks, yields, since; /* (Triforce) the waits that ended in the spin and those that slept, the time spun, the spins left out (HOS_VCORE_SPIN_MAX threads spinning already); grant to run: from vc_grant until the waiter holds vc_lock again, after a spin and after a sleep (NX2navi's "core grant to run", runtime/kernel.c grant_at) */ int64_t spun, slept, spin_ns, spin_capped, run_spin_ns, run_spin_n, run_sleep_ns, run_sleep_n, run_max_ns; } vc_st; int hos_vcores_on(void) { static int on = -1; if (on < 0) { const char *e = getenv("HOS_VCORES"); on = e ? atoi(e) != 0 : 1; /* (mkw-recomp: on unless HOS_VCORES=0; the game's job queues need it) */ } return on; } static int vc_cores(void) { return hos_cores() > 8 ? 8 : hos_cores(); } static unsigned vc_allowed(HosThread *t) { const unsigned all = (1u << vc_cores()) - 1; /* (Triforce, 11-14, the lab) HOS_VCORE_ANY=1: a thread takes any free core, not only the ones its affinity mask names. What the game needs of the cores is that no two threads run under one core number at a time (its per-core data has no locks); the masks are the console's placement. OoT pins nearly every thread to one core (six BaseProc executors, five module workers, the main and Presentation threads on six cores), and here a thread waits for ITS core while others stand free: at the Market ~61 waits a frame of ~0.1 ms each, 7 ms of waiting a frame summed over the threads, on the path the frame now waits for (the game's own CPU work, 13-14 ms a frame with the replay on its own thread). The risk: a thread that reads its core number, blocks, and goes on under another */ static int any = -1; if (any < 0) any = getenv("HOS_VCORE_ANY") && atoi(getenv("HOS_VCORE_ANY")) != 0; if (any) return all; unsigned m = (unsigned)(t->core_mask & all); if (!m) m = 1u << hos_thread_core(t); return m; } /* a runs before b */ static int vc_better(HosThread *a, HosThread *b) { return a->priority != b->priority ? a->priority < b->priority : a->vc_since < b->vc_since; } /* a free core t may run on (its last one, its ideal one, then the lowest), or -1; holding vc_lock */ static int vc_free_for(HosThread *t) { const unsigned m = vc_allowed(t); if (t->vc_last > 0 && (m >> (t->vc_last - 1) & 1) && !vc_owner[t->vc_last - 1]) return t->vc_last - 1; const int ideal = hos_thread_core(t); if ((m >> ideal & 1) && !vc_owner[ideal]) return ideal; for (int c = 0; c < vc_cores(); c++) if ((m >> c & 1) && !vc_owner[c]) return c; return -1; } static void vc_take(HosThread *t, int c) { vc_owner[c] = t; __atomic_store_n(&t->vc_held, c + 1, __ATOMIC_RELEASE); /* ((Triforce) a waiter in vc_spin reads it without vc_lock) */ t->vc_last = c + 1; t->vc_kick = 0; } static void vc_dequeue(HosThread *t) { for (int i = 0; i < vc_nq; i++) if (vc_queue[i] == t) { vc_queue[i] = vc_queue[--vc_nq]; break; } t->vc_queued = 0; } /* core c is free: to the best waiter allowed on it; holding vc_lock */ static void vc_grant(int c) { HosThread *best = NULL; for (int i = 0; i < vc_nq; i++) if ((vc_allowed(vc_queue[i]) >> c & 1) && (!best || vc_better(vc_queue[i], best))) best = vc_queue[i]; if (!best) return; vc_dequeue(best); best->vc_grant_at = hos_now_ns(); /* ((Triforce) grant to run, taken where it holds vc_lock again: vc_acquire) */ vc_take(best, c); WakeConditionVariable(&best->vc_cv); /* (no one sleeps there when it spins: vc_spin reads vc_held) */ } static void vc_stats_locked(int64_t now) { if (!vc_st.since) vc_st.since = now; if (now - vc_st.since < 5000000000LL) return; hos_log("vcores: %lld takes, %lld waits (mean %.0f us, longest %.1f ms), %lld kicks, %lld yields, %lld escapes in %.0f s; " "%lld of the waits ended in the spin (%.2f ms spun, %lld spins left out with HOS_VCORE_SPIN_MAX threads spinning), " "%lld slept; a granted core ran %.1f us after its grant after a spin (%lld), %.1f us after a sleep (%lld), longest " "%.0f us", (long long)vc_st.takes, (long long)vc_st.waits, vc_st.waits ? vc_st.wait_ns / 1e3 / vc_st.waits : 0.0, vc_st.max_ns / 1e6, (long long)vc_st.kicks, (long long)vc_st.yields, (long long)vc_st.escapes, (now - vc_st.since) / 1e9, (long long)vc_st.spun, vc_st.spin_ns / 1e6, (long long)vc_st.spin_capped, (long long)vc_st.slept, vc_st.run_spin_n ? vc_st.run_spin_ns / 1e3 / vc_st.run_spin_n : 0.0, (long long)vc_st.run_spin_n, vc_st.run_sleep_n ? vc_st.run_sleep_ns / 1e3 / vc_st.run_sleep_n : 0.0, (long long)vc_st.run_sleep_n, vc_st.run_max_ns / 1e3); memset(&vc_st, 0, sizeof vc_st); vc_st.since = now; } static int64_t vc_env_ns(const char *name, int64_t us) { const char *e = getenv(name); return (e ? atoll(e) : us) * 1000; } /* (Triforce, the third performance audit's P1: NX2navi's spin before the sleep, runtime/kernel.c core_take, which came from totk-pc-recomp's runtime/kernel.c, MIT, Copyright (c) 2026 the totk-pc-recomp authors) A waiter that asked a holder to give its core up spins for it before it sleeps: up to HOS_VCORE_SPIN_US (250; 0: it sleeps at once, as before), with vc_lock let go, until a release grants it the core (vc_grant sets its vc_held), then takes the lock again. The holder gives the core up at its next nn::os call, SVC or loop poll, mostly within microseconds; a sleep adds an OS wake to each such hand-off (NX2navi's log on this PC: 1,391,252 waits ended in its spin against 530 that did not; Triforce at the Market: ~66,000 core waits per 5 s, 67 us mean). Who gets a core does not change: the waiter stays queued, and vc_grant picks the best waiter as before; only how soon the one it picks runs. At most HOS_VCORE_SPIN_MAX threads spin at once (vc_spin_most: by default a quarter of the logical processors beyond 8, at most 4: none on 8 or fewer, 1 on 12, 2 on 16), so that the spin takes only processors the game leaves idle (5.4-6.1 of 16 busy at the Market) and a busy PC does not lose. tests/test_vcore_spin.py */ static int vc_spin_most(int logical) { const int n = logical > 8 ? (logical - 8) / 4 : 0; return n > 4 ? 4 : n; } static int vc_spinning; /* (threads in vc_spin now) */ static int vc_spin_max(void) { static int most = -1; if (most < 0) { const char *e = getenv("HOS_VCORE_SPIN_MAX"); SYSTEM_INFO si; GetSystemInfo(&si); const int n = e ? atoi(e) : vc_spin_most((int)si.dwNumberOfProcessors); most = n < 0 ? 0 : n; } return most; } /* holding vc_lock: lets it go and spins until t holds a core or ns pass, then takes it again; 0: it did not spin (HOS_VCORE_SPIN_MAX threads spin already) */ static int vc_spin(HosThread *t, int64_t ns) { if (__atomic_add_fetch(&vc_spinning, 1, __ATOMIC_RELAXED) > vc_spin_max()) { __atomic_sub_fetch(&vc_spinning, 1, __ATOMIC_RELAXED); vc_st.spin_capped++; return 0; } ReleaseSRWLockExclusive(&vc_lock); const int64_t s0 = hos_now_ns(), end = s0 + ns; while (!__atomic_load_n(&t->vc_held, __ATOMIC_ACQUIRE) && hos_now_ns() < end) _mm_pause(); const int64_t spun = hos_now_ns() - s0; __atomic_sub_fetch(&vc_spinning, 1, __ATOMIC_RELAXED); AcquireSRWLockExclusive(&vc_lock); vc_st.spin_ns += spun; return 1; } static void vc_acquire(HosThread *t) { static int64_t esc_better = -1, esc_eq, slice, spin; if (esc_better < 0) { esc_eq = vc_env_ns("HOS_VCORE_ESCAPE_EQ_US", 250000); { /* ((Triforce) the spin, vc_spin: HOS_VCORE_SPIN_US, up to a second; off where no thread may spin) */ const int64_t s = vc_env_ns("HOS_VCORE_SPIN_US", 250); spin = !vc_spin_max() || s <= 0 ? 0 : s > 1000000000LL ? 1000000000LL : s; static int told; if (!told++) hos_log("vcores: a waiter spins up to %lld us for a core it asked a holder for before it sleeps " "(HOS_VCORE_SPIN_US; 0: off), %d threads at most at once (HOS_VCORE_SPIN_MAX)", (long long)(spin / 1000), vc_spin_max()); } esc_better = vc_env_ns("HOS_VCORE_ESCAPE_US", 250000); slice = vc_env_ns("HOS_VCORE_SLICE_US", 2000); } AcquireSRWLockExclusive(&vc_lock); vc_st.takes++; int c = vc_free_for(t); if (c >= 0) { vc_take(t, c); ReleaseSRWLockExclusive(&vc_lock); return; } const int64_t t0 = hos_now_ns(); t->vc_since = t0; t->vc_queued = 1; if (vc_nq < 1024) vc_queue[vc_nq++] = t; vc_st.waits++; int spun = 0, slept = 0, ended = 0; /* ((Triforce) the spin tried, a sleep taken, the wait ended in the spin) */ for (;;) { /* the worst holder worse than t on a core t may use gives it up at its next checkpoint; after HOS_VCORE_SLICE_US (2000) of waiting, a holder as good as t does too (a time slice: Horizon would leave it running, but its threads never run two at once on one core, which the game's per-core data needs). The holder sees the request at its next nn::os call or SVC, or at a loop header (the interrupt polls) */ const unsigned m = vc_allowed(t); HosThread *worst = NULL; for (int k = 0; k < vc_cores(); k++) { HosThread *h = vc_owner[k]; if ((m >> k & 1) && h && h->priority > t->priority && (!worst || h->priority > worst->priority)) worst = h; } if (!worst && hos_now_ns() - t0 > slice) for (int k = 0; k < vc_cores() && !worst; k++) if ((m >> k & 1) && vc_owner[k] && vc_owner[k]->priority == t->priority) worst = vc_owner[k]; if (worst && !worst->vc_kick) { worst->vc_kick = worst->priority > t->priority ? 1 : 2; Ctx *rc = worst->run_ctx; /* (a fiber's context while it runs one) */ __atomic_store_n(rc ? &rc->poll : &worst->ctx.poll, 1, __ATOMIC_RELAXED); vc_st.kicks++; } /* (Triforce) a holder was asked for a core t may use: a spin for it first, once a wait (vc_spin). A release in the spin granted t its core (vc_held), the only one it then holds; else a core freed that no waiter took */ if (worst && spin && !spun) { spun = 1; if (vc_spin(t, spin)) { if (t->vc_held) { ended = 1; break; } if ((c = vc_free_for(t)) >= 0) { vc_dequeue(t); vc_take(t, c); ended = 1; break; } } } slept = 1; SleepConditionVariableSRW(&t->vc_cv, &vc_lock, 1, 0); if (t->vc_held) break; /* granted by a release */ if ((c = vc_free_for(t)) >= 0) { vc_dequeue(t); vc_take(t, c); break; } const int64_t waited = hos_now_ns() - t0; if (waited > (worst ? esc_better : esc_eq)) { vc_dequeue(t); vc_st.escapes++; static int logged; if (logged < 40) { logged++; char who[256] = ""; int len = 0; for (int k = 0; k < vc_cores() && len < 200; k++) if (m >> k & 1) len += snprintf(who + len, sizeof who - len, " core %d: %s (%d)%s;", k, vc_owner[k] ? vc_owner[k]->name : "-", vc_owner[k] ? vc_owner[k]->priority : 0, vc_owner[k] && vc_owner[k]->vc_kick ? " asked" : ""); hos_log("vcores: %s (priority %d) ran without a core after %.1f ms:%s", t->name, t->priority, waited / 1e6, who); } break; } } const int64_t now = hos_now_ns(), waited = now - t0; vc_st.wait_ns += waited; if (waited > vc_st.max_ns) vc_st.max_ns = waited; if (ended) vc_st.spun++; /* ((Triforce) every wait ends in the spin or after a sleep) */ else if (slept) vc_st.slept++; if (t->vc_grant_at) { /* ((Triforce) granted by a release: from the grant until it runs) */ const int64_t run = now - t->vc_grant_at; if (ended) vc_st.run_spin_ns += run, vc_st.run_spin_n++; else vc_st.run_sleep_ns += run, vc_st.run_sleep_n++; if (run > vc_st.run_max_ns) vc_st.run_max_ns = run; t->vc_grant_at = 0; } vc_stats_locked(now); ReleaseSRWLockExclusive(&vc_lock); } static void vc_release(HosThread *t) { if (!t->vc_held) return; AcquireSRWLockExclusive(&vc_lock); const int c = t->vc_held - 1; vc_owner[c] = NULL; t->vc_held = 0; t->vc_kick = 0; vc_grant(c); ReleaseSRWLockExclusive(&vc_lock); } /* gives the core to a waiter allowed on it that is better than t (strict) or as good (a yield) */ static void vc_yield(HosThread *t, int strict) { if (!t->vc_held) return; /* (Triforce, NX2navi's yield fast path: runtime/kernel.c SleepThread's yield, chrg6 745d48bc) nobody waits for a core and no waiter asked t for its own: no lock taken, as most yields find (the game's yield loops pay this thousands of times a second). The look is racy as NX2navi's: a thread that queues right after it is seen by t's next yield, and asks t itself meanwhile (vc_acquire's kick: at once when better, after HOS_VCORE_SLICE_US when as good); a request already made (vc_kick) takes the lock as before */ if (!__atomic_load_n(&vc_nq, __ATOMIC_RELAXED) && !t->vc_kick) return; AcquireSRWLockExclusive(&vc_lock); const int c = t->vc_held - 1; int any = 0; for (int i = 0; i < vc_nq && !any; i++) { HosThread *w = vc_queue[i]; any = (vc_allowed(w) >> c & 1) && (strict ? w->priority < t->priority : w->priority <= t->priority); } t->vc_kick = 0; if (any) vc_st.yields++; ReleaseSRWLockExclusive(&vc_lock); if (!any) return; vc_release(t); vc_acquire(t); } int hos_vcore_away(void) { HosThread *t = cur; if (!hos_vcores_on() || !t || !t->vc_held) return 0; vc_release(t); return 1; } void hos_vcore_back(void) { HosThread *t = cur; if (hos_vcores_on() && t && !t->vc_held) vc_acquire(t); } /* Lifted code calls this from its interrupt polls with the loop's guest registers still in host registers (a64_poll_cur, mkw_glue.c: the poll hands over no context, so the compiler need not spill them). It, and everything it calls, must never read or write the thread's guest context (Ctx) or run guest code: what it would see there may be older than the loop. A stall dump of a thread spinning in a polled loop shows such older registers for the same reason. */ void hos_vcore_checkpoint(void) { HosThread *t = cur; if (t && t->vc_kick) vc_yield(t, t->vc_kick == 1); } void hos_vcore_yield(void) { HosThread *t = cur; if (hos_vcores_on() && t) vc_yield(t, 0); } /* SleepConditionVariableSRW for guest threads: the core given up for the wait; after it the lock is let go while a core is taken again (waiting for a core while holding it could stall the thread that will wake the next waiter), so callers see a spurious wake-up and look at their condition again, as they do anyway */ int hos_cv_sleep(void *cv, void *lk, unsigned ms) { const int away = hos_vcore_away(); const BOOL r = SleepConditionVariableSRW((PCONDITION_VARIABLE)cv, (PSRWLOCK)lk, ms, 0); if (away) { ReleaseSRWLockExclusive((PSRWLOCK)lk); hos_vcore_back(); AcquireSRWLockExclusive((PSRWLOCK)lk); } return r; } #else int hos_vcores_on(void) { return 0; } int hos_vcore_away(void) { return 0; } void hos_vcore_back(void) {} void hos_vcore_checkpoint(void) {} void hos_vcore_yield(void) {} int hos_cv_sleep(void *cv, void *lk, unsigned ms) { return SleepConditionVariableSRW((PCONDITION_VARIABLE)cv, (PSRWLOCK)lk, ms, 0); } #endif