mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 02:49:57 -04:00
Give back what a host thread keeps for itself when it ends.
Every guest thread is a host thread, and the game makes and ends them all through play. A thread that slept through plat_wait_ns kept a waitable timer, one that read or wrote a file an event for each, and one that decoded zstd through the host a decoder context of about 100 KiB; nothing closed or freed them when the thread ended, and a detached thread's own record stayed allocated. They are now given back when the thread's function returns: the handles in win32.c thread_main, the decoder context through plat_thread_end_hook, and the record by whichever of the thread and its detach comes last. tests/thread_end_check.py runs 416 such threads and compares the process's handle count before and after. tests/sync_wake_check.c gets the two stand-ins kernel.c has needed since the service wait log, so that check builds and passes again. The Linux file only gains the hook call and was not compiled here. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -395,6 +395,11 @@ waits on its event and leaves the others asleep, a wait on several handles repor
|
||||
the wait on it, timed waits keep their time, a cancel ends any wait, and 4,000 signals to eight looping waiters are
|
||||
each seen. It runs a second time with `NX2NAVI_SYNC_WAKE_ALL=1`. No game files.
|
||||
|
||||
A change to what a host thread gives back when it ends (`runtime/platform/win32.c` `thread_main`,
|
||||
`plat_thread_end_hook`) passes `python tests\thread_end_check.py`, which builds the platform layer alone: 416 threads
|
||||
sleep, read a file and end, half joined and half detached; the end hook runs once on each, and on Windows the process
|
||||
holds as many handles afterwards as before (each thread's sleep timer and read event are closed). No game files.
|
||||
|
||||
A change to the log queue (`runtime/logq.c`) passes `python tests\logq_check.py`, which builds it with the platform
|
||||
layer and a destination of its own: 160,000 lines from eight threads arrive whole and in each thread's order, a line
|
||||
logged while the queue's lock is taken waits and does not pass the lines queued before it, a destination that takes
|
||||
|
||||
@@ -113,10 +113,19 @@ static ZSTD_DDict *dict_for(uint64_t guest, uint64_t content, uint64_t size) {
|
||||
return dd;
|
||||
}
|
||||
|
||||
// A thread's one-shot decoder context (host_decode), freed when the thread ends (plat_thread_end_hook): the game
|
||||
// makes and ends threads all through play, and each that had decoded left its context behind, about 100 KiB.
|
||||
static __thread ZSTD_DCtx *dctx;
|
||||
static volatile int32_t dctx_hooked;
|
||||
|
||||
static void dctx_thread_end(void) {
|
||||
if (dctx) ZSTD_freeDCtx(dctx);
|
||||
dctx = NULL;
|
||||
}
|
||||
|
||||
// Decodes on this thread's host context. Returns libzstd's result; *ok 0 when the inputs cannot be taken.
|
||||
static size_t host_decode(uint64_t dst, uint64_t cap, uint64_t src, uint64_t size, uint64_t dict, uint64_t dict_size,
|
||||
uint64_t ddict, int *ok) {
|
||||
static __thread ZSTD_DCtx *dctx;
|
||||
*ok = 0;
|
||||
if ((cap && !dst) || !src || src + size > RT_ADDRESS_LIMIT) return 0; // dst: decode() checks a guest one
|
||||
ZSTD_DDict *dd = NULL;
|
||||
@@ -139,7 +148,10 @@ static size_t host_decode(uint64_t dst, uint64_t cap, uint64_t src, uint64_t siz
|
||||
} else if (dict && dict_size) {
|
||||
if (!(dd = dict_for(0, dict, dict_size))) return 0;
|
||||
}
|
||||
if (!dctx && !(dctx = ZSTD_createDCtx())) return 0;
|
||||
if (!dctx) {
|
||||
if (!plat_xchg32(&dctx_hooked, 1)) plat_thread_end_hook(dctx_thread_end);
|
||||
if (!(dctx = ZSTD_createDCtx())) return 0;
|
||||
}
|
||||
*ok = 1;
|
||||
return dd ? ZSTD_decompress_usingDDict(dctx, gp(dst), (size_t)cap, gp(src), (size_t)size, dd)
|
||||
: ZSTD_decompressDCtx(dctx, gp(dst), (size_t)cap, gp(src), (size_t)size);
|
||||
|
||||
@@ -159,3 +159,20 @@ int plat_addr_str(const PlatAddr *a, char *out, size_t cap) {
|
||||
memcpy(out, s, (size_t)n + 1);
|
||||
return n;
|
||||
}
|
||||
|
||||
// ---- what a thread keeps for itself (platform.h plat_thread_end_hook)
|
||||
static void (*end_hooks[PLAT_THREAD_END_HOOKS])(void);
|
||||
static volatile int32_t n_end_hooks;
|
||||
|
||||
void plat_thread_end_hook(void (*fn)(void)) {
|
||||
int32_t i = __atomic_fetch_add(&n_end_hooks, 1, __ATOMIC_ACQ_REL);
|
||||
if (i < PLAT_THREAD_END_HOOKS) __atomic_store_n(&end_hooks[i], fn, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
void plat_thread_end_run(void) {
|
||||
int32_t n = __atomic_load_n(&n_end_hooks, __ATOMIC_ACQUIRE);
|
||||
for (int32_t i = 0; i < n && i < PLAT_THREAD_END_HOOKS; i++) {
|
||||
void (*fn)(void) = __atomic_load_n(&end_hooks[i], __ATOMIC_ACQUIRE);
|
||||
if (fn) fn();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,6 +114,7 @@ static void *thread_main(void *p) {
|
||||
__atomic_store_n(&t->tid, (int32_t)syscall(SYS_gettid), __ATOMIC_RELEASE);
|
||||
if (t->name[0]) pthread_setname_np(pthread_self(), t->name);
|
||||
t->fn(t->arg);
|
||||
plat_thread_end_run(); // what the other files keep per thread (platform.h plat_thread_end_hook)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,11 @@ void plat_thread_join(PlatThread *t); // waits for fn to return, then frees
|
||||
// Waits at most ns for fn to return: 1 when it did (t is freed), 0 when it still runs (t stays valid).
|
||||
int plat_thread_join_ns(PlatThread *t, uint64_t ns);
|
||||
void plat_thread_detach(PlatThread *t); // frees t; the thread runs on
|
||||
// fn runs on every thread these calls make, when its function has returned and before the thread ends: for what a
|
||||
// thread keeps for itself alone (a decoder context). At most PLAT_THREAD_END_HOOKS, each added once.
|
||||
#define PLAT_THREAD_END_HOOKS 4
|
||||
void plat_thread_end_hook(void (*fn)(void));
|
||||
void plat_thread_end_run(void); // the platform files' thread start functions call it
|
||||
// Restricts t (NULL: the calling thread) to the logical processors in mask (bit n = processor n of the first group).
|
||||
// Returns the previous mask, 0 on failure.
|
||||
uint64_t plat_thread_affinity(PlatThread *t, uint64_t mask);
|
||||
|
||||
@@ -31,11 +31,23 @@ struct PlatThread {
|
||||
HANDLE h;
|
||||
PlatThreadFn fn;
|
||||
void *arg;
|
||||
volatile int32_t state; // 0 running and owned by its maker, 1 detached, 2 its function has returned
|
||||
};
|
||||
|
||||
// What a thread made here keeps for itself is given back when its function returns: the handles below (the sleep
|
||||
// timer of plat_wait_ns, the events of plat_file_pread and plat_file_pwrite), what the other files keep per thread
|
||||
// (plat_thread_end_hook) and, for a detached thread, its PlatThread. The game makes and ends threads all through
|
||||
// play, each a host thread: nothing closed them, so every ended thread that had slept, read or written left up
|
||||
// to three kernel handles behind for the rest of the run. A guest thread ends by returning here too (kernel.c:
|
||||
// ExitThread jumps back to the thread's start).
|
||||
static void thread_handles_close(void);
|
||||
|
||||
static DWORD WINAPI thread_main(LPVOID p) {
|
||||
PlatThread *t = p;
|
||||
t->fn(t->arg);
|
||||
plat_thread_end_run();
|
||||
thread_handles_close();
|
||||
if (plat_xchg32(&t->state, 2) == 1) free(t); // detached while it ran: nobody else frees it
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -73,10 +85,12 @@ int plat_thread_join_ns(PlatThread *t, uint64_t ns) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// The thread keeps using t->fn and t->arg, so the struct stays allocated (a few bytes per detached thread).
|
||||
// The thread still reads t->fn and t->arg when it starts, so the struct goes when the thread's function has
|
||||
// returned (thread_main), or here when it already has.
|
||||
void plat_thread_detach(PlatThread *t) {
|
||||
CloseHandle(t->h);
|
||||
t->h = NULL;
|
||||
if (plat_xchg32(&t->state, 1) == 2) free(t);
|
||||
}
|
||||
|
||||
int plat_cpu_cores(uint64_t *all, uint64_t *first) {
|
||||
@@ -474,13 +488,22 @@ uint64_t plat_now_ns(void) {
|
||||
return plat_scaled(qpc_ns, (uint64_t)t.QuadPart);
|
||||
}
|
||||
|
||||
// A high-resolution waitable timer per sleeping thread (Windows 10 1803+; one handle per thread, kept until exit)
|
||||
// A high-resolution waitable timer per sleeping thread (Windows 10 1803+; one handle per thread, closed when the
|
||||
// thread's function returns, thread_handles_close)
|
||||
// wakes within about 0.1-0.5 ms; the last PLAT_SPIN_NS are spent yielding so the deadline is met closely.
|
||||
// Without that timer kind, Sleep(1) granularity applies.
|
||||
#define PLAT_SPIN_NS 300000u
|
||||
|
||||
static __thread HANDLE sleep_timer;
|
||||
static __thread int sleep_timer_tried;
|
||||
static __thread HANDLE pread_ev, pwrite_ev; // plat_file_pread, plat_file_pwrite: the event of their overlapped call
|
||||
|
||||
static void thread_handles_close(void) {
|
||||
if (sleep_timer) CloseHandle(sleep_timer);
|
||||
if (pread_ev) CloseHandle(pread_ev);
|
||||
if (pwrite_ev) CloseHandle(pwrite_ev);
|
||||
sleep_timer = pread_ev = pwrite_ev = NULL, sleep_timer_tried = 0;
|
||||
}
|
||||
|
||||
void plat_wait_ns(uint64_t ns) {
|
||||
if (!sleep_timer_tried) {
|
||||
@@ -1302,8 +1325,8 @@ void plat_file_close(PlatFile *f) {
|
||||
}
|
||||
|
||||
int64_t plat_file_pread(PlatFile *f, void *buf, uint32_t n, uint64_t off) {
|
||||
static __thread HANDLE ev;
|
||||
if (!ev && !(ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1;
|
||||
HANDLE ev = pread_ev;
|
||||
if (!ev && !(ev = pread_ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1;
|
||||
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev};
|
||||
DWORD got = 0;
|
||||
if (ReadFile((HANDLE)f, buf, n, &got, &ov)) return got;
|
||||
@@ -1561,8 +1584,8 @@ PlatFile *plat_fs_open(const char *path, int write, int *err) {
|
||||
PlatFile *plat_fs_create(const char *path, int *err) { return fs_open(path, GENERIC_WRITE, 0, CREATE_NEW, err); }
|
||||
|
||||
int64_t plat_file_pwrite(PlatFile *f, const void *buf, uint32_t n, uint64_t off) {
|
||||
static __thread HANDLE ev;
|
||||
if (!ev && !(ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1;
|
||||
HANDLE ev = pwrite_ev;
|
||||
if (!ev && !(ev = pwrite_ev = CreateEventW(NULL, TRUE, FALSE, NULL))) return -1;
|
||||
OVERLAPPED ov = {.Offset = (DWORD)off, .OffsetHigh = (DWORD)(off >> 32), .hEvent = ev};
|
||||
DWORD put = 0;
|
||||
if (WriteFile((HANDLE)f, buf, n, &put, &ov)) return put;
|
||||
|
||||
@@ -12,7 +12,8 @@ static int fails, wake_all;
|
||||
// ---- the runtime kernel.c reaches
|
||||
unsigned rt_trace;
|
||||
uint64_t rt_guest_start_ns;
|
||||
volatile uint32_t prof_replay_tid;
|
||||
volatile uint32_t prof_replay_tid, prof_main_tid;
|
||||
int mem_readable(uint64_t a, uint64_t n) { (void)a, (void)n; return 0; } // NX2NAVI_SVC_WAITS walks guest frames: off here
|
||||
const char *cfg_get(const char *name) { return !strcmp(name, "NX2NAVI_SYNC_WAKE_ALL") && wake_all ? "1" : NULL; }
|
||||
const char *cfg_value(const char *name) { (void)name; return "0"; } // no asynchronous preemption, no slow-service log
|
||||
int cfg_bool(const char *name) { (void)name; return 0; }
|
||||
|
||||
91
tests/thread_end_check.c
Normal file
91
tests/thread_end_check.c
Normal file
@@ -0,0 +1,91 @@
|
||||
// runtime/platform: what a thread keeps for itself is given back when its function returns (win32.c thread_main,
|
||||
// platform.h plat_thread_end_hook). Threads that sleep through plat_wait_ns and read a file are made and ended by the
|
||||
// hundred, joined and detached; the end hook must run once on each, and on Windows the process must not hold more
|
||||
// handles afterwards than a few threads' worth. Built and run by tests/thread_end_check.py.
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "platform/platform.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
static long handles(void) {
|
||||
DWORD n = 0;
|
||||
return GetProcessHandleCount(GetCurrentProcess(), &n) ? (long)n : -1;
|
||||
}
|
||||
#else
|
||||
static long handles(void) { return 0; } // no handle per thread there; the hook is what is checked
|
||||
#endif
|
||||
|
||||
static int fails;
|
||||
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
|
||||
|
||||
void rt_log(const char *fmt, ...) { // the platform layer's lines
|
||||
va_list ap;
|
||||
va_start(ap, fmt);
|
||||
vfprintf(stderr, fmt, ap);
|
||||
va_end(ap);
|
||||
fputc('\n', stderr);
|
||||
}
|
||||
|
||||
static const char *path;
|
||||
static volatile int32_t ended, hook_runs, hook_other;
|
||||
static __thread int worked;
|
||||
|
||||
static void hook(void) {
|
||||
if (worked) plat_inc32(&hook_runs);
|
||||
else plat_inc32(&hook_other);
|
||||
worked = 0;
|
||||
}
|
||||
|
||||
static void worker(void *arg) {
|
||||
(void)arg;
|
||||
worked = 1;
|
||||
plat_wait_ns(1200000); // over a millisecond: the thread's own timer
|
||||
PlatFile *f = plat_file_open(path);
|
||||
char b[16];
|
||||
if (!f || plat_file_pread(f, b, sizeof b, 0) != (int64_t)sizeof b) plat_inc32(&hook_other); // counted as a failure
|
||||
if (f) plat_file_close(f);
|
||||
plat_inc32(&ended);
|
||||
}
|
||||
|
||||
static void wait_ended(int32_t want) {
|
||||
for (int i = 0; i < 20000 && __atomic_load_n(&ended, __ATOMIC_ACQUIRE) < want; i++) plat_sleep_ms(1);
|
||||
plat_sleep_ms(100); // the last ones leave their start function and the system lets go of them
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc != 2) {
|
||||
printf("usage: thread_end_check FILE_OF_16_BYTES_OR_MORE\n");
|
||||
return 2;
|
||||
}
|
||||
path = argv[1];
|
||||
plat_thread_end_hook(hook);
|
||||
enum { WARM = 16, ROUNDS = 25, PER = 16 };
|
||||
for (int i = 0; i < WARM; i++) plat_thread_spawn(worker, NULL, "warm", 0);
|
||||
wait_ended(WARM);
|
||||
long h0 = handles();
|
||||
int32_t made = WARM;
|
||||
for (int r = 0; r < ROUNDS; r++) {
|
||||
PlatThread *t[PER / 2];
|
||||
for (int i = 0; i < PER / 2; i++) {
|
||||
t[i] = plat_thread_new(worker, NULL, "joined");
|
||||
made += t[i] != NULL;
|
||||
made += plat_thread_spawn(worker, NULL, "detached", 0);
|
||||
}
|
||||
for (int i = 0; i < PER / 2; i++)
|
||||
if (t[i]) plat_thread_join(t[i]);
|
||||
wait_ended(made);
|
||||
}
|
||||
long h1 = handles();
|
||||
CHECK(made == WARM + ROUNDS * PER, "%d threads made of %d", (int)made, WARM + ROUNDS * PER);
|
||||
CHECK(__atomic_load_n(&ended, __ATOMIC_ACQUIRE) == made, "%d of %d threads ended", (int)ended, (int)made);
|
||||
CHECK(hook_runs == made, "the end hook ran on %d of %d threads", (int)hook_runs, (int)made);
|
||||
CHECK(!hook_other, "%d threads could not read the file, or ran the hook without having worked", (int)hook_other);
|
||||
// each thread made a timer and a read event: left open, 400 threads would hold 800 more handles
|
||||
CHECK(h1 - h0 < 64, "%ld handles more after %d ended threads (%ld before, %ld after)", h1 - h0, ROUNDS * PER, h0, h1);
|
||||
printf("thread_end_check: %s (%d threads, the hook ran on %d, handles %ld before and %ld after)\n",
|
||||
fails ? "FAILED" : "ok", (int)made, (int)hook_runs, h0, h1);
|
||||
return fails ? 1 : 0;
|
||||
}
|
||||
48
tests/thread_end_check.py
Normal file
48
tests/thread_end_check.py
Normal file
@@ -0,0 +1,48 @@
|
||||
"""Build and run tests/thread_end_check.c: what a thread keeps for itself is given back when it ends
|
||||
(runtime/platform/win32.c thread_main, platform.h plat_thread_end_hook). 416 threads sleep, read a file and end, half
|
||||
joined and half detached; the end hook must run once on each, and on Windows the process's handle count must be what
|
||||
it was. No game files, no GPU.
|
||||
|
||||
Usage: thread_end_check.py
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
|
||||
def compiler():
|
||||
if os.environ.get('CC'):
|
||||
return os.environ['CC']
|
||||
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
|
||||
return str(MSYS_BIN / 'clang.exe')
|
||||
return shutil.which('clang') or 'clang'
|
||||
|
||||
|
||||
def main():
|
||||
work = ROOT / 'build' / 'test' / 'thread_end'
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
win = os.name == 'nt'
|
||||
exe = work / ('thread_end_check.exe' if win else 'thread_end_check')
|
||||
env = dict(os.environ)
|
||||
if win and MSYS_BIN.exists():
|
||||
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
|
||||
rt = ROOT / 'runtime'
|
||||
plat = [rt / 'platform' / ('win32.c' if win else 'linux.c'), rt / 'platform' / 'common.c']
|
||||
libs = ['-lbcrypt', '-lws2_32', '-lpsapi', '-lwinmm', '-ldbghelp', '-lshell32', '-lole32', '-luser32', '-ladvapi32'] if win \
|
||||
else ['-lpthread']
|
||||
subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-fno-strict-aliasing',
|
||||
'-march=x86-64-v3', '-I', str(rt),
|
||||
str(ROOT / 'tests' / 'thread_end_check.c'), *map(str, plat), '-o', str(exe), '-lm', *libs],
|
||||
env=env)
|
||||
data = work / 'read.bin'
|
||||
data.write_bytes(bytes(range(64)))
|
||||
return subprocess.run([str(exe), str(data)], env=env, timeout=300).returncode
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user