Put the process's OS handle count on the sizes line and add a handle census by kind behind NX2NAVI_HANDLE_CENSUS.

A session of 2026-11-13 grew from 683 to 1095 OS handles in 11 minutes, rising at every area load and never falling,
while the guest's own handles stayed at 116-122 and the threads flat; the VMA block count fell back while the handle
count did not. The log could not show it: the 600-frame "mem: sizes" line had only the guest's handles, and nothing
told which kind of handle grows.

kernel.c kern_sizes now adds "process handles" (plat_handle_count, one system call every 600 frames under trace=perf)
beside the guest's "handles". platform.h gets plat_handle_census: on Windows the system's extended handle table
filtered by our process id, with the type names from NtQueryObject's ObjectTypesInformation; on Linux the
/proc/self/fd links grouped by kind. It never opens, duplicates or names a handle (a name query can block on a pipe).
With NX2NAVI_HANDLE_CENSUS=1 (default 0, read at start) and trace=perf, perf.c starts one long-lived thread that the
600-frame report wakes; when the count grew by 16 or more since its last census it logs "mem: handles frame N: T in
all (+D since the last census): Event a (+x), ...". The census reads a few MiB of the system's table in 2-5 ms, off
the drawing threads; one thread for the run keeps a new thread's own handle out of what it counts. New cfg rows:
HANDLE_CENSUS in known[], sections[] (diagnostics) and metas[] ("0", "0|1", restart), and an nx2navi.ini.example
entry.

New tests/handle_census_check.py builds the platform layer alone: 20 events and 5 sections show as Event +20 and
Section +5, the total stays within 2 of GetProcessHandleCount, the figures return once they are closed, and kinds
past the room given go into the last entry; docs/building.md describes it. Not reproduced before the change (the
growth needs a game run). handle_census_check, sync_wake_check and the cfg row check pass; every runtime file compiles
with the dev flags, and perf.c, kernel.c, win32.c and cfg.c add no -Wall -Wextra warnings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-13 17:11:52 -05:00
parent baf83fda6a
commit 26b7cfd658
10 changed files with 360 additions and 2 deletions

View File

@@ -61,6 +61,7 @@ static const char *const known[] = {
"SKIN_MOTION", // vk_skin_motion.inc
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
"MV_SURVEY",
"HANDLE_CENSUS",
"PIPE_SEED_BOOT", "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_OCCLUSION_OFF", "PATCH_WIDE_SAVE", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_SAME_BIG", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_HARD_ORDER", "TEX_STALE_KEPT", "ASTC_SCRATCH_TRIM", "PRESENT_HELD", "HELD_SYNC",
"PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_FULL_S", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW",
"POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS",
@@ -167,6 +168,7 @@ static const struct { const char *key, *section; } sections[] = {
{"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MEM_CENSUS", "diagnostics"}, {"WB_NO_REUP", "diagnostics"}, {"WB_PROBE", "diagnostics"}, {"WB_SPLIT", "performance"}, {"SHADOW_BULK_WRITE", "performance"}, {"WB_LAZY", "performance"}, {"HEAP_LAZY", "performance"}, {"BARRIER_GAME", "performance"}, {"MESH_AHEAD", "performance"}, {"MESH_FALLBACK", "performance"}, {"MESH_FALLBACK_MB", "performance"}, {"MESH_FALLBACK_WORKERS", "performance"}, {"WB_LAZY_TOUCH", "diagnostics"}, {"WB_LAZY_CHUNK", "performance"}, {"FLUSH_UPLOAD", "diagnostics"}, {"GPA_TWIN_MAX_KB", "diagnostics"}, {"MIRROR_CAPTURE_KB", "diagnostics"}, {"WB_SKIP_MB", "diagnostics"},
{"WB_MERGE_MAX", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, {"TEXUP_LOG_MS", "diagnostics"},
{"WAIT_WATCH_S", "diagnostics"},
{"HANDLE_CENSUS", "diagnostics"},
{"VMA_BLOCK_MB", "diagnostics"}, {"DEVICE_LOST_AT", "diagnostics"}, {"STAGING_FAIL_AT", "diagnostics"}, {"VRAM_BALLAST_MB", "diagnostics"},
{"MEM_BUDGET_TEST_VRAM_MB", "diagnostics"}, {"MEM_BUDGET_TEST_RAM_MB", "diagnostics"},
{"TEX_COMPACT", "performance"}, {"TEX_COMPACT_MIN_MB", "performance"}, {"TEX_COMPACT_FRAME_MB", "performance"},
@@ -422,6 +424,8 @@ static const CfgMeta metas[] = {
{"MIRROR_CAPTURE_KB", "0", "0..1048576", RESTART}, {"WB_SKIP_MB", "0", "0..4096", RESTART}, // vk_memory.c capture_cap, wb_skip
{"WB_MERGE_MAX", "32", "1..256", RESTART}, {"TEXUP_LOG_MS", "0", "0..60000", RESTART}, // vk_texture.c vk_texstat_frame
{"WAIT_WATCH_S", "0", "0..3600", RESTART}, // kernel.c wait_watch: a lock or thread-end wait this long is logged; 0 off
// perf.c hc_main: with NX2NAVI_TRACE=perf, the process's handles by kind when their count grew
{"HANDLE_CENSUS", "0", "0|1", RESTART},
{"VMA_BLOCK_MB", "64", "16..256", RESTART}, // vk_headroom.c vk_vma_block_size
// vk_texture.c compact_on: idle textures move out of sparsely used blocks of video memory; the free space inside
// the blocks from which a pass starts, and the bytes moved per frame (also the largest texture that moves)

View File

@@ -2447,7 +2447,8 @@ int kern_svc(Cpu *c, uint32_t imm) {
}
// NX2NAVI_TRACE=perf "mem: sizes" (perf.c): guest threads, the handle table, the system-event table and fibers, each
// against its fixed size (the tables are fatal when full).
// against its fixed size (the tables are fatal when full). Beside the guest's handles, the OS handles of the process
// (platform.h plat_handle_count): they grew at every area load of a session while the guest's stayed flat.
void kern_sizes(PerfSizes *s) {
uint64_t threads = 0, handles = 0, fibers = 0, ses;
plat_mutex_lock_shared(&G);
@@ -2462,6 +2463,7 @@ void kern_sizes(PerfSizes *s) {
plat_mutex_unlock_shared(&g_fiber_lock);
perf_size(s, "guest threads", threads, 0, PERF_SIZE_NONE);
perf_size(s, "handles", handles, MAX_H, PERF_SIZE_NONE);
perf_size(s, "process handles", plat_handle_count(), 0, PERF_SIZE_NONE);
perf_size(s, "system events", ses, sizeof g_se / sizeof *g_se, PERF_SIZE_NONE);
perf_size(s, "fibers", fibers, FIBER_MAX, PERF_SIZE_NONE);
}

View File

@@ -57,6 +57,7 @@ uint64_t perf_gpu_busy_total(void) { return (uint64_t)__atomic_load_n(&gpu_busy_
// through exit() rather than plat_exit, whose log sync this is, gets the lines of the frame-time summary, the stutter
// map and everything queued before them into the log instead of losing them with the log thread's queue.
static void log_sync_at_exit(void) { rt_log_sync(); }
static void hc_start(void);
void prof_start(void) {
flight_init();
@@ -65,6 +66,7 @@ void prof_start(void) {
atexit(perf_summary_flush); // boot.c's _exit path goes through prof_write instead
perf_base = (rt_trace & RT_TRACE_PERF) || flight_on;
perf_on = perf_base || overlay == 2;
hc_start();
if (prof_sampler_start()) atexit(prof_write); // a --seconds run ends in _exit: boot.c calls prof_write itself first
}
@@ -242,6 +244,75 @@ static void sizes_log(uint64_t frame) {
}
}
// ---- NX2NAVI_HANDLE_CENSUS=1 with NX2NAVI_TRACE=perf: the process's OS handles by kind (platform.h plat_handle_census)
// at the 600-frame report, when their count (the sizes line's "process handles") grew by HC_GROWTH or more since the
// last census. In a session of 2026-11-13 that count rose at every area load, 683 to 1095 in 11 minutes, and never
// fell, while the guest's handles stayed at 116-122: this says which kind grows. The census reads the system's whole
// handle table (2-5 ms and a buffer of a few MiB), so it runs on one thread of its own, started with the trace and
// woken by the report; a new thread per census would add its own Thread handle to what it counts.
#define HC_GROWTH 16
#define HC_KINDS 64
static PlatEvent hc_ev;
static volatile uint64_t hc_frame; // the report's frame, for the line
static int hc_on;
static int hc_cmp(const void *x, const void *y) {
const PlatHandleKind *a = x, *b = y;
return a->n < b->n ? 1 : a->n > b->n ? -1 : strcmp(a->name, b->name);
}
static void hc_main(void *arg) {
(void)arg;
static PlatHandleKind now[HC_KINDS], last[HC_KINDS];
int n_last = -1;
uint32_t count_last = 0, total_last = 0;
for (;;) {
plat_event_wait(&hc_ev);
uint64_t frame = __atomic_load_n(&hc_frame, __ATOMIC_RELAXED);
uint32_t count = plat_handle_count();
if (n_last >= 0 && count < count_last + HC_GROWTH) continue;
uint64_t t0 = kern_now_ns();
int n = plat_handle_census(now, HC_KINDS);
if (n < 0) {
rt_log("mem: handles frame %llu: the census by kind failed; not tried again", (unsigned long long)frame);
return;
}
double ms = (kern_now_ns() - t0) / 1e6;
qsort(now, (size_t)n, sizeof *now, hc_cmp);
uint32_t total = 0;
for (int i = 0; i < n; i++) total += now[i].n;
char line[1600];
int o = snprintf(line, sizeof line, "mem: handles frame %llu: %u in all", (unsigned long long)frame, total);
if (n_last >= 0) o += snprintf(line + o, sizeof line - (size_t)o, " (%+d since the last census)", (int)(total - total_last));
o += snprintf(line + o, sizeof line - (size_t)o, ":");
for (int i = 0; i < n && o < (int)sizeof line - 80; i++) {
o += snprintf(line + o, sizeof line - (size_t)o, "%s %s %u", i ? "," : "", now[i].name, now[i].n);
if (n_last < 0) continue;
int64_t was = 0;
for (int k = 0; k < n_last; k++)
if (!strcmp(last[k].name, now[i].name)) was = last[k].n;
if (now[i].n != was) o += snprintf(line + o, sizeof line - (size_t)o, " (%+lld)", (long long)now[i].n - was);
}
if (o < (int)sizeof line - 40) snprintf(line + o, sizeof line - (size_t)o, "; the census took %.1f ms", ms);
rt_log("%s", line);
memcpy(last, now, (size_t)n * sizeof *now);
n_last = n, count_last = count, total_last = total;
}
}
static void hc_start(void) {
if (!(rt_trace & RT_TRACE_PERF) || !cfg_bool("NX2NAVI_HANDLE_CENSUS")) return;
plat_event_init(&hc_ev, 0, 0);
hc_on = plat_thread_spawn(hc_main, NULL, "handle census", 0);
}
// The 600-frame report (perf_present): wakes the census thread, which decides whether the count grew enough.
static void hc_report(uint64_t frame) {
if (!hc_on) return;
__atomic_store_n(&hc_frame, frame, __ATOMIC_RELAXED);
plat_event_set(&hc_ev);
}
// The memory lines of a segment's end or a settled scene: the back end's (vk_mem_snapshot), then the sizes.
static void mem_snapshot(uint64_t frame) {
vk_mem_snapshot(frame);
@@ -1012,5 +1083,5 @@ void perf_present(uint64_t frame) {
sum = max = n = ab_b = 0;
memset(wait_ns, 0, sizeof wait_ns);
win_start = now;
if (frame % 600 == 0) sizes_log(frame); // beside vk_memory.c's 600-frame memory lines
if (frame % 600 == 0) sizes_log(frame), hc_report(frame); // beside vk_memory.c's 600-frame memory lines
}

View File

@@ -2257,6 +2257,47 @@ uint32_t plat_handle_count(void) {
return n ? n - 1 : 0; // less the descriptor of this walk
}
// plat_handle_census: each descriptor's link, by kind: "socket:[N]" and "pipe:[N]" by their prefix, an anonymous
// inode by its kind ("anon_inode:[eventfd]" is eventfd), a device by its /dev folder, any other path as file.
int plat_handle_census(PlatHandleKind *out, int max) {
DIR *d = max > 0 ? opendir("/proc/self/fd") : NULL;
if (!d) return -1;
int n = 0, self = dirfd(d);
for (struct dirent *e; (e = readdir(d));) {
if (e->d_name[0] == '.' || atoi(e->d_name) == self) continue;
char link[64], target[256], kind[32];
snprintf(link, sizeof link, "/proc/self/fd/%s", e->d_name);
ssize_t len = readlink(link, target, sizeof target - 1);
if (len < 0) continue; // closed meanwhile
target[len] = 0;
if (!strncmp(target, "/dev/", 5)) {
size_t k = strcspn(target + 5, "/");
snprintf(kind, sizeof kind, "/dev/%.*s", (int)(k < 20 ? k : 20), target + 5);
} else if (target[0] == '/') {
snprintf(kind, sizeof kind, "%s", "file");
} else if (!strncmp(target, "anon_inode:", 11)) {
const char *k = target + 11 + (target[11] == '[');
snprintf(kind, sizeof kind, "%.*s", (int)strcspn(k, "]"), k);
} else {
snprintf(kind, sizeof kind, "%.*s", (int)strcspn(target, ":"), target);
}
int i = 0;
while (i < n && strcmp(out[i].name, kind)) i++;
if (i == n) {
if (n == max) { // no room: the last entry takes the rest
i = max - 1;
snprintf(out[i].name, sizeof out[i].name, "%s", "(more kinds)");
} else {
snprintf(out[n].name, sizeof out[n].name, "%s", kind);
out[n++].n = 0;
}
}
out[i].n++;
}
closedir(d);
return n;
}
// glibc's mallinfo2 (2.33 and later) sums every arena: arena is what the arenas took with brk and mmap, hblkhd the
// blocks mmapped one by one, uordblks what the arenas handed out. Other C libraries leave the figures unknown.
void plat_heap_info(PlatHeapInfo *h) {

View File

@@ -538,6 +538,13 @@ int plat_page_file_managed(void);
void plat_process_memory(uint64_t *resident, uint64_t *private_bytes);
// The handles (Windows) or file descriptors (Linux) this process holds; 0 when unknown.
uint32_t plat_handle_count(void);
// The same by kind: Windows the object type of each handle (Event, Section, Thread, File...), Linux what each file
// descriptor's link names (socket, pipe, an anon_inode kind, a /dev folder, file). Fills out with up to max kinds,
// unsorted, and returns how many, -1 on failure; the kinds past max are counted in the last entry, named "(more
// kinds)". Never opens, duplicates or names a handle. Windows reads the system's whole handle table into a buffer of a
// few MiB kept between calls: one thread calls it at a time.
typedef struct { char name[32]; uint32_t n; } PlatHandleKind;
int plat_handle_census(PlatHandleKind *out, int max);
// The C runtime's malloc heap in bytes, for the memory report (vk_memory.c system_log): what it took from the system
// (committed) and what of that it handed out (used), and how many other heaps the process has (Windows: those DLLs made
// with HeapCreate, the graphics driver's among them; their sizes are not asked, as a heap made without serialization

View File

@@ -2188,6 +2188,106 @@ uint32_t plat_handle_count(void) {
return GetProcessHandleCount(GetCurrentProcess(), &n) ? (uint32_t)n : 0;
}
// plat_handle_census: the system's handle table (NtQuerySystemInformation, class 64, the extended handle information)
// filtered by this process's id, and the object types' names from NtQueryObject's ObjectTypesInformation, whose entries
// carry the type index the table uses (Windows 8.1 and later). The table holds every process's handles; a census took
// 1.5-4.7 ms over 3.6 MiB of it on the development PC.
typedef struct {
PVOID object;
ULONG_PTR pid, handle;
ULONG access;
USHORT backtrace, type;
ULONG attributes, reserved;
} HcEntry;
typedef struct {
ULONG_PTR n, reserved;
HcEntry e[1];
} HcTable;
typedef struct {
USHORT len, max;
PWSTR buf;
} HcName;
typedef struct { // one entry of ObjectTypesInformation; its name's characters follow it
HcName name;
ULONG totals[13]; // object and handle counts, pool use, their high-water marks, invalid attributes: not read
GENERIC_MAPPING mapping;
ULONG valid_access;
BOOLEAN security, maintain;
UCHAR index;
CHAR reserved;
ULONG pool, paged, nonpaged;
} HcType;
static void hc_type_names(char names[256][32]) {
typedef LONG(NTAPI * Qo)(HANDLE, ULONG, PVOID, ULONG, PULONG);
HMODULE nt = GetModuleHandleW(L"ntdll.dll");
Qo qo = nt ? (Qo)(void *)GetProcAddress(nt, "NtQueryObject") : NULL;
if (!qo) return;
size_t cap = 1u << 16;
uint8_t *buf = NULL;
LONG st = -1;
for (int tries = 0; tries < 6; tries++, cap *= 2) {
uint8_t *b = realloc(buf, cap);
if (!b) break;
buf = b;
ULONG need = 0;
st = qo(NULL, 3, buf, (ULONG)cap, &need); // ObjectTypesInformation
if (st != (LONG)0xC0000004L) break; // not STATUS_INFO_LENGTH_MISMATCH: done, or failed
}
if (st >= 0 && buf) {
ULONG count = *(ULONG *)buf;
const uint8_t *p = buf + sizeof(void *), *end = buf + cap; // the count, then entries at pointer alignment
for (ULONG i = 0; i < count && p + sizeof(HcType) <= end; i++) {
const HcType *t = (const HcType *)p;
WideCharToMultiByte(CP_UTF8, 0, t->name.buf, t->name.len / 2, names[t->index], 31, NULL, NULL);
p += sizeof(HcType) + ((t->name.max + sizeof(void *) - 1) & ~(sizeof(void *) - 1));
}
}
free(buf);
}
int plat_handle_census(PlatHandleKind *out, int max) {
typedef LONG(NTAPI * Qsi)(ULONG, PVOID, ULONG, PULONG);
static char names[256][32];
static int named;
static void *table;
static size_t cap;
HMODULE nt = GetModuleHandleW(L"ntdll.dll");
Qsi qsi = nt ? (Qsi)(void *)GetProcAddress(nt, "NtQuerySystemInformation") : NULL;
if (!qsi || max <= 0) return -1;
if (!named) hc_type_names(names), named = 1;
LONG st = -1;
for (int tries = 0; tries < 6; tries++) {
if (!table && !(table = malloc(cap = 4u << 20))) return -1;
ULONG need = 0;
st = qsi(64, table, (ULONG)cap, &need); // SystemExtendedHandleInformation
if (st != (LONG)0xC0000004L) break; // the table outgrew the buffer: a larger one
size_t want = need > cap ? need + (1u << 20) : cap * 2;
void *b = realloc(table, want);
if (!b) return -1;
table = b, cap = want;
}
if (st < 0) return -1;
uint32_t count[256] = {0};
const HcTable *t = table;
ULONG_PTR pid = GetCurrentProcessId();
for (ULONG_PTR i = 0; i < t->n && offsetof(HcTable, e) + (i + 1) * sizeof(HcEntry) <= cap; i++)
if (t->e[i].pid == pid) count[t->e[i].type & 255]++;
int n = 0;
for (int k = 0; k < 256; k++) {
if (!count[k]) continue;
if (n == max) { // no room: the last entry takes the rest
snprintf(out[max - 1].name, sizeof out[0].name, "%s", "(more kinds)");
out[max - 1].n += count[k];
continue;
}
if (names[k][0]) snprintf(out[n].name, sizeof out[0].name, "%s", names[k]);
else snprintf(out[n].name, sizeof out[0].name, "type %d", k);
out[n++].n = count[k];
}
return n;
}
// The UCRT's malloc takes its memory from the heap _get_heap_handle names (the process heap); HeapSummary sums it. The
// other heaps are only counted: one made with HEAP_NO_SERIALIZE has no lock to take.
void plat_heap_info(PlatHeapInfo *h) {