From 8106fdaafd5bffa2baa33a0963a5cdbfceeb0da1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:42 +0100 Subject: [PATCH 01/64] Keep the movie converter's helper threads instead of starting ten new ones per picture. run_stripes started NSTRIPES - 1 threads for each of its two phases, about 300 new threads a second while a movie played. Every DLL in the process sets itself up for each new thread; on an RX 580 with AMD's 23.10 driver the process commit grew by about 300 MB a second during the title movie until Windows ran out of commit (45.7 GB, FATAL "movie: out of memory for a 1920x1080 frame"). The helpers now start once and wait for each batch; one caller at a time. Co-Authored-By: Claude Opus 5.5 --- runtime/ov_movie2.c | 56 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/runtime/ov_movie2.c b/runtime/ov_movie2.c index 7766869f..4b89087c 100644 --- a/runtime/ov_movie2.c +++ b/runtime/ov_movie2.c @@ -289,20 +289,64 @@ static void stripe_run(void *arg) { to_nv12(t); } } +// The NSTRIPES - 1 helper threads are made once and kept. Made per call (two calls per picture, 30-60 pictures a +// second) they were about 300 new threads a second for as long as a movie played, and every DLL in the process (the +// GPU driver's, overlays and audio hooks) sets itself up for each new thread: a cost per thread, and with an AMD driver +// on an RX 580 the process commit grew by about 300 MB a second during the title movie until the system ran out +// (FATAL "movie: out of memory for a 1920x1080 frame" at 45.7 GB). One caller at a time (call_lock); a worker that +// could not be started leaves its stripe to the caller. +static struct { + PlatMutex call_lock; // one run_stripes at a time (several decoders may exist) + PlatMutex lock; // the fields below + PlatCond go, done; + Stripe job[NSTRIPES]; + uint64_t gen; // bumped per batch + int pending; // helper stripes of the batch not finished + int nworkers; // started helpers (stripes 1..nworkers); -1 not tried yet +} SP = {PLAT_MUTEX_INIT, PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT, {{0}}, 0, 0, -1}; + +static void stripe_worker(void *arg) { + int i = (int)(intptr_t)arg; + uint64_t seen = 0; + plat_mutex_lock(&SP.lock); + for (;;) { + while (SP.gen == seen) plat_cond_wait(&SP.go, &SP.lock); + seen = SP.gen; + Stripe t = SP.job[i]; + plat_mutex_unlock(&SP.lock); + stripe_run(&t); + plat_mutex_lock(&SP.lock); + if (!--SP.pending) plat_cond_broadcast(&SP.done); + } +} + static void run_stripes(Stripe base) { - Stripe st[NSTRIPES]; - PlatThread *th[NSTRIPES] = {0}; + plat_mutex_lock(&SP.call_lock); + if (SP.nworkers < 0) { + SP.nworkers = 0; + for (int i = 1; i < NSTRIPES; i++, SP.nworkers++) + if (!plat_thread_spawn(stripe_worker, (void *)(intptr_t)i, "movie convert", 0)) break; + if (SP.nworkers < NSTRIPES - 1) rt_log("movie: %d of %d convert threads started", SP.nworkers, NSTRIPES - 1); + } int rows = ((base.h + NSTRIPES - 1) / NSTRIPES + 1) & ~1; // even: chroma rows split at the same places + Stripe st[NSTRIPES]; for (int i = 0; i < NSTRIPES; i++) { st[i] = base; st[i].y0 = i * rows < base.h ? i * rows : base.h; st[i].y1 = (i + 1) * rows < base.h ? (i + 1) * rows : base.h; - if (i) th[i] = plat_thread_new(stripe_run, &st[i], "movie convert"); - if (i && !th[i]) stripe_run(&st[i]); } + plat_mutex_lock(&SP.lock); + memcpy(SP.job, st, sizeof st); + SP.pending = SP.nworkers; + SP.gen++; + plat_cond_broadcast(&SP.go); + plat_mutex_unlock(&SP.lock); stripe_run(&st[0]); - for (int i = 1; i < NSTRIPES; i++) - if (th[i]) plat_thread_join(th[i]); + for (int i = SP.nworkers + 1; i < NSTRIPES; i++) stripe_run(&st[i]); // helpers that could not start + plat_mutex_lock(&SP.lock); + while (SP.pending) plat_cond_wait(&SP.done, &SP.lock); + plat_mutex_unlock(&SP.lock); + plat_mutex_unlock(&SP.call_lock); } // Out of memory for a decoded frame: the frame is dropped (the game shows the previous one a little longer) and From 96f82791c8a0eb7bc9cd34e7a85d5c4c76a13b80 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:43 +0100 Subject: [PATCH 02/64] Submit a large upload batch early, and log every heap after an out-of-memory submit. A save load made thousands of textures between two replay submissions, and all of them went into one upload batch: about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission, which AMD's 23.10 driver on an RX 580 refused with VK_ERROR_OUT_OF_DEVICE_MEMORY ("vk: upload submit failed (-2)"). The outermost vk_upload_end now submits the open batch once its staging holds NX2NAVI_UPLOAD_BATCH_MB (256 by default; 0 = no limit). The batch still precedes the replay submission that uses its textures, and its staging goes as it retires. When a submit fails for lack of memory, vk_mem_oom_report logs each heap's use and budget as the driver counts them, VMA's blocks and allocations there, the open and in-flight upload batches, the VRAM mirrors and pending releases. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 1 + runtime/vk_memory.c | 60 ++++++++++++++++++++++++++++++++++++++++++++- runtime/vk_sync.c | 1 + 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/runtime/vk.h b/runtime/vk.h index 292f4453..77faf919 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -483,6 +483,7 @@ int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls void vk_astc_time_summary(void); // logs the totals void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value +void vk_mem_oom_report(const char *what, VkResult r); // logs heaps and upload batches after an out-of-memory failure void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 1571b8b8..eed11d2d 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -4150,6 +4150,7 @@ typedef struct { StgChunk *chunk; // arena chunks this batch's uploads took ranges of (vk_staging_at), the last one open int nchunk, cap_chunk; VkDeviceSize chunk_used; // bytes of the last chunk handed out + uint64_t bytes; // staging bytes of its buffers (upload_cap) } Batch; enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; @@ -4222,6 +4223,7 @@ static void free_staging(VkCtx *c, Batch *b) { else vmaDestroyBuffer(c->vma, b->chunk[i].buf, b->chunk[i].alloc); } b->nchunk = 0, b->chunk_used = 0; + b->bytes = 0; } // The cached staging buffer that fits n with the least waste: at most four times the request, so a small upload does not @@ -4391,7 +4393,60 @@ VkCommandBuffer vk_upload_begin(void) { return open_b.cb; } -void vk_upload_end(void) { plat_rmutex_unlock(&cmd_lock); } +// NX2NAVI_UPLOAD_BATCH_MB (default 256; 0 = no limit): once the open batch's staging buffers hold that much, the +// outermost vk_upload_end submits it. A save load made thousands of textures between two replay submissions, and every +// one of them went into one batch: about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission, +// which AMD's 23.10 driver on an RX 580 refused with VK_ERROR_OUT_OF_DEVICE_MEMORY. Submitted early, a batch still +// precedes the replay submission that uses its textures (the queue keeps order), and its staging is freed as it retires. +static uint64_t upload_cap = UINT64_MAX - 1; // UINT64_MAX - 1: not read yet +static uint64_t n_early; + +void vk_upload_end(void) { + if (upload_cap == UINT64_MAX - 1) { + const char *e = cfg_get("NX2NAVI_UPLOAD_BATCH_MB"); + long mb = e && *e ? atol(e) : 256; + upload_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX; + if (mb > 0) rt_log("vk: upload batches submitted early above %ld MiB of staging (NX2NAVI_UPLOAD_BATCH_MB)", mb); + else rt_log("vk: upload batches not limited (NX2NAVI_UPLOAD_BATCH_MB=0)"); + } + if (open_active && open_b.bytes >= upload_cap && cmd_lock.depth == 1) { + n_early++; + if (n_early <= 4 || !(n_early & (n_early - 1))) + rt_log("vk: upload batch submitted early with %.0f MiB of staging in %d buffers and %d arena chunks (%llu so far)", + open_b.bytes / 1048576.0, open_b.nstg, open_b.nchunk, (unsigned long long)n_early); + vk_upload_flush(0); + } + plat_rmutex_unlock(&cmd_lock); +} + +// An out-of-memory failure of a submit: every heap's use against its budget (the driver's numbers with +// VK_EXT_memory_budget), VMA's blocks and allocations on it, and the upload batches, so the log names what ran out. +// Takes no lock but the allocator's own: it may run with cmd_lock held. +void vk_mem_oom_report(const char *what, VkResult r) { + if (r != VK_ERROR_OUT_OF_DEVICE_MEMORY && r != VK_ERROR_OUT_OF_HOST_MEMORY) return; + VkCtx *c = vk_ctx(); + if (!c) return; + rt_log("vk: %s failed: out of %s memory (%d)", what, r == VK_ERROR_OUT_OF_DEVICE_MEMORY ? "device" : "host", (int)r); + VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, + .pNext = c->has_memory_budget ? &b : NULL}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + const double M = 1048576.0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + rt_log("vk: heap %u (%s, %.0f MiB): driver use %.0f of budget %.0f MiB%s; vma %u blocks %.0f MiB, %u allocations " + "%.0f MiB", h, c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "device-local" : "host", + c->mem_props.memoryHeaps[h].size / M, c->has_memory_budget ? b.heapUsage[h] / M : vb[h].usage / M, + c->has_memory_budget ? b.heapBudget[h] / M : vb[h].budget / M, + c->has_memory_budget ? "" : " (VMA's estimate)", vb[h].statistics.blockCount, + vb[h].statistics.blockBytes / M, vb[h].statistics.allocationCount, vb[h].statistics.allocationBytes / M); + uint64_t ib = 0; + for (int i = 0; i < ninflight; i++) ib += inflight[i].bytes; + rt_log("vk: upload batch open %d (%d staging buffers, %.0f MiB), %d in flight (%.0f MiB of staging), %llu submitted " + "early; VRAM mirrors %lld (%.0f MiB); objects waiting for release %d", open_active, open_b.nstg, + open_b.bytes / M, ninflight, ib / M, (unsigned long long)n_early, (long long)m_live_n, m_live_bytes / M, ndead); +} // Staging memory comes from a pool of its own that keeps STG_MIN_BLOCKS blocks from its first use (at boot). In the // default pools VMA creates a block under the lock of its memory type (VmaBlockVector::Allocate), so a block made by @@ -4503,6 +4558,7 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { else { if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg); open_b.stg[open_b.nstg++] = (Staging){b, a, bi.size}; + open_b.bytes += bi.size; } return info.pMappedData; } @@ -4541,6 +4597,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) { } if (open_b.nchunk == open_b.cap_chunk) grow((void **)&open_b.chunk, &open_b.cap_chunk, sizeof *open_b.chunk); open_b.chunk[open_b.nchunk++] = k; + open_b.bytes += ARENA_CHUNK; // the staging the batch holds (upload_cap) at = 0; } StgChunk *k = &open_b.chunk[open_b.nchunk - 1]; @@ -4568,6 +4625,7 @@ uint64_t vk_upload_flush(int wait) { VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); if (r != VK_SUCCESS) { + vk_mem_oom_report("upload submit", r); vk_device_lost_check(r, "the submit of an upload batch"); rt_fatal("vk: upload submit failed (%d)", (int)r); } diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 11b3db00..f9109378 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -238,6 +238,7 @@ static void hold_flush_locked(VkCtx *c) { } VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); if (r != VK_SUCCESS) { + vk_mem_oom_report("replay submit", r); vk_device_lost_check(r, "the submit of the replay's command buffers"); rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n, (unsigned long long)H.last, (int)r); From 3de5577512d1ee6be99c290ed4f872abe65fc81b Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:45 +0100 Subject: [PATCH 03/64] Allocate texture images within the heaps' budgets, moving on to the next heap with room. An RX 580 (8 GiB, AMD 23.10) shows its memory as a 6 GiB device-local heap and a 2 GiB device-local, host-visible one. Asked for device-local memory without a budget, VMA chose the first heap every time: a save load put 5.6 GiB there against a budget of 5.1 GiB while the second heap held 69 MiB, and the next submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. create_tex now allocates with VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT: device-local memory of any heap first, then any memory type the image allows, and only then past the budget as before. The log names images placed outside the first device-local heap. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_texture.c | 65 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 940f2ad8..399434d2 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1057,11 +1057,56 @@ static uint32_t detail_skip(const tex_t *t, const TexKey *k, const TexVkFormat * return most >= 2 && levels >= 4 && w >= DETAIL_LOW_MIN && h >= DETAIL_LOW_MIN ? 2 : 1; } +// Where texture images go when video memory runs short. An RX 580 (8 GiB, AMD 23.10) shows its memory as a 6 GiB +// device-local heap and a 2 GiB device-local, host-visible one. Asked for device-local memory without a budget, VMA +// took the first heap every time: a save load put 5.6 GiB into it against a budget of 5.1 GiB while the other heap +// held 69 MiB, and the next submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. Each step below allocates within the +// heaps' budgets (VK_EXT_memory_budget): device-local memory of any heap, then any memory the image allows (system +// memory, slower to sample); only the last step allocates past the budget, as before. +static uint64_t placed[VK_MAX_MEMORY_HEAPS], n_spilled; + +static void note_placement(VkCtx *c, VmaAllocation a, int step) { // tex_lock held + VmaAllocationInfo info; + vmaGetAllocationInfo(c->vma, a, &info); + uint32_t heap = c->mem_props.memoryTypes[info.memoryType].heapIndex; + if (heap >= VK_MAX_MEMORY_HEAPS) return; + uint64_t n = ++placed[heap]; + int first_heap = 1; + for (uint32_t h = 0; h < heap; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) first_heap = 0; + if (step == 0 && first_heap && (c->mem_props.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)) return; + if (step) n_spilled++; + if (n <= 4 || !(n & (n - 1))) + rt_log("vk: texture memory: %llu images in heap %u (%s)%s", (unsigned long long)n, heap, + c->mem_props.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "video memory" : "system memory", + step == 2 ? ", past the budget" : ""); +} + +static VkResult image_alloc(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocationCreateFlags own, VkImage *img, + VmaAllocation *a) { + static const struct { VkMemoryPropertyFlags req; VmaAllocationCreateFlags flags; } step[3] = { + {VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT}, + {0, VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT}, + {0, 0}}; + VkResult r = VK_ERROR_OUT_OF_DEVICE_MEMORY; + for (int i = 0; i < 3; i++) { + VmaAllocationCreateInfo ai = {.flags = step[i].flags | own, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .requiredFlags = step[i].req}; + r = vmaCreateImage(c->vma, ci, &ai, img, a, NULL); + if (r == VK_SUCCESS) { + note_placement(c, *a, i); + return r; + } + } + return r; +} + // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom -// thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference (vma_image). +// thread keeps ahead (vk_headroom.c), within the budget: past it, the next device-local memory type that has room +// (image_alloc in vma_image). Render targets keep vmaCreateImage and the driver's preference (vma_image). // VK_NULL_HANDLE when the image does not take that memory. enum { SUBALLOC_MAX = 32 << 20 }; static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocation *alloc) { @@ -1073,9 +1118,11 @@ static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocati VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; vkGetImageMemoryRequirements2(c->device, &ri, &mr2); VkMemoryRequirements mr = mr2.memoryRequirements; - VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && vmaAllocateMemory(c->vma, &mr, &ai, alloc, NULL) == VK_SUCCESS; + if (ok) note_placement(c, *alloc, 0); if (ok && vmaBindImageMemory(c->vma, *alloc, img) != VK_SUCCESS) { vmaFreeMemory(c->vma, *alloc); ok = 0; @@ -1087,15 +1134,15 @@ static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocati return img; } -// An image made with vmaCreateImage. One of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's -// 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, against the budget and the -// system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the block emptied; its own -// memory goes back when it does. +// An image made with vmaCreateImage, by image_alloc's steps. One of DEDICATED_MIN or more gets memory of its own: +// freed inside one of VMA's 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, +// against the budget and the system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the +// block emptied; its own memory goes back when it does. static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *ik, VkImage *img, VmaAllocation *alloc) { - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, ik); - if ((known ? known->size : image_size(ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; - return vmaCreateImage(c->vma, ci, &ai, img, alloc, NULL); + VmaAllocationCreateFlags own = (known ? known->size : image_size(ci)) >= DEDICATED_MIN + ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; + return image_alloc(c, ci, own, img, alloc); } // defer_init: the caller records the image's first transition (init_image or alias_copy). From 28d7cdbc5111d4eeec87d72e578d1c2bd9eaf110 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:47 +0100 Subject: [PATCH 04/64] Limit VMA's use of each device-local heap to its budget where the GPU has another one. An RX 580 (8 GiB, AMD 23.10) shows its memory as a 6 GiB device-local heap and a 2 GiB device-local, host-visible one, and Windows granted the game 5.1 GiB of the first. A save load put 5.6 GiB of textures, VRAM mirrors and pools into the first heap while the second held 69 MiB, and the next submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY; budget flags on the texture allocations alone did not change that. VMA now gets heap limits (pHeapSizeLimit) when the device has two or more device-local heaps: each heap's budget at device creation less a tenth (at least 256 MiB) for memory VMA does not see. Every allocation past a limit goes to the next memory type that allows it. NX2NAVI_VRAM_HEAP_LIMIT: auto, 0 (off) or MiB for the largest heap. A physical pool that finds no video memory left takes any memory type it may live in instead of failing. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_device.c | 59 ++++++++++++++++++++++++++++++++++++++++++--- runtime/vk_memory.c | 13 ++++++++-- 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/runtime/vk_device.c b/runtime/vk_device.c index b790a10c..02ca353a 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -283,6 +283,53 @@ retry: vk_mem_class_log(&g_vk.props, &g_vk.mem_props); return 1; } +// Heap limits for VMA where the GPU has more than one device-local heap. An RX 580 (8 GiB, AMD 23.10) shows a 6 GiB +// heap and a 2 GiB host-visible one; Windows granted the game 5.1 GiB of the first. VMA, asked for device-local memory, +// takes the first heap whatever its budget, and a save load put 5.6 GiB there while the second held 69 MiB: the next +// submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. With a limit VMA counts a heap full at it and places what follows in +// the next memory type that allows it (the other device-local heap first), for every allocation: textures, render +// targets, VRAM mirrors, pools. The limit is each device-local heap's budget when the device is made (VK_EXT_memory_budget, +// what other programs leave) less a margin for memory VMA does not see (swapchain, FidelityFX): a tenth, at least +// 256 MiB. NX2NAVI_VRAM_HEAP_LIMIT: auto (default), 0 (no limits), or a number of MiB for the largest device-local heap. +static int vram_heap_limits(VkDeviceSize *limit) { + const char *e = cfg_get("NX2NAVI_VRAM_HEAP_LIMIT"); + long mb = e && *e && strcmp(e, "auto") ? atol(e) : -1; + if (!mb) return 0; + VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, + .pNext = g_vk.has_memory_budget ? &b : NULL}; + vkGetPhysicalDeviceMemoryProperties2(g_vk.phys, &p2); + const VkPhysicalDeviceMemoryProperties *mp = &p2.memoryProperties; + int nlocal = 0; + uint32_t largest = 0; + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) { + if (!nlocal++ || mp->memoryHeaps[h].size > mp->memoryHeaps[largest].size) largest = h; + } + if (mb < 0 && (nlocal < 2 || !g_vk.has_memory_budget)) return 0; // auto: only where another heap can take the rest + for (uint32_t h = 0; h < VK_MAX_MEMORY_HEAPS; h++) limit[h] = VK_WHOLE_SIZE; + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) { + if (!(mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)) continue; + VkDeviceSize lim; + if (mb > 0) { + if (h != largest) continue; + lim = (VkDeviceSize)mb << 20; + } else { + VkDeviceSize budget = b.heapBudget[h] && b.heapBudget[h] < mp->memoryHeaps[h].size ? b.heapBudget[h] + : mp->memoryHeaps[h].size; + VkDeviceSize margin = budget / 10 > (256ull << 20) ? budget / 10 : (256ull << 20); + if (budget <= margin * 2) continue; // too small to limit + lim = budget - margin; + } + if (lim >= mp->memoryHeaps[h].size) continue; + limit[h] = lim; + rt_log("vk: memory heap %u (%llu MiB, budget %llu MiB): the allocator uses at most %llu MiB, then the next heap " + "(NX2NAVI_VRAM_HEAP_LIMIT)", h, (unsigned long long)(mp->memoryHeaps[h].size >> 20), + (unsigned long long)(b.heapBudget[h] >> 20), (unsigned long long)(lim >> 20)); + } + return 1; +} + typedef struct { VkPhysicalDeviceFeatures2 f2; @@ -467,9 +514,13 @@ static int create_device(void) { rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "the extension is there, its task or mesh feature is not"); } // vkGetDeviceFaultInfoEXT needs the feature, not just the extension { // mesh_fallback: auto (on a device without mesh shaders), 1 (on every device, for testing) or 0 - const char *mf = cfg_value("NX2NAVI_MESH_FALLBACK"); - g_vk.mesh_fallback = !strcmp(mf, "1") || (strcmp(mf, "0") && !g_vk.has_mesh_shader); - if (g_vk.mesh_fallback) + // without mesh shaders the mesh emulation (NX2NAVI_MESH_EMULATION, on unless 0) draws them; mesh_emulation = 0 + // leaves them to this fallback + const char *mf = cfg_value("NX2NAVI_MESH_FALLBACK"), *me = cfg_value("NX2NAVI_MESH_EMULATION"); + int emu = !g_vk.has_mesh_shader && strcmp(me, "0"); + g_vk.mesh_fallback = !emu && (!strcmp(mf, "1") || (strcmp(mf, "0") && !g_vk.has_mesh_shader)); + if (emu) rt_log("vk: mesh shaders: emulated with compute (mesh_emulation = 1); mesh_fallback is not used"); + else if (g_vk.mesh_fallback) rt_log("vk: mesh fallback: task and mesh programs run as compute passes and are drawn from their buffers (%s)", g_vk.has_mesh_shader ? "mesh_fallback = 1" : "this device has no mesh shaders"); else if (!g_vk.has_mesh_shader) @@ -536,6 +587,8 @@ static int create_device(void) { (g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0), .preferredLargeHeapBlockSize = vk_vma_block_size(), .pDeviceMemoryCallbacks = vk_headroom_callbacks()}; + VkDeviceSize heap_limit[VK_MAX_MEMORY_HEAPS]; + if (vram_heap_limits(heap_limit)) ai.pHeapSizeLimit = heap_limit; if ((r = vmaCreateAllocator(&ai, &g_vk.vma)) != VK_SUCCESS) { rt_log("vk: vmaCreateAllocator failed (%d)", (int)r); return 0; } vk_mem_ballast(g_vk.vma); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index eed11d2d..6ea6d530 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -237,8 +237,17 @@ static int alloc_physical(VkCtx *c, PoolRec *r) { VkMemoryRequirements req = {.size = (rb.size + align - 1) & ~(align - 1), .alignment = align, .memoryTypeBits = bits}; VmaAllocationCreateInfo aci = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; - if ((res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL)) != VK_SUCCESS || - (res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) { + res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL); + if (res != VK_SUCCESS) { // video memory full (VMA heap limits, vk_device.c): any memory the pool can live in + VmaAllocationCreateInfo any = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, + .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + r->alloc = NULL; + res = vmaAllocateMemory(c->vma, &req, &any, &r->alloc, NULL); + if (res == VK_SUCCESS) + rt_log("vk: physical pool of %#llx bytes: no video memory left, placed in system memory", + (unsigned long long)r->size); + } + if (res != VK_SUCCESS || (res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) { rt_log("vk: physical pool of %#llx bytes: device-local allocation failed (%d)", (unsigned long long)r->size, (int)res); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); vkDestroyBuffer(c->device, r->buf, NULL); From 3b345c519086c96bc26b5479302231951e191516 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:49 +0100 Subject: [PATCH 05/64] Add NX2NAVI_PASS_TIMES: GPU time per render pass and dispatch in the log. No vendor profiler covers an RX 580 (RGP supports RDNA only) and RenderDoc cannot capture this program, so nothing showed where 38 ms of GPU time per frame went. With NX2NAVI_PASS_TIMES=1 every pass mark also writes a timestamp at its two points into a query slot pair of its ring slot. The replay thread takes in the marks whose submission has finished, resets their slots on the host (hostQueryReset) and sums their time per kind of pass (size, attachment formats, first fragment or compute shader); every 600 frames the log lists the 15 with the most GPU time per frame. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_crumbs.c | 154 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index eef9ebaa..6f45bf2f 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -15,6 +15,7 @@ #include "cfg.h" #include "host.h" +#include "nvn_api.h" #include "vk.h" static PFN_vkCmdWriteBufferMarkerAMD write_marker; @@ -27,6 +28,11 @@ int vk_pass_marks_on; static int pm_amd; // marks into words 2 and 3 static PFN_vkCmdSetCheckpointNV set_checkpoint; static void pass_marks_init(VkCtx *c); +static void pass_times_init(VkCtx *c); +static void pass_times_poll(void); +static VkQueryPool pt_pool; // pass times (below): two timestamps per ring slot +static volatile uint8_t pt_timed[VK_PASS_MARKS]; // the slot's mark writes timestamps; cleared once they are read + static void buffer_init(VkCtx *c) { write_marker = (PFN_vkCmdWriteBufferMarkerAMD)vk_dpa(c->device, "vkCmdWriteBufferMarkerAMD"); @@ -49,11 +55,14 @@ static void buffer_init(VkCtx *c) { void vk_crumbs_init(VkCtx *c) { if (c->has_buffer_marker) buffer_init(c); pass_marks_init(c); + pass_times_init(c); } void vk_crumbs_release(void) { VkCtx *c = vk_ctx(); if (buf && c) vmaDestroyBuffer(c->vma, buf, alloc); + if (pt_pool && c) vkDestroyQueryPool(c->device, pt_pool, NULL); + pt_pool = VK_NULL_HANDLE; buf = VK_NULL_HANDLE, alloc = NULL, words = NULL, write_marker = NULL; vk_pass_marks_on = pm_amd = 0, set_checkpoint = NULL; } @@ -120,7 +129,11 @@ static void pass_marks_init(VkCtx *c) { : "no VK_AMD_buffer_marker; NX2NAVI_PASS_CRUMBS=1 asks for NVIDIA checkpoints"); } + void vk_pass_mark_cmd(VkCommandBuffer cb, uint32_t id, int end) { + if (pt_pool && pt_timed[id & (VK_PASS_MARKS - 1)] == 1) + vkCmdWriteTimestamp(cb, end ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pt_pool, + (id & (VK_PASS_MARKS - 1)) * 2 + (end != 0)); if (pm_amd) write_marker(cb, end ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, buf, end ? 12 : 8, id); @@ -447,10 +460,13 @@ void vk_gpu_passes_frame(uint64_t frame) { VkPassMark *vk_pass_mark_begin(VkCommandBuffer cb, int kind, uint64_t fence) { if (!vk_pass_marks_on || !cb) return NULL; + if (pt_pool) pass_times_poll(); uint32_t id = ++seq; if (!id) id = ++seq; // 0 means no mark VkPassMark *m = &ring[id & (VK_PASS_MARKS - 1)]; m->id = id, m->kind = (uint8_t)kind, m->fence = fence, m->draws = UINT32_MAX; + // timed unless the slot's previous mark is still waiting to be read (its queries are not reset yet) + if (pt_pool) pt_timed[id & (VK_PASS_MARKS - 1)] = !pt_timed[id & (VK_PASS_MARKS - 1)] ? 1 : 2; rec_pass_mark(cb, id, 0); gp_begin(cb, m); gp_in_mark = 1; @@ -591,3 +607,141 @@ void vk_pass_marks_report(void) { } nv_report(); } + +// ---- pass times (NX2NAVI_PASS_TIMES=1): GPU time per render pass and dispatch, for GPUs no vendor profiler supports +// (RGP covers RDNA only, so an RX 580 has none, and RenderDoc cannot capture this program). Every pass mark also +// writes a timestamp at its two points (TOP_OF_PIPE before the pass, BOTTOM_OF_PIPE after it), into the query slot +// pair of its ring slot. pass_times_poll, on the replay thread at each new mark, takes in the marks whose submission +// has finished, resets their slots on the host (hostQueryReset) and sums their time per kind of pass: its size, +// attachment formats and its first fragment (or compute) shader. Every 600 presented frames the log lists the passes +// with the most GPU time per frame. Passes overlap on the GPU, so the times are each pass's span, not its exclusive +// share; their order is what matters. A slot whose mark was not read yet when the ring comes round is not timed +// (pt_timed 2) for that turn. +#define PT_SUMS 1024 +#define PT_REPORT_FRAMES 600 +#define PT_TOP 15 +typedef struct { + uint64_t key; + uint8_t kind, ncolor; + uint32_t w, h, d; + VkFormat c0, depth; + unsigned long long sid; + double us; + uint64_t n, draws; +} PtSum; +static PtSum pt_sum[PT_SUMS]; +static uint32_t pt_read; // the newest mark id taken in +static double pt_ns_per_tick, pt_total_us; +static uint64_t pt_mask, pt_frame0, pt_marks, pt_lost; + +static void pass_times_init(VkCtx *c) { + const char *e = cfg_get("NX2NAVI_PASS_TIMES"); + if (!e || !*e || !strcmp(e, "0")) return; + if (!vk_pass_marks_on) { + rt_log("vk: NX2NAVI_PASS_TIMES: pass marks are off (NX2NAVI_PASS_CRUMBS=0 or no marker extension): no pass times"); + return; + } + VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES}; + VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12}; + vkGetPhysicalDeviceFeatures2(c->phys, &f2); + uint32_t nq = 0; + vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL); + VkQueueFamilyProperties qp[16]; + if (nq > 16) nq = 16; + vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp); + uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0; + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * VK_PASS_MARKS}; + if (!f12.hostQueryReset || !bits || c->props.limits.timestampPeriod <= 0 || + vkCreateQueryPool(c->device, &qi, NULL, &pt_pool) != VK_SUCCESS) { + rt_log("vk: NX2NAVI_PASS_TIMES: no hostQueryReset or no timestamps on the queue: no pass times"); + pt_pool = VK_NULL_HANDLE; + return; + } + vkResetQueryPool(c->device, pt_pool, 0, 2 * VK_PASS_MARKS); + pt_ns_per_tick = c->props.limits.timestampPeriod; + pt_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1; + pt_read = seq; + pt_frame0 = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + rt_log("vk: pass times on (NX2NAVI_PASS_TIMES): GPU time per render pass and dispatch, the top %d every %d frames", + PT_TOP, PT_REPORT_FRAMES); +} + +static void pt_add(const VkPassMark *m, double us) { + unsigned long long sid = shader_id(m->kind == VK_MARK_DISPATCH ? m->first[0] : m->first[1]); + uint64_t key = sid * 0x9E3779B97F4A7C15ull ^ (uint64_t)m->kind << 56 ^ (uint64_t)m->w << 32 ^ (uint64_t)m->h << 16 ^ + (uint64_t)m->ncolor << 8 ^ (uint64_t)m->color[0] * 0x100000001b3ull ^ (uint64_t)m->depth << 40; + if (!key) key = 1; + for (uint32_t i = (uint32_t)(key % PT_SUMS), k = 0; k < PT_SUMS; k++, i = (i + 1) % PT_SUMS) { + PtSum *e = &pt_sum[i]; + if (e->key && e->key != key) continue; + if (!e->key) + *e = (PtSum){key, m->kind, m->ncolor, m->w, m->h, m->d, m->ncolor ? m->color[0] : VK_FORMAT_UNDEFINED, + e->us += us, e->n++, e->draws += m->draws == UINT32_MAX ? 0 : m->draws; + return; + } + pt_lost++; // table full: counted in the total only +} + +static int pt_cmp(const void *a, const void *b) { + double x = ((const PtSum *)a)->us, y = ((const PtSum *)b)->us; + return x < y ? 1 : x > y ? -1 : 0; +} + +static void pt_report(uint64_t frames) { + static PtSum top[PT_SUMS]; + int n = 0; + for (int i = 0; i < PT_SUMS; i++) + if (pt_sum[i].key) top[n++] = pt_sum[i]; + qsort(top, (size_t)n, sizeof *top, pt_cmp); + double f = frames ? (double)frames : 1; + rt_log("vk: pass times over %llu frames: %.2f ms of pass spans per frame in %.0f passes and dispatches (%d kinds%s)", + (unsigned long long)frames, pt_total_us / f / 1e3, pt_marks / f, n, pt_lost ? ", table full" : ""); + for (int i = 0; i < n && i < PT_TOP; i++) { + const PtSum *e = &top[i]; + if (e->kind == VK_MARK_DISPATCH) + rt_log("vk: %5.2f ms/frame %5.1f%% dispatch %ux%ux%u groups, cs %016llx: %.1f per frame, %.3f ms each", + e->us / f / 1e3, pt_total_us > 0 ? e->us * 100 / pt_total_us : 0, e->w, e->h, e->d, e->sid, e->n / f, + e->us / (double)e->n / 1e3); + else + rt_log("vk: %5.2f ms/frame %5.1f%% pass %ux%u x%u, %u colour (first fmt %d), depth fmt %d, fs %016llx: %.1f per " + } + memset(pt_sum, 0, sizeof pt_sum); + pt_total_us = 0, pt_marks = 0, pt_lost = 0; +} + +// Replay thread (vk_pass_mark_begin). +static void pass_times_poll(void) { + VkCtx *c = vk_ctx(); + uint64_t done = vk_sync_fence_done(); + while (pt_read != seq) { + uint32_t id = pt_read + 1; + if (!id) { pt_read = id; continue; } + uint32_t slot = id & (VK_PASS_MARKS - 1); + const VkPassMark *m = &ring[slot]; + if (m->id == id) { + if (!m->fence || m->fence > done) break; // its submission is still being recorded or running + // the submission finished: both timestamps are in, or one was never written (a pass whose end was not + // recorded), which leaves that mark out + if (pt_timed[slot] == 1 && m->draws != UINT32_MAX) { + uint64_t r[4] = {0}; + VkResult q = vkGetQueryPoolResults(c->device, pt_pool, slot * 2, 2, sizeof r, r, 16, + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + if ((q == VK_SUCCESS || q == VK_NOT_READY) && r[1] && r[3]) { + double us = (double)((r[2] - r[0]) & pt_mask) * pt_ns_per_tick / 1e3; + if (us < 1e6) pt_add(m, us), pt_total_us += us, pt_marks++; + } + } + if (pt_timed[slot]) { + vkResetQueryPool(c->device, pt_pool, slot * 2, 2); + pt_timed[slot] = 0; + } + } + pt_read = id; + } + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (now - pt_frame0 >= PT_REPORT_FRAMES) { + pt_report(now - pt_frame0); + pt_frame0 = now; + } +} From 28d8d47eb38b9e82a51b69a6c4d4da98d7b102ea Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:50 +0100 Subject: [PATCH 06/64] Add NX2NAVI_FLUSH_SKIP_CLEAN: flushed ranges upload only the pages the CPU wrote. On 0.6.1 the mirror sync already leaves flush ranges of write-watched pools uncopied (0101a87), which covers what this did; the key stays known and does nothing. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 6ea6d530..0781cb0d 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2042,6 +2042,9 @@ static int cregions; static PlatMutex flush_lock = PLAT_MUTEX_INIT; static int flush_cap = FLUSH_RANGES; // ranges a pool keeps; 1 with NX2NAVI_FLUSH_SPAN=1 static int flush_upload; // NX2NAVI_FLUSH_UPLOAD=1: flush ranges are copied where the write watch sees them too +// NX2NAVI_FLUSH_SKIP_CLEAN (flush_clean_bytes in the 0.5.0 and 0.6.0 fixes: flushed ranges upload only the pages the CPU +// wrote) is this default since 0.6.1: a flush range of a write-watched pool is not copied at all; the key stays known so an +// nx2navi.ini that sets it reads without a warning, and does nothing. typedef struct { uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; From 496c163d86396ce1c80fcb8e8ac3776a5ca40d36 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:52 +0100 Subject: [PATCH 07/64] Map the "Nyx Joy-Con" USB HID joystick as a gamepad, and log joysticks SDL cannot use. A Switch can present its Joy-Con over USB from its boot menu's tools as a plain HID joystick, "Nyx Joy-Con" (VID 0x11EC, PID 0xA7E1): X, Y, Z, Rz for the sticks, a hat for the D-pad and twelve buttons. SDL has no gamepad mapping for it, so it was a joystick only and input.c, which opens gamepads, did not see it ("input: no gamepad"). scan_pads now gives every known joystick without a mapping one built from its own GUID (buttons 1-4 B, A, Y, X as south, east, west, north, then L, R, ZL, ZR, Minus, Plus and the stick clicks), and logs any other unmapped joystick once with its name, ids and GUID. (cherry picked from commit ed4c68c3e8460e247bab3ad4f3f02e2672868ddf) Co-Authored-By: Claude Opus 5.5 --- runtime/input.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/runtime/input.c b/runtime/input.c index e1547b95..01d5b28e 100644 --- a/runtime/input.c +++ b/runtime/input.c @@ -533,6 +533,50 @@ static void pad_primary(SDL_Gamepad *p, const char *why) { pad_east_a ? "labelled A" : "not labelled A"); } +// Joysticks SDL has no gamepad mapping for are not gamepads to it, and this file opens gamepads only. Known ones get a +// mapping built from their own GUID (so it matches whatever bus, version and name checksum SDL put in it); any other is +// logged once with its name, ids and GUID. +// - hekate's USB HID Joy-Con mode (Nyx "USB Tools" > "Joy-Con mode", bdk/usb/usb_descriptors.c, usb_gadget_hid.c): +// "Nyx Joy-Con", VID 0x11EC, PID 0xA7E1, a plain HID joystick: X, Y (left stick), Z, Rz (right stick) from 0 to 255 +// with up at 0, a hat for the D-pad, and buttons 1-12 B, A, Y, X (south, east, west, north), L, R, ZL, ZR, Minus, +// Plus, left and right stick clicks. Capture and Home are not sent. +static const struct { uint16_t vid, pid; const char *name, *map; } known_joysticks[] = { + {0x11EC, 0xA7E1, "hekate Joy-Con (USB HID)", + "a:b0,b:b1,x:b2,y:b3,leftshoulder:b4,rightshoulder:b5,lefttrigger:b6,righttrigger:b7,back:b8,start:b9,leftstick:b10," + "rightstick:b11,dpup:h0.1,dpright:h0.2,dpdown:h0.4,dpleft:h0.8,leftx:a0,lefty:a1,rightx:a2,righty:a3"}, +}; + +static void map_joysticks(void) { + static SDL_JoystickID seen[32]; + static int nseen; + int n = 0; + SDL_JoystickID *ids = SDL_GetJoysticks(&n); + for (int i = 0; ids && i < n; i++) { + int done = 0; + for (int k = 0; k < nseen && !done; k++) done = seen[k] == ids[i]; + if (done || SDL_IsGamepad(ids[i])) continue; + if (nseen < (int)(sizeof seen / sizeof *seen)) seen[nseen++] = ids[i]; + uint16_t vid = SDL_GetJoystickVendorForID(ids[i]), pid = SDL_GetJoystickProductForID(ids[i]); + char guid[64]; + SDL_GUIDToString(SDL_GetJoystickGUIDForID(ids[i]), guid, (int)sizeof guid); + const char *jname = SDL_GetJoystickNameForID(ids[i]); + int mapped = 0; + for (size_t k = 0; k < sizeof known_joysticks / sizeof *known_joysticks; k++) { + if (known_joysticks[k].vid != vid || known_joysticks[k].pid != pid) continue; + char m[512]; + snprintf(m, sizeof m, "%s,%s,%s", guid, known_joysticks[k].name, known_joysticks[k].map); + int r = SDL_AddGamepadMapping(m); + rt_log("input: joystick \"%s\" (%04x:%04x, GUID %s) mapped as a gamepad (%s)%s%s", jname ? jname : "?", vid, pid, + guid, known_joysticks[k].name, r < 0 ? ": mapping refused, " : "", r < 0 ? SDL_GetError() : ""); + mapped = 1; + } + if (!mapped) + rt_log("input: joystick \"%s\" (%04x:%04x, GUID %s) has no gamepad mapping in SDL: not used (an SDL mapping " + "in the SDL_GAMECONTROLLERCONFIG environment variable adds one)", jname ? jname : "?", vid, pid, guid); + } + SDL_free(ids); +} + // Under lock: closes the gamepads that went away and opens the ones that came; the primary falls back to the first // open one when it left. Logs each change. // The game is always told one controller is connected, so a disconnect read as nothing pressed and play went on @@ -540,6 +584,7 @@ static void pad_primary(SDL_Gamepad *p, const char *why) { // its place, the menu opens: the player sees it, and it takes the keyboard and mouse until it is closed. static void scan_pads(void) { int had = pad != NULL; + map_joysticks(); for (int i = 0; i < npads;) { if (SDL_GamepadConnected(pads[i])) { i++; continue; } rt_log("input: gamepad \"%s\" disconnected", SDL_GetGamepadName(pads[i]) ? SDL_GetGamepadName(pads[i]) : "?"); From 1bfe90e3e2701b51567552e0ec042e317c8e152d Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:54 +0100 Subject: [PATCH 08/64] Pass times: log the largest viewport each kind of pass draws with. A dynamic resolution that keeps its targets at the largest size and draws the scene into part of them shows up as a viewport smaller than the pass; one that resizes its targets shows up in the pass size. NX2NAVI_SCENE_SCALE=33 left the handheld scene pass at 1440x810, so this tells which of the two the game does there. (cherry picked from commit af7abe36b0b15c5a63b354d2870543987d1f4b1a) Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 1 + runtime/vk_crumbs.c | 7 +++++++ runtime/vk_exec.c | 1 + runtime/vk_exec_draw.c | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/runtime/vk.h b/runtime/vk.h index 77faf919..b1756797 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -128,6 +128,7 @@ typedef struct { uint32_t step; // index of the pass, or of the dispatch, within its submission, from 0 uint32_t w, h, d; // render area and layers; a dispatch: group counts (0 0 0: indirect) uint32_t draws; // draws recorded in the pass; UINT32_MAX until its end is recorded + uint32_t vw, vh; // the largest viewport a draw of the pass set (dynamic resolution draws in part) uint64_t fence; // the submission: the timeline value it signals VkFormat color[8], depth; // attachment formats (VK_FORMAT_UNDEFINED: none) const GfxShader *first[2], *last[2];// vertex and fragment shader of the first and last draw; a dispatch: first[0] diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index 6f45bf2f..86fd6ba4 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -628,6 +628,7 @@ typedef struct { unsigned long long sid; double us; uint64_t n, draws; + uint32_t vw, vh; // the largest viewport its draws set } PtSum; static PtSum pt_sum[PT_SUMS]; static uint32_t pt_read; // the newest mark id taken in @@ -677,6 +678,9 @@ static void pt_add(const VkPassMark *m, double us) { if (e->key && e->key != key) continue; if (!e->key) *e = (PtSum){key, m->kind, m->ncolor, m->w, m->h, m->d, m->ncolor ? m->color[0] : VK_FORMAT_UNDEFINED, + m->depth, sid, 0, 0, 0, 0, 0}; + if (m->vw > e->vw) e->vw = m->vw; + if (m->vh > e->vh) e->vh = m->vh; e->us += us, e->n++, e->draws += m->draws == UINT32_MAX ? 0 : m->draws; return; } @@ -705,6 +709,9 @@ static void pt_report(uint64_t frames) { e->us / (double)e->n / 1e3); else rt_log("vk: %5.2f ms/frame %5.1f%% pass %ux%u x%u, %u colour (first fmt %d), depth fmt %d, fs %016llx: %.1f per " + "frame, %.3f ms and %.0f draws each, viewport up to %ux%u", e->us / f / 1e3, + pt_total_us > 0 ? e->us * 100 / pt_total_us : 0, e->w, e->h, e->d, e->ncolor, (int)e->c0, (int)e->depth, + e->sid, e->n / f, e->us / (double)e->n / 1e3, (double)e->draws / (double)e->n, e->vw, e->vh); } memset(pt_sum, 0, sizeof pt_sum); pt_total_us = 0, pt_marks = 0, pt_lost = 0; diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 7e139eff..74a6f35a 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -860,6 +860,7 @@ void vk_exec_pass_begin(VkExec *ex) { m->step = ex->mark_passes++; m->w = ex->pass_w, m->h = ex->pass_h, m->d = ex->pass_layers; m->layer = ex->depth.image ? ex->depth.base_layer : 0; + m->vw = m->vh = 0; m->ncolor = (uint8_t)(ex->pass_ncolor < 8 ? ex->pass_ncolor : 8); for (int i = 0; i < m->ncolor; i++) m->color[i] = ex->pass_color[i]; m->depth = ex->pass_depth ? ex->pass_depth : ex->pass_stencil; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 0c588229..8936f2de 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -3682,6 +3682,10 @@ static VkViewport draw_viewport(VkExec *ex) { if (w < 1) w = 1; if (h < 1) h = 1; VkViewport v = {x, y + h, w, -h, ex->depth_range[0], ex->depth_range[1]}; // the y flip: negative height + if (ex->mark) { // NX2NAVI_PASS_TIMES: how much of the pass the draws cover + if ((uint32_t)w > ex->mark->vw) ex->mark->vw = (uint32_t)w; + if ((uint32_t)h > ex->mark->vh) ex->mark->vh = (uint32_t)h; + } vk_dlaa_viewport(ex, &v); // the scene's draws move by the DLAA jitter return v; } From 4ba78a7b313f20cea8c86df521018dd2577fa1d1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:56 +0100 Subject: [PATCH 09/64] Add NX2NAVI_MIRROR_HOT: pools copied through their VRAM mirror every frame go back to system memory. On an RX 580 with the scene at 854x480 the render passes take 11 ms of a 32 ms frame. The rest is the mirrors: 66 MiB of CPU-written pages up and 76 MiB of compute-shader writes back each frame, mostly through six 6-16 MiB pools the game refills every frame, with the GPU idle 7.8 ms a frame while those copies wait. With mirror_hot = N a mirrored pool whose uploads, write-backs and merges average N percent of its size (and at least 1 MiB) a frame over 120 frames leaves the mirror at the start of a submission: it comes off the mirrored list, the device address table points at the imported buffer again, and the mirror buffer is freed after the next submission. Default 0, off. (cherry picked from commit 82e5dcb682375fa48bbaffa1d61f825272544e36) Co-Authored-By: Claude Opus 5.5 --- nx2navi.ini.example | 7 +++++ runtime/vk_memory.c | 63 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 99ca6beb..267b8d03 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -284,6 +284,9 @@ # which game memory gets no copy in video memory (memory = mirror), which costs far more frame time than a texture # uploaded again. 0: textures never give way for video memory. # tex_vram_pressure = 80 +# Textures the game released that are kept as they are (1 to 16384); past this count only those that lose nothing +# when dropped go. Default: one per 2 MB of the video memory budget, 2048 to 8192. Read at start. +# tex_released_max = 4096 # Megabytes of this process's commit above which the log says so. Windows counts video memory against the commit; # with a discrete GPU the video memory in use is left out of the comparison, since it is not in RAM. Textures no longer # give way at this bound: dropping one lowers both sides of the comparison alike, so nothing they could do brought the @@ -414,6 +417,10 @@ # 3D render scale: the scene's size relative to the game resolution, game (5/6, the game's own 1600x900 at 1080p) or # a percentage (25 to 400, at most 4096 pixels a side); above 100 supersamples the scene (anti-aliasing). Read at start. # render_scale = game +# Discrete GPUs: a game memory pool whose copies to and from video memory each frame reach this percentage of its size +# is read and written by the GPU in system memory instead (0: every pool keeps its video memory copy). For GPUs and +# buses short of bandwidth; on an RX 580 the pools the game refills every frame cost about 8 ms a frame in copies. +# mirror_hot = 0 # Scaling to the window when it is larger than the game resolution: fsr (AMD FSR 1) or bilinear, and FSR's sharpening # from 0 (off) to 100. Changes while the game runs take effect at once. # upscaler = fsr diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 0781cb0d..26d5f4cb 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -82,6 +82,7 @@ typedef struct PoolRec { FlushRange fl[FLUSH_RANGES + 1]; // flush_lock: nvnMemoryPoolFlushMappedRange pages not copied yet (flush_add) int fl_n; // flush_lock: ranges in fl; read without the lock to skip an empty list uint64_t fl_calls, fl_sum; // flush_lock: the calls and bytes behind fl, for mirror_sync's statistics + uint64_t hot_bytes; // replay: uploads, write-backs and merges since the last hot check (mirror_hot) // Shadow storage (POOL_IMPORT in shadow mode, "shadow storage" below): buf is a host-visible copy of the guest pages. VmaAllocation salloc; uint8_t *smap; // its mapping; offsets as in buf (from import_lo) @@ -3257,6 +3258,7 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; r->vd_wrote_sub = msub, r->vd_wrote_n++; + r->hot_bytes += size; // A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies. uint64_t o = 0; if (shader && !(sp->offset & 3) && !(size & 3) && size <= merge_max() && merging()) @@ -3404,7 +3406,68 @@ static void freeze_pass(void) { } } +// NX2NAVI_MIRROR_HOT (percent, 0 off): a mirrored pool whose copies each frame (CPU pages up, GPU writes back) reach this +// share of its size goes back to being read and written by the GPU in system memory. On an RX 580 in Kokiri Forest a +// handful of 6-16 MiB pools the CPU refills every frame and compute shaders write took 66 MiB of uploads and 76 MiB of +// write-backs a frame through the mirrors, the GPU idle 7.8 ms a frame with those copies queued; read in place, only +// the bytes the shaders touch cross the bus. Checked every HOT_FRAMES frames at the start of a submission, when the +// previous one has queued its write-backs and nothing is pending for the next: the pool leaves the mirrored list, the +// device address table points at its imported buffer again, and the mirror is freed once the submissions that may still +// use it are done (vk_defer_buffer, as mirror_evict). Pools under HOT_MIN a frame stay mirrored. +#define HOT_FRAMES 120 +#define HOT_MIN (1ull << 20) +static int mirror_hot = -1; +static uint64_t hot_frame0, n_demoted; + +static void hot_check(void) { + if (mirror_hot < 0) { + mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p; + if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot); + if (!mirror_hot) return; + uint64_t fr = mirror_frame; + if (!hot_frame0 || fr < hot_frame0) { + hot_frame0 = fr ? fr : 1; + return; + if (fr - hot_frame0 < HOT_FRAMES || npend || nwbl) return; // nsnl: the previous submission's, recorded already + double frames = (double)(fr - hot_frame0); + hot_frame0 = fr; + PoolRec *out[8]; + VkBuffer dead[8]; + VmaAllocation dead_a[8]; + int nout = 0; + plat_mutex_lock(&pool_lock); + for (int i = 0; i < nm;) { + PoolRec *r = mlist[i]; + r->hot_bytes = (uint64_t)per; // kept for the log line below + r->hot_bytes = 0, i++; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict) + continue; + mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first + // as mirror_evict, under the exclusive lock (vk_pool_fini takes pools off the list under it), but for good: + // MS_REFUSED with no retry (retry_sub 0), so neither vk_mem_use nor the budget retries mirror it again + mirror_forget(r); // takes r off mlist: i now names the next pool + r->mstate = MS_REFUSED, r->retry_sub = 0; + plat_mutex_lock(&flush_lock); // flushes not copied yet: the pool is read in place from now on + __atomic_store_n(&r->fl_n, 0, __ATOMIC_RELEASE); + r->fl_calls = r->fl_sum = 0; + plat_mutex_unlock(&flush_lock); + // the pages name the imported buffer alone again (the twin was that buffer) + if (r->gpa_rec) r->gpa_rec = vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->buf, r->import_lo); + dead[nout] = r->mbuf, dead_a[nout] = r->malloc_; + r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL; + out[nout++] = r; + plat_mutex_unlock(&pool_lock); + for (int k = 0; k < nout; k++) { + PoolRec *r = out[k]; + uint64_t n = r->import_hi - r->import_lo, per = r->hot_bytes; + r->hot_bytes = 0; + vk_defer_buffer(dead[k], dead_a[k]); // after the submission being recorded + char f[160]; + flag_names(r->flags, f, sizeof f); + if (++n_demoted <= 32) + rt_log("mem: pool %#llx (%.1f MiB, %s) copied %.1f MiB a frame through its mirror: system memory from now on", + (unsigned long long)r->lo, n / 1048576.0, f, per / 1048576.0); void vk_mem_mirror_begin(void) { + if (vk_mirror_on) hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions if (!freeze_idle) return; plat_mutex_lock(&gen_lock); From e576c6b1df94ff777cbca18726791ff53cfa96b5 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:21:58 +0100 Subject: [PATCH 10/64] make_game.py: write nx2navi.map next to the exe, without the lifted functions. A NX2NAVI_PROFILE profile names Rips; tools/prof_report.py needs the link map to name them, and the launcher's build wrote none. The link now writes the map and keeps its non-lifted lines (the runtime, libraries; the f_ game functions are hundreds of MiB of it) as nx2navi.map in the game folder. The link key changes once, so the next build links again. Co-Authored-By: Claude Opus 5.5 --- tools/make_game.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tools/make_game.py b/tools/make_game.py index 4a4c228d..8698ae0d 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1681,6 +1681,20 @@ def retune_due(obj, msys): return True +def write_runtime_map(full, dst): + """The linker's map (the build cache's, which tools/prof_report.py reads) without the lines of the lifted game + functions (f_): the runtime's and the libraries' symbols, a few MiB, next to the exe to send with a crash log + or a NX2NAVI_PROFILE profile of the render thread.""" + lifted = re.compile(r'\bf_[0-9a-f]{4,}\b') # any line naming a lifted function: its symbol or its section + try: + with open(full, encoding='utf-8', errors='replace') as i, open(dst, 'w', encoding='utf-8') as o: + for line in i: + if not lifted.search(line): + o.write(line) + except OSError: + pass + + def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): """romfs_job: the RomFS step still running beside this one (a first build). Everything that does not include romfs_config.h compiles meanwhile; the RomFS index and the few files that include it follow once it is done. @@ -1794,7 +1808,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): stamp = [[o.name, o.stat().st_size, o.stat().st_mtime_ns] for o in objs] if prebuilt: stamp.append([prebuilt[0].name, prebuilt[0].stat().st_size, sha256_file(prebuilt[0])]) - link_key = sha256_bytes(json.dumps([LINK_LIBS, ld, cid] + stamp).encode()) + link_key = sha256_bytes(json.dumps([LINK_LIBS, ld, cid, 'map'] + stamp).encode()) state_link = cache_dir / 'link.key' if exe.is_file() and state_link.is_file() and state_link.read_text(encoding='utf-8') == link_key: say(f' {EXE_NAME} is current, not linked again') @@ -1814,6 +1828,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): except PermissionError: tmp.unlink(missing_ok=True) fail(f'{exe} is in use: close the game and try again') + write_runtime_map(cache_dir / 'ocarina.map', out / 'nx2navi.map') # next to the exe, to send with a crash log state_link.write_text(link_key, encoding='utf-8') copy_if_changed(gen / 'romfs-index.bin', out / 'romfs-index.bin') xlate = xlate_source() From f3a9d2967f20cd6c30ca0734a3ce44e1d37c97da Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:00 +0100 Subject: [PATCH 11/64] Profiler: name the exe's addresses from its own symbol table, and sample the replay thread alone. A NX2NAVI_PROFILE profile needed the link map to name its Rips, and the launcher's builds keep none. A MinGW link keeps the COFF symbol table in the exe, so prof_sampler_write now reads it (runtime/prof_syms.c, plain file reads) and appends `sym
+` for every exe address in the Rip and chain lists. NX2NAVI_PROFILE_REPLAY=1 samples only the replay thread: with NX2NAVI_REPLAY_ASYNC=1 the render thread spends 98% of a 37 ms frame waiting, and the replay thread (22 ms a frame recording Vulkan commands) is the one to look at. (cherry picked from commit 9d3126ca780d7f0f0b91468d42b8ee70572747c8) Co-Authored-By: Claude Opus 5.5 --- runtime/prof_syms.c | 112 ++++++++++++++++++++++++++++++++++++++++++++ runtime/prof_syms.h | 9 ++++ runtime/prof_win.c | 15 ++++++ 3 files changed, 136 insertions(+) create mode 100644 runtime/prof_syms.c create mode 100644 runtime/prof_syms.h diff --git a/runtime/prof_syms.c b/runtime/prof_syms.c new file mode 100644 index 00000000..dbf098df --- /dev/null +++ b/runtime/prof_syms.c @@ -0,0 +1,112 @@ +// Names for a sampling profile's exe addresses from the exe's own COFF symbol table (runtime/prof_win.c). A MinGW link +// keeps the table in the file (PointerToSymbolTable), so a profile names its functions without the link map: for each +// address the profile lists, `sym
+`. Plain file reads, no Windows headers, so the parser +// builds and is tested anywhere. +#include +#include +#include +#include + +#include "prof_syms.h" + +typedef struct { uint32_t rva; uint32_t str; char sname[8]; } CSym; // str: string-table offset, 0 short name + +static int cmp_sym(const void *a, const void *b) { + uint32_t x = ((const CSym *)a)->rva, y = ((const CSym *)b)->rva; + return x < y ? -1 : x > y; +} + +static int cmp_u64(const void *a, const void *b) { + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + return x < y ? -1 : x > y; +} + +static uint32_t rd32(const uint8_t *p) { return p[0] | p[1] << 8 | p[2] << 16 | (uint32_t)p[3] << 24; } +static uint16_t rd16(const uint8_t *p) { return (uint16_t)(p[0] | p[1] << 8); } + +int prof_syms_write(FILE *out, const char *exe_path, uint64_t base, const uint64_t *addr, size_t n) { + FILE *f = fopen(exe_path, "rb"); + if (!f) return -1; + uint8_t h[0x400]; + int named = -1; + CSym *syms = NULL; + char *strtab = NULL; + uint8_t *raw = NULL; + uint64_t *a = NULL; + if (fread(h, 1, 0x40, f) != 0x40 || h[0] != 'M' || h[1] != 'Z') goto done; + uint32_t pe = rd32(h + 0x3c); + if (fseek(f, pe, SEEK_SET) || fread(h, 1, 24, f) != 24 || memcmp(h, "PE\0\0", 4)) goto done; + uint16_t nsec = rd16(h + 6), opt = rd16(h + 20); + uint32_t symptr = rd32(h + 12), nsym = rd32(h + 16); + if (!symptr || !nsym || nsec > 96) goto done; + uint32_t sec_va[96], sec_exec[96]; + if (fseek(f, pe + 24 + opt, SEEK_SET)) goto done; + for (int i = 0; i < nsec; i++) { + if (fread(h, 1, 40, f) != 40) goto done; + sec_va[i] = rd32(h + 12); + sec_exec[i] = (rd32(h + 36) & 0x20000020) != 0; // CNT_CODE or MEM_EXECUTE + } + // the string table follows the symbols: its first 4 bytes are its size + uint8_t szb[4]; + if (fseek(f, (long)symptr + (long)nsym * 18, SEEK_SET) || fread(szb, 1, 4, f) != 4) goto done; + uint32_t strsz = rd32(szb); + strtab = malloc(strsz + 1u); + if (!strtab) goto done; + memcpy(strtab, szb, 4); + if (strsz > 4 && fread(strtab + 4, 1, strsz - 4, f) != strsz - 4) goto done; + strtab[strsz] = 0; + // the symbols in chunks; functions in executable sections + size_t ns = 0, cs = 1 << 16; + syms = malloc(cs * sizeof *syms); + enum { CH = 1 << 16 }; + raw = malloc(CH * 18); + if (!syms || !raw || fseek(f, symptr, SEEK_SET)) goto done; + uint32_t skip = 0; + for (uint32_t done_n = 0; done_n < nsym;) { + uint32_t k = nsym - done_n < CH ? nsym - done_n : CH; + if (fread(raw, 18, k, f) != k) goto done; + for (uint32_t i = 0; i < k; i++) { + const uint8_t *s = raw + i * 18; + if (skip) { skip--; continue; } // auxiliary records + skip = s[17]; + int16_t sec = (int16_t)rd16(s + 12); + uint8_t cls = s[16]; + if (sec <= 0 || sec > nsec || !sec_exec[sec - 1] || (cls != 2 && cls != 3)) continue; + CSym c = {sec_va[sec - 1] + rd32(s + 8), 0, {0}}; + if (rd32(s)) memcpy(c.sname, s, 8); + else c.str = rd32(s + 4); + const char *nm = c.str ? (c.str < strsz ? strtab + c.str : "") : c.sname; + if (nm[0] == '.' || !nm[0]) continue; // section symbols + if (ns == cs) { + CSym *g = realloc(syms, (cs *= 2) * sizeof *syms); + if (!g) goto done; + syms = g; + } + syms[ns++] = c; + } + done_n += k; + } + qsort(syms, ns, sizeof *syms, cmp_sym); + a = malloc((n ? n : 1) * sizeof *a); + if (!a) goto done; + memcpy(a, addr, n * sizeof *a); + qsort(a, n, sizeof *a, cmp_u64); + named = 0; + size_t j = 0; + for (size_t i = 0; i < n; i++) { + if (i && a[i] == a[i - 1]) continue; + if (a[i] < base || a[i] - base >= 0xffffffffull) continue; + uint32_t rva = (uint32_t)(a[i] - base); + while (j < ns && syms[j].rva <= rva) j++; // a is sorted, so j only grows + if (!j) continue; + const CSym *c = &syms[j - 1]; + char sn[9] = {0}; + const char *nm = c->str ? (c->str < strsz ? strtab + c->str : "?") : (memcpy(sn, c->sname, 8), sn); + fprintf(out, "sym %llx %s+0x%x\n", (unsigned long long)a[i], nm, rva - c->rva); + named++; + } +done: + free(a), free(raw), free(syms), free(strtab); + fclose(f); + return named; +} diff --git a/runtime/prof_syms.h b/runtime/prof_syms.h new file mode 100644 index 00000000..c583e8f4 --- /dev/null +++ b/runtime/prof_syms.h @@ -0,0 +1,9 @@ +#pragma once +#include +#include +#include + +// Writes `sym
+0x` for each of the n addresses (any order, repeats allowed) that falls in +// the image loaded at base, from the COFF symbol table of the file at exe_path. Returns the lines written, -1 when the +// file has no symbol table or cannot be read. +int prof_syms_write(FILE *out, const char *exe_path, uint64_t base, const uint64_t *addr, size_t n); diff --git a/runtime/prof_win.c b/runtime/prof_win.c index 95470d9c..a8df4008 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -29,6 +29,7 @@ #include "host.h" #include "perfmon.h" #include "prof.h" +#include "prof_syms.h" #define render_tid prof_render_tid #define main_tid prof_main_tid @@ -355,7 +356,21 @@ void prof_sampler_write(void) { fputc('\n', f); } if (stacks_dropped) fprintf(f, "stacks_dropped %llu\n", (unsigned long long)stacks_dropped); + // the exe's own symbol table names the addresses above, so the profile needs no link map + size_t na = 0, ca = (size_t)cnt + 1; + for (int i = 0; i < ns && i < PROF_STACKS; i++) ca += stacks[i].depth; + uint64_t *addrs = malloc(ca * sizeof *addrs); + char exe[MAX_PATH]; + int named = -1; + if (addrs && GetModuleFileNameA(NULL, exe, sizeof exe)) { + for (int i = 0; i < cnt && i < PROF_TOP; i++) addrs[na++] = table[i].rip; + for (int i = 0; i < ns && i < PROF_STACKS; i++) + for (uint32_t d = 0; d < stacks[i].depth; d++) addrs[na++] = stacks[i].rip[d]; + named = prof_syms_write(f, exe, (uint64_t)(uintptr_t)GetModuleHandleW(NULL), addrs, na); + } + free(addrs); fclose(f); + if (named < 0) rt_log("prof: no symbol table in the exe: the addresses stay unnamed"); rt_log("prof: %llu samples, %d Rips -> %s", (unsigned long long)total, cnt, out_path); } From 8913ce17365ab012f0232644fe55891aaa9664bc Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:01 +0100 Subject: [PATCH 12/64] make_game.py: NX2NAVI_GUEST_OPT=2 builds the lifted code at -O2. With the lifted code built for the CPU's level, a profile on an RX 580 PC (Haswell-class CPU) has two game threads running it nearly all the time and the replay thread idle a quarter of the time: the game's own code bounds the frame. The lifted code is built at -O1 for build time; the environment variable NX2NAVI_GUEST_OPT=2 builds it at -O2 for a player who will wait for the longer build. Part of each object's key, so changing it rebuilds the lifted code. Co-Authored-By: Claude Opus 5.5 --- tools/make_game.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make_game.py b/tools/make_game.py index 8698ae0d..5343e50e 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1440,7 +1440,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): cc = [str(fast), f'--sysroot={msys}'] if guest and fast else [str(clangxx if cpp else clang)] # -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally declspec = ['-fdeclspec'] if cpp and not WIN else [] - return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2', + return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', guest_opt if guest else '-O2', *march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)] return cmd_for From aec5b5a8401610029a9bf9ed55a58c95baf4558e Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:03 +0100 Subject: [PATCH 13/64] Emulate mesh shaders with compute on GPUs without VK_EXT_mesh_shader (NX2NAVI_MESH_EMULATION, off by default). Without the extension every mesh draw was skipped ("mesh draws need VK_EXT_mesh_shader: skipped"): on an RX 580 the game's grass and foliage were missing. The translator already builds each task and mesh program as compute code that MeshSpirv.cs wraps into TaskEXT / MeshEXT modules; runtime/mesh_emu_spirv.c turns those modules into GLCompute ones: outputs and payload move to workgroup memory, SetMeshOutputsEXT's counts are kept, and the entry function's end reserves vertex slots and indices in an arena with atomics and copies the vertices and triangles out (task: the payload and the mesh group count go to a task record, and the mesh dispatch to the draw record). A generated vertex shader reads the vertices back for the game's fragment shader. Checked with spirv-val (Vulkan 1.3) on the 1016 task and mesh programs of Ocarina of Time 1.1.1: 1008 convert and validate; the 8 point-list meshes stay unemulated. vk_pipeline.c gives an emulated mesh pipeline a layout whose task and mesh bindings are visible to compute, with push constants and a compute-side descriptor template, builds the generated vertex shader's graphics pipeline and the two compute pipelines; vk_exec_draw.c prepares each mesh draw as before, then outside the pass zeroes the arena counters (first emulated draw of a submission), fills a draw record, pushes the same descriptors to the compute bind point, dispatches the task and mesh stages (direct, or from the indirect ring entry) and draws DrawIndexedIndirect from the record. mesh_emulation = 1 turns it on; mesh_emu_mb sizes the arena (160 MiB). (cherry picked from commit efc43b177694d8f8ce5700fcf28d80e889a45589) Co-Authored-By: Claude Opus 5.5 --- nx2navi.ini.example | 3 + runtime/mesh_emu_spirv.c | 798 +++++++++++++++++++++++++++++++++++++++ runtime/mesh_emu_spirv.h | 37 ++ runtime/vk.h | 8 + runtime/vk_exec_draw.c | 109 +++++- runtime/vk_pipeline.c | 166 +++++++- 6 files changed, 1105 insertions(+), 16 deletions(-) create mode 100644 runtime/mesh_emu_spirv.c create mode 100644 runtime/mesh_emu_spirv.h diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 267b8d03..261e56ed 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -421,6 +421,9 @@ # is read and written by the GPU in system memory instead (0: every pool keeps its video memory copy). For GPUs and # buses short of bandwidth; on an RX 580 the pools the game refills every frame cost about 8 ms a frame in copies. # mirror_hot = 0 +# GPUs without mesh shaders (VK_EXT_mesh_shader: AMD before RDNA 2, NVIDIA before Turing): 1 runs the game's mesh +# and task shaders (grass and other foliage) as compute shaders and draws what they make; 0 skips those draws. The +# mesh_emulation = 0 # Scaling to the window when it is larger than the game resolution: fsr (AMD FSR 1) or bilinear, and FSR's sharpening # from 0 (off) to 100. Changes while the game runs take effect at once. # upscaler = fsr diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c new file mode 100644 index 00000000..df312373 --- /dev/null +++ b/runtime/mesh_emu_spirv.c @@ -0,0 +1,798 @@ +// Mesh shader emulation: SPIR-V rewriting (mesh_emu_spirv.h). The input is what tools/xlate/MeshSpirv.cs makes: the +// translator's compute main, called from an entry function that copies its shared-memory stage buffer into the mesh +// outputs after SetMeshOutputsEXT (mesh), or into the task payload before EmitMeshTasksEXT (task). +// +// Mesh -> GLCompute: the output arrays and the task payload move to Workgroup memory; SetMeshOutputsEXT is dropped (its +// counts are kept); at the entry function's end thread 0 reserves vertex slots and indices in the arena with atomics, +// and every thread copies vertices and triangles out (indices made absolute vertex slots, relative to the vertex area). +// With a task stage mesh workgroup (x, y) is mesh group x of task record (task base + y): a prologue returns when x is +// past the record's count, loads the payload, and the module's WorkgroupId reads become (x, 0, 0). +// Task -> GLCompute: the payload moves to Workgroup memory; EmitMeshTasksEXT becomes a store of the payload and the +// mesh group count into the task record, and workgroup 0 writes the mesh dispatch {max_mesh, task workgroups, 1} +// into the draw record. +#include "mesh_emu_spirv.h" + +#include +#include +#include + +enum { + OpSourceContinued = 2, OpSource = 3, OpSourceExtension = 4, OpName = 5, OpMemberName = 6, OpString = 7, OpLine = 8, + OpExtension = 10, OpExtInstImport = 11, OpExtInst = 12, OpMemoryModel = 14, OpEntryPoint = 15, OpExecutionMode = 16, + OpCapability = 17, OpTypeVoid = 19, OpTypeBool = 20, OpTypeInt = 21, OpTypeFloat = 22, OpTypeVector = 23, + OpTypeArray = 28, OpTypeRuntimeArray = 29, OpTypeStruct = 30, OpTypePointer = 32, OpTypeFunction = 33, + OpConstant = 43, OpConstantComposite = 44, OpFunction = 54, OpFunctionEnd = 56, OpVariable = 59, OpLoad = 61, + OpStore = 62, OpAccessChain = 65, OpInBoundsAccessChain = 66, OpDecorate = 71, OpMemberDecorate = 72, + OpCompositeConstruct = 80, OpCompositeExtract = 81, OpUConvert = 113, OpConvertUToPtr = 120, OpBitcast = 124, + OpIAdd = 128, OpIMul = 132, OpLogicalAnd = 167, OpSelect = 169, OpIEqual = 170, OpINotEqual = 171, + OpULessThan = 176, OpULessThanEqual = 178, OpControlBarrier = 224, OpAtomicIAdd = 234, OpAtomicUMax = 239, + OpSelectionMerge = 247, OpLabel = 248, OpBranch = 249, OpBranchConditional = 250, OpReturn = 253, + OpNoLine = 317, OpModuleProcessed = 330, OpExecutionModeId = 331, OpConstantNull = 46, OpCopyObject = 83, + OpEmitMeshTasksEXT = 5294, OpSetMeshOutputsEXT = 5295, +}; +enum { ScInput = 1, ScOutput = 3, ScWorkgroup = 4, ScPrivate = 6, ScFunction = 7, ScPushConstant = 9, + ScPhysicalStorageBuffer = 5349, ScTaskPayload = 5402 }; +enum { DecBlock = 2, DecArrayStride = 6, DecBuiltIn = 11, DecLocation = 30, DecOffset = 35, DecPerPrimitive = 5271 }; + BiPrimTri = 5296 }; +enum { CapShader = 1, CapInt64 = 11, CapMeshShading = 5283, CapPSBAddresses = 5347 }; +enum { ModelVertex = 0, ModelGLCompute = 5, ModelTask = 5364, ModelMesh = 5365 }; +enum { ModeLocalSize = 17, ModeOutputVertices = 26, ModeOutputPoints = 27, ModeOutputLines = 5269, + ModeOutputPrimitives = 5270, ModeOutputTriangles = 5298 }; +enum { GlslUMin = 38 }; +enum { MaxIns = 4096 }; + +typedef struct { uint32_t *w; size_t n, cap; int bad; } Buf; + +static void put(Buf *b, uint32_t x) { + if (b->n == b->cap) { + size_t c = b->cap ? b->cap * 2 : 1024; + uint32_t *w = realloc(b->w, c * sizeof *w); + if (!w) { b->bad = 1; return; } + b->w = w, b->cap = c; + } + b->w[b->n++] = x; +} +static void putn(Buf *b, const uint32_t *w, size_t n) { for (size_t i = 0; i < n; i++) put(b, w[i]); } +static void op(Buf *b, uint32_t opc, int n, ...) { + va_list ap; + va_start(ap, n); + put(b, (uint32_t)(n + 1) << 16 | opc); + for (int i = 0; i < n; i++) put(b, va_arg(ap, uint32_t)); + va_end(ap); +} +static void op_str(Buf *b, uint32_t opc, const uint32_t *pre, int npre, const char *s, const uint32_t *post, int npost) { + size_t sl = strlen(s) + 1, sw = (sl + 3) / 4; + put(b, (uint32_t)(1 + npre + sw + npost) << 16 | opc); + for (int i = 0; i < npre; i++) put(b, pre[i]); + for (size_t i = 0; i < sw; i++) { + uint32_t x = 0; + for (size_t k = 0; k < 4 && i * 4 + k < sl; k++) x |= (uint32_t)(uint8_t)s[i * 4 + k] << (8 * k); + put(b, x); + } + for (int i = 0; i < npost; i++) put(b, post[i]); +} + +// ---- the module +typedef struct { + const uint32_t *w; + size_t n; + uint32_t bound; + uint32_t t_void, t_bool, t_u32, t_u64, t_f32, t_v3u, t_v4u, t_v4f; + uint32_t glsl, ep_fn, model, wgid, lii, nwg; + int has_int64, has_psb, has_psb_ext; + struct { uint32_t id, sc, to; } ptr[4096]; + int nptr; + struct { uint32_t id, type, sc; } var[2048]; + int nvar; + struct { uint32_t id, type, val; } cst[8192]; + int ncst; +} Mod; + +static uint32_t find_ptr(const Mod *m, uint32_t sc, uint32_t to) { + for (int i = 0; i < m->nptr; i++) + if (m->ptr[i].sc == sc && m->ptr[i].to == to) return m->ptr[i].id; + return 0; +} +static uint32_t ptr_to(const Mod *m, uint32_t id) { + for (int i = 0; i < m->nptr; i++) + if (m->ptr[i].id == id) return m->ptr[i].to; + return 0; +} +static const uint32_t *find_type(const Mod *m, uint32_t id) { + for (size_t i = 5; i < m->n;) { + uint32_t o = m->w[i] & 0xffff, len = m->w[i] >> 16; + if (!len || o == OpFunction) break; + if ((o == OpTypeArray || o == OpTypeRuntimeArray || o == OpTypeVector || o == OpTypeStruct) && m->w[i + 1] == id) + return m->w + i; + i += len; + } + return NULL; +} +static int const_u32(const Mod *m, uint32_t id, uint32_t *v) { + for (int i = 0; i < m->ncst; i++) + if (m->cst[i].id == id && m->cst[i].type == m->t_u32) return *v = m->cst[i].val, 1; + return 0; +} +static int is_debug(uint32_t o) { + return o == OpSourceContinued || o == OpSource || o == OpSourceExtension || o == OpName || o == OpMemberName || + o == OpString || o == OpModuleProcessed; +} +static int is_annot(uint32_t o) { return o == OpDecorate || o == OpMemberDecorate || o == 73 || o == 74 || o == 75 || o == 332 || o == 5632 || o == 5633; } + +static int parse(Mod *m, const uint32_t *w, size_t n, const char **why) { + m->w = w, m->n = n; + if (n < 5 || w[0] != 0x07230203) return *why = "not SPIR-V", 0; + m->bound = w[3]; + for (size_t i = 5; i < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) return *why = "malformed", 0; + if (len > MaxIns) return *why = "an instruction too long", 0; + const uint32_t *a = w + i + 1; + if (o == OpFunction) break; + switch (o) { + case OpCapability: + if (a[0] == CapInt64) m->has_int64 = 1; + if (a[0] == CapPSBAddresses) m->has_psb = 1; + break; + case OpExtension: if (!strcmp((const char *)a, "SPV_KHR_physical_storage_buffer")) m->has_psb_ext = 1; break; + case OpExtInstImport: if (!strcmp((const char *)(a + 1), "GLSL.std.450")) m->glsl = a[0]; break; + case OpEntryPoint: + if (m->ep_fn) return *why = "several entry points", 0; + m->model = a[0], m->ep_fn = a[1]; + break; + case OpTypeVoid: if (!m->t_void) m->t_void = a[0]; break; + case OpTypeBool: if (!m->t_bool) m->t_bool = a[0]; break; + case OpTypeInt: + if (a[1] == 32 && !a[2] && !m->t_u32) m->t_u32 = a[0]; + if (a[1] == 64 && !a[2] && !m->t_u64) m->t_u64 = a[0]; + break; + case OpTypeFloat: if (a[1] == 32 && !m->t_f32) m->t_f32 = a[0]; break; + case OpTypeVector: + if (a[1] == m->t_u32 && a[2] == 3 && !m->t_v3u) m->t_v3u = a[0]; + if (a[1] == m->t_u32 && a[2] == 4 && !m->t_v4u) m->t_v4u = a[0]; + if (a[1] == m->t_f32 && m->t_f32 && a[2] == 4 && !m->t_v4f) m->t_v4f = a[0]; + break; + case OpTypePointer: + if (m->nptr == 4096) return *why = "too many pointer types", 0; + m->ptr[m->nptr].id = a[0], m->ptr[m->nptr].sc = a[1], m->ptr[m->nptr++].to = a[2]; + break; + case OpVariable: + if (m->nvar == 2048) return *why = "too many variables", 0; + m->var[m->nvar].id = a[1], m->var[m->nvar].type = a[0], m->var[m->nvar++].sc = a[2]; + break; + case OpConstant: + if (len == 4 && m->ncst < 8192) m->cst[m->ncst].id = a[1], m->cst[m->ncst].type = a[0], m->cst[m->ncst++].val = a[2]; + break; + case OpDecorate: + if (len >= 4 && a[1] == DecBuiltIn) { + if (a[2] == BiWorkgroupId) m->wgid = a[0]; + if (a[2] == BiLocalInvocationIndex) m->lii = a[0]; + if (a[2] == BiNumWorkgroups) m->nwg = a[0]; + } + break; + } + i += len; + } + if (!m->ep_fn || (m->model != ModelMesh && m->model != ModelTask)) return *why = "not a task or mesh module", 0; + if (!m->t_u32 || !m->t_void || !m->glsl || !m->lii || !m->wgid) return *why = "missing basic types or built-ins", 0; + return 1; +} + +// ---- emission over new ids +typedef struct { + Mod *m; + Buf *types, *annot, *code; + uint32_t p_u32_psb, p_v4u_psb, p_words, p_slots, p_pc_u64, p_pc_u32, pc_var, c64_16; + struct { uint32_t id, val; } cc[256]; + int ncc; +} Em; + +static uint32_t nid(Mod *m) { return m->bound++; } +static uint32_t cu(Em *e, uint32_t v) { + for (int i = 0; i < e->m->ncst; i++) + if (e->m->cst[i].type == e->m->t_u32 && e->m->cst[i].val == v) return e->m->cst[i].id; + for (int i = 0; i < e->ncc; i++) + if (e->cc[i].val == v) return e->cc[i].id; + uint32_t id = nid(e->m); + op(e->types, OpConstant, 3, e->m->t_u32, id, v); + if (e->ncc < 256) e->cc[e->ncc].id = id, e->cc[e->ncc++].val = v; + return id; +} +static uint32_t E1(Em *e, uint32_t o, uint32_t t, uint32_t a) { uint32_t r = nid(e->m); op(e->code, o, 3, t, r, a); return r; } +static uint32_t E2(Em *e, uint32_t o, uint32_t t, uint32_t a, uint32_t b) { uint32_t r = nid(e->m); op(e->code, o, 4, t, r, a, b); return r; } +static uint32_t E3(Em *e, uint32_t o, uint32_t t, uint32_t a, uint32_t b, uint32_t c) { uint32_t r = nid(e->m); op(e->code, o, 5, t, r, a, b, c); return r; } +static uint32_t add(Em *e, uint32_t a, uint32_t b) { return E2(e, OpIAdd, e->m->t_u32, a, b); } +static uint32_t mul(Em *e, uint32_t a, uint32_t b) { return E2(e, OpIMul, e->m->t_u32, a, b); } +static uint32_t ult(Em *e, uint32_t a, uint32_t b) { return E2(e, OpULessThan, e->m->t_bool, a, b); } +static uint32_t ld(Em *e, uint32_t t, uint32_t p) { return E1(e, OpLoad, t, p); } +static uint32_t pc_u64(Em *e) { return ld(e, e->m->t_u64, E2(e, OpAccessChain, e->p_pc_u64, e->pc_var, cu(e, 0))); } +static uint32_t pc_u32(Em *e, uint32_t k) { return ld(e, e->m->t_u32, E2(e, OpAccessChain, e->p_pc_u32, e->pc_var, cu(e, k))); } +static uint32_t words_base(Em *e) { return E1(e, OpConvertUToPtr, e->p_words, pc_u64(e)); } +static uint32_t word_ptr(Em *e, uint32_t base, uint32_t idx) { return E3(e, OpAccessChain, e->p_u32_psb, base, cu(e, 0), idx); } +static uint32_t word_load(Em *e, uint32_t base, uint32_t idx) { + uint32_t p = word_ptr(e, base, idx), r = nid(e->m); + op(e->code, OpLoad, 5, e->m->t_u32, r, p, 2u, 4u); // Aligned 4 + return r; +} +static void word_store(Em *e, uint32_t base, uint32_t idx, uint32_t v) { op(e->code, OpStore, 4, word_ptr(e, base, idx), v, 2u, 4u); } +static uint32_t atomic(Em *e, uint32_t o, uint32_t base, uint32_t idx, uint32_t v) { + uint32_t p = word_ptr(e, base, idx), r = nid(e->m); + op(e->code, o, 6, e->m->t_u32, r, p, cu(e, 1), cu(e, 0), v); // Device scope, relaxed + return r; +} +static void barrier(Em *e) { op(e->code, OpControlBarrier, 3, cu(e, 2), cu(e, 2), cu(e, 264)); } // Workgroup, AcqRel|WorkgroupMemory +static void if_begin(Em *e, uint32_t cond, uint32_t *merge) { + uint32_t t = nid(e->m); + *merge = nid(e->m); + op(e->code, OpSelectionMerge, 2, *merge, 0u); + op(e->code, OpBranchConditional, 3, cond, t, *merge); + op(e->code, OpLabel, 1, t); +} +static void if_end(Em *e, uint32_t merge) { + op(e->code, OpBranch, 1, merge); + op(e->code, OpLabel, 1, merge); +} + + +// The mesh epilogue (replaces the entry function's return) +static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int nov, uint32_t vcount, uint32_t pcount, + Mod *m = e->m; + barrier(e); + uint32_t lid = ld(e, m->t_u32, m->lii), base = words_base(e); + uint32_t mz; + if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); + { + uint32_t vb = atomic(e, OpAtomicIAdd, base, cu(e, 0), vsz); + uint32_t iok = E2(e, OpULessThanEqual, m->t_bool, add(e, ib, isz), icap); + uint32_t vok = E2(e, OpULessThanEqual, m->t_bool, add(e, vb, vsz), vcap); + uint32_t mi; + if_begin(e, iok, &mi); + atomic(e, OpAtomicIAdd, base, drw, isz); // the draw record's index count + if_end(e, mi); + uint32_t all = E2(e, OpLogicalAnd, m->t_bool, iok, vok); + atomic(e, OpAtomicIAdd, base, cu(e, 3), E3(e, OpSelect, m->t_u32, all, cu(e, 0), cu(e, 1))); + op(e->code, OpStore, 2, sh_vb, E3(e, OpSelect, m->t_u32, vok, vb, cu(e, 0xffffffffu))); + op(e->code, OpStore, 2, sh_ib, E3(e, OpSelect, m->t_u32, iok, ib, cu(e, 0xffffffffu))); + } + if_end(e, mz); + barrier(e); + uint32_t VB = ld(e, m->t_u32, sh_vb), IB = ld(e, m->t_u32, sh_ib); + uint32_t m3; + if_begin(e, E2(e, OpINotEqual, m->t_bool, IB, cu(e, 0xffffffffu)), &m3); + { + uint32_t vbok = E2(e, OpINotEqual, m->t_bool, VB, cu(e, 0xffffffffu)); + uint32_t m4; + if_begin(e, vbok, &m4); + { + uint32_t vw = pc_u32(e, 4); + uint32_t a64 = E2(e, OpIAdd, m->t_u64, pc_u64(e), E2(e, OpIMul, m->t_u64, E1(e, OpUConvert, m->t_u64, vw), e->c64_16)); + uint32_t sbase = E1(e, OpConvertUToPtr, e->p_slots, a64); + for (uint32_t r = 0; r * T < info->max_v; r++) { + uint32_t v = add(e, lid, cu(e, r * T)), mv; + if_begin(e, ult(e, v, vcount), &mv); + uint32_t s0 = add(e, VB, mul(e, v, cu(e, S))); + for (int k = 0; k < nov; k++) { + if (ov[k].role == 2) continue; + uint32_t dst = E3(e, OpAccessChain, e->p_v4u_psb, sbase, cu(e, 0), add(e, s0, cu(e, ov[k].slot))); + op(e->code, OpStore, 4, dst, bits, 2u, 16u); + } + if_end(e, mv); + } + } + if_end(e, m4); + uint32_t iw = pc_u32(e, 2), idxv = 0; + for (int k = 0; k < nov; k++) + if (ov[k].role == 2) idxv = ov[k].var; + for (uint32_t r = 0; r * T < info->max_p; r++) { + uint32_t p = add(e, lid, cu(e, r * T)), mp; + if_begin(e, ult(e, p, pcount), &mp); + uint32_t val = E3(e, OpSelect, m->t_u32, vbok, add(e, VB, mul(e, li, cu(e, S))), cu(e, 0)); + word_store(e, base, add(e, at, cu(e, c)), val); + } + if_end(e, mp); + } + } + if_end(e, m3); + op(e->code, OpReturn, 0); +} + +// The task epilogue (replaces EmitMeshTasksEXT x y z) +static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint32_t p_wg_u32, uint32_t nwg, + uint32_t ex, uint32_t ey, uint32_t ez) { + Mod *m = e->m; + uint32_t T = info->threads; + uint32_t cnt = mul(e, ex, mul(e, ey, ez)); + uint32_t nc = E3(e, OpSelect, m->t_u32, ult(e, cnt, cu(e, info->max_mesh)), cnt, cu(e, info->max_mesh)); + barrier(e); + uint32_t lid = ld(e, m->t_u32, m->lii), wg = ld(e, m->t_v3u, m->wgid), nw = ld(e, m->t_v3u, nwg); + uint32_t wx = E2(e, OpCompositeExtract, m->t_u32, wg, 0u), wy = E2(e, OpCompositeExtract, m->t_u32, wg, 1u), + wz = E2(e, OpCompositeExtract, m->t_u32, wg, 2u); + uint32_t nx = E2(e, OpCompositeExtract, m->t_u32, nw, 0u), ny = E2(e, OpCompositeExtract, m->t_u32, nw, 1u), + nz = E2(e, OpCompositeExtract, m->t_u32, nw, 2u); + uint32_t nxy = mul(e, nx, ny); + uint32_t t = add(e, wx, add(e, mul(e, wy, nx), mul(e, wz, nxy))); + uint32_t base = words_base(e), drw = pc_u32(e, 1); + uint32_t tb = word_load(e, base, add(e, drw, cu(e, 8))); + { + uint32_t recw = add(e, cu(e, MESH_EMU_TOFF / 4), mul(e, slot, cu(e, MESH_EMU_TWORDS))); + uint32_t v = ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, payload, k)); + word_store(e, base, add(e, add(e, recw, cu(e, 1)), k), v); + } + uint32_t m2; + if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2); + word_store(e, base, recw, nc); + if_end(e, m2); + } + if_end(e, m1); + uint32_t first = E2(e, OpLogicalAnd, m->t_bool, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), + E2(e, OpIEqual, m->t_bool, t, cu(e, 0))); + uint32_t m3; + if_begin(e, first, &m3); + { + uint32_t total = mul(e, nxy, nz); + // task records past the area are not written: their mesh groups are not dispatched + uint32_t ty = E3(e, OpSelect, m->t_u32, ult(e, total, room), total, room); + word_store(e, base, add(e, drw, cu(e, 5)), cu(e, info->max_mesh)); + word_store(e, base, add(e, drw, cu(e, 6)), ty); + word_store(e, base, add(e, drw, cu(e, 7)), cu(e, 1)); + } + if_end(e, m3); + op(e->code, OpReturn, 0); +} + +uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why) { + static const char *dummy; + if (!why) why = &dummy; + *why = NULL; + Mod *m = calloc(1, sizeof *m); + uint32_t *ins = malloc(MaxIns * 4), *ret = NULL; + Buf caps = {0}, exts = {0}, imps = {0}, modes = {0}, debug = {0}, annot = {0}, types = {0}, ntypes = {0}, nannot = {0}, + code = {0}, out = {0}; + if (!m || !ins) { *why = "out of memory"; goto done; } + if (!parse(m, w, n, why)) goto done; + memset(info, 0, sizeof *info); + int mesh = m->model == ModelMesh; + info->task = !mesh; + for (size_t i = 5; i < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + const uint32_t *a = w + i + 1; + if (o == OpFunction) break; + if (o == OpExecutionMode && a[0] == m->ep_fn) { + if (a[1] == ModeLocalSize) info->threads = a[2] * a[3] * a[4]; + if (a[1] == ModeOutputVertices) info->max_v = a[2]; + if (a[1] == ModeOutputPrimitives) info->max_p = a[2]; + if (a[1] == ModeOutputTriangles) info->prim = 3; + if (a[1] == ModeOutputLines) info->prim = 2; + if (a[1] == ModeOutputPoints) info->prim = 1; + } + i += len; + } + if (!info->threads) { *why = "no LocalSize"; goto done; } + if (!m->t_f32 || (mesh && !m->t_v4f)) { *why = "no float types"; goto done; } + + // output variables and the payload + OutVar ov[40]; + int nov = 0; + uint32_t payload = 0, payload_len = 0; + for (int i = 0; i < m->nvar; i++) { + uint32_t sc = m->var[i].sc; + if (sc != ScOutput && sc != ScTaskPayload) continue; + const uint32_t *arr = find_type(m, ptr_to(m, m->var[i].type)); + uint32_t len = 0; + if (!arr || (arr[0] & 0xffff) != OpTypeArray || !const_u32(m, arr[3], &len)) { *why = "an output or payload is not a sized array"; goto done; } + if (sc == ScTaskPayload) { + if (arr[2] != m->t_u32) { *why = "a payload that is not uint[]"; goto done; } + payload = m->var[i].id, payload_len = len; + continue; + } + if (nov == 40) { *why = "too many outputs"; goto done; } + ov[nov] = (OutVar){m->var[i].id, len, 9, 0, 0}; + for (size_t k = 5; k < n;) { + uint32_t o = w[k] & 0xffff, l2 = w[k] >> 16; + const uint32_t *a = w + k + 1; + if (o == OpFunction) break; + if (o == OpDecorate && a[0] == m->var[i].id) { + if (a[1] == DecBuiltIn && a[2] == BiPosition) ov[nov].role = 0; + else if (a[1] == DecLocation) ov[nov].role = 1, ov[nov].loc = a[2]; + else if (a[1] == DecBuiltIn) { *why = "an unsupported mesh output built-in"; goto done; } + else if (a[1] == DecPerPrimitive) { *why = "per-primitive outputs are not emulated"; goto done; } + } + k += l2; + } + if (ov[nov].role == 9) { *why = "an output without a role"; goto done; } + nov++; + } + info->payload_words = payload_len; + if (payload_len > MESH_EMU_TWORDS - 1) { *why = "payload too large"; goto done; } + if (mesh) { + uint32_t locs = 0; + int have_pos = 0, have_idx = 0; + for (int i = 0; i < nov; i++) { + if (ov[i].role == 1) { + if (ov[i].loc >= 32 || locs >> ov[i].loc & 1) { *why = "bad generic location"; goto done; } + locs |= 1u << ov[i].loc; + } + have_pos |= ov[i].role == 0, have_idx |= ov[i].role == 2; + } + if (!have_pos || !have_idx) { *why = "no position or no indices"; goto done; } + for (uint32_t l = 0; l < 32; l++) + if (locs >> l & 1) info->generic_loc[info->n_generic++] = (uint8_t)l; + info->slots = 1 + info->n_generic; + for (int i = 0; i < nov; i++) { + if (ov[i].role == 0) ov[i].slot = 0; + if (ov[i].role == 1) + for (uint32_t g = 0; g < info->n_generic; g++) + if (info->generic_loc[g] == ov[i].loc) ov[i].slot = 1 + g; + } + } + uint32_t emit_x = 0, emit_y = 0, emit_z = 0; + if (!mesh) { + size_t i = 5; + while (i < n && (w[i] & 0xffff) != OpFunction) i += w[i] >> 16; + for (; i < n; i += w[i] >> 16) + if ((w[i] & 0xffff) == OpEmitMeshTasksEXT) emit_x = w[i + 1], emit_y = w[i + 2], emit_z = w[i + 3]; + if (!emit_x || !payload) { *why = "no EmitMeshTasksEXT or no payload"; goto done; } + info->max_mesh = 32; + i = 5; + while (i < n && (w[i] & 0xffff) != OpFunction) i += w[i] >> 16; + for (; i < n; i += w[i] >> 16) { + uint32_t v; + if ((w[i] & 0xffff) == OpExtInst && (w[i] >> 16) == 7 && w[i + 2] == emit_x && w[i + 3] == m->glsl && + w[i + 4] == GlslUMin && const_u32(m, w[i + 6], &v) && v && v <= 1024) + info->max_mesh = v; + } + } + + // ---- new types, constants and variables + Em e = {.m = m, .types = &ntypes, .annot = &nannot, .code = &code}; + if (!m->t_bool) m->t_bool = nid(m), op(&ntypes, OpTypeBool, 1, m->t_bool); + if (!m->t_u64) m->t_u64 = nid(m), op(&ntypes, OpTypeInt, 3, m->t_u64, 64u, 0u); + if (!m->t_v3u) m->t_v3u = nid(m), op(&ntypes, OpTypeVector, 3, m->t_v3u, m->t_u32, 3u); + if (!m->t_v4u) m->t_v4u = nid(m), op(&ntypes, OpTypeVector, 3, m->t_v4u, m->t_u32, 4u); + uint32_t rt_u = nid(m), s_words = nid(m), rt_v4 = nid(m), s_slots = nid(m), pc_struct = nid(m); + op(&ntypes, OpTypeRuntimeArray, 2, rt_u, m->t_u32); + op(&ntypes, OpTypeStruct, 2, s_words, rt_u); + op(&ntypes, OpTypeRuntimeArray, 2, rt_v4, m->t_v4u); + op(&ntypes, OpTypeStruct, 2, s_slots, rt_v4); + op(&ntypes, OpTypeStruct, 8, pc_struct, m->t_u64, m->t_u32, m->t_u32, m->t_u32, m->t_u32, m->t_u32, m->t_u32); + e.p_words = nid(m), op(&ntypes, OpTypePointer, 3, e.p_words, (uint32_t)ScPhysicalStorageBuffer, s_words); + e.p_slots = nid(m), op(&ntypes, OpTypePointer, 3, e.p_slots, (uint32_t)ScPhysicalStorageBuffer, s_slots); +#define NEED_PTR(DST, SC, TO) do { DST = find_ptr(m, SC, TO); if (!DST) { DST = nid(m); op(&ntypes, OpTypePointer, 3, DST, (uint32_t)(SC), TO); \ + m->ptr[m->nptr].id = DST, m->ptr[m->nptr].sc = SC, m->ptr[m->nptr++].to = TO; } } while (0) + NEED_PTR(e.p_u32_psb, ScPhysicalStorageBuffer, m->t_u32); + NEED_PTR(e.p_v4u_psb, ScPhysicalStorageBuffer, m->t_v4u); + uint32_t p_pc = nid(m); + op(&ntypes, OpTypePointer, 3, p_pc, (uint32_t)ScPushConstant, pc_struct); + NEED_PTR(e.p_pc_u64, ScPushConstant, m->t_u64); + NEED_PTR(e.p_pc_u32, ScPushConstant, m->t_u32); + e.pc_var = nid(m), op(&ntypes, OpVariable, 3, p_pc, e.pc_var, (uint32_t)ScPushConstant); + e.c64_16 = nid(m), op(&ntypes, OpConstant, 4, m->t_u64, e.c64_16, 16u, 0u); + op(&nannot, OpDecorate, 3, rt_u, (uint32_t)DecArrayStride, 4u); + op(&nannot, OpDecorate, 2, s_words, (uint32_t)DecBlock); + op(&nannot, OpMemberDecorate, 4, s_words, 0u, (uint32_t)DecOffset, 0u); + op(&nannot, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u); + op(&nannot, OpDecorate, 2, s_slots, (uint32_t)DecBlock); + op(&nannot, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u); + op(&nannot, OpDecorate, 2, pc_struct, (uint32_t)DecBlock); + for (uint32_t k = 0; k < 7; k++) op(&nannot, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u); + + // Output and payload pointers become Workgroup pointers: in place when no such pointer exists yet, else mapped + struct { uint32_t from, to; } pmap[512]; + int npmap = 0; + for (int i = 0; i < m->nptr; i++) { + if (m->ptr[i].sc != ScOutput && m->ptr[i].sc != ScTaskPayload) continue; + uint32_t have = find_ptr(m, ScWorkgroup, m->ptr[i].to); + if (have) { + if (npmap == 512) { *why = "too many pointer types"; goto done; } + pmap[npmap].from = m->ptr[i].id, pmap[npmap++].to = have; + m->ptr[i].sc = 0xffffffffu; // dropped + } else { + m->ptr[i].sc = ScWorkgroup; // rewritten in place below + } + } + uint32_t p_wg_u32, p_wg_v4f = 0, p_wg_v3u = 0; + NEED_PTR(p_wg_u32, ScWorkgroup, m->t_u32); + uint32_t sh_vb = 0, sh_ib = 0, wgemu = 0, p_priv_v3 = 0, p_priv_u32 = 0, p_in_u32 = find_ptr(m, ScInput, m->t_u32); + if (mesh) { + sh_vb = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_vb, (uint32_t)ScWorkgroup); + sh_ib = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_ib, (uint32_t)ScWorkgroup); + } + int remap_wg = mesh && payload; + if (remap_wg) { + NEED_PTR(p_priv_v3, ScPrivate, m->t_v3u); + NEED_PTR(p_priv_u32, ScPrivate, m->t_u32); + wgemu = nid(m), op(&ntypes, OpVariable, 3, p_priv_v3, wgemu, (uint32_t)ScPrivate); + } + uint32_t nwg = m->nwg; + if (!mesh && !nwg) { + uint32_t p_in_v3; + NEED_PTR(p_in_v3, ScInput, m->t_v3u); + nwg = nid(m), op(&ntypes, OpVariable, 3, p_in_v3, nwg, (uint32_t)ScInput); + op(&nannot, OpDecorate, 3, nwg, (uint32_t)DecBuiltIn, (uint32_t)BiNumWorkgroups); + } +#undef NEED_PTR + + // ---- one pass over the module + uint32_t moved[48]; + int nmoved = 0; + for (int i = 0; i < nov; i++) moved[nmoved++] = ov[i].var; + if (payload) moved[nmoved++] = payload; + uint32_t vcount = 0, pcount = 0; + int in_fn = 0, in_ep = 0, ep_started = 0, ep_done = 0; + for (size_t i = 5; i < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + memcpy(ins, w + i, len * 4); + const uint32_t *a = w + i + 1; + if (o == OpFunction) in_fn = 1, in_ep = a[1] == m->ep_fn; + if (!in_fn) { + if (o == OpCapability) { if (a[0] != CapMeshShading) putn(&caps, ins, len); } + else if (o == OpExtension) { if (strcmp((const char *)a, "SPV_EXT_mesh_shader")) putn(&exts, ins, len); } + else if (o == OpExtInstImport) putn(&imps, ins, len); + else if (o == OpMemoryModel || o == OpEntryPoint) {} + else if (o == OpExecutionMode || o == OpExecutionModeId) { + if (!(o == OpExecutionMode && (a[1] == ModeOutputVertices || a[1] == ModeOutputPrimitives || + a[1] == ModeOutputTriangles || a[1] == ModeOutputLines || a[1] == ModeOutputPoints))) + putn(&modes, ins, len); + } else if (is_debug(o)) putn(&debug, ins, len); + else if (is_annot(o)) { + int drop = 0; + for (int k = 0; k < nmoved; k++) drop |= o == OpDecorate && a[0] == moved[k]; + if (!drop) putn(&annot, ins, len); + } else { // types, constants, global variables + int drop = 0; + if (o == OpTypePointer) { + for (int k = 0; k < npmap; k++) drop |= pmap[k].from == a[0]; + if (a[1] == ScOutput || a[1] == ScTaskPayload) ins[2] = ScWorkgroup; + } + if (o == OpVariable && (a[2] == ScOutput || a[2] == ScTaskPayload)) ins[3] = ScWorkgroup; + for (uint32_t k = 1; k < len; k++) { + int idpos = o == OpTypeFunction || o == OpTypeStruct || o == OpTypeArray || o == OpTypeRuntimeArray || + (o == OpTypePointer && k == 3) || k == 1; + if (o == OpTypePointer && k == 1) idpos = 0; + for (int q = 0; idpos && q < npmap; q++) + if (ins[k] == pmap[q].from) ins[k] = pmap[q].to; + } + if (!drop) putn(&types, ins, len); + } + i += len; + continue; + } + // function code + if (len >= 2) + for (int q = 0; q < npmap; q++) + if (ins[1] == pmap[q].from) ins[1] = pmap[q].to; + if (remap_wg && len >= 4 && (o == OpLoad || o == OpAccessChain || o == OpInBoundsAccessChain) && ins[3] == m->wgid) { + ins[3] = wgemu; + if (o != OpLoad) { + if (!p_in_u32 || ins[1] != p_in_u32) { *why = "an unexpected WorkgroupId access"; goto done; } + ins[1] = p_priv_u32; + } + } else if (remap_wg) { + for (uint32_t k = 1; k < len; k++) { // id operands only: literals of these instructions may equal the id + int lit = (o == OpExtInst && k == 4) || (o == 81 && k >= 4) || (o == 82 && k >= 5) || (o == 79 && k >= 5) || + (o == OpLoad && k >= 4) || (o == OpStore && k >= 3) || o == 251 /* OpSwitch */ || o == OpSelectionMerge || + o == 246 /* OpLoopMerge */; + if (!lit && ins[k] == m->wgid) { *why = "an unexpected WorkgroupId use"; goto done; } + } + } + if (o == OpFunctionEnd) in_fn = 0, in_ep = 0; + if (in_ep && o == OpLabel && !ep_started) { + putn(&code, ins, len); + size_t j = i + len; // function variables stay first + while (j < n && (w[j] & 0xffff) == OpVariable) putn(&code, w + j, w[j] >> 16), j += w[j] >> 16; + ep_started = 1; + i = j; + if (remap_wg) { + uint32_t lid = ld(&e, m->t_u32, m->lii), wg = ld(&e, m->t_v3u, m->wgid); + uint32_t x = E2(&e, OpCompositeExtract, m->t_u32, wg, 0u), y = E2(&e, OpCompositeExtract, m->t_u32, wg, 1u); + uint32_t base = words_base(&e), drw = pc_u32(&e, 1); + uint32_t tb = word_load(&e, base, add(&e, drw, cu(&e, 8))); + uint32_t slotc = E3(&e, OpSelect, m->t_u32, ok, slot, cu(&e, 0)); + uint32_t recw = add(&e, cu(&e, MESH_EMU_TOFF / 4), mul(&e, slotc, cu(&e, MESH_EMU_TWORDS))); + uint32_t cnt = word_load(&e, base, recw); + uint32_t act = E2(&e, OpLogicalAnd, m->t_bool, ok, ult(&e, x, cnt)); + uint32_t cont = nid(m), retl = nid(m); + op(&code, OpSelectionMerge, 2, cont, 0u); + op(&code, OpBranchConditional, 3, act, cont, retl); + op(&code, OpLabel, 1, retl); + op(&code, OpReturn, 0); + op(&code, OpLabel, 1, cont); + uint32_t v3 = nid(m); + op(&code, OpCompositeConstruct, 5, m->t_v3u, v3, x, cu(&e, 0), cu(&e, 0)); + op(&code, OpStore, 2, wgemu, v3); + uint32_t v = word_load(&e, base, add(&e, add(&e, recw, cu(&e, 1)), k)); + op(&code, OpStore, 2, E2(&e, OpAccessChain, p_wg_u32, payload, k), v); + } + barrier(&e); + } + continue; + } + if (in_ep && o == OpSetMeshOutputsEXT) { + vcount = a[0], pcount = a[1]; + i += len; + continue; + } + if (in_ep && mesh && o == OpReturn) { + if (!vcount || ep_done) { *why = "no SetMeshOutputsEXT before the return, or several returns"; goto done; } + ep_done = 1; + i += len; + continue; + } + if (in_ep && !mesh && o == OpEmitMeshTasksEXT) { + task_epilogue(&e, info, payload, p_wg_u32, nwg, emit_x, emit_y, emit_z); + ep_done = 1; + i += len; + continue; + } + putn(&code, ins, len); + i += len; + } + if (!ep_done) { *why = "the entry function's end was not found"; goto done; } + + // ---- assemble + put(&out, 0x07230203); + put(&out, w[1] < 0x00010500 ? 0x00010500 : w[1]); // SPIR-V 1.5: physical storage buffers are core + put(&out, w[2]); + put(&out, m->bound); + put(&out, 0); + putn(&out, caps.w, caps.n); + if (!m->has_int64) op(&out, OpCapability, 1, (uint32_t)CapInt64); + if (!m->has_psb) op(&out, OpCapability, 1, (uint32_t)CapPSBAddresses); + putn(&out, exts.w, exts.n); + putn(&out, imps.w, imps.n); + op(&out, OpMemoryModel, 2, 5348u, 1u); // PhysicalStorageBuffer64, GLSL450 + { // entry point: every global variable + Buf ep = {0}; + uint32_t pre[2] = {ModelGLCompute, m->ep_fn}; + uint32_t ifc[2048 + 16]; + int ni = 0; + for (int i = 0; i < m->nvar && ni < 2048; i++) { + int dropped = 0; + if (!dropped && m->var[i].sc != ScFunction) ifc[ni++] = m->var[i].id; + } + // the new ones: scan ntypes for OpVariable + for (size_t i = 0; i < ntypes.n;) { + uint32_t o = ntypes.w[i] & 0xffff, len = ntypes.w[i] >> 16; + if (o == OpVariable && ni < 2048 + 16) ifc[ni++] = ntypes.w[i + 2]; + i += len; + } + op_str(&ep, OpEntryPoint, pre, 2, "main", ifc, ni); + putn(&out, ep.w, ep.n); + free(ep.w); + } + putn(&out, modes.w, modes.n); + putn(&out, debug.w, debug.n); + putn(&out, annot.w, annot.n); + putn(&out, nannot.w, nannot.n); + putn(&out, types.w, types.n); + putn(&out, ntypes.w, ntypes.n); + putn(&out, code.w, code.n); + out.w[3] = m->bound; + if (caps.bad || exts.bad || imps.bad || modes.bad || debug.bad || annot.bad || types.bad || ntypes.bad || + nannot.bad || code.bad || out.bad) { *why = "out of memory"; goto done; } + ret = out.w, *out_n = out.n, out.w = NULL; +done: + free(caps.w), free(exts.w), free(imps.w), free(modes.w), free(debug.w), free(annot.w), free(types.w), free(ntypes.w), + free(nannot.w), free(code.w), free(out.w); + free(m), free(ins); + return ret; +} + +// ---- the vertex shader of an emulated mesh pipeline + Buf b = {0}, t = {0}, an = {0}, c = {0}; + uint32_t id = 1; +#define N() (id++) + uint32_t glsl = N(), fn = N(), t_void = N(), t_fn = N(), t_u32 = N(), t_i32 = N(), t_f32 = N(), t_u64 = N(), + t_v4u = N(), t_v4f = N(), t_v4i = N(), rt_v4 = N(), s_slots = N(), p_slots = N(), p_v4u_psb = N(), + pc_struct = N(), p_pc = N(), pc_var = N(), p_pc_u64 = N(), p_pc_u32 = N(), p_in_i32 = N(), + (void)glsl; + uint32_t vecs[3][5] = {{0}}; // [type][comps] vector ids (comps 2..4), [type][1] the scalar + vecs[0][1] = t_f32, vecs[1][1] = t_i32, vecs[2][1] = t_u32, vecs[0][4] = t_v4f, vecs[1][4] = t_v4i, vecs[2][4] = t_v4u; + op(&t, OpTypeVoid, 1, t_void); + op(&t, OpTypeFunction, 2, t_fn, t_void); + op(&t, OpTypeInt, 3, t_u32, 32u, 0u); + op(&t, OpTypeInt, 3, t_i32, 32u, 1u); + op(&t, OpTypeFloat, 2, t_f32, 32u); + op(&t, OpTypeInt, 3, t_u64, 64u, 0u); + op(&t, OpTypeVector, 3, t_v4u, t_u32, 4u); + op(&t, OpTypeVector, 3, t_v4f, t_f32, 4u); + op(&t, OpTypeVector, 3, t_v4i, t_i32, 4u); + for (int ty = 0; ty < 3; ty++) + for (int k = 2; k < 4; k++) { + vecs[ty][k] = N(); + op(&t, OpTypeVector, 3, vecs[ty][k], vecs[ty][1], (uint32_t)k); + } + op(&t, OpTypeRuntimeArray, 2, rt_v4, t_v4u); + op(&t, OpTypeStruct, 2, s_slots, rt_v4); + op(&t, OpTypePointer, 3, p_slots, (uint32_t)ScPhysicalStorageBuffer, s_slots); + op(&t, OpTypePointer, 3, p_v4u_psb, (uint32_t)ScPhysicalStorageBuffer, t_v4u); + op(&t, OpTypeStruct, 8, pc_struct, t_u64, t_u32, t_u32, t_u32, t_u32, t_u32, t_u32); + op(&t, OpTypePointer, 3, p_pc, (uint32_t)ScPushConstant, pc_struct); + op(&t, OpTypePointer, 3, p_pc_u64, (uint32_t)ScPushConstant, t_u64); + op(&t, OpTypePointer, 3, p_pc_u32, (uint32_t)ScPushConstant, t_u32); + op(&t, OpTypePointer, 3, p_in_i32, (uint32_t)ScInput, t_i32); + op(&t, OpTypePointer, 3, p_out_v4f, (uint32_t)ScOutput, t_v4f); + op(&t, OpConstant, 3, t_u32, c0, 0u); + op(&t, OpConstant, 3, t_u32, c4, 4u); + op(&t, OpConstant, 4, t_u64, c16, 16u, 0u); + op(&t, OpVariable, 3, p_pc, pc_var, (uint32_t)ScPushConstant); + op(&t, OpVariable, 3, p_in_i32, vidx, (uint32_t)ScInput); + op(&t, OpVariable, 3, p_out_v4f, pos, (uint32_t)ScOutput); + op(&an, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u); + op(&an, OpDecorate, 2, s_slots, (uint32_t)DecBlock); + op(&an, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u); + op(&an, OpDecorate, 2, pc_struct, (uint32_t)DecBlock); + for (uint32_t k = 0; k < 7; k++) op(&an, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u); + op(&an, OpDecorate, 3, vidx, (uint32_t)DecBuiltIn, (uint32_t)BiVertexIndex); + op(&an, OpDecorate, 3, pos, (uint32_t)DecBuiltIn, (uint32_t)BiPosition); + ifc[ni++] = pc_var, ifc[ni++] = vidx, ifc[ni++] = pos; + uint32_t p = N(), v = N(); + op(&t, OpVariable, 3, p, v, (uint32_t)ScOutput); + ifc[ni++] = v; + } + // code: slot = vertex area + VertexIndex; each output from slot + its index + uint32_t lab = N(); + op(&c, OpFunction, 4, t_void, fn, 0u, t_fn); + op(&c, OpLabel, 1, lab); + uint32_t e64 = N(), e64p = N(), vw = N(), vwp = N(), vi = N(), viu = N(), s = N(), s64 = N(), off = N(), a = N(), base = N(); + op(&c, OpAccessChain, 4, p_pc_u64, e64p, pc_var, c0); + op(&c, OpLoad, 3, t_u64, e64, e64p); + op(&c, OpAccessChain, 4, p_pc_u32, vwp, pc_var, c4); + op(&c, OpLoad, 3, t_u32, vw, vwp); + op(&c, OpLoad, 3, t_i32, vi, vidx); + op(&c, OpBitcast, 3, t_u32, viu, vi); + op(&c, OpIAdd, 4, t_u32, s, vw, viu); + op(&c, OpUConvert, 3, t_u64, s64, s); + op(&c, OpIMul, 4, t_u64, off, s64, c16); + op(&c, OpIAdd, 4, t_u64, a, e64, off); + op(&c, OpConvertUToPtr, 3, p_slots, base, a); + uint32_t slot_of[33]; + for (int k = 0; k < 33; k++) slot_of[k] = 0xffffffffu; + slot_of[32] = 0; // position + for (uint32_t g = 0; g < mi->n_generic; g++) slot_of[mi->generic_loc[g]] = 1 + g; + uint32_t val; + if (slot_of[l] == 0xffffffffu) { + val = 0; + } else { + uint32_t ci = N(), p = N(); + val = N(); + op(&t, OpConstant, 3, t_u32, ci, slot_of[l]); + op(&c, OpAccessChain, 5, p_v4u_psb, p, base, c0, ci); + op(&c, OpLoad, 5, t_v4u, val, p, 2u, 16u); + } + uint32_t f = N(); + op(&c, OpBitcast, 3, t_v4f, f, val); + op(&c, OpStore, 2, pos, f); + continue; + } + if (!val) { // not written by the mesh stage: zero + uint32_t z = N(); + r = N(); + } else { + uint32_t cv = val; + if (ty != 2) cv = N(), op(&c, OpBitcast, 3, v4t, cv, val); + if (k == 4) r = cv; + else { + r = N(); + } + } + } + op(&c, OpReturn, 0); + op(&c, OpFunctionEnd, 0); + // assemble + put(&b, 0x07230203), put(&b, 0x00010500), put(&b, 0), put(&b, id), put(&b, 0); + op(&b, OpCapability, 1, (uint32_t)CapShader); + op(&b, OpCapability, 1, (uint32_t)CapInt64); + op(&b, OpCapability, 1, (uint32_t)CapPSBAddresses); + op(&b, OpMemoryModel, 2, 5348u, 1u); + uint32_t pre[2] = {ModelVertex, fn}; + op_str(&b, OpEntryPoint, pre, 2, "main", ifc, (int)ni); + putn(&b, an.w, an.n); + putn(&b, t.w, t.n); + putn(&b, c.w, c.n); + b.w[3] = id; + int bad = b.bad || t.bad || an.bad || c.bad; + free(t.w), free(an.w), free(c.w); + if (bad) { free(b.w); return NULL; } + *out_n = b.n; + return b.w; \ No newline at end of file diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h new file mode 100644 index 00000000..24d316e4 --- /dev/null +++ b/runtime/mesh_emu_spirv.h @@ -0,0 +1,37 @@ +#pragma once +// Mesh shader emulation for GPUs without VK_EXT_mesh_shader (NX2NAVI_MESH_EMULATION, vk_pipeline.c / vk_exec_draw.c): +// the translator's TaskEXT and MeshEXT modules (tools/xlate/MeshSpirv.cs) become GLCompute modules that write their +// vertices, triangles and task payloads into one GPU buffer (the arena), and a generated vertex shader reads the +// vertices back for the game's fragment shader. Plain C over SPIR-V words; no Vulkan types. +#include +#include + +// The arena: one buffer, its address in the push constants of every emulation stage. +// MESH_EMU_DOFF: draw records, MESH_EMU_DWORDS words each: DrawIndexedIndirect {indexCount, instanceCount, +// MESH_EMU_TOFF: task records, MESH_EMU_TWORDS words each: mesh workgroup count, then the payload +// index area (uint32 indices), vertex area (16-byte slots: position, then each generic output) at offsets the +// replay passes in the push constants. +#define MESH_EMU_DOFF 256u +#define MESH_EMU_DWORDS 16u +#define MESH_EMU_DRECS 4096u +#define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +// Push constants of every emulation stage (compute and the vertex shader), 32 bytes: +// u64 arena address; uint draw record (word index); uint index area (word index); uint index capacity; +#define MESH_EMU_PC_BYTES 32u + +typedef struct MeshEmuInfo { + int task; // 1 TaskEXT, 0 MeshEXT + uint32_t threads; // LocalSize x + uint32_t max_v, max_p; // mesh: OutputVertices, OutputPrimitivesEXT + uint32_t payload_words; // the task payload's size (mesh: what it reads, 0 without a task stage) + uint32_t max_mesh; // task: the most mesh workgroups one task workgroup launches (dispatch width) + uint32_t slots; // mesh: 16-byte slots per vertex: position and each generic output + uint32_t n_generic; // mesh: generic outputs, in slot order 1..n + uint8_t generic_loc[32]; // their locations +} MeshEmuInfo; + +// The compute module for a TaskEXT or MeshEXT module (malloc'd, *out_n words); NULL with why set when the module is not +// one the converter knows (it then stays unemulated: its draws are skipped as before). +uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why); + +// The vertex shader of an emulated mesh pipeline: position and the generic outputs from the vertex slots, as types[l] \ No newline at end of file diff --git a/runtime/vk.h b/runtime/vk.h index b1756797..57fcd927 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -830,6 +830,10 @@ typedef struct { // from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs. VkDescriptorUpdateTemplate tpl[2]; int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools) + // NX2NAVI_MESH_EMULATION: an emulated mesh pipeline (a generated vertex shader and the fragment stage; the task and + // mesh stages are compute pipelines of the same layout, vk_pipeline_mesh_emu), and set 0's compute-side template. + int emu; + VkDescriptorUpdateTemplate emu_tpl_c; } VkPipeInfo; // One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the // bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte. @@ -866,6 +870,10 @@ typedef struct { } VkPrewarmProgress; void vk_pipeline_prewarm_progress(VkPrewarmProgress *p); // any thread, while vk_pipeline_prewarm runs or after VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info); +// NX2NAVI_MESH_EMULATION: the compute pipelines of an emulated mesh pipeline (built with it); 0 when it has none. +struct MeshEmuInfo; +int vk_pipeline_mesh_emu(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, VkPipeline *mesh, + VkPipeline *task, struct MeshEmuInfo *mi); // Records every dynamic state that VkPipeState determines, after binding a graphics pipeline: cull mode, front face, // topology, depth and stencil tests and ops, depth bias enable, rasterizer discard, vertex input, and, where the GPU // has them dynamic, depth clamp, polygon mode, blend enables, equations, write masks and alpha to coverage. The replay diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 8936f2de..765f1cea 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -33,6 +33,7 @@ #include "vk_mesh_fb.h" #include "platform/platform.h" #include "vk_dlaa.h" +#include "mesh_emu_spirv.h" #define MAX_BINDINGS 128 #define MAX_DESC 1024 @@ -3422,10 +3423,12 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded. // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. +static uint32_t emu_d_bytes; static void bind_resources(VkExec *ex, const VkPipeInfo *info) { DrawCtx *const cx = ex->dc; VkWriteDescriptorSet *w = cx->bind_w; VkDescSlot *d = cx->bind_d; + emu_d = d; VkBufferView *bv = cx->bind_bv; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(info); @@ -3567,6 +3570,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { // An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which // the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors if (n_sets == 2 && lo[1] == hi[1]) d0[1] = nd; + emu_d_bytes = (uint32_t)((n_sets > 1 ? d0[1] : nd) * sizeof *d); // Under the validation layer, a new graphics layout binds set 1 before set 0 is pushed or bound. The layer (SDK // 1.4.363, PipelineLayoutCompatDef::operator==) takes any two INDEPENDENT_SETS layouts as compatible, so // vkCmdPushDescriptorSetKHR keeps the push set it made for an earlier layout and drops writes to bindings that @@ -4126,6 +4130,7 @@ static void prog_lead(const prog_t *p, int mesh) { (unsigned long long)PLD.n[m][4], (unsigned long long)PLD.n[m][5]); } +static const VkPipeInfo *PD_info; // the pipeline of the last draw prepare_draw_ex prepared // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of // the mesh fallback (mfb_draw): the vertex stage is DrawCtx.mfb_cur_vs, in an ordinary pipeline without vertex input. @@ -4328,7 +4333,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int delta = cfg_bool("NX2NAVI_DYN_DELTA"); if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)"); } - if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb, + // an emulated mesh draw (NX2NAVI_MESH_EMULATION) binds a vertex pipeline: its topology, primitive restart and + // vertex input are dynamic and are set as for any vertex draw + if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb && !info->emu, &vk_exec_frame.dyn_left); else vk_pipeline_set_dynamic(ex->cb, &s); ex->recorded++; @@ -4359,6 +4366,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int vk_exec_oq_draw(ex); DRS.issued[mesh]++; draw_stats_tick(); + PD_info = info; return 1; } @@ -5561,7 +5569,17 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { ex->recorded++; } +static int mesh_emu_on(void) { + static int on = -1; + if (on < 0) { + if (on) rt_log("vk: mesh shaders emulated with compute (NX2NAVI_MESH_EMULATION)"); + } + return on; +} + static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { + PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); + if (!fn && mesh_emu_on()) { static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) if (!d->count || !on) return; @@ -5570,8 +5588,7 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { mfb_draw(ex, d->count, NULL); return; } - PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); - if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader (mesh_emulation = 1 emulates them): skipped"); ex->dc->c0_binst = 0; if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); @@ -5690,6 +5707,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy vk_gpu_gap_note(VK_GAP_MESH_RING, 12); vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1); + } VkBufferCopy cp = {src->offset, *at, 4}; vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); mesh_count_note(ex, src); @@ -5719,8 +5737,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan .dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, .dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; - vkCmdPipelineBarrier2(ex->cb, &di); - ex->recorded += 3; + vkCmdPipelineBarrier2(cb, &di); return 1; } @@ -5746,7 +5763,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { return; } VkDeviceSize at; - if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + if (!mesh_ring_entry(ex, ex->cb, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; vk_exec_frame.draws++; static uint32_t n_logged; @@ -5764,6 +5781,86 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { VK_MEM_BOUND(ip); } +// ---- NX2NAVI_MESH_EMULATION: mesh draws on GPUs without VK_EXT_mesh_shader +// Each mesh draw (prepared like one, prepare_draw_ex, for its layout, descriptors and graphics state) runs its task +// and mesh stages as compute pipelines (vk_pipeline_mesh_emu) outside the pass: they write vertices and triangles into +// the arena (mesh_emu_spirv.h) and the draw record's DrawIndexedIndirect, which the generated vertex shader and the +// game's fragment shader then draw. The arena's counters are zeroed at the first emulated draw of each submission; + int failed; + uint64_t draws, skipped; +} ME; + + return 1; +} + +static int me_arena(void) { + if (ME.buf) return 1; + if (ME.failed) return 0; + VkDeviceSize size = (VkDeviceSize)mb << 20; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + return 0; + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) { + ME.buf = VK_NULL_HANDLE, ME.failed = 1; + rt_log("vk: mesh emulation: the %d MiB arena could not be created: mesh draws skipped", mb); + return 0; + VkBufferDeviceAddressInfo dai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = ME.buf}; + ME.addr = vkGetBufferDeviceAddress(vk_ctx()->device, &dai); + uint64_t voff = ioff + ibytes; + ME.iw = (uint32_t)(ioff / 4), ME.icap = (uint32_t)(ibytes / 4), ME.vw = (uint32_t)(voff / 16), ME.vcap = (uint32_t)((size - voff) / 16); + return 1; +static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, VkAccessFlags2 da) { + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); + } + } +// count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from + if (!me_arena()) return 0; + ex->dc->c0_binst = 0; + const VkPipeInfo *info = PD_info; + const GfxShader *ms = ex->dc->B.st.vs, *ts = ex->dc->B.st.ts; + VkPipeline mp = VK_NULL_HANDLE, tp = VK_NULL_HANDLE; + MeshEmuInfo mi; + if (!info || !info->emu || !ex->dc->B.st_valid || !vk_pipeline_mesh_emu(ms, ts, info->layout, &mp, &tp, &mi) || !!ts != !!tp || + (info->push && !info->emu_tpl_c) || (!info->push && !ex->dc->B.ds_bound[0])) { + ME.skipped++; + return 0; + VkPipelineLayout layout = info->layout; + int push = info->push, pool = info->pool; + VkDescriptorUpdateTemplate tpl_c = info->emu_tpl_c; + VkDescriptorSet set0 = ex->dc->B.ds_bound[0]; + } + } + uint32_t ep = vk_texture_epoch(); + if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over + ME.epoch = ep; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + uint32_t rec = ME.rec++ % MESH_EMU_DRECS; + VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4; + vkCmdFillBuffer(cb, ME.buf, ro, 4, 0); + vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); + cb = ex->cb; + vkCmdBindIndexBuffer(cb, ME.buf, (VkDeviceSize)ME.iw * 4, VK_INDEX_TYPE_UINT32); + vkCmdPushConstants(cb, layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, pc); + vkCmdDrawIndexedIndirect(cb, ME.buf, ro, 1, 20); + ex->recorded += 3; + vk_exec_frame.draws++; + // the index buffer and push constants are the emulation's: the next draw binds everything again + vk_exec_draw_new_cb(); + ex->dirty = VK_EXEC_DIRTY_ALL; + ME.draws++; + static uint64_t log_frame; + uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if ((rt_trace & RT_TRACE_PERF) && f / 600 != log_frame / 600) { + log_frame = f; + ME.draws = ME.skipped = 0; + return 1; +} + static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) { VkBuffer b; VkDeviceSize o; diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 15a41f5e..110c67b6 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -43,6 +43,7 @@ #include #include "cfg.h" +#include "mesh_emu_spirv.h" #include "gfx.h" #include "host.h" #include "nvn_hmap.h" @@ -130,6 +131,8 @@ typedef struct Layout { uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1 VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use) VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs + uint8_t emu; // NX2NAVI_MESH_EMULATION: a mesh layout whose task and mesh stages run as compute + VkDescriptorUpdateTemplate tpl_c; // emu: set 0's template for the compute bind point VkDescriptorSetLayoutBinding b[]; } Layout; @@ -2588,8 +2591,8 @@ static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off, // layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set // without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write // structs. -static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, - int push, VkPipelineLayout pl, uint32_t set) { +static VkDescriptorUpdateTemplate make_tpl_bp(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, + int push, VkPipelineLayout pl, uint32_t set, int bp) { if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE; VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS]; size_t off = 0; @@ -2604,7 +2607,8 @@ static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b .pDescriptorUpdateEntries = e, .templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET, .descriptorSetLayout = sl, - .pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, + .pipelineBindPoint = bp >= 0 ? (VkPipelineBindPoint)bp + : stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, .pipelineLayout = pl, .set = set}; VkDescriptorUpdateTemplate t; if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) { @@ -2613,6 +2617,10 @@ static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b } return t; } +static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, + int push, VkPipelineLayout pl, uint32_t set) { + return make_tpl_bp(b, n, sl, push, pl, set, -1); +} // Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock. // Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats. @@ -2667,6 +2675,124 @@ static Layout *layout_get(Bindings *B) { return l; } +// NX2NAVI_MESH_EMULATION (GPUs without VK_EXT_mesh_shader): the layout of an emulated mesh pipeline. The task and +// mesh stages' bindings are visible to the compute stage (they run as compute pipelines of this layout), the +// fragment stage's stay; push constants (MESH_EMU_PC_BYTES) for the compute stages and the generated vertex shader; +// set 0 gets a template per bind point. Caller holds P.lock. +static int mesh_emu_on(void) { + static int on = -1; + return on; +} + +static Layout *layout_get_emu(Bindings *B) { + for (uint32_t i = 0; i < B->n; i++) { + VkShaderStageFlags f = B->b[i].stageFlags; + if (f & (VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT)) + f = (f & ~(VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT)) | VK_SHADER_STAGE_COMPUTE_BIT; + B->b[i].stageFlags = f; + B->pb[i].stages = f; + } + uint64_t h = fnv(FNV0, "mesh-emu", 8); + int64_t total = bindings_sort(B, &h); + if (total < 0) return NULL; + Layout **head = &layouts[h % LAYOUT_BUCKETS]; + for (Layout *l = *head; l; l = l->next) + if (l->hash == h && l->emu && l->n == B->n && !memcmp(l->b, B->b, B->n * sizeof *B->b)) return l; + Layout *l = calloc(1, sizeof *l + B->n * sizeof *B->b); + if (!l) return NULL; + l->hash = h, l->n = B->n, l->emu = 1; + memcpy(l->b, B->b, B->n * sizeof *B->b); + l->push = total <= (int64_t)P.max_push; + VkDescriptorSetLayoutCreateInfo si = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0, + .bindingCount = l->n, .pBindings = l->b}; + if (vkCreateDescriptorSetLayout(P.device, &si, NULL, &l->set) != VK_SUCCESS) { + free(l); + return NULL; + } + VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()}; + VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES}; + VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3, + .pSetLayouts = sets3, .pushConstantRangeCount = 1, .pPushConstantRanges = &pcr}; + if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) { + vkDestroyDescriptorSetLayout(P.device, l->set, NULL); + free(l); + return NULL; + } + l->tpl[0] = make_tpl_bp(l->b, l->n, l->set, l->push, l->layout, 0, VK_PIPELINE_BIND_POINT_GRAPHICS); + l->tpl_c = make_tpl_bp(l->b, l->n, l->set, l->push, l->layout, 0, VK_PIPELINE_BIND_POINT_COMPUTE); + l->next = *head; + *head = l; + P.stats.layouts++; + return l; +} + +// The compute pipelines of an emulated mesh pipeline, per (mesh, task, layout): made by the pipeline's build +// (create_graphics, any thread) and found by its draws. ok 0: the stages could not be converted or built. +typedef struct MeshEmuEnt { + struct MeshEmuEnt *next; + const GfxShader *ms, *ts; + VkPipelineLayout layout; + int ok; + VkPipeline mesh, task; + MeshEmuInfo mi, ti; +} MeshEmuEnt; +static PlatMutex emu_lock = PLAT_MUTEX_INIT; + + return e; +} + +static VkPipeline emu_compute(const uint32_t *w, size_t n, VkPipelineLayout layout) { + VkShaderModule m = make_module(w, n, 0); + if (!m) return VK_NULL_HANDLE; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = layout}; + VkPipeline p = VK_NULL_HANDLE; + if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &p) != VK_SUCCESS) p = VK_NULL_HANDLE; + vkDestroyShaderModule(P.device, m, NULL); + return p; +} + +static const MeshEmuEnt *emu_get(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, uint32_t toff, int create) { + e = calloc(1, sizeof *e); + e->ms = ms, e->ts = ts, e->layout = layout; + const char *why = NULL; + uint64_t n = 0; + size_t cn = 0; + const uint32_t *w = gfx_shader_spirv(ms, &n); + uint32_t *c = w ? mesh_emu_compute(w, (size_t)n, &e->mi, &cn, &why) : NULL; + if (c && (e->mesh = emu_compute(c, cn, layout))) e->ok = 1; + else if (!why) why = "the mesh stage's compute pipeline was not created"; + free(c); + if (e->ok && e->mi.payload_words && !ts) e->ok = 0, why = "a mesh stage that reads a task payload without a task stage"; + if (e->ok && ts) { + w = gfx_shader_spirv(ts, &n); + c = w ? mesh_emu_compute(w, (size_t)n, &e->ti, &cn, &why) : NULL; + uint32_t *r = c && toff ? spirv_rebind(c, cn, toff) : NULL; // the task stage's bindings follow the fragment stage's + if (c && (e->task = emu_compute(r ? r : c, cn, layout))) { + if (e->ti.payload_words != e->mi.payload_words) e->ok = 0, why = "task and mesh payload sizes differ"; + } else { + e->ok = 0; + if (!why) why = "the task stage's compute pipeline was not created"; + } + free(r), free(c); + } + plat_mutex_unlock(&emu_lock); + return e; +} + +int vk_pipeline_mesh_emu(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, VkPipeline *mesh, + VkPipeline *task, MeshEmuInfo *mi) { + const MeshEmuEnt *e = emu_get(ms, ts, layout, 0, 0); + if (!e || !e->ok) return 0; + *mesh = e->mesh, *task = e->task; + *mi = e->mi; + if (ts) mi->max_mesh = e->ti.max_mesh; + return 1; +} + // Independent layouts (P.indep): the vertex stage's bindings in set 0, the fragment stage's in set 1 (empty without a // fragment shader), each with its own support buffer at binding 0, the pipeline layout created with // VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT so stage libraries can use it. Only the P.push_stage set is a @@ -2793,6 +2919,7 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel p->info.n_sets = l && l->set1 ? 2 : 1; p->info.pool = B ? B->pool : 0; if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1]; + if (l && l->emu) p->info.emu = 1, p->info.emu_tpl_c = l->tpl_c; p->lay = l; Pipe **head = &pipes[h % PIPE_BUCKETS]; p->next = *head; @@ -3111,11 +3238,28 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw); uint32_t *vpatched = NULL, *fpatched = NULL; - int mesh = is_mesh(s); + int mesh = is_mesh(s), emu = mesh && l->emu; + // the water pair's vertex edit is for a translated vertex stage: an emulated mesh draw's generated one has none int wmode = mesh || !l->set1 ? 0 : water_mode(s, k); // the edited fragment modules are in their set 1 form uint64_t wn = 0; const uint32_t *vwater = water_pair_vertex(vcode, vw, wmode, &wn); if (vwater) vcode = vwater, vw = wn; + if (emu) { // NX2NAVI_MESH_EMULATION: the task and mesh stages as compute pipelines, a generated vertex shader here + uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0); + const MeshEmuEnt *me = emu_get(s->vs, s->ts, l->layout, toff, 1); + if (!me || !me->ok) return VK_NULL_HANDLE; + GfxShaderIo fin[32]; + uint8_t ty[32], cp[32]; + uint32_t need = s->fs ? gfx_shader_inputs(s->fs, fin) : 0; + for (int i = 0; i < 32; i++) { + ty[i] = need >> i & 1 && !fin[i].packed && fin[i].type <= GFX_TYPE_UINT ? (uint8_t)fin[i].type : 0; + cp[i] = need >> i & 1 && !fin[i].packed && fin[i].comps ? (uint8_t)fin[i].comps : 4; + } + size_t vn = 0; + if (!vpatched) return VK_NULL_HANDLE; + vcode = vpatched, vw = vn; + mesh = 0; // a vertex pipeline from here on + } VkShaderModule tm = VK_NULL_HANDLE; if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss) uint64_t tw; @@ -3137,14 +3281,15 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay } if (s->fs) { GfxShaderIo fin[32], vout[32]; - // a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none + // a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none; an emulated one + // (mesh_emu) writes what the fragment stage reads int points = 0; uint32_t need = gfx_shader_inputs(s->fs, fin), missing = 0, - have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : gfx_shader_outputs(s->vs, vout); + have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : emu ? ~0u : gfx_shader_outputs(s->vs, vout); for (int i = 0; i < 32; i++) { if (!(need >> i & 1) || fin[i].packed) continue; if (!(have >> i & 1)) missing |= 1u << i; - else if (!mesh && vout[i].type != fin[i].type) + else if (!mesh && !emu && vout[i].type != fin[i].type) LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type); } if (missing && !l->set1 && !mesh) { @@ -3193,7 +3338,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf; uint64_t sw; const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now - pin_subgroup32(&stages[0], &rqv, scode, sw); + if (!emu) pin_subgroup32(&stages[0], &rqv, scode, sw); if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw); VkVertexInputAttributeDescription va[32]; VkVertexInputBindingDescription vb[17]; @@ -4509,11 +4654,12 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode // vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its // whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note). uint32_t fspan = s->fs ? binding_span(s->fs) : 1; - if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders"; + int emu = !vk_ctx()->has_mesh_shader && mesh_emu_on(); + if (!vk_ctx()->has_mesh_shader && !emu) why = "the GPU has no mesh shaders (mesh_emulation = 1 emulates them)"; else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch"; else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) || (s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings"; - else if (!(l = layout_get(&B))) why = "descriptor layout creation failed"; + else if (!(l = emu ? layout_get_emu(&B) : layout_get(&B))) why = "descriptor layout creation failed"; } else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch"; else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated"; else if (P.indep) { From 1534dc570a3a50d61f7f9da09ab47105804decf2 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:05 +0100 Subject: [PATCH 14/64] Mesh emulation: size the task records from the arena, and log what each submission asked for. The task records were a fixed 4096 per submission (3 KiB each): a dense patch of grass that launches more task workgroups in one submission lost the rest silently. Their count is now a push constant (NX2NAVI_MESH_EMU_TASKS, default 16384) and the arena defaults to 256 MiB. The header counts what the stages asked for past the capacities too; before the counters start over they go to a readback ring, and the perf line reports the most one submission used of the vertex slots, indices and task records, and the mesh workgroups that did not fit. The first draws the emulation skips are logged with the reason. (cherry picked from commit 81cfd91bf468f13e4debf287d080f96637a4c382) Co-Authored-By: Claude Opus 5.5 --- nx2navi.ini.example | 4 +++ runtime/mesh_emu_spirv.c | 8 ++++++ runtime/mesh_emu_spirv.h | 5 ++++ runtime/vk_exec_draw.c | 56 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 261e56ed..15ed5851 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -423,7 +423,11 @@ # mirror_hot = 0 # GPUs without mesh shaders (VK_EXT_mesh_shader: AMD before RDNA 2, NVIDIA before Turing): 1 runs the game's mesh # and task shaders (grass and other foliage) as compute shaders and draws what they make; 0 skips those draws. The +# arena for their vertices and triangles is mesh_emu_mb MiB of video memory, with mesh_emu_tasks task records (3 KiB +# each) per submission; with trace = perf the log says how much a submission used. Read at start. # mesh_emulation = 0 +# mesh_emu_mb = 256 +# mesh_emu_tasks = 16384 # Scaling to the window when it is larger than the game resolution: fsr (AMD FSR 1) or bilinear, and FSR's sharpening # from 0 (off) to 100. Changes while the game runs take effect at once. # upscaler = fsr diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index df312373..17d5cd23 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -313,6 +313,12 @@ static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint uint32_t t = add(e, wx, add(e, mul(e, wy, nx), mul(e, wz, nxy))); uint32_t base = words_base(e), drw = pc_u32(e, 1); uint32_t tb = word_load(e, base, add(e, drw, cu(e, 8))); + uint32_t trecs = pc_u32(e, 6), slot = add(e, tb, t), m1; + uint32_t mz; + if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); + atomic(e, OpAtomicUMax, base, cu(e, 2), add(e, slot, cu(e, 1))); // records asked for, past the capacity too + if_end(e, mz); + if_begin(e, ult(e, slot, trecs), &m1); { uint32_t recw = add(e, cu(e, MESH_EMU_TOFF / 4), mul(e, slot, cu(e, MESH_EMU_TWORDS))); uint32_t v = ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, payload, k)); @@ -331,6 +337,7 @@ static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint { uint32_t total = mul(e, nxy, nz); // task records past the area are not written: their mesh groups are not dispatched + uint32_t room = E3(e, OpSelect, m->t_u32, ult(e, tb, trecs), E2(e, 130 /* OpISub */, m->t_u32, trecs, tb), cu(e, 0)); uint32_t ty = E3(e, OpSelect, m->t_u32, ult(e, total, room), total, room); word_store(e, base, add(e, drw, cu(e, 5)), cu(e, info->max_mesh)); word_store(e, base, add(e, drw, cu(e, 6)), ty); @@ -587,6 +594,7 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ uint32_t x = E2(&e, OpCompositeExtract, m->t_u32, wg, 0u), y = E2(&e, OpCompositeExtract, m->t_u32, wg, 1u); uint32_t base = words_base(&e), drw = pc_u32(&e, 1); uint32_t tb = word_load(&e, base, add(&e, drw, cu(&e, 8))); + uint32_t slot = add(&e, tb, y), ok = ult(&e, slot, pc_u32(&e, 6)); uint32_t slotc = E3(&e, OpSelect, m->t_u32, ok, slot, cu(&e, 0)); uint32_t recw = add(&e, cu(&e, MESH_EMU_TOFF / 4), mul(&e, slotc, cu(&e, MESH_EMU_TWORDS))); uint32_t cnt = word_load(&e, base, recw); diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index 24d316e4..38350b9f 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -15,8 +15,13 @@ #define MESH_EMU_DWORDS 16u #define MESH_EMU_DRECS 4096u #define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +// task records: as many as the push constants say (the replay sizes them, NX2NAVI_MESH_EMU_TASKS); the index area +// follows them // Push constants of every emulation stage (compute and the vertex shader), 32 bytes: // u64 arena address; uint draw record (word index); uint index area (word index); uint index capacity; +// uint vertex area (16-byte slot index); uint vertex capacity (slots); uint task record capacity +// Header words 0-2 count what the stages asked for, beyond the capacities too (the replay logs the most per +// submission); word 3 counts mesh workgroups that did not fit. #define MESH_EMU_PC_BYTES 32u typedef struct MeshEmuInfo { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 765f1cea..1839426f 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5786,8 +5786,18 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { // and mesh stages as compute pipelines (vk_pipeline_mesh_emu) outside the pass: they write vertices and triangles into // the arena (mesh_emu_spirv.h) and the draw record's DrawIndexedIndirect, which the generated vertex shader and the // game's fragment shader then draw. The arena's counters are zeroed at the first emulated draw of each submission; +// the draw records go round a ring of MESH_EMU_DRECS. NX2NAVI_MESH_EMU_MB sizes the arena (default 256 MiB), +// NX2NAVI_MESH_EMU_TASKS its task records per submission (default 16384, 3 KiB each). Before the counters start over +// they are copied to a readback ring; the perf log reports the most a submission asked for against the capacities. +#define ME_RB 64 + VkBuffer buf, rb; + VmaAllocation alloc, rb_alloc; + uint32_t *rb_map; + uint64_t addr, resets; + uint32_t iw, icap, vw, vcap, trecs, epoch, rec; int failed; uint64_t draws, skipped; + uint32_t max[4]; } ME; return 1; @@ -5796,6 +5806,13 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { static int me_arena(void) { if (ME.buf) return 1; if (ME.failed) return 0; + const char *v = cfg_get("NX2NAVI_MESH_EMU_MB"), *t = cfg_get("NX2NAVI_MESH_EMU_TASKS"); + int mb = v && atoi(v) >= 32 ? atoi(v) : 256; + if (mb > 2048) mb = 2048; + ME.trecs = t && atoi(t) >= 256 ? (uint32_t)atoi(t) : 16384; + if (ME.trecs > 262144) ME.trecs = 262144; + uint64_t tbytes = (uint64_t)MESH_EMU_TOFF + (uint64_t)ME.trecs * MESH_EMU_TWORDS * 4; + if (tbytes + (32ull << 20) > ((uint64_t)mb << 20)) mb = (int)((tbytes >> 20) + 64); // room for vertices and indices VkDeviceSize size = (VkDeviceSize)mb << 20; VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | @@ -5803,14 +5820,26 @@ static int me_arena(void) { VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; return 0; + } if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) { ME.buf = VK_NULL_HANDLE, ME.failed = 1; rt_log("vk: mesh emulation: the %d MiB arena could not be created: mesh draws skipped", mb); return 0; + } VkBufferDeviceAddressInfo dai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = ME.buf}; ME.addr = vkGetBufferDeviceAddress(vk_ctx()->device, &dai); + uint64_t ioff = (tbytes + 4095) & ~4095ull, ibytes = ((size - ioff) / 4) & ~4095ull; uint64_t voff = ioff + ibytes; ME.iw = (uint32_t)(ioff / 4), ME.icap = (uint32_t)(ibytes / 4), ME.vw = (uint32_t)(voff / 16), ME.vcap = (uint32_t)((size - voff) / 16); + rt_log("vk: mesh emulation arena: %d MiB, %u task records, %u indices, %u vertex slots (16 bytes)", mb, ME.trecs, + ME.icap, ME.vcap); + VkBufferCreateInfo rbi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ME_RB * 16, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo rai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo rinfo; + if (vmaCreateBuffer(vk_ctx()->vma, &rbi, &rai, &ME.rb, &ME.rb_alloc, &rinfo) == VK_SUCCESS) ME.rb_map = rinfo.pMappedData; + else ME.rb = VK_NULL_HANDLE; return 1; static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, VkAccessFlags2 da) { VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; @@ -5826,6 +5855,12 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla MeshEmuInfo mi; if (!info || !info->emu || !ex->dc->B.st_valid || !vk_pipeline_mesh_emu(ms, ts, info->layout, &mp, &tp, &mi) || !!ts != !!tp || (info->push && !info->emu_tpl_c) || (!info->push && !ex->dc->B.ds_bound[0])) { + static int logged; + if (logged++ < 8) + rt_log("vk: mesh emulation: a draw of program %llu skipped: %s", ex->prog ? (unsigned long long)ex->prog->serial : 0ull, + !info || !info->emu ? "its pipeline is not an emulated one" + : !mp ? "its stages are not emulated (see vk_pipeline: mesh emulation)" + : "its descriptors cannot be bound for compute"); ME.skipped++; return 0; VkPipelineLayout layout = info->layout; @@ -5833,16 +5868,31 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla VkDescriptorUpdateTemplate tpl_c = info->emu_tpl_c; VkDescriptorSet set0 = ex->dc->B.ds_bound[0]; } + } } uint32_t ep = vk_texture_epoch(); if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over ME.epoch = ep; + if (ME.rb && ME.rb_map) { // the last submission's counters to the readback ring; one 32 resets back is read + if (ME.resets >= 32) { + vmaInvalidateAllocation(vk_ctx()->vma, ME.rb_alloc, 0, VK_WHOLE_SIZE); + const uint32_t *h = ME.rb_map + 4 * ((ME.resets - 32) % ME_RB); + for (int k = 0; k < 4; k++) + if (h[k] > ME.max[k]) ME.max[k] = h[k]; + VkBufferCopy rc = {0, 16 * (ME.resets % ME_RB), 16}; + vkCmdCopyBuffer(cb, ME.buf, ME.rb, 1, &rc); + ME.resets++; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT); me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); uint32_t rec = ME.rec++ % MESH_EMU_DRECS; VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4; vkCmdFillBuffer(cb, ME.buf, ro, 4, 0); vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); + uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; + } + } cb = ex->cb; vkCmdBindIndexBuffer(cb, ME.buf, (VkDeviceSize)ME.iw * 4, VK_INDEX_TYPE_UINT32); vkCmdPushConstants(cb, layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, pc); @@ -5857,7 +5907,13 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); if ((rt_trace & RT_TRACE_PERF) && f / 600 != log_frame / 600) { log_frame = f; + rt_log("perf: frame %llu: mesh emulation: %llu draws, %llu skipped (not emulated or not ready); most in one submission: " + "%u of %u vertex slots, %u of %u indices, %u of %u task records, %u mesh workgroups that did not fit", + (unsigned long long)f, (unsigned long long)ME.draws, (unsigned long long)ME.skipped, ME.max[0], ME.vcap, + ME.max[1], ME.icap, ME.max[2], ME.trecs, ME.max[3]); ME.draws = ME.skipped = 0; + memset(ME.max, 0, sizeof ME.max); + } return 1; } From 8b78e645393a17a458599e3f90fc059640a79ff1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:07 +0100 Subject: [PATCH 15/64] Mesh emulation: point meshes too, and a warning for draws from a nonzero workgroup. The 8 point-list mesh programs of Ocarina 1.1.1 (32 points, position and point size) were left unemulated. Their indices are single uints and the point size goes to an extra vertex slot, which the generated vertex shader writes to PointSize; the draw asks for a point list (the pipeline key's topology class follows the draw's primitive for mesh draws), chosen from the mesh stage's OutputPoints. A mesh draw from a nonzero first workgroup is drawn from 0 as before, now with a warning in the emulated path too. All 1016 programs convert and pass spirv-val. (cherry picked from commit 8900aeaffa3264e4cc655ffd487cb2d2f3e3dd17) Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.c | 64 ++++++++++++++++++++++++++++++++++++++++ runtime/mesh_emu_spirv.h | 9 +++++- runtime/vk_exec_draw.c | 17 +++++++++++ runtime/vk_pipeline.c | 3 +- 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index 17d5cd23..4afec639 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -33,6 +33,7 @@ enum { enum { ScInput = 1, ScOutput = 3, ScWorkgroup = 4, ScPrivate = 6, ScFunction = 7, ScPushConstant = 9, ScPhysicalStorageBuffer = 5349, ScTaskPayload = 5402 }; enum { DecBlock = 2, DecArrayStride = 6, DecBuiltIn = 11, DecLocation = 30, DecOffset = 35, DecPerPrimitive = 5271 }; +enum { BiPosition = 0, BiPointSize = 1, BiPrimPoint = 5294, BiVertexIndex = 42, BiNumWorkgroups = 24, BiWorkgroupId = 26, BiLocalInvocationIndex = 29, BiPrimTri = 5296 }; enum { CapShader = 1, CapInt64 = 11, CapMeshShading = 5283, CapPSBAddresses = 5347 }; enum { ModelVertex = 0, ModelGLCompute = 5, ModelTask = 5364, ModelMesh = 5365 }; @@ -233,12 +234,17 @@ static void if_end(Em *e, uint32_t merge) { op(e->code, OpLabel, 1, merge); } +typedef struct { uint32_t var, len, role, loc, slot; } OutVar; // role 0 position, 1 generic, 2 indices, 3 point size // The mesh epilogue (replaces the entry function's return) static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int nov, uint32_t vcount, uint32_t pcount, + uint32_t sh_vb, uint32_t sh_ib, uint32_t p_wg_v4f, uint32_t p_wg_v3u, uint32_t p_wg_f32, + uint32_t p_wg_u32) { Mod *m = e->m; + uint32_t S = info->slots, T = info->threads, K = info->prim; barrier(e); uint32_t lid = ld(e, m->t_u32, m->lii), base = words_base(e); + uint32_t isz = mul(e, pcount, cu(e, K)), vsz = mul(e, vcount, cu(e, S)); uint32_t mz; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); { @@ -273,6 +279,16 @@ static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int uint32_t s0 = add(e, VB, mul(e, v, cu(e, S))); for (int k = 0; k < nov; k++) { if (ov[k].role == 2) continue; + uint32_t bits; + if (ov[k].role == 3) { // the point size, in x + uint32_t ps = ld(e, m->t_f32, E2(e, OpAccessChain, p_wg_f32, ov[k].var, v)); + uint32_t pb = E1(e, OpBitcast, m->t_u32, ps); + bits = nid(m); + op(e->code, OpCompositeConstruct, 6, m->t_v4u, bits, pb, cu(e, 0), cu(e, 0), cu(e, 0)); + } else { + uint32_t src = ld(e, m->t_v4f, E2(e, OpAccessChain, p_wg_v4f, ov[k].var, v)); + bits = E1(e, OpBitcast, m->t_v4u, src); + } uint32_t dst = E3(e, OpAccessChain, e->p_v4u_psb, sbase, cu(e, 0), add(e, s0, cu(e, ov[k].slot))); op(e->code, OpStore, 4, dst, bits, 2u, 16u); } @@ -286,6 +302,11 @@ static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int for (uint32_t r = 0; r * T < info->max_p; r++) { uint32_t p = add(e, lid, cu(e, r * T)), mp; if_begin(e, ult(e, p, pcount), &mp); + uint32_t tri = K == 3 ? ld(e, m->t_v3u, E2(e, OpAccessChain, p_wg_v3u, idxv, p)) + : ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, idxv, p)); + uint32_t at = add(e, add(e, iw, IB), mul(e, p, cu(e, K))); + for (uint32_t c = 0; c < K; c++) { + uint32_t li = K == 3 ? E2(e, OpCompositeExtract, m->t_u32, tri, c) : tri; uint32_t val = E3(e, OpSelect, m->t_u32, vbok, add(e, VB, mul(e, li, cu(e, S))), cu(e, 0)); word_store(e, base, add(e, at, cu(e, c)), val); } @@ -375,6 +396,7 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ i += len; } if (!info->threads) { *why = "no LocalSize"; goto done; } + if (mesh && info->prim != 3 && info->prim != 1) { *why = "only triangle and point meshes are emulated"; goto done; } if (!m->t_f32 || (mesh && !m->t_v4f)) { *why = "no float types"; goto done; } // output variables and the payload @@ -400,6 +422,8 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ if (o == OpFunction) break; if (o == OpDecorate && a[0] == m->var[i].id) { if (a[1] == DecBuiltIn && a[2] == BiPosition) ov[nov].role = 0; + else if (a[1] == DecBuiltIn && (a[2] == BiPrimTri || a[2] == BiPrimPoint)) ov[nov].role = 2; + else if (a[1] == DecBuiltIn && a[2] == BiPointSize) ov[nov].role = 3; else if (a[1] == DecLocation) ov[nov].role = 1, ov[nov].loc = a[2]; else if (a[1] == DecBuiltIn) { *why = "an unsupported mesh output built-in"; goto done; } else if (a[1] == DecPerPrimitive) { *why = "per-primitive outputs are not emulated"; goto done; } @@ -407,6 +431,9 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ k += l2; } if (ov[nov].role == 9) { *why = "an output without a role"; goto done; } + if (ov[nov].role < 2 && arr[2] != m->t_v4f) { *why = "a vertex output that is not a vec4"; goto done; } + if (ov[nov].role == 3 && arr[2] != m->t_f32) { *why = "a point size that is not a float"; goto done; } + if (ov[nov].role == 2 && arr[2] != (info->prim == 3 ? m->t_v3u : m->t_u32)) { *why = "indices of an unexpected type"; goto done; } nov++; } info->payload_words = payload_len; @@ -425,6 +452,8 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ for (uint32_t l = 0; l < 32; l++) if (locs >> l & 1) info->generic_loc[info->n_generic++] = (uint8_t)l; info->slots = 1 + info->n_generic; + for (int i = 0; i < nov; i++) + if (ov[i].role == 3) info->psize_slot = info->slots, ov[i].slot = info->slots++; for (int i = 0; i < nov; i++) { if (ov[i].role == 0) ov[i].slot = 0; if (ov[i].role == 1) @@ -499,6 +528,8 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ } uint32_t p_wg_u32, p_wg_v4f = 0, p_wg_v3u = 0; NEED_PTR(p_wg_u32, ScWorkgroup, m->t_u32); + uint32_t p_wg_f32 = 0; + if (mesh) { NEED_PTR(p_wg_v4f, ScWorkgroup, m->t_v4f); NEED_PTR(p_wg_v3u, ScWorkgroup, m->t_v3u); NEED_PTR(p_wg_f32, ScWorkgroup, m->t_f32); } uint32_t sh_vb = 0, sh_ib = 0, wgemu = 0, p_priv_v3 = 0, p_priv_u32 = 0, p_in_u32 = find_ptr(m, ScInput, m->t_u32); if (mesh) { sh_vb = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_vb, (uint32_t)ScWorkgroup); @@ -622,6 +653,7 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ } if (in_ep && mesh && o == OpReturn) { if (!vcount || ep_done) { *why = "no SetMeshOutputsEXT before the return, or several returns"; goto done; } + mesh_epilogue(&e, info, ov, nov, vcount, pcount, sh_vb, sh_ib, p_wg_v4f, p_wg_v3u, p_wg_f32, p_wg_u32); ep_done = 1; i += len; continue; @@ -686,6 +718,22 @@ done: return ret; } +int mesh_emu_prim(const uint32_t *w, size_t n) { + uint32_t ep = 0; + for (size_t i = 5; i + 1 < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + if (!len || o == OpFunction) break; + if (o == OpEntryPoint && len >= 3) ep = w[i + 1] == ModelMesh || w[i + 1] == ModelTask; + if (o == OpExecutionMode && len >= 3 && ep) { + if (w[i + 2] == ModeOutputTriangles) return 3; + if (w[i + 2] == ModeOutputLines) return 2; + if (w[i + 2] == ModeOutputPoints) return 1; + } + i += len; + } + return 0; +} + // ---- the vertex shader of an emulated mesh pipeline Buf b = {0}, t = {0}, an = {0}, c = {0}; uint32_t id = 1; @@ -693,6 +741,7 @@ done: uint32_t glsl = N(), fn = N(), t_void = N(), t_fn = N(), t_u32 = N(), t_i32 = N(), t_f32 = N(), t_u64 = N(), t_v4u = N(), t_v4f = N(), t_v4i = N(), rt_v4 = N(), s_slots = N(), p_slots = N(), p_v4u_psb = N(), pc_struct = N(), p_pc = N(), pc_var = N(), p_pc_u64 = N(), p_pc_u32 = N(), p_in_i32 = N(), + vidx = N(), p_out_v4f = N(), pos = N(), c0 = N(), c4 = N(), c16 = N(), p_out_f32 = N(), psz = N(); (void)glsl; uint32_t vecs[3][5] = {{0}}; // [type][comps] vector ids (comps 2..4), [type][1] the scalar vecs[0][1] = t_f32, vecs[1][1] = t_i32, vecs[2][1] = t_u32, vecs[0][4] = t_v4f, vecs[1][4] = t_v4i, vecs[2][4] = t_v4u; @@ -726,6 +775,11 @@ done: op(&t, OpVariable, 3, p_pc, pc_var, (uint32_t)ScPushConstant); op(&t, OpVariable, 3, p_in_i32, vidx, (uint32_t)ScInput); op(&t, OpVariable, 3, p_out_v4f, pos, (uint32_t)ScOutput); + if (mi->psize_slot) { + op(&t, OpTypePointer, 3, p_out_f32, (uint32_t)ScOutput, t_f32); + op(&t, OpVariable, 3, p_out_f32, psz, (uint32_t)ScOutput); + op(&an, OpDecorate, 3, psz, (uint32_t)DecBuiltIn, (uint32_t)BiPointSize); + } op(&an, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u); op(&an, OpDecorate, 2, s_slots, (uint32_t)DecBlock); op(&an, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u); @@ -734,6 +788,7 @@ done: op(&an, OpDecorate, 3, vidx, (uint32_t)DecBuiltIn, (uint32_t)BiVertexIndex); op(&an, OpDecorate, 3, pos, (uint32_t)DecBuiltIn, (uint32_t)BiPosition); ifc[ni++] = pc_var, ifc[ni++] = vidx, ifc[ni++] = pos; + if (mi->psize_slot) ifc[ni++] = psz; uint32_t p = N(), v = N(); op(&t, OpVariable, 3, p, v, (uint32_t)ScOutput); ifc[ni++] = v; @@ -754,6 +809,15 @@ done: op(&c, OpIMul, 4, t_u64, off, s64, c16); op(&c, OpIAdd, 4, t_u64, a, e64, off); op(&c, OpConvertUToPtr, 3, p_slots, base, a); + if (mi->psize_slot) { + uint32_t ci = N(), p = N(), v = N(), x = N(), f = N(); + op(&t, OpConstant, 3, t_u32, ci, mi->psize_slot); + op(&c, OpAccessChain, 5, p_v4u_psb, p, base, c0, ci); + op(&c, OpLoad, 5, t_v4u, v, p, 2u, 16u); + op(&c, OpCompositeExtract, 4, t_u32, x, v, 0u); + op(&c, OpBitcast, 3, t_f32, f, x); + op(&c, OpStore, 2, psz, f); + } uint32_t slot_of[33]; for (int k = 0; k < 33; k++) slot_of[k] = 0xffffffffu; slot_of[32] = 0; // position diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index 38350b9f..dd7eeb58 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -28,6 +28,8 @@ typedef struct MeshEmuInfo { int task; // 1 TaskEXT, 0 MeshEXT uint32_t threads; // LocalSize x uint32_t max_v, max_p; // mesh: OutputVertices, OutputPrimitivesEXT + uint32_t prim; // mesh: 3 triangles, 2 lines, 1 points (triangles and points are emulated) + uint32_t psize_slot; // mesh: the slot holding the point size (x), 0 without one uint32_t payload_words; // the task payload's size (mesh: what it reads, 0 without a task stage) uint32_t max_mesh; // task: the most mesh workgroups one task workgroup launches (dispatch width) uint32_t slots; // mesh: 16-byte slots per vertex: position and each generic output @@ -39,4 +41,9 @@ typedef struct MeshEmuInfo { // one the converter knows (it then stays unemulated: its draws are skipped as before). uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why); -// The vertex shader of an emulated mesh pipeline: position and the generic outputs from the vertex slots, as types[l] \ No newline at end of file +// The vertex shader of an emulated mesh pipeline: position and the generic outputs from the vertex slots, as types[l] +// (0 float, 1 signed, 2 unsigned) with comps[l] components for every location l in need (the fragment stage's +// inputs); a location the mesh stage does not write is zero. malloc'd, *out_n words. +// A task or mesh module's output primitive: 3 triangles, 2 lines, 1 points, 0 not a mesh module. +int mesh_emu_prim(const uint32_t *w, size_t n); + diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 1839426f..db456d07 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5569,6 +5569,20 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { ex->recorded++; } +// The primitive an emulated draw's mesh stage outputs: point meshes draw point lists, the rest triangles. +static int mesh_emu_draw_prim(VkExec *ex) { + static struct { const GfxShader *s; int prim; } memo[256]; + int present = 0, pending = 0; + const GfxShader *ms = ex->prog ? shader_for(ex, NVN_STAGE_MESH, &present, &pending, 0) : NULL; + if (!ms) return NVN_DRAW_PRIMITIVE_TRIANGLES; + unsigned h = (unsigned)((uintptr_t)ms >> 4) & 255; + if (memo[h].s != ms) { + uint64_t n = 0; + const uint32_t *w = gfx_shader_spirv(ms, &n); + memo[h].s = ms, memo[h].prim = w && mesh_emu_prim(w, (size_t)n) == 1 ? NVN_DRAW_PRIMITIVE_POINTS : NVN_DRAW_PRIMITIVE_TRIANGLES; + } + return memo[h].prim; +} static int mesh_emu_on(void) { static int on = -1; if (on < 0) { @@ -5580,6 +5594,7 @@ static int mesh_emu_on(void) { static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); if (!fn && mesh_emu_on()) { + if (d->first) warn_once(W_MESH, "an emulated mesh draw from workgroup %u: drawn from workgroup 0", d->first); static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) if (!d->count || !on) return; @@ -5849,6 +5864,8 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla // count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from if (!me_arena()) return 0; ex->dc->c0_binst = 0; + int prim = mesh_emu_draw_prim(ex); + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1)) return 0; const VkPipeInfo *info = PD_info; const GfxShader *ms = ex->dc->B.st.vs, *ts = ex->dc->B.st.ts; VkPipeline mp = VK_NULL_HANDLE, tp = VK_NULL_HANDLE; diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 110c67b6..d0ac9eeb 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3091,7 +3091,8 @@ static void gkey_fill(const VkPipeState *s, GKey *k) { o[3] = (uint8_t)blend_factor(b->src_a), o[4] = (uint8_t)blend_factor(b->dst_a), o[5] = (uint8_t)blend_op(b->eq_a); } } - if (is_mesh(s)) k->topo_class = 2; // mesh draws: no vertex input, the mesh stage declares its primitives + // mesh draws: no vertex input, the mesh stage declares its primitives (emulated point meshes draw point lists) + if (is_mesh(s)) k->topo_class = s->prim == NVN_DRAW_PRIMITIVE_POINTS ? 0 : 2; if (!P.dyn_vi && !is_mesh(s)) { VtxInput v; vertex_input(s, &v); From 1768d5f60761e6ee898d6fa80c335f899aeb49f2 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:08 +0100 Subject: [PATCH 16/64] Mesh emulation: the vertex shader declares the fragment shader's inputs one for one. The generated vertex shader wrote one vector per location. A fragment shader that packs two inputs into one location (Component decorations: a vec2 at component 0 and another at component 2) then had nothing matching its second input, whose value is undefined: wrong colours on the emulated foliage. The vertex shader now declares an output for every input variable of the fragment shader, with its Location, Component, type and Flat, filled from those components of the mesh stage's vector at that location (zero where the mesh stage writes none). (cherry picked from commit c38eb31889314ebfb164ff9dffad5796eba46ed4) Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.c | 68 ++++++++++++++++++++++++++++++++++++++++ runtime/mesh_emu_spirv.h | 4 +++ runtime/vk_pipeline.c | 3 ++ 3 files changed, 75 insertions(+) diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index 4afec639..e4cd78d9 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -735,7 +735,50 @@ int mesh_emu_prim(const uint32_t *w, size_t n) { } // ---- the vertex shader of an emulated mesh pipeline +// The fragment shader's input variables as declared: Location, Component, scalar type and width, Flat. 0 when one +// is not a plain 32-bit scalar or vector (the caller then goes by the reflection's per-location summary). +typedef struct { uint32_t loc, comp, type, comps, flat; } FsIn; +static int fs_inputs(const uint32_t *w, size_t n, FsIn *o, int max) { + if (!w || n < 5 || w[0] != 0x07230203) return 0; + uint32_t bound = w[3]; + if (bound > (1u << 22)) return 0; + uint32_t *loc = calloc(bound, 4), *comp = calloc(bound, 4), *ty = calloc(bound, 4), *cn = calloc(bound, 4), *pt = calloc(bound, 4); + uint8_t *flat = calloc(bound, 1), *bi = calloc(bound, 1), *hl = calloc(bound, 1); + int k = 0, ok = loc && comp && ty && cn && pt && flat && bi && hl; + for (size_t i = 5; ok && i < n;) { + uint32_t o2 = w[i] & 0xffff, len = w[i] >> 16; + const uint32_t *a = w + i + 1; + if (!len || i + len > n || o2 == OpFunction) break; + if (o2 == OpDecorate && len >= 3 && a[0] < bound) { + if (a[1] == DecLocation && len >= 4) loc[a[0]] = a[2], hl[a[0]] = 1; + if (a[1] == 31 && len >= 4) comp[a[0]] = a[2]; // Component + if (a[1] == 14) flat[a[0]] = 1; // Flat + if (a[1] == DecBuiltIn) bi[a[0]] = 1; + } + if (o2 == OpTypeFloat && len >= 3 && a[0] < bound && a[1] == 32) ty[a[0]] = 1, cn[a[0]] = 1; + if (o2 == OpTypeInt && len >= 4 && a[0] < bound && a[1] == 32) ty[a[0]] = a[2] ? 2 : 3, cn[a[0]] = 1; + if (o2 == OpTypeVector && len >= 4 && a[0] < bound && a[1] < bound && ty[a[1]]) ty[a[0]] = ty[a[1]], cn[a[0]] = a[2]; + if (o2 == OpTypePointer && len >= 4 && a[0] < bound && a[1] == ScInput) pt[a[0]] = a[2] + 1; + if (o2 == OpVariable && len >= 4 && a[2] == ScInput && a[1] < bound && a[0] < bound && !bi[a[1]] && hl[a[1]]) { + uint32_t p = pt[a[0]] ? pt[a[0]] - 1 : 0; + if (!p || p >= bound || !ty[p] || cn[p] > 4 || loc[a[1]] >= 32 || k == max) { k = 0, ok = 0; break; } + o[k++] = (FsIn){loc[a[1]], comp[a[1]], ty[p] - 1, cn[p], flat[a[1]]}; + } + i += len; + } + free(loc), free(comp), free(ty), free(cn), free(pt), free(flat), free(bi), free(hl); + return ok ? k : 0; +} + +uint32_t *mesh_emu_vs(const MeshEmuInfo *mi, uint32_t need, const uint8_t types[32], const uint8_t comps[32], + const uint32_t *fsw, size_t fsn, size_t *out_n) { Buf b = {0}, t = {0}, an = {0}, c = {0}; + FsIn in[64]; + int nin = fs_inputs(fsw, fsn, in, 64); + if (!nin) // the per-location summary: one vector per location + for (uint32_t l = 0; l < 32; l++) + if (need >> l & 1) + in[nin++] = (FsIn){l, 0, types[l] <= 2 ? types[l] : 0, comps[l] >= 1 && comps[l] <= 4 ? comps[l] : 4, types[l] == 1 || types[l] == 2}; uint32_t id = 1; #define N() (id++) uint32_t glsl = N(), fn = N(), t_void = N(), t_fn = N(), t_u32 = N(), t_i32 = N(), t_f32 = N(), t_u64 = N(), @@ -787,10 +830,23 @@ int mesh_emu_prim(const uint32_t *w, size_t n) { for (uint32_t k = 0; k < 7; k++) op(&an, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u); op(&an, OpDecorate, 3, vidx, (uint32_t)DecBuiltIn, (uint32_t)BiVertexIndex); op(&an, OpDecorate, 3, pos, (uint32_t)DecBuiltIn, (uint32_t)BiPosition); + uint32_t ifc[80], ni = 0; ifc[ni++] = pc_var, ifc[ni++] = vidx, ifc[ni++] = pos; if (mi->psize_slot) ifc[ni++] = psz; + // outputs: one per fragment input, same Location, Component and type + uint32_t outv[64] = {0}, outt[64] = {0}; + for (int q = 0; q < nin; q++) { + int ty = in[q].type <= 2 ? (int)in[q].type : 0, k = in[q].comps >= 1 && in[q].comps <= 4 ? (int)in[q].comps : 4; + if (in[q].comp + (uint32_t)k > 4) k = (int)(4 - in[q].comp); + in[q].type = (uint32_t)ty, in[q].comps = (uint32_t)k; + outt[q] = vecs[ty][k]; uint32_t p = N(), v = N(); + op(&t, OpTypePointer, 3, p, (uint32_t)ScOutput, outt[q]); op(&t, OpVariable, 3, p, v, (uint32_t)ScOutput); + op(&an, OpDecorate, 3, v, (uint32_t)DecLocation, in[q].loc); + if (in[q].comp) op(&an, OpDecorate, 3, v, 31u, in[q].comp); // Component + if (in[q].flat || ty) op(&an, OpDecorate, 2, v, 14u); // Flat + outv[q] = v; ifc[ni++] = v; } // code: slot = vertex area + VertexIndex; each output from slot + its index @@ -822,6 +878,8 @@ int mesh_emu_prim(const uint32_t *w, size_t n) { for (int k = 0; k < 33; k++) slot_of[k] = 0xffffffffu; slot_of[32] = 0; // position for (uint32_t g = 0; g < mi->n_generic; g++) slot_of[mi->generic_loc[g]] = 1 + g; + for (int q = -1; q < nin; q++) { // -1: the position + uint32_t l = q < 0 ? 32 : in[q].loc; uint32_t val; if (slot_of[l] == 0xffffffffu) { val = 0; @@ -832,22 +890,32 @@ int mesh_emu_prim(const uint32_t *w, size_t n) { op(&c, OpAccessChain, 5, p_v4u_psb, p, base, c0, ci); op(&c, OpLoad, 5, t_v4u, val, p, 2u, 16u); } + if (q < 0) { uint32_t f = N(); op(&c, OpBitcast, 3, t_v4f, f, val); op(&c, OpStore, 2, pos, f); continue; } + int ty = (int)in[q].type, k = (int)in[q].comps; + uint32_t v4t = vecs[ty][4], r, c0i = in[q].comp; if (!val) { // not written by the mesh stage: zero uint32_t z = N(); r = N(); + op(&t, OpConstantNull, 2, outt[q], z); + op(&c, OpCopyObject, 3, outt[q], r, z); } else { uint32_t cv = val; if (ty != 2) cv = N(), op(&c, OpBitcast, 3, v4t, cv, val); if (k == 4) r = cv; + else if (k == 1) r = N(), op(&c, OpCompositeExtract, 4, outt[q], r, cv, c0i); else { r = N(); + put(&c, (uint32_t)(5 + k) << 16 | 79u); // OpVectorShuffle: components comp..comp+k-1 + put(&c, outt[q]), put(&c, r), put(&c, cv), put(&c, cv); + for (int z = 0; z < k; z++) put(&c, c0i + (uint32_t)z); } } + op(&c, OpStore, 2, outv[q], r); } op(&c, OpReturn, 0); op(&c, OpFunctionEnd, 0); diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index dd7eeb58..ffa99fa4 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -47,3 +47,7 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ // A task or mesh module's output primitive: 3 triangles, 2 lines, 1 points, 0 not a mesh module. int mesh_emu_prim(const uint32_t *w, size_t n); +// fsw: the fragment shader's SPIR-V; its input variables are matched one for one (Location, Component, type, Flat), +// else (NULL, or a declaration the parser does not take) one vector per location of need. +uint32_t *mesh_emu_vs(const MeshEmuInfo *m, uint32_t need, const uint8_t types[32], const uint8_t comps[32], + const uint32_t *fsw, size_t fsn, size_t *out_n); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index d0ac9eeb..ef7c32f5 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3257,6 +3257,9 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay cp[i] = need >> i & 1 && !fin[i].packed && fin[i].comps ? (uint8_t)fin[i].comps : 4; } size_t vn = 0; + uint64_t fsw = 0; + const uint32_t *fcode0 = s->fs ? gfx_shader_spirv(s->fs, &fsw) : NULL; + vpatched = mesh_emu_vs(&me->mi, need, ty, cp, fcode0, (size_t)fsw, &vn); if (!vpatched) return VK_NULL_HANDLE; vcode = vpatched, vw = vn; mesh = 0; // a vertex pipeline from here on From 2b11e322508f0848e38076eff3c8b594c02262f8 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:10 +0100 Subject: [PATCH 17/64] NX2NAVI_PASS_TIMES: write the timestamps straight into the command buffer, not through the recorder. vk_pass_mark_cmd runs on the recorder thread for a pass mark packet; through vk.h's macro its vkCmdWriteTimestamp became a packet of its own, put into the ring by the recorder thread while the replay thread was writing it. The ring broke ("FATAL: vk_rec: unknown packet") on NVIDIA GPUs, where the recorder is on by default. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_crumbs.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index 86fd6ba4..72bc56c5 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -18,6 +18,11 @@ #include "nvn_api.h" #include "vk.h" +// vk_pass_mark_cmd also runs on the recorder thread (vk_rec.c R_PassMark), where vk.h's macro would turn the timestamp +// into a new packet for the part being recorded: a second producer on the ring, which corrupts it ("vk_rec: unknown +// packet" with NX2NAVI_PASS_TIMES=1 on NVIDIA, where the recorder is on). The command goes straight into cb. +#undef vkCmdWriteTimestamp + static PFN_vkCmdWriteBufferMarkerAMD write_marker; static VkBuffer buf; static VmaAllocation alloc; From 131ee6f275c9b410bd8a8283ffeb52072494241e Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:11 +0100 Subject: [PATCH 18/64] NX2NAVI_DUMP_KEY: F10 dumps the next frame. A frame dump needed its frame number in advance (NX2NAVI_DUMP) or a depth target address (NX2NAVI_DUMP_AFTER); with dump_key = 1 each press of F10 in the game window dumps the next frame to /f, as often as wanted. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_dump.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index 0835c482..1903961a 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -44,6 +44,9 @@ uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r); // uint64_t vk_exec_storage_addr(const VkExec *ex, int st, const GfxShaderRes *r); // vk_exec_draw.c int vk_exec_raw_9e5(VkImage img, VkFormat f); // vk_exec_draw.c #include "platform/platform.h" +#ifdef _WIN32 +#include +#endif #define MAX_PASSES 400 #define PNG_CAP (2ull << 30) @@ -1162,7 +1165,7 @@ void vk_dump_init(void) { if (dump_white || dump_after) { dump_frame = UINT64_MAX; vk_thumbs_enable(16); - } else if (movie_k || key_mode || dump_file[0]) { + } else if (movie_k || key_mode || dump_file[0] || dump_key) { dump_frame = UINT64_MAX; } else if (!e || !*e || !(dump_frame = strtoull(e, NULL, 10))) return; const char *lf = cfg_get("NX2NAVI_DUMP_LOG"); @@ -1207,10 +1210,25 @@ void vk_dump_init(void) { if (!bd_w || !bd_h) bd_stride = 0; snprintf(path, sizeof path, "%s/bufs.txt", dir); // a dump named as the run goes opens it with its folder (open_files); a named frame's folder exists already - if (bd_size && !dump_white && !movie_k && !key_mode && !dump_file[0] && !(bufs_f = fopen(path, "w"))) bd_size = 0; + if (bd_size && !dump_white && !movie_k && !key_mode && !dump_file[0] && !dump_key && !(bufs_f = fopen(path, "w"))) bd_size = 0; } vk_dump_armed = 1; rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); + else rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); +} + +static int key_pressed(void) { +#ifdef _WIN32 + static int was; + HWND fg = GetForegroundWindow(); + DWORD pid = 0; + if (fg) GetWindowThreadProcessId(fg, &pid); + int pressed = down && !was; + was = down; + return pressed; +#else + return 0; +#endif } // The thumbnail of frame f is blank white left of the overlay and above the dialogue box: -1 not available, 0 no, 1 yes. @@ -1279,6 +1297,12 @@ void vk_dump_present(uint64_t frame) { dump_frame = frame + 1; rt_log("vk_dump: %s found: frame %llu goes to %s", dump_file, (unsigned long long)dump_frame, dir); } + if (dump_key && dump_frame == UINT64_MAX && key_pressed()) { // NX2NAVI_DUMP_KEY: our key names the next frame + file_more = 0; + snprintf(dir, sizeof dir, "%s/f%llu", base_dir, (unsigned long long)(frame + 1)); + open_files(); // before the frame is named: its passes write to them + dump_frame = frame + 1; + } if (frame != dump_frame) return; plat_mutex_lock(&rec_lock); finished = 1; @@ -1294,7 +1318,7 @@ void vk_dump_present(uint64_t frame) { passes_f = NULL; rt_log("vk_dump: frame %llu: %d passes and dispatches, about %llu MB of PNGs in %s", (unsigned long long)frame, seq, (unsigned long long)(png_bytes >> 20), dir); - if (dump_white || movie_k || key_mode || dump_file[0]) { // ready for another dump (vk_dump_present checks a white one's thumbnail 3 presents on) + if (dump_white || movie_k || key_mode || dump_file[0] || dump_key) { // ready for another dump (vk_dump_present checks a white one's thumbnail 3 presents on) if (dump_white) white_last = frame; plat_mutex_lock(&rec_lock); seq = copied = 0, last_targets = 0, pass_skip = NULL, png_bytes = 0; From 6169d823858e995e0b906b81addc04465f0dbb4e Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:13 +0100 Subject: [PATCH 19/64] NX2NAVI_DUMP_KEY takes a key name; Pause by default. F10 is the performance overlay's key, so a press dumped a frame and changed the overlay. dump_key = 1 or pause now means the Pause key; scrolllock, insert, home, end, pageup, pagedown, f2-f7, f9 and f12 can be chosen instead. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_dump.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index 1903961a..7dd48d74 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -26,6 +26,9 @@ // (the terrain decals draw one pass each into the G-buffer and would use up both caps). // NX2NAVI_DUMP_SHADERS=,...: copy images only for passes using those shader hashes; metadata covers the whole frame. // NX2NAVI_DUMP_RAW=1: also save each copied image's native texel bytes as .bin, for numerical analysis. +// NX2NAVI_DUMP_KEY= (Windows): each press of that key in the game window dumps the next frame to +// NX2NAVI_DUMP_DIR/f. Keys: pause (also 1), scrolllock, insert, home, end, pageup, pagedown, f2-f7, f9, f12 (F1, F8, +// F10 and F11 are the menu, the flight recorder, the overlay and fullscreen). The game still gets a key it has bound. #include #include #include @@ -62,6 +65,8 @@ static char base_dir[400]; // NX2NAVI_DUMP_DIR; with NX2NAVI_DUMP_WH static uint64_t white_last; // NX2NAVI_DUMP_WHITE: the frame dumped last (0: none), checked 3 presents on static int white_tries, white_caught; static uint64_t dump_after; // NX2NAVI_DUMP_AFTER +static int dump_key; // NX2NAVI_DUMP_KEY: the Windows virtual-key code, 0 off +static const char *dump_key_name; static FILE *passes_f, *images_f; static int seq, finished, copied; // copied: passes and dispatches of the dump frame whose images were copied static int skip_runs, dump_ubo; // NX2NAVI_DUMP_SKIP_RUNS, NX2NAVI_DUMP_UBO (vec4 per block, 0 for none) @@ -1162,6 +1167,17 @@ void vk_dump_init(void) { key_mode = e && !strcmp(e, "key"); const char *df = cfg_get("NX2NAVI_DUMP_FILE"); if (df && *df) snprintf(dump_file, sizeof dump_file, "%s", df); + // NX2NAVI_DUMP_KEY: a key of ours, also outside NX2NAVI_DUMP=key + { + static const struct { const char *n; int vk; } keys[] = { + {"1", 0x13}, {"pause", 0x13}, {"scrolllock", 0x91}, {"insert", 0x2d}, {"home", 0x24}, {"end", 0x23}, + {"pageup", 0x21}, {"pagedown", 0x22}, {"f2", 0x71}, {"f3", 0x72}, {"f4", 0x73}, {"f5", 0x74}, {"f6", 0x75}, + {"f7", 0x76}, {"f9", 0x78}, {"f12", 0x7b}}; + const char *k = cfg_get("NX2NAVI_DUMP_KEY"); + for (size_t i = 0; k && *k && i < sizeof keys / sizeof *keys; i++) + if (!strcasecmp(k, keys[i].n)) dump_key = keys[i].vk, dump_key_name = i ? keys[i].n : "pause"; + if (k && *k && strcmp(k, "0") && !dump_key) rt_log("vk_dump: NX2NAVI_DUMP_KEY=%s: not a key this knows (pause, scrolllock, insert, home, end, pageup, pagedown, f2-f7, f9, f12)", k); + } if (dump_white || dump_after) { dump_frame = UINT64_MAX; vk_thumbs_enable(16); @@ -1213,16 +1229,18 @@ void vk_dump_init(void) { if (bd_size && !dump_white && !movie_k && !key_mode && !dump_file[0] && !dump_key && !(bufs_f = fopen(path, "w"))) bd_size = 0; } vk_dump_armed = 1; - rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); + if (dump_key) rt_log("vk_dump: %s dumps the next frame to %s/f", dump_key_name, base_dir); else rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); } +// NX2NAVI_DUMP_KEY: its key went down since the last present (the game window has the focus). static int key_pressed(void) { #ifdef _WIN32 static int was; HWND fg = GetForegroundWindow(); DWORD pid = 0; if (fg) GetWindowThreadProcessId(fg, &pid); + int down = pid == GetCurrentProcessId() && (GetAsyncKeyState(dump_key) & 0x8000) != 0; int pressed = down && !was; was = down; return pressed; @@ -1302,6 +1320,7 @@ void vk_dump_present(uint64_t frame) { snprintf(dir, sizeof dir, "%s/f%llu", base_dir, (unsigned long long)(frame + 1)); open_files(); // before the frame is named: its passes write to them dump_frame = frame + 1; + rt_log("vk_dump: %s: frame %llu goes to %s", dump_key_name, (unsigned long long)dump_frame, dir); } if (frame != dump_frame) return; plat_mutex_lock(&rec_lock); From 3b90353c5be65acd67ec8e2c035a102ebadb45d9 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:15 +0100 Subject: [PATCH 20/64] Optional (NX2NAVI_VEC_INDEX_FIX): uniform vector components at computed indices loaded whole and selected. AMD's Windows drivers can read the wrong component through an access chain into a uniform block vector whose component index is computed (the translator's c[n][i >> 2][i & 3] for an LDC with a register offset), the bug Ryujinx works around with HasVectorIndexingBug. With NX2NAVI_VEC_INDEX_FIX=1 (or auto: AMD's own drivers) each shader is patched in memory as it loads: the chain without its last index loads the whole vector, CompositeExtract takes each component and OpSelect picks the indexed one. Off by default: it did not change the volumetric fog's teal blotches on an RX 580, whose per-volume extinction reads its colour this way. Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_shader.c | 180 +++++++++++++++++++++++++++++++++++++++++++ runtime/vk_caps.c | 7 ++ 2 files changed, 187 insertions(+) diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 77be1d9f..7fd3bdf5 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1175,6 +1175,185 @@ out: free(arrayed), free(need), free(bind), free(org), free(query); } +// Every load of one component of a uniform block vector at a computed component index (OpAccessChain into a Uniform +// block whose last index, into a vector, is not a constant: the translator's c[n][i >> 2][i & 3] for an LDC with a +// register offset) becomes a load of the whole vector, its components extracted with constant indices and the one +// wanted picked with OpSelect. AMD's Windows drivers can read the wrong component through such an access chain, the bug +// Ryujinx works around with HasVectorIndexingBug (VectorComponentSelect). The volumetric fog's per-volume +// extinction reads its colour this way. It did not change the fog's teal blotches on an RX 580 (23.10.01.14), so it is +// off by default. NX2NAVI_VEC_INDEX_FIX: 1 everywhere, auto on AMD's own drivers, unset or 0 off. In memory only. +int vk_caps_vec_index_bug(void); // vk_caps.c + +enum { VX_TYPE_BOOL = 20, VX_TYPE_INT = 21, VX_TYPE_FLOAT = 22, VX_TYPE_VECTOR = 23, VX_TYPE_ARRAY = 28, + VX_TYPE_RTARRAY = 29, VX_TYPE_STRUCT = 30, VX_TYPE_POINTER = 32, VX_CONSTANT = 43, VX_FUNCTION = 54, + VX_VARIABLE = 59, VX_LOAD = 61, VX_ACCESS_CHAIN = 65, VX_IN_BOUNDS_ACCESS_CHAIN = 66, VX_COMPOSITE_EXTRACT = 81, + VX_SELECT = 169, VX_IEQUAL = 170, VX_SC_UNIFORM = 2 }; + +static void patch_vec_index(GfxShader *s) { + static int on = -1; + static volatile int32_t n_logged; + if (on < 0) { + const char *e = cfg_get("NX2NAVI_VEC_INDEX_FIX"); + on = e && *e ? (strcmp(e, "auto") ? *e != '0' : vk_caps_vec_index_bug()) : 0; + if (on) rt_log("gfx_shader: uniform vector components at computed indices are read whole and selected " + "(NX2NAVI_VEC_INDEX_FIX=0 turns it off)"); + } + uint32_t *w = s->spirv; + uint64_t n = s->words; + if (!on || n < 5) return; + uint32_t bound = w[3]; + if (bound > (1u << 22)) return; + uint64_t *def = calloc(bound, sizeof *def); // word index of the instruction defining each id + uint32_t *ptr_v = calloc(bound, sizeof *ptr_v); // per vector type: its Uniform pointer type, if any + uint32_t *target = calloc(bound, sizeof *target); // per access chain id to rewrite: the vector access chain's id + uint32_t *acv = calloc(bound, sizeof *acv); // per access chain id to rewrite: the vector type + if (!def || !ptr_v || !target || !acv) goto out; + uint64_t first_fn = 0; + uint32_t bool_t = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) goto out; + if (op == VX_FUNCTION && !first_fn) first_fn = i; + uint32_t rid = 0; + switch (op) { + // no result id (or none at word 2): debug, annotations, mode setting, stores, barriers, control flow + case 3: case 4: case 5: case 6: case 8: case 10: case 14: case 15: case 16: case 17: case 56: case 62: case 63: + case 71: case 72: case 73: case 74: case 75: case 99: case 224: case 225: case 228: case 246: case 247: case 248: + case 249: case 250: case 251: case 252: case 253: case 254: case 255: case 317: case 330: case 331: case 332: + case 4416: case 5294: case 5295: case 5632: case 5633: break; + case 7: case 11: // OpString, OpExtInstImport: result at word 1 + case VX_TYPE_BOOL: case VX_TYPE_INT: case VX_TYPE_FLOAT: case VX_TYPE_VECTOR: case VX_TYPE_ARRAY: + case VX_TYPE_RTARRAY: case VX_TYPE_STRUCT: case VX_TYPE_POINTER: rid = len >= 2 ? w[i + 1] : 0; break; + default: + if (op >= 19 && op <= 39) rid = len >= 2 ? w[i + 1] : 0; // the other OpType* + else if (len >= 3) rid = w[i + 2]; + } + if (op == VX_TYPE_BOOL && !bool_t) bool_t = w[i + 1]; + if (op == VX_TYPE_POINTER && len == 4 && w[i + 2] == VX_SC_UNIFORM && w[i + 3] < bound && !ptr_v[w[i + 3]]) + ptr_v[w[i + 3]] = w[i + 1]; + if (rid && rid < bound && !def[rid]) def[rid] = i; + i += len; + } + if (!first_fn) goto out; +#define VX_OP(id) ((id) < bound && def[id] ? w[def[id]] & 0xffff : 0) + // the access chains to rewrite and what they need + int found = 0; + uint32_t next = bound; + uint32_t need_ptr[64], need_ptr_v[64], nneed = 0; // new Uniform pointer types: id, vector type + uint32_t cint[8][4] = {{0}}, cint_t[8] = {0}, ncint = 0; // constants 0..3 per index type + for (uint64_t i = first_fn; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if ((op == VX_ACCESS_CHAIN || op == VX_IN_BOUNDS_ACCESS_CHAIN) && len >= 6) { + uint32_t rt = w[i + 1], id = w[i + 2], base = w[i + 3], nidx = len - 4, last = w[i + len - 1]; + uint32_t vt = 0; + if (VX_OP(rt) == VX_TYPE_POINTER && w[def[rt] + 2] == VX_SC_UNIFORM && VX_OP(base) == VX_VARIABLE && + VX_OP(last) != VX_CONSTANT && VX_OP(w[def[base] + 1]) == VX_TYPE_POINTER) { + uint32_t t = w[def[w[def[base] + 1]] + 3]; // the variable's pointee + int ok = 1; + for (uint32_t k = 0; k + 1 < nidx && ok; k++) { + uint32_t top = VX_OP(t), ix = w[i + 4 + k]; + if (top == VX_TYPE_STRUCT) { + if (VX_OP(ix) != VX_CONSTANT) { ok = 0; break; } + uint32_t m = w[def[ix] + 3], slen = w[def[t]] >> 16; + if (2 + m >= slen) { ok = 0; break; } + t = w[def[t] + 2 + m]; + } else if (top == VX_TYPE_ARRAY || top == VX_TYPE_RTARRAY) { + t = w[def[t] + 2]; + } else ok = 0; + } + uint32_t sc = w[def[rt] + 3]; + if (ok && VX_OP(t) == VX_TYPE_VECTOR && w[def[t] + 2] == sc && w[def[t] + 3] >= 2 && w[def[t] + 3] <= 4 && + def[last] && (w[def[last]] >> 16) >= 3) { + uint32_t it = w[def[last] + 1]; // the index's type + if (VX_OP(it) == VX_TYPE_INT) vt = t; + if (vt && !ptr_v[vt]) { + if (nneed == 64) vt = 0; + else need_ptr[nneed] = next++, need_ptr_v[nneed++] = vt, ptr_v[vt] = need_ptr[nneed - 1]; + } + if (vt) { + uint32_t c = 0; + while (c < ncint && cint_t[c] != it) c++; + if (c == ncint) { + if (ncint == 8) vt = 0; + else cint_t[ncint++] = it; + } + } + } + } + if (vt && id < bound) target[id] = next++, acv[id] = vt, found++; + } + i += len; + } + if (!found) goto out; + // constants 0..3 of each index type: existing ones where the module has them + for (uint64_t i = 5; i < first_fn;) { + uint32_t len = w[i] >> 16; + if ((w[i] & 0xffff) == VX_CONSTANT && len == 4 && w[i + 3] < 4) + for (uint32_t c = 0; c < ncint; c++) + if (cint_t[c] == w[i + 1] && !cint[c][w[i + 3]]) cint[c][w[i + 3]] = w[i + 2]; + i += len; + } + uint32_t newc[8][4] = {{0}}; + for (uint32_t c = 0; c < ncint; c++) + for (int v = 0; v < 4; v++) + if (!cint[c][v]) cint[c][v] = newc[c][v] = next++; + uint32_t new_bool = bool_t ? 0 : next++; + if (!bool_t) bool_t = new_bool; + // count the loads to size the output: each becomes 1 + 4 + 3 + 3 instructions of up to 5 words + uint64_t loads = 0; + for (uint64_t i = first_fn; i < n;) { + uint32_t len = w[i] >> 16; + if ((w[i] & 0xffff) == VX_LOAD && len >= 4 && w[i + 3] < bound && target[w[i + 3]]) loads++; + i += len; + } + uint32_t *o = malloc((n + 64 * 4 + 32 * 4 + 4 + (uint64_t)found * 16 + loads * 64) * sizeof *o); + if (!o) goto out; + uint64_t k = 0; + for (uint64_t i = 0; i < 5; i++) o[k++] = w[i]; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (i == first_fn) { // the new declarations, last in the global section + for (uint32_t j = 0; j < nneed; j++) + o[k++] = 4u << 16 | VX_TYPE_POINTER, o[k++] = need_ptr[j], o[k++] = VX_SC_UNIFORM, o[k++] = need_ptr_v[j]; + if (new_bool) o[k++] = 2u << 16 | VX_TYPE_BOOL, o[k++] = new_bool; + for (uint32_t c = 0; c < ncint; c++) + for (uint32_t v = 0; v < 4; v++) + if (newc[c][v]) o[k++] = 4u << 16 | VX_CONSTANT, o[k++] = cint_t[c], o[k++] = newc[c][v], o[k++] = v; + } + for (uint32_t j = 0; j < len; j++) o[k++] = w[i + j]; + if ((op == VX_ACCESS_CHAIN || op == VX_IN_BOUNDS_ACCESS_CHAIN) && len >= 6 && w[i + 2] < bound && target[w[i + 2]]) { + // the same chain without its last index: a pointer to the vector + o[k++] = (len - 1) << 16 | op, o[k++] = ptr_v[acv[w[i + 2]]], o[k++] = target[w[i + 2]]; + for (uint32_t j = 3; j < len - 1; j++) o[k++] = w[i + j]; + } else if (op == VX_LOAD && len >= 4 && w[i + 3] < bound && target[w[i + 3]]) { + k -= len; // replaced + uint32_t st = w[i + 1], res = w[i + 2], ac = w[i + 3], vt = acv[ac], nc = w[def[vt] + 3]; + uint32_t idx = w[def[ac] + (w[def[ac]] >> 16) - 1], it = w[def[idx] + 1], c = 0; + while (cint_t[c] != it) c++; + uint32_t vec = next++, e[4], sel = 0; + o[k++] = 4u << 16 | VX_LOAD, o[k++] = vt, o[k++] = vec, o[k++] = target[ac]; + for (uint32_t m = 0; m < nc; m++) + e[m] = next++, o[k++] = 5u << 16 | VX_COMPOSITE_EXTRACT, o[k++] = st, o[k++] = e[m], o[k++] = vec, o[k++] = m; + sel = e[0]; + for (uint32_t m = 1; m < nc; m++) { + uint32_t eq = next++, r = m + 1 == nc ? res : next++; + o[k++] = 5u << 16 | VX_IEQUAL, o[k++] = bool_t, o[k++] = eq, o[k++] = idx, o[k++] = cint[c][m]; + o[k++] = 6u << 16 | VX_SELECT, o[k++] = st, o[k++] = r, o[k++] = eq, o[k++] = e[m], o[k++] = sel; + sel = r; + } + } + i += len; + } + o[3] = next; + free(s->spirv); + s->spirv = o, s->words = k; + if (plat_inc32(&n_logged) <= 4) + rt_log("gfx_shader: %d uniform vector component reads at computed indices now select the component", found); +#undef VX_OP +out: + free(def), free(ptr_v), free(target), free(acv); +} + static void reflect_stage(GfxShader *s) { const char *js = s->reflection; char stage[8]; @@ -1197,6 +1376,7 @@ static void reflect(GfxShader *s) { reflect_spirv(s); reflect_exact(s); patch_txd_transpose(s); + patch_vec_index(s); } // ---- use masks (see the cache key at the top) diff --git a/runtime/vk_caps.c b/runtime/vk_caps.c index bcd0bde2..5945b135 100644 --- a/runtime/vk_caps.c +++ b/runtime/vk_caps.c @@ -41,6 +41,12 @@ void vk_caps_features(VkPhysicalDeviceFeatures *f) { vk_caps.float64 = f->shaderFloat64; } +static int amd_proprietary; // AMD's own Windows driver (any branch): gfx_shader.c patch_vec_index + +// gfx_shader.c patch_vec_index: the driver reads uniform vector components at computed indices wrongly (Ryujinx's +// HasVectorIndexingBug, which it sets for AMD's Windows drivers). +int vk_caps_vec_index_bug(void) { return amd_proprietary; } + void vk_caps_driver(VkPhysicalDevice p) { VkPhysicalDeviceDriverProperties drv = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &drv}; @@ -51,6 +57,7 @@ void vk_caps_driver(VkPhysicalDevice p) { // "26.10.1 (AMD proprietary shader compiler)" where the RDNA 3 and 4 driver names LLPC. vk_caps.amd_sc = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY && strstr(drv.driverInfo, "shader compiler"); vk_caps.amd_llpc = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY && !vk_caps.amd_sc; + amd_proprietary = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY; vk_caps.mesa = drv.driverID == VK_DRIVER_ID_MESA_RADV || drv.driverID == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA || drv.driverID == VK_DRIVER_ID_MESA_NVK; } From 703e9fecd962e5fbba718c840f10cb46177f58e6 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:16 +0100 Subject: [PATCH 21/64] Textures read after another image drew over their pool memory get memory of their own. The game overlaps transient render targets in its memory pools, and the replay binds their images into that memory. An image of another format or size drawn over an image's memory leaves the GPU's tiled, compressed texels of the other image there. Ocarina's volumetric fog reads its 360x202 RGBA32F fog-volume texture at the start of a frame, before drawing it again, and the 1440x810 R11G11B10F bloom target had been drawn over that memory at the end of the previous frame: the fog lighting took those texels as volume distances and types (values from 1e-12 to 3e33 in a frame dump where the texture holds 9 to 50000), and drew teal splotches in Kokiri Forest's fog on AMD and NVIDIA. Each pool-bound image the GPU writes marks the pool-bound images of another layout that overlap it; one read while marked is dropped and comes back with memory of its own from then on, so it keeps what it last drew (the read that found it gets an empty image once). NX2NAVI_TEX_CLOBBER=0 keeps such images in pool memory. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 + runtime/vk_texture.c | 111 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index eb36aad7..c0ea134d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -137,6 +137,7 @@ static const struct { const char *key, *section; } sections[] = { {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"WATER_PROJECTIVE", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"FFX_PIPE_CACHE", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"TEX_CLOBBER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, @@ -295,6 +296,7 @@ static const CfgMeta metas[] = { // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, {"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_SPLIT_FILL", "1", "0|1|2", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_CLOBBER", "1", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "1", "0|1", RESTART}, {"ASTC_ARENA", "1", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 399434d2..3ef98963 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -112,6 +112,9 @@ typedef struct Tex { VmaAllocation flat3d_alloc; uint32_t flat3d_up, flat3d_layer; // n_up and the layer of the copy's last refresh uint64_t flat3d_wr; // wr_fence then + uint64_t used_frame; // presented frame (gfx_counters.frames) when a draw binding or lookup last named it + uint8_t in_rt; // in rts[] (bound into pool memory, the GPU wrote it) + uint8_t clobbered; // another image of another layout wrote over its pool memory since its own last write } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the @@ -1146,6 +1149,79 @@ static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *i } // defer_init: the caller records the image's first transition (init_image or alias_copy). +// ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER) +// The game places transient render targets over each other in its memory pools, and the replay binds their images +// into the same memory (bind_pool_memory). An image of another format or size that renders over an image's memory +// leaves it holding the other image's texels in the GPU's own tiled and compressed layout, which reads back as +// unrelated numbers. Ocarina's volumetric fog reads its 360x202 RGBA32F fog-volume texture at the start of a frame, +// before it draws it again; at the end of the previous frame the 1440x810 R11G11B10F bloom target had been drawn over +// that memory, and the fog lighting took those numbers as volume distances and types: teal splotches in the fog of +// Kokiri Forest, on AMD and NVIDIA. An image read while in that state gets memory of its own from then on +// (private_keys), so it keeps what it last drew; the read that found it takes an empty image once. +// NX2NAVI_TEX_CLOBBER=0 keeps such images in pool memory. +static Tex **rts; // pool-bound images the GPU wrote, sorted by address +static int nrts, crts; +static uint64_t max_rt; // largest storage size in rts[] +static nvn_hmap private_keys = {.klen = sizeof(TexKey)}; +static int clobber_fix = -1; +static volatile int32_t n_unaliased; + +static int rts_lower(uint64_t a) { + int lo = 0, hi = nrts; + while (lo < hi) { + int m = (lo + hi) / 2; + if (rts[m]->pub.addr < a) lo = m + 1; + else hi = m; + } + return lo; +} + +static void rt_add(Tex *e) { + if (e->in_rt) return; + if (nrts == crts) { + crts = crts ? 2 * crts : 256; + if (!(rts = realloc(rts, sizeof *rts * (size_t)crts))) rt_fatal("vk: out of memory for the render target list"); + } + int i = rts_lower(e->pub.addr); + memmove(rts + i + 1, rts + i, sizeof *rts * (size_t)(nrts - i)); + rts[i] = e; + nrts++; + if (e->t.L.size > max_rt) max_rt = e->t.L.size; + e->in_rt = 1; +} + +static void rt_remove(Tex *e) { + if (!e->in_rt) return; + for (int i = rts_lower(e->pub.addr); i < nrts && rts[i]->pub.addr == e->pub.addr; i++) + if (rts[i] == e) { + memmove(rts + i, rts + i + 1, sizeof *rts * (size_t)(nrts - i - 1)); + nrts--; + break; + } + e->in_rt = 0; +} + +static int alias_compatible(const Tex *e, const Tex *src, const char **why); + +// e, bound into pool memory, was just written by the GPU: its own content is current, and the pool-bound images of +// another layout whose storage overlaps it no longer hold theirs. +static void mark_clobbered(Tex *e) { + if (clobber_fix < 0) clobber_fix = cfg_bool("NX2NAVI_TEX_CLOBBER"); + e->clobbered = 0; + if (!clobber_fix || !e->bound_to) return; + rt_add(e); + uint64_t lo = e->pub.addr, hi = lo + e->t.L.size; + for (int i = rts_lower(lo > max_rt ? lo - max_rt : 0); i < nrts && rts[i]->pub.addr < hi; i++) { + Tex *o = rts[i]; + if (o == e || o->clobbered || o->bound_to != e->bound_to || o->pub.addr + o->t.L.size <= lo) continue; + const char *why; // the same texels through both: a compatible image of the same shape (another size tiles apart) + if (o->pub.image == e->pub.image || (alias_compatible(o, e, &why) && o->pub.width == e->pub.width && + o->pub.height == e->pub.height && o->pub.depth == e->pub.depth)) + continue; + o->clobbered = 1; + } +} + static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick; VkCtx *c = vk_ctx(); @@ -1257,7 +1333,9 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { e->pub.addr = k->addr; img_key_of(&ci, &e->ik); VkImage img = VK_NULL_HANDLE; - if (scale == 1 && nvn_gpu_only(k->addr)) { // storage in a physical or virtual pool: alias the pool memory (section 3.2) + // storage in a physical or virtual pool: alias the pool memory (section 3.2), unless an image of another layout + // overwrote it there before it was read (private_keys) + if (scale == 1 && nvn_gpu_only(k->addr) && !nvn_hm_get(&private_keys, k)) { const VkMemoryRequirements *q = nvn_hm_get(&img_reqs, &e->ik); VkMemSpan sp; if (q && !pool_fits(c, e, q, L.size, &sp)) { // it would not bind there (img_reqs) @@ -1963,6 +2041,32 @@ static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { return k < (int)n ? k : (int)n - 1; } + } +} + + rt_remove(e); + n_site[0]++; + tex_gen_bump(e->key.addr); + nvn_hm_del(&by_key, &e->key); + Tex *head = nvn_hm_get(&by_addr, &e->key.addr); + if (head == e) { + if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at); + else nvn_hm_del(&by_addr, &e->key.addr); + } else { + for (Tex *p = head; p; p = p->next_at) + if (p->next_at == e) { p->next_at = e->next_at; break; } + if (e->released) nreleased--, rel_unlink(e); + if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) + vk_defer_image(e->pub.image, e->alloc); + free(e->snap); + // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a + // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on + // whichever thread released the texture. Freed here, a pass that began again read the freed record, cached a view of + // the dead image (vk_exec_target_view), and a later image given the same handle drew through it into freed memory. + e->pub.gone = 1; + vk_defer_free(e); +} + // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries one // goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). NX2navi // finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames. @@ -3999,6 +4103,7 @@ void vk_texture_gpu_written(const tex_t *t) { } pool_gpu_owned(e); e->pub.gpu_owned = 1; + mark_clobbered(e); e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged" if (e->pub.skip) { // texture detail leaves out what the GPU can write (detail_skip); said if it ever does static volatile int32_t n_skip_written; @@ -4339,6 +4444,7 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; +again:; uint32_t g = agen[sinfo[id] & 0xffff]; // resolved again: an entry at its address was created, dropped or released, or the slot registered (s->ce // cleared); an empty result is not kept, as image creation may succeed later, except a held one for the pool @@ -4359,6 +4465,9 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { // a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission // (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too) if (e && !pool_ctx) touch(e); + s->ce = NULL; + goto again; + } // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A From 4ed51cf7ffc7b684b63e5ad76fb55ae6eda1835e Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:18 +0100 Subject: [PATCH 22/64] Overwritten textures: swizzle siblings follow the image that gets its own memory; other descriptors keep pool memory. The game renders through one descriptor and samples through another that differs only in swizzle, and both read the same pool memory. When one of them got memory of its own (read after an overwrite), the other kept reading the pool bytes, which the first no longer updated. Now the image and its swizzle siblings become one image in memory of its own: entry_of hands that image out for any swizzle of its key. If a live descriptor of another format or geometry reads the same storage, the image stays in pool memory as before. The perf line counts both outcomes. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_texture.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 3ef98963..f62001a6 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1165,6 +1165,7 @@ static uint64_t max_rt; // largest storage size in rts[] static nvn_hmap private_keys = {.klen = sizeof(TexKey)}; static int clobber_fix = -1; static volatile int32_t n_unaliased; +static uint64_t n_unaliased_period, n_unalias_kept; // since the last perf line: given own memory, left in pool memory static int rts_lower(uint64_t a) { int lo = 0, hi = nrts; @@ -3975,6 +3976,12 @@ static Tex *entry_of(const tex_t *t) { if (pool_ctx) return NULL; retire_cancel(e); } + if (!e && private_keys.n) // another swizzle of an image that has memory of its own (unalias): the same image + for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) + if (!p->released && !p->bound_to && nvn_hm_get(&private_keys, &p->key) && same_but_swizzle(&p->key, &k)) { + touch(p); + return p; + } if (!e) { if (held_out(&k) || create_held(&k)) return NULL; uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition @@ -4434,6 +4441,33 @@ uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAX int vk_texture_slot_count_locked(void) { return ntslots; } int vk_sampler_slot_count_locked(void) { return nsslots; } +// e was read while marked clobbered (mark_clobbered). It and the descriptors that differ from it only in swizzle (the +// same texels: the game renders through one and samples through another) are dropped, and come back as one image in +// memory of its own (create_tex, entry_of). A live descriptor of another format or geometry over its storage reads +// those bytes through the pool memory, which an image of its own would no longer update: e then stays in pool memory +// (the clobbered read is what it was). 1 when e was dropped. +static int unalias(Tex *e) { + e->clobbered = 0; + Tex *sib[16]; + int n = 0; + for (Tex *p = nvn_hm_get(&by_addr, &e->key.addr); p; p = p->next_at) { + if (p == e || p->released) continue; + if (!same_but_swizzle(&p->key, &e->key) || n == 16) { + n_unalias_kept++; + return 0; + } + sib[n++] = p; + } + if (n_unaliased++ < 32) + rt_log("vk: texture %#llx (%ux%u nvn fmt %#x) was read after another image drew over its memory: it gets memory " + "of its own", (unsigned long long)e->pub.addr, e->pub.width, e->pub.height, e->key.d.format); + n_unaliased_period++; + nvn_hm_put(&private_keys, &e->key, (void *)1); + for (int i = 0; i < n; i++) destroy_tex(sib[i]); + destroy_tex(e); // bumps the address generation: every slot naming it or a sibling resolves again + return 1; +} + const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { lock(); const VkTexImage *r = vk_texture_by_id_locked(id, view, has_view); @@ -4465,6 +4499,7 @@ again:; // a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission // (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too) if (e && !pool_ctx) touch(e); + if (e && e->clobbered && unalias(e)) { // read after another image overwrote its pool memory: own memory from now on s->ce = NULL; goto again; } @@ -4536,9 +4571,11 @@ void vk_texture_new_submit(void) { if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { log_at = f; rt_log("perf: frame %llu: texture aliases: content copied %llu (%llu from a released texture), filled from storage " - "%llu, left uninitialised %llu; taken back with their own content %llu", + "%llu, left uninitialised %llu; taken back with their own content %llu; read after an overwrite: given own " + "memory %llu, kept in pool memory %llu", (unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_released, - (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept); + (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept, + (unsigned long long)n_unaliased_period, (unsigned long long)n_unalias_kept); n_alias_copied = n_alias_upload = n_alias_uninit = n_alias_kept = n_alias_released = 0; if (n_dinit_inline || n_dinit_upload) rt_log("perf: frame %llu: new depth images started at the replay's position %llu (%llu ended an open pass), " @@ -4550,6 +4587,7 @@ void vk_texture_new_submit(void) { "holds dropped by a mark that another entry at the address had taken %llu", (unsigned long long)f * 60, (unsigned long long)n_disuse_shared, (unsigned long long)n_disuse_diverted); n_disuse_shared = n_disuse_diverted = 0; + n_unaliased_period = n_unalias_kept = 0; rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms); " "skipped with unchanged bytes %llu (%.1f MiB)", (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, From bd69291262df1d7456b12b807f7aaf01d68a0d06 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:19 +0100 Subject: [PATCH 23/64] Task and mesh stages get their own c0 bank. c0_info keeps the last bank per stage and reuses it when the inputs (storage and uniform bindings, base instance) are equal, but task and mesh shared the compute entry and the inputs do not name the stage, while the storage table sits at 0x670 for the task stage and at 0x770 for the mesh stage. When a program's task and mesh stages bind the same storage buffers, the second stage got the first one's bank and read zero storage addresses. In a Kokiri Forest frame every draw with a task stage (moss foliage, Link's house door) launched mesh groups that made no vertices, while mesh draws without a task stage drew. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index db456d07..c26b27cc 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -154,11 +154,11 @@ typedef struct { VkBuffer sup_buf; VkDeviceSize sup_off; int sup_valid; - uint8_t c0[3][VK_C0_BYTES]; // last c0 bank per stage (vertex, fragment, compute) and where it lives - VkBuffer c0_buf[3]; - VkDeviceSize c0_off[3]; - int c0_valid[3]; - struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; uint8_t push[1024]; } c0_in[3]; // what vk_c0_fill read for c0[k] + uint8_t c0[5][VK_C0_BYTES]; // last c0 bank per stage (vertex, fragment, compute, task, mesh) and where it lives + VkBuffer c0_buf[5]; + VkDeviceSize c0_off[5]; + int c0_valid[5]; + struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; uint8_t push[1024]; } c0_in[5]; // what vk_c0_fill read for c0[k] VkPipelineLayout ds_layout; // graphics layout the sets below were last pushed or bound with VkDescriptorSet ds_bound[2]; // allocated set bound per set number (0: none or pushed since) VkDescriptorSet ds_last[2]; // last allocated and written set per set number, contents in DS @@ -1277,7 +1277,11 @@ static uint32_t nvn2_cs_tables_at(const uint8_t *ctl); static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { DrawCtx *const cx = ex->dc; - int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; + // One bank per stage: the inputs below do not say which stage they are for, and the stages lay the storage table out + // at different offsets (c0_ssbo_base). Task and mesh shared the compute entry: the foliage's task and mesh stages + // bind the same storage buffers, so the task stage was handed the mesh stage's bank (its table at 0x770), read + // zero addresses at 0x670, culled every instance and launched no mesh workgroups. + int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : st == NVN_STAGE_TASK ? 3 : st == NVN_STAGE_MESH ? 4 : 2; // vk_c0_fill reads the storage slots, and the uniform slots and dispatch size for compute or the base instance for // graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed. // A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record From 888f4ca17a37e9d8e8a5b7328a1341ddd6a9fcbc Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:21 +0100 Subject: [PATCH 24/64] Mesh emulation: the arena counts mesh workgroups started, past the task-record check and at the output copy. Draws with a task stage drew nothing on an RX 580 although their task records ask for mesh groups (mesh.txt), and the same converted stages produce vertices on lavapipe. Header words 4-7 now count, per submission, the emulated mesh workgroups that started, those whose task record covers them, those that reached the output copy and the primitives those declared; mesh.txt prints them after each draw. Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.c | 11 +++++++++++ runtime/vk_exec_draw.c | 1 + 2 files changed, 12 insertions(+) diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index e4cd78d9..3cc8bed9 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -245,6 +245,12 @@ static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int barrier(e); uint32_t lid = ld(e, m->t_u32, m->lii), base = words_base(e); uint32_t isz = mul(e, pcount, cu(e, K)), vsz = mul(e, vcount, cu(e, S)); + // header word 6: mesh workgroups that reached the output copy; word 7: the primitives they declared (diagnosis) + { + uint32_t l0 = E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)); + atomic(e, OpAtomicIAdd, base, cu(e, 6), E3(e, OpSelect, m->t_u32, l0, cu(e, 1), cu(e, 0))); + atomic(e, OpAtomicIAdd, base, cu(e, 7), E3(e, OpSelect, m->t_u32, l0, pcount, cu(e, 0))); + } uint32_t mz; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); { @@ -630,6 +636,11 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ uint32_t recw = add(&e, cu(&e, MESH_EMU_TOFF / 4), mul(&e, slotc, cu(&e, MESH_EMU_TWORDS))); uint32_t cnt = word_load(&e, base, recw); uint32_t act = E2(&e, OpLogicalAnd, m->t_bool, ok, ult(&e, x, cnt)); + // header words 4 and 5: mesh workgroups started and past this check (diagnosis: vk_dump mesh.txt) + uint32_t l0 = E2(&e, OpIEqual, m->t_bool, lid, cu(&e, 0)); + atomic(&e, OpAtomicIAdd, base, cu(&e, 4), E3(&e, OpSelect, m->t_u32, l0, cu(&e, 1), cu(&e, 0))); + atomic(&e, OpAtomicIAdd, base, cu(&e, 5), + E3(&e, OpSelect, m->t_u32, E2(&e, OpLogicalAnd, m->t_bool, l0, act), cu(&e, 1), cu(&e, 0))); uint32_t cont = nid(m), retl = nid(m); op(&code, OpSelectionMerge, 2, cont, 0u); op(&code, OpBranchConditional, 3, act, cont, retl); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index c26b27cc..de18ac57 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5905,6 +5905,7 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla ME.resets++; me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); + vkCmdFillBuffer(cb, ME.buf, 0, 32, 0); // the counters, and the diagnosis words 4-7 (mesh_emu_spirv.c) me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); uint32_t rec = ME.rec++ % MESH_EMU_DRECS; From 7118dd426d201ed8d44e979cc8c3297f089de556 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:22 +0100 Subject: [PATCH 25/64] Mesh emulation barriers carry MEMORY_READ/WRITE: the task stage's mesh dispatch was read as zeros on AMD Polaris. The emulation's barriers named only synchronization2's SHADER_STORAGE_READ/WRITE bits (above bit 31). With AMD's driver for the RX 580 (23.10.01) the indirect dispatch of the mesh stage, which reads the {groups, task workgroups, 1} the task stage had just written, launched no workgroup: mesh.txt showed 0 mesh groups started for every draw with a task stage while its record held 32 x 12 x 1, and Kokiri's moss foliage and Link's door were missing. The masks now also carry MEMORY_WRITE and MEMORY_READ, as the replay's general barrier does. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index de18ac57..3f0e2a48 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5860,7 +5860,16 @@ static int me_arena(void) { if (vmaCreateBuffer(vk_ctx()->vma, &rbi, &rai, &ME.rb, &ME.rb_alloc, &rinfo) == VK_SUCCESS) ME.rb_map = rinfo.pMappedData; else ME.rb = VK_NULL_HANDLE; return 1; +// The access masks also carry the generic MEMORY_WRITE / MEMORY_READ bits. SHADER_STORAGE_READ/WRITE exist only in +// synchronization2 (bits above 31), and AMD's driver for Polaris (23.10.01, RX 580) did not act on them: the mesh +// dispatch that reads the {groups, task workgroups, 1} the task stage had just written read zeros and launched no +// workgroup (mesh.txt: "mesh groups started 0" with the record holding 32 x 12 x 1), so every draw with a task stage +// (Kokiri's moss foliage, Link's door) drew nothing. Mesh draws without a task stage drew, because their record is +// read by the draw after a command buffer boundary. The general replay barrier (vk_exec.c barrier_to) uses those bits. static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, VkAccessFlags2 da) { + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, + .srcAccessMask = sa | VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = ds, + .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); } From bb654470b2990b2d2dd88ecfc4c4f58426552545 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:24 +0100 Subject: [PATCH 26/64] Crash report: the log names what the crashing thread was building, and a library link names its case. Co-Authored-By: Claude Opus 5.5 --- runtime/main.c | 3 +++ runtime/platform/linux.c | 1 + runtime/platform/platform.h | 1 + runtime/platform/win32.c | 1 + runtime/vk_pipeline.c | 15 +++++++++++---- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/runtime/main.c b/runtime/main.c index 9e376322..a6e67e07 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -291,6 +291,9 @@ static void crash_handler(const PlatCrash *x) { if (foreign_hook(x->module)) rt_logf(" %s is not part of the game: a recording, overlay or capture program put it into the game. Close that " "program or turn off its Vulkan capture, then start the game again\n", x->module); + // what the thread was doing (a pipeline build names its shaders): the stderr note crash_veh writes misses the log + const char *note = plat_crash_context_get(); + if (note) rt_logf(" the thread was %s\n", note); uint64_t f = x->module ? 0 : lifted_function(rip); if (f) rt_logf(" in lifted function f_%llx = %s\n", (unsigned long long)f, ldr_describe(f, d, sizeof d)); // abort and invalid parameter: where the host code called it; a fault in another module: the host calls that led diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index da703894..8c531f28 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -1294,6 +1294,7 @@ void plat_mem_access_hook(int (*fn)(uint64_t addr, int write, uint64_t ip)) { void plat_crash_context(const char *s) { (void)s; } // the Windows crash notes only int plat_crash_filter_reassert(char *who, size_t n) { (void)who, (void)n; return 0; } // Windows only +const char *plat_crash_context_get(void) { return NULL; } // plat_jmp_save / plat_jmp_go (platform.h): leaf code that does not move rsp before it leaves, as on Windows. _Static_assert(sizeof(PlatJmp) == 72 && __builtin_offsetof(PlatJmp, mxcsr) == 64, "plat_jmp_save layout"); diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 0722cf08..64c0b056 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -296,6 +296,7 @@ void plat_crash_context(const char *s); // Puts the crash hook's last-chance exception filter back in place. 1 when a DLL loaded later had set its own in its // place (who: that filter's module and offset), 0 when it was still there. Windows only; 0 elsewhere. int plat_crash_filter_reassert(char *who, size_t n); +const char *plat_crash_context_get(void); // the calling thread's note, for the crash report (NULL: none) int plat_mem_read(uint64_t addr, void *out, size_t n); // copies n bytes, 0 instead of faulting on unmapped memory int plat_mem_readable(uint64_t addr, uint64_t n); // every page of [addr, addr + n) can be read; no fault void *plat_aligned_alloc(size_t size, size_t align); // free with plat_aligned_free; NULL on failure diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index fa742a84..cfaa5c9e 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -908,6 +908,7 @@ static void (*crash_fn)(const PlatCrash *c); static const char *image_name(const char *base); static __thread const char *crash_ctx; void plat_crash_context(const char *s) { crash_ctx = s; } +const char *plat_crash_context_get(void) { return crash_ctx; } static void raw_note(const char *tag, DWORD code, uintptr_t rip, uintptr_t addr) { char b[256]; void *base = NULL; diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index ef7c32f5..2fc476ce 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2495,9 +2495,9 @@ static const char *origin_name(const GfxShader *s) { // The shaders of the pipeline this thread builds, for the crash notes (platform.h plat_crash_context): the first 8 // bytes of each cache key, as gfx_shader.c names its cache files. static void crash_note(const char *what, const GfxShader *a, const GfxShader *b, const GfxShader *c) { - static __thread char buf[160]; + static __thread char buf[320]; char *o = buf; - o += snprintf(o, 40, "building %s", what); + o += snprintf(o, 160, "building %s", what); const GfxShader *sh[3] = {a, b, c}; for (int i = 0; i < 3; i++) if (sh[i]) { @@ -3607,7 +3607,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; pi.pNext = fb_chain(&fci, &fb, pi.pNext); - crash_note("a pipeline library", s->vs, s->fs, NULL); + static const char *const kind_name[4] = {"a vertex input library", "a pre-rasterization library", + "a fragment shader library", "a fragment output library"}; + crash_note(kind_name[kind], s->vs, s->fs, NULL); if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing @@ -3838,7 +3840,12 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc .layout = l->layout}; if (build) plat_mutex_unlock(&P.lock); uint64_t t1 = qpc(); - crash_note("a library link", s->vs, s->fs, NULL); + char what[160]; // the parts that differ between links: a link that crashes the driver names its case + snprintf(what, sizeof what, + "a library link (fragment shader %s, %u colour targets, depth format %u, stencil format %u, %u samples, " + "vertex outputs added %#x)", s->fs ? "yes" : "none", k->n_color, k->depth, k->stencil, k->samples, + pk.missing); + crash_note(what, s->vs, s->fs, NULL); if (vkCreateGraphicsPipelines(P.device, VK_NULL_HANDLE, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); double ms = ms_since(t1); // the link alone: library builds are counted apart From a2656f850184d56fa2e129b5c686e9b4fbaad4f1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:26 +0100 Subject: [PATCH 27/64] Mesh emulation: task records hold payloads of up to 776 words. Since 0.6.0 the translator sizes a task payload 32 bytes past the program header's size, for the header before the program's data (b6b76b7): Ocarina's foliage task, 3072 bytes, now declares 3104. A record held 768 payload words, so the emulation refused those programs ("payload too large") and the foliage went undrawn. Records are 780 words. Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index ffa99fa4..430ca76a 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -15,6 +15,8 @@ #define MESH_EMU_DWORDS 16u #define MESH_EMU_DRECS 4096u #define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +#define MESH_EMU_TWORDS 780u // 1 + payloads of up to 776 words (3072 bytes and the 32 the + // translator adds for the payload header since 0.6.0), padded // task records: as many as the push constants say (the replay sizes them, NX2NAVI_MESH_EMU_TASKS); the index area // follows them // Push constants of every emulation stage (compute and the vertex shader), 32 bytes: From 68c259c7b3491071218b5ca8c60f294d2029990a Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:27 +0100 Subject: [PATCH 28/64] make_game.py: the build log says when the lifted code is built at -O2. Co-Authored-By: Claude Opus 5.5 --- tools/make_game.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/make_game.py b/tools/make_game.py index 5343e50e..2df2f550 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1721,6 +1721,8 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): cid, cid_old = compiler_ids(clang, env) # fast_clang's compiler writes the same objects: not part of the key now = cpu_level(clang, env) level = lifted_level(obj, now, update) + if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2': # runtime_cmd's guest_opt + say(' lifted code at -O2 (NX2NAVI_GUEST_OPT): a long build') if level == now: say(f' the lifted code is built for x86-64-v{level}, this processor\'s level' if level else ' the lifted code is built for generic x86-64: the processor\'s level could not be read') From f18639e055b6ff303355602a28c1c111f394ddb3 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:29 +0100 Subject: [PATCH 29/64] Mesh emulation: payloads that are not a multiple of the workgroup size. Since 0.6.0 a task payload is 32 bytes longer than the program header's size (b6b76b7): Ocarina's foliage task payload is 776 words, which 32 threads do not divide. The emulation refused those programs ("payload not a multiple of the workgroup size"), their pipelines failed, and the task draws (grass on the moss, the door) went undrawn again. The copies of the payload into and out of the task record take a last, partial round, where threads past the payload idle. Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index 3cc8bed9..280b1f2e 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -348,8 +348,12 @@ static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint if_begin(e, ult(e, slot, trecs), &m1); { uint32_t recw = add(e, cu(e, MESH_EMU_TOFF / 4), mul(e, slot, cu(e, MESH_EMU_TWORDS))); + for (uint32_t r = 0; r * T < info->payload_words; r++) { // the last round's threads past the payload idle + uint32_t k = add(e, lid, cu(e, r * T)), mk = 0; + if ((r + 1) * T > info->payload_words) if_begin(e, ult(e, k, cu(e, info->payload_words)), &mk); uint32_t v = ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, payload, k)); word_store(e, base, add(e, add(e, recw, cu(e, 1)), k), v); + if (mk) if_end(e, mk); } uint32_t m2; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2); @@ -650,8 +654,12 @@ uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_ uint32_t v3 = nid(m); op(&code, OpCompositeConstruct, 5, m->t_v3u, v3, x, cu(&e, 0), cu(&e, 0)); op(&code, OpStore, 2, wgemu, v3); + for (uint32_t r = 0; r * info->threads < payload_len; r++) { // as task_epilogue's copy + uint32_t k = add(&e, lid, cu(&e, r * info->threads)), mk = 0; + if ((r + 1) * info->threads > payload_len) if_begin(&e, ult(&e, k, cu(&e, payload_len)), &mk); uint32_t v = word_load(&e, base, add(&e, add(&e, recw, cu(&e, 1)), k)); op(&code, OpStore, 2, E2(&e, OpAccessChain, p_wg_u32, payload, k), v); + if (mk) if_end(&e, mk); } barrier(&e); } From 9259149527e698f5b027cc4cf821ccb634b047bc Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:31 +0100 Subject: [PATCH 30/64] Settings menu: a Lama patches tab with the settings of the RX 580 patches Released textures kept, top texture levels left out, hot pools in system memory, the heap limit, the upload batch limit, the kept ASTC buffer, mesh emulation and its arena and task records (greyed where the GPU has mesh shaders), the overwritten-texture fix, the vector index fix, -O2 game code, pass times and the frame dump key. Each key gets a row in the settings table; make_game.py reads guest_opt from nx2navi.ini when the environment does not set it. Co-Authored-By: Claude Opus 5.5 --- nx2navi.ini.example | 3 +++ runtime/cfg.c | 10 +++++++++ runtime/ui/menu.cpp | 54 +++++++++++++++++++++++++++++++++++++++++---- runtime/ui_text.c | 53 ++++++++++++++++++++++++++++++++++++++++++++ runtime/ui_text.h | 14 ++++++++++++ runtime/vk_device.c | 3 +++ tools/make_game.py | 22 +++++++++++++++++- 7 files changed, 154 insertions(+), 5 deletions(-) diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 15ed5851..9408b580 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -421,6 +421,9 @@ # is read and written by the GPU in system memory instead (0: every pool keeps its video memory copy). For GPUs and # buses short of bandwidth; on an RX 580 the pools the game refills every frame cost about 8 ms a frame in copies. # mirror_hot = 0 +# 2: the next Rebuild the Game compiles the game's code at -O2 instead of -O1 (faster game threads on a slower CPU, a +# build several times longer). Read by tools/make_game.py; the environment variable NX2NAVI_GUEST_OPT wins. +# guest_opt = 1 # GPUs without mesh shaders (VK_EXT_mesh_shader: AMD before RDNA 2, NVIDIA before Turing): 1 runs the game's mesh # and task shaders (grass and other foliage) as compute shaders and draws what they make; 0 skips those draws. The # arena for their vertices and triangles is mesh_emu_mb MiB of video memory, with mesh_emu_tasks task records (3 KiB diff --git a/runtime/cfg.c b/runtime/cfg.c index c0ea134d..e546128b 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -61,6 +61,7 @@ static const char *const known[] = { "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", + "SUBMIT_FINISH_GPU", "SVC_WAITS", "SNAP_WIDEN", "SUBMIT_FINISH_RESULTS", "SUBMIT_FINISH_KINDS", "FOLIAGE", "MIRROR_STAGE", "FLUSH_SKIP_CLEAN", "MIRROR_HOT", "MESH_EMULATION", "MESH_EMU_MB", "MESH_EMU_TASKS", "MESH_EMU_BATCH", "GUEST_OPT", "LOG_TEX_NEW", "DUMP_KEY", "VEC_INDEX_FIX", "PASS_TIMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", @@ -73,6 +74,7 @@ static const char *const known[] = { "SET_AT", "SWKBD_TEST", "TAA", "DBG_TEST", "DEBUG_MENU", "MENU_TEST_DEBUG", "FSR4", "FSR4_FG", "FG4_VERSION", "FG4_SWAPCHAIN", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_RELEASED_MAX", "TEX_REUSE", "TEX_SPLIT_FILL", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WATER_PROJECTIVE", "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "UPLOAD_BATCH_MB", "VRAM_HEAP_LIMIT", "VRAM_RESERVE_MB", "TEX_CLOBBER", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", @@ -88,6 +90,8 @@ static const char *const known[] = { // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). static const struct { const char *key, *section; } sections[] = { + // the menu's Lama patches tab (ui/menu.cpp tab_lama) + {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, {"FPS_LIMIT", "frame rate"}, {"FPS60", "frame rate"}, {"RTSS", "frame rate"}, {"REALTIME", "frame rate"}, {"CLOTH_REALTIME", "frame rate"}, {"HIGHFPS_FIX", "frame rate"}, {"PHYSICS_STEPS", "frame rate"}, {"PICKUP_COUNT", "frame rate"}, {"STEP_STEADY", "frame rate"}, @@ -293,6 +297,12 @@ static const CfgMeta metas[] = { // all|: the compute pipelines the boot keeps, those dispatched most recently (vk_pipeline.c, compute-used.bin) {"COMPUTE_KEEP", "512", NULL, RESTART}, {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again + // the menu's Lama patches tab (ui/menu.cpp tab_lama): the settings of the RX 580 patches + {"TEX_RELEASED_MAX", "", NULL, RESTART}, // vk_texture.c released_max: "" by the video memory budget, else 1..16384 + {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, + {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB + {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit + {"PASS_TIMES", "0", "0|1", RESTART}, {"DUMP_KEY", "", NULL, RESTART}, // vk_crumbs.c, vk_dump.c // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, {"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_SPLIT_FILL", "1", "0|1|2", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index e318b28e..eabd4e64 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -83,6 +83,7 @@ int vk_fg4_failed(void); // ... it failed in this run, FSR 3.1's int vk_fg4_active(void); // ... its swapchain is in place const char *vk_fg4_name(void); // ... the version the library runs here ("3.1.6", "4.0.2") int vk_texture_detail_auto(void); // vk_texture.c: the texture detail auto picks on this card, 0 full to 2 low +int vk_mesh_shader_native(void); // vk_device.c: the GPU has mesh shaders (the mesh emulation does nothing) double frame_step_limit_hz(void); int frame_limit_adjustable(void); int rtss_limit_state(void); @@ -504,7 +505,7 @@ static int fonts_up(float scale) { // what the title face shows in the CJK languages: the menu's title and tabs (Simplified Chinese menu text) and // the boot screen's title (the game's language) static const UiTextId titles[] = {UI_MENU_TITLE, UI_MENU_TAB_SCREEN, UI_MENU_TAB_GAME, UI_MENU_TAB_SOUND, - UI_MENU_TAB_CONTROLS, UI_MENU_TAB_COOP, UI_BOOT_TITLE}; + UI_MENU_TAB_CONTROLS, UI_MENU_TAB_LAMA, UI_MENU_TAB_COOP, UI_BOOT_TITLE}; for (int i = 0; i < N; i++) { if (!data[i]) continue; ImFontGlyphRangesBuilder b; @@ -572,8 +573,8 @@ extern "C" void menu_vk_release(void) { // NX2NAVI_ prefix optional) puts the cursor on that setting's row and scrolls it into view when the row is next drawn // (opt_focus_key), so a page's shot shows a row below the first screenful; it does not switch tabs. COOP_PAGE, COOP_JOIN // and COOP_ACCEPT_OFFER drive the Co-op tab (ui/coop_page.h coop_test_set). -// Ocarina of Time: the options this port has and co-op (ui/coop_page.cpp) -enum { TAB_SCREEN, TAB_GAME, TAB_SOUND, TAB_CONTROLS, TAB_COOP, TAB_COUNT }; +// Ocarina of Time: the options this port has, co-op (ui/coop_page.cpp) and the Lama patches tab (the RX 580 patches) +enum { TAB_SCREEN, TAB_GAME, TAB_SOUND, TAB_CONTROLS, TAB_LAMA, TAB_COOP, TAB_COUNT }; // co-op last: tabs_shown enum { TEST_PAGE_FRAMES = 30, TEST_SETS = 16, TEST_OPENS = 8 }; struct TestSet { uint64_t frame; char key[64], value[64]; }; static struct { @@ -1623,8 +1624,52 @@ static void tab_coop(float scale) { coop_after_page(scale); } +// ---- the Lama patches tab: the settings the RX 580 patches add (apply_rx580_fix), each read where its patch reads it +static void tab_lama(float scale) { + opt_begin("##lama", scale); + opt_text(UI_MENU_LAMA_INTRO); + opt_group(UI_MENU_LAMA_MEMORY); + { // vk_texture.c sweep_released + static const char *const values[] = {"", "512", "1024", "2048", "4096", "8192"}; + const char *names[] = {menu_text(UI_MENU_AUTO), "512", "1024", "2048", "4096", "8192"}; + opt_choice_list("NX2NAVI_TEX_RELEASED_MAX", UI_MENU_LAMA_RELEASED, values, names, 6, UI_MENU_LAMA_RELEASED_DESC); + opt_slider("NX2NAVI_MIRROR_HOT", UI_MENU_LAMA_MIRROR_HOT, 0, 100, UI_MENU_LAMA_MIRROR_HOT_DESC, 5, "%.0f %%"); + { // vk_device.c vram_heap_limits + static const char *const values[] = {"auto", "0"}; + const char *names[] = {menu_text(UI_MENU_AUTO), menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_VRAM_HEAP_LIMIT", UI_MENU_LAMA_HEAP_LIMIT, values, names, 2, UI_MENU_LAMA_HEAP_LIMIT_DESC); + { // vk_memory.c upload_batch_cap + static const char *const values[] = {"128", "256", "512", "1024", "0"}; + const char *names[] = {"128 MB", "256 MB", "512 MB", "1024 MB", menu_text(UI_MENU_LAMA_NO_LIMIT)}; + opt_choice_list("NX2NAVI_UPLOAD_BATCH_MB", UI_MENU_LAMA_UPLOAD_BATCH, values, names, 5, UI_MENU_LAMA_UPLOAD_BATCH_DESC); + opt_toggle("NX2NAVI_ASTC_SCRATCH_KEEP", UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC); + opt_group(UI_MENU_LAMA_RENDER); + const bool native = vk_mesh_shader_native() != 0; // vk_exec_draw.c: the emulation only where the GPU lacks them + opt_toggle("NX2NAVI_MESH_EMULATION", UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC); + } + opt_slider("NX2NAVI_MESH_EMU_MB", UI_MENU_LAMA_MESH_MB, 64, 2048, UI_MENU_LAMA_MESH_MB_DESC, 64, "%.0f MB"); + opt_slider("NX2NAVI_MESH_EMU_TASKS", UI_MENU_LAMA_MESH_TASKS, 4096, 65536, UI_MENU_LAMA_MESH_TASKS_DESC, 4096, "%.0f"); + opt_toggle("NX2NAVI_TEX_CLOBBER", UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC); + static const UiTextId vec[] = {UI_MENU_OFF, UI_MENU_LAMA_VEC_AUTO, UI_MENU_ON}; // gfx_shader.c patch_vec_index + opt_choice("NX2NAVI_VEC_INDEX_FIX", UI_MENU_LAMA_VEC_INDEX, vec, 3, UI_MENU_LAMA_VEC_INDEX_DESC); + opt_group(UI_MENU_LAMA_BUILD); + // tools/make_game.py reads it from nx2navi.ini at the next Rebuild the Game (guest_opt_ini) + opt_toggle_value("NX2NAVI_GUEST_OPT", UI_MENU_LAMA_GUEST_OPT, UI_MENU_LAMA_GUEST_OPT_DESC, "2", "1", + menu_text(UI_MENU_LAMA_GUEST_OPT_NOTE)); + opt_group(UI_MENU_LAMA_DIAG); + opt_toggle("NX2NAVI_PASS_TIMES", UI_MENU_LAMA_PASS_TIMES, UI_MENU_LAMA_PASS_TIMES_DESC); // vk_crumbs.c +#ifdef _WIN32 + { // vk_dump.c: the keys it knows + static const char *const values[] = {"", "pause", "scrolllock", "insert", "f12"}; + const char *names[] = {menu_text(UI_MENU_OFF), "Pause", "Scroll Lock", "Insert", "F12"}; + opt_choice_list("NX2NAVI_DUMP_KEY", UI_MENU_LAMA_DUMP_KEY, values, names, 5, UI_MENU_LAMA_DUMP_KEY_DESC); + } +#endif + opt_end(); +} + static const UiTextId kTabNames[TAB_COUNT] = {UI_MENU_TAB_SCREEN, UI_MENU_TAB_GAME, UI_MENU_TAB_SOUND, - UI_MENU_TAB_CONTROLS, UI_MENU_TAB_COOP}; + UI_MENU_TAB_CONTROLS, UI_MENU_TAB_LAMA, UI_MENU_TAB_COOP}; static int g_tab; // the tab shown // The tabs the menu shows: the Co-op tab is a beta, shown only with coop_beta = 1 in nx2navi.ini (NX2NAVI_COOP_BETA), @@ -1751,6 +1796,7 @@ static void layout_menu(uint32_t w, uint32_t h, float scale) { case TAB_SOUND: tab_sound(scale); break; case TAB_CONTROLS: tab_controls(scale); break; case TAB_COOP: tab_coop(scale); break; + case TAB_LAMA: tab_lama(scale); break; } ImGui::EndTabItem(); } diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 9f7402bc..ed89bbe4 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -89,6 +89,59 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_TEX_DETAIL_DESC] = "Sharpness of large surface textures. Reduced and Low leave out their largest versions, so that an area fits the video memory of 6 and 8 GB cards; only surfaces seen up close look softer. Automatic picks by the card's video memory.", [UI_MENU_TEX_DETAIL_AUTO] = "Automatic: %s", [UI_MENU_TEX_DETAIL_FULL] = "Full", [UI_MENU_TEX_DETAIL_REDUCED] = "Reduced", [UI_MENU_TEX_DETAIL_LOW] = "Low", + [UI_MENU_TAB_LAMA] = "Lama patches", + [UI_MENU_LAMA_INTRO] = "Settings of the RX 580 patches (apply_rx580_fix). Most take effect after a restart.", + [UI_MENU_LAMA_MEMORY] = "Video memory and textures", + [UI_MENU_LAMA_RENDER] = "Rendering", + [UI_MENU_LAMA_BUILD] = "Build", + [UI_MENU_LAMA_DIAG] = "Diagnostics", + [UI_MENU_LAMA_RELEASED] = "Released textures kept", + [UI_MENU_LAMA_RELEASED_DESC] = "How many textures the game has released are kept as they are; past it only those that " + "lose nothing when dropped go. Fewer leaves more video memory for the rest; more " + "saves making them again; too few lets some blink black for a moment. Auto: one per 2 MB " + "of the video memory budget, 2048 to 8192.", + [UI_MENU_LAMA_CLOBBER] = "Fix overwritten textures", + [UI_MENU_LAMA_CLOBBER_DESC] = "A texture the game reads after another image was drawn over its memory gets memory of " + "its own. Fixes the teal splotches in Kokiri Forest's fog.", + [UI_MENU_LAMA_MIRROR_HOT] = "Pools copied every frame stay in system memory", + [UI_MENU_LAMA_MIRROR_HOT_DESC] = "A game memory pool whose copies to and from video memory each frame reach this " + [UI_MENU_LAMA_HEAP_LIMIT] = "Video memory heap limit", + [UI_MENU_LAMA_HEAP_LIMIT_DESC] = "Auto: on GPUs with a second video memory heap (an RX 580 shows 6 GB and 2 GB), " + "each heap is filled only to its budget before the next one is used, so the " + "driver does not run out. Off: no limit.", + [UI_MENU_LAMA_UPLOAD_BATCH] = "Upload batch limit", + [UI_MENU_LAMA_UPLOAD_BATCH_DESC] = "Texture uploads waiting in staging memory are sent to the GPU early once they " + "hold this much, so an area load cannot run video memory out.", + [UI_MENU_LAMA_NO_LIMIT] = "No limit", + [UI_MENU_LAMA_ASTC_KEEP] = "Keep the ASTC decode buffer", + [UI_MENU_LAMA_ASTC_KEEP_DESC] = "The buffer that GPU-decodes compressed textures is kept between upload batches " + "instead of being allocated again for each one.", + [UI_MENU_LAMA_MESH_EMU] = "Mesh shader emulation", + [UI_MENU_LAMA_MESH_EMU_DESC] = "Draws grass, moss and other foliage on GPUs without mesh shaders (AMD RX 400/500 " + "and Vega, GTX 10-series and older), with compute shaders. GPUs with mesh shaders " + "ignore it.", + [UI_MENU_LAMA_MESH_NATIVE] = "this GPU has mesh shaders", + [UI_MENU_LAMA_MESH_MB] = "Mesh emulation memory", + [UI_MENU_LAMA_MESH_MB_DESC] = "Video memory for the emulated meshes' vertices and indices, in MB.", + [UI_MENU_LAMA_MESH_TASKS] = "Mesh emulation task records", + [UI_MENU_LAMA_MESH_TASKS_DESC] = "Task records per submission for the emulated meshes. Raise it when the log reports " + "that a submission asked for more.", + [UI_MENU_LAMA_VEC_INDEX] = "Vector index fix", + [UI_MENU_LAMA_VEC_INDEX_DESC] = "Shaders read uniform vector components at computed indices whole and select the " + "component, for drivers that get such reads wrong. Auto: on AMD's own drivers.", + [UI_MENU_LAMA_VEC_AUTO] = "Auto (AMD)", + [UI_MENU_LAMA_GUEST_OPT] = "Optimised game code (-O2)", + [UI_MENU_LAMA_GUEST_OPT_DESC] = "The next Rebuild the Game in the launcher compiles the game's code at -O2 instead " + "of -O1: the game's own threads run faster on a slower CPU. The build takes " + "several times longer.", + [UI_MENU_LAMA_GUEST_OPT_NOTE] = "at the next rebuild", + [UI_MENU_LAMA_PASS_TIMES] = "Pass times in the log", + [UI_MENU_LAMA_PASS_TIMES_DESC] = "GPU time of every render pass and dispatch goes to the log, with the largest " + "viewport each kind of pass draws with. Costs a little frame time.", + [UI_MENU_LAMA_DUMP_KEY] = "Frame dump key", + [UI_MENU_LAMA_DUMP_KEY_DESC] = "Each press of this key in the game window dumps the next frame (images and the " + "frame's passes) to the dump folder, for a glitch caught while it is on screen. " + "Windows only.", [UI_MENU_SHARPNESS] = "FSR sharpness", [UI_MENU_SHARPNESS_DESC] = "How sharp FSR makes the picture. 0 turns it off.", [UI_MENU_SHARPNESS_NEEDS_FSR] = "only with AMD FSR 1 or FSR 3.1", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 9d098b1f..7857fbf3 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -332,6 +332,20 @@ typedef enum { UI_MENU_FRAMEGEN_MULT_UPTO, // printf: "%d", the highest multiplier this GPU and driver allow UI_MENU_FRAMEGEN_MULT_OLD_DLL, // nx2navi_dlssg.dll is older than multi frame generation UI_MENU_FRAMEGEN_UNAVAILABLE, UI_MENU_FRAMEGEN_OFF_FILES, + // the Lama patches tab (ui/menu.cpp tab_lama): the settings of the RX 580 patches, English only + UI_MENU_TAB_LAMA, UI_MENU_LAMA_INTRO, UI_MENU_LAMA_MEMORY, UI_MENU_LAMA_RENDER, UI_MENU_LAMA_BUILD, + UI_MENU_LAMA_DIAG, + UI_MENU_LAMA_RELEASED, UI_MENU_LAMA_RELEASED_DESC, + UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC, + UI_MENU_LAMA_MIRROR_HOT, UI_MENU_LAMA_MIRROR_HOT_DESC, + UI_MENU_LAMA_UPLOAD_BATCH, UI_MENU_LAMA_UPLOAD_BATCH_DESC, UI_MENU_LAMA_NO_LIMIT, + UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC, + UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC, UI_MENU_LAMA_MESH_NATIVE, + UI_MENU_LAMA_MESH_MB, UI_MENU_LAMA_MESH_MB_DESC, UI_MENU_LAMA_MESH_TASKS, UI_MENU_LAMA_MESH_TASKS_DESC, + UI_MENU_LAMA_VEC_INDEX, UI_MENU_LAMA_VEC_INDEX_DESC, UI_MENU_LAMA_VEC_AUTO, + UI_MENU_LAMA_GUEST_OPT, UI_MENU_LAMA_GUEST_OPT_DESC, UI_MENU_LAMA_GUEST_OPT_NOTE, + UI_MENU_LAMA_PASS_TIMES, UI_MENU_LAMA_PASS_TIMES_DESC, + UI_MENU_LAMA_DUMP_KEY, UI_MENU_LAMA_DUMP_KEY_DESC, UI_TEXT_COUNT } UiTextId; diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 02ca353a..9b547287 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -794,3 +794,6 @@ int probe_vkdevice(void) { printf("probe-vkdevice: %s (validation errors %ld)\n", g_errors_total ? "FAILED" : "passed", g_errors_total); return g_errors_total ? 1 : 0; } + +// The GPU has mesh shaders: the mesh emulation (NX2NAVI_MESH_EMULATION) does nothing there (the menu's Lama patches tab). +int vk_mesh_shader_native(void) { return g_vk.has_mesh_shader; } diff --git a/tools/make_game.py b/tools/make_game.py index 2df2f550..0f5cbe26 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1425,10 +1425,29 @@ def fast_clang(msys, clang, env): return fast if mine and clang_ident(fast, env) == mine else None +def guest_opt_ini(out): + """NX2NAVI_GUEST_OPT from nx2navi.ini next to the exe (the menu's Lama patches tab writes it there) when the + environment does not set it: the next build of the game folder out takes it.""" + if os.environ.get('NX2NAVI_GUEST_OPT', '').strip(): + return + try: + for line in (Path(out) / 'nx2navi.ini').read_text(encoding='utf-8', errors='replace').splitlines(): + key, eq, value = line.partition('=') + key = key.strip().lower() + if eq and key in ('guest_opt', 'nx2navi_guest_opt'): + value = value.split('#')[0].split(';')[0].strip().strip('"') + if value: + os.environ['NX2NAVI_GUEST_OPT'] = value + except OSError: + pass + + def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): """fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers, - and its command, which is part of its objects' keys, stays as it was.""" + and its command, which is part of its objects' keys, stays as it was. NX2NAVI_GUEST_OPT (an environment variable, + or guest_opt in nx2navi.ini) 1 builds the lifted code at -O1; by default -O2: the game's own threads run it and + bound the frame on a slower CPU; the build takes several times longer.""" def cmd_for(src, dst): cpp = src.suffix in ('.cpp', '.cxx') # the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict @@ -1721,6 +1740,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): cid, cid_old = compiler_ids(clang, env) # fast_clang's compiler writes the same objects: not part of the key now = cpu_level(clang, env) level = lifted_level(obj, now, update) + guest_opt_ini(out) if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2': # runtime_cmd's guest_opt say(' lifted code at -O2 (NX2NAVI_GUEST_OPT): a long build') if level == now: From 01d81811a14967c01ba3d594d881d5e5388f807c Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:32 +0100 Subject: [PATCH 31/64] NX2NAVI_LOG_TEX_NEW: log the textures made from a frame on, with what destroyed the same key before Co-Authored-By: Claude Opus 5.5 --- runtime/vk_texture.c | 91 ++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index f62001a6..259a6c44 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -1713,41 +1713,6 @@ int vk_texture_slot_retry_locked(int id) { static void retire_remove(Tex *e); -static void destroy_tex(Tex *e) { - if (e->retiring) retire_remove(e); - pool_unlist(e); - pool_forget(e); - own_remove(e); - gwin_remove(e); - n_site[0]++; - tex_gen_bump(e->key.addr); - nvn_hm_del(&by_key, &e->key); - Tex *head = nvn_hm_get(&by_addr, &e->key.addr); - if (head == e) { - if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at); - else nvn_hm_del(&by_addr, &e->key.addr); - } else { - for (Tex *p = head; p; p = p->next_at) - if (p->next_at == e) { p->next_at = e->next_at; break; } - } - if (e->released) nreleased--, rel_unlink(e); - if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) - vk_defer_image(e->pub.image, e->alloc); - if (e->flat3d) { // its 3D copy goes the same way - vk_exec_image_gone(e->flat3d->image); - vk_defer_image(e->flat3d->image, e->flat3d_alloc); - e->flat3d->gone = 1; - vk_defer_free(e->flat3d); - } - free(e->snap); - // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a - // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on - // whichever thread released the texture. Freed here, a pass that began again read the freed record, cached a view of - // the dead image (vk_exec_target_view), and a later image given the same handle drew through it into freed memory. - e->pub.gone = 1; - vk_defer_free(e); -} - // ---- a flat texture sampled as a 3D one // Maxwell samples a texture by its own descriptor, whatever the shader's instruction says: a 2D texture under a 3D // sample gives its texel at (u, v), and the third coordinate does nothing. Vulkan has no 3D view of a 2D image, so @@ -1821,7 +1786,6 @@ const VkTexImage *vk_texture_flat3d_locked(const VkTexImage *img, uint32_t layer if (cb) vk_upload_end(); return e->flat3d; } - // ---- origins (NX2NAVI_TRACE=perf; a run without it tests the flag and counts the drops, nothing more) // A texture drawn without its content leaves nothing in the log: no call fails and the image exists. What can still be // told afterwards is where the entry came from, so each entry keeps how it was made and how its content started @@ -2042,9 +2006,55 @@ static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { return k < (int)n ? k : (int)n - 1; } +// NX2NAVI_LOG_TEX_NEW= (diagnostics): from that presented frame on, the first LOG_NEW_MAX textures made are +// logged with what was at their address and, when the same key was destroyed before, how many frames ago and from +// where (destroy_tex's caller, an address the map file names: "destroy_tex at" gives the reference). For the steady +// creations a still scene shows (about seven a frame in Kokiri Forest), each with its image, memory and upload. +#define LOG_NEW_MAX 400 +#define GONE_N 4096 +static struct { uint64_t h, frame; void *ra; } gone[GONE_N]; +static uint32_t gone_n; +static long long log_new_at = -2; +static int log_new_n; + +static void log_new(const Tex *e, const TexKey *k) { // tex_lock held + if (log_new_at == -2) { + const char *v = cfg_get("NX2NAVI_LOG_TEX_NEW"); + log_new_at = v && *v ? atoll(v) : -1; } + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (log_new_at < 0 || now < (uint64_t)log_new_at || log_new_n >= LOG_NEW_MAX) return; + log_new_n++; + int live = 0, rel = 0; + for (const Tex *p = nvn_hm_get(&by_addr, &k->addr); p; p = p->next_at) + if (p != e) p->released ? rel++ : live++; + uint64_t h = key_hash(k); + char was[160] = "never destroyed before"; + for (uint32_t i = 0; i < GONE_N && i < gone_n; i++) { + uint32_t j = (gone_n - 1 - i) & (GONE_N - 1); + if (gone[j].h == h) { + snprintf(was, sizeof was, "the same key destroyed %llu frames ago from %p (destroy_tex at %p)", + (unsigned long long)(now - gone[j].frame), gone[j].ra, (void *)destroy_tex); + break; + } + } + rt_log("vk: new texture frame %llu: %#llx %ux%ux%u nvn fmt %#x target %d levels %u%s%s; at its address %d live, %d " + "released; %s", (unsigned long long)now, (unsigned long long)k->addr, e->pub.width, e->pub.height, + e->pub.depth, k->d.format, k->d.target, e->pub.levels, e->pub.mirrored ? ", mirrored" : "", + e->alloc ? ", own memory" : "", live, rel, was); } +__attribute__((noinline)) static void destroy_tex(Tex *e) { + if (log_new_at >= 0) { // NX2NAVI_LOG_TEX_NEW: who destroyed which key, for the log of new textures + uint32_t j = gone_n++ & (GONE_N - 1); + gone[j].h = key_hash(&e->key), gone[j].frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + gone[j].ra = __builtin_return_address(0); + } + if (e->retiring) retire_remove(e); + pool_unlist(e); + pool_forget(e); + own_remove(e); + gwin_remove(e); rt_remove(e); n_site[0]++; tex_gen_bump(e->key.addr); @@ -2056,9 +2066,16 @@ static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { } else { for (Tex *p = head; p; p = p->next_at) if (p->next_at == e) { p->next_at = e->next_at; break; } + } if (e->released) nreleased--, rel_unlink(e); if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) vk_defer_image(e->pub.image, e->alloc); + if (e->flat3d) { // its 3D copy goes the same way + vk_exec_image_gone(e->flat3d->image); + vk_defer_image(e->flat3d->image, e->flat3d_alloc); + e->flat3d->gone = 1; + vk_defer_free(e->flat3d); + } free(e->snap); // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on @@ -2132,7 +2149,6 @@ static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { #define RELEASED_HARD 16384 #define SWEEP_LOOK 64 #define SWEEP_UNSAMPLED PRESENTS(120) - static int comes_back_intact(const Tex *e); static uint64_t stale_after(const Tex *e); @@ -3999,6 +4015,7 @@ static Tex *entry_of(const tex_t *t) { nvn_hm_put(&by_key, &k, e); n_site[1]++; tex_gen_bump(k.addr); + log_new(e, &k); vk_texstat_time(VK_TS_CREATE, c0); e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // new: not disused before it is drawn } From 93e68d57f11a0bd1dcd153d8953ae8d33006705b Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:34 +0100 Subject: [PATCH 32/64] Pool marks: a fragment shader stores its marks where it ends, not between a sample's coordinates and the sample. On AMD's driver for Polaris (RX 580, 23.10) a store between a pool texture's access chain and its sample left the quad's helper lanes out of the code in between: derivatives at a triangle's edge were garbage and edge pixels came out dark, the dotted lines along the paths' triangles since 0.7.0 (frame dump f1714: the colour pass covers every pixel, the depth test is not what drops them). Each access now notes its slot in a Private variable; before each return of the entry function and before a kill, terminate or demote every noted slot is marked, an unset one going to a spare slot past the pool. Accesses in loops or outside the entry function note theirs in a ring of 8, marked in a loop at the way out. Checked with spirv-val on the 45 marked shaders of the dump. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 6 +- runtime/vk_pipeline.c | 192 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 179 insertions(+), 19 deletions(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 3f0e2a48..21093d83 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2068,13 +2068,15 @@ static VkDescriptorSetLayout pool_layout_make(void) { static void pool_marks_make(void) { VkCtx *c = vk_ctx(); - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)VK_POOL_TEX * 4, + // a uint past the slots: the spare that a fragment shader's marks of accesses it did not reach go to + // (vk_pipeline.c spirv_pool_mark); it is never read back + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)(VK_POOL_TEX + 1) * 4, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; if (vmaCreateBuffer(c->vma, &bi, &ai, &PL.marks, &PL.marks_alloc, NULL) != VK_SUCCESS) rt_fatal("vk: the texture pool marks buffer failed"); - bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + bi.size = (VkDeviceSize)VK_POOL_TEX * 4, bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; VmaAllocationCreateInfo hi = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; for (int i = 0; i < MARKS_RB; i++) { diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 2fc476ce..887a5134 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2209,6 +2209,14 @@ int vk_pool_marks_enabled(void) { // its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it // does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the // shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows). +// A fragment shader's marks are stored where it ends, not where it samples. A store between a sample's coordinates and +// the sample takes the quad's helper lanes out of the code in between on AMD's driver for Polaris: their coordinates +// stayed stale, the derivatives at a triangle's edge were garbage, and edge pixels came out dark (the lines along the +// paths' triangles since 0.7.0). Each access only notes its slot in a Private variable (a register); before each +// return of the entry function, and before a kill, terminate or demote anywhere, every variable is stored to its slot, +// a variable no access set going to the spare slot past the pool (vk_exec_draw.c pool_marks_make). An access in a loop +// or outside the entry function can run more than once with other slots: it notes them in a ring of RING Private +// variables, so a loop's last RING slots are all marked. // *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked (*why says what stopped it). // NULL unless marked. static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status, @@ -2219,7 +2227,10 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227, OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380, EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014, - SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 }; + SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35, OP_TYPE_BOOL = 20, + OP_LOAD_ = 61, OP_BITCAST = 124, OP_SELECT_ = 169, OP_INOT_EQUAL = 171, OP_LOOP_MERGE = 246, OP_LABEL_ = 248, + OP_RETURN = 253, OP_RETURN_VALUE = 254, SC_PRIVATE = 6, LOOP_DEPTH = 64, OP_TYPE_ARRAY_ = 28, OP_IADD = 128, + OP_BITWISE_AND = 199, OP_ULESS_THAN = 176, OP_BRANCH = 249, OP_BRANCH_CONDITIONAL = 250, OP_PHI = 245 }; *status = 0, *why = NULL; if (n < 5) return NULL; uint32_t bound = w[3]; @@ -2228,11 +2239,19 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui uint8_t *set2 = calloc(bound, 1); if (!bind || !set2) return free(bind), free(set2), *status = -1, *why = "no memory", NULL; uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes - uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0; + uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0, int_t = 0, bool_t = 0; int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok; + uint32_t *rtype = calloc(bound, sizeof *rtype); // per id: its result type, where the defer path needs it + uint8_t *is_type = calloc(bound, 1); + if (!rtype || !is_type) + return free(bind), free(set2), free(rtype), free(is_type), *status = -1, *why = "no memory", NULL; +#define POOL_MARK_FREE() (free(bind), free(set2), free(rtype), free(is_type), free(site_def)) + // per site: 0 stored in place; else deferred, bit 1 the index is signed (a bitcast), bit 2 a ring (in a loop or + // outside the entry function) + uint8_t *site_def = NULL; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; - if (!l || i + l > n) return free(bind), free(set2), *status = -1, *why = "an instruction past the module's end", NULL; + if (!l || i + l > n) return POOL_MARK_FREE(), *status = -1, *why = "an instruction past the module's end", NULL; if (op == SPV_OP_FUNCTION) { fn_start = i; break; } if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l; if (op == SPV_OP_ENTRY_POINT && l >= 3) { @@ -2251,6 +2270,10 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui early = 0; // the guest's own storage buffers: its stores would change under early tests } else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) { uint_t = w[i + 1]; + } else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 1) { + int_t = w[i + 1]; + } else if (op == OP_TYPE_BOOL && l == 2) { + bool_t = w[i + 1]; } else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) { if (w[i + 3] == 0 && !c0) c0 = w[i + 2]; if (w[i + 3] == 1 && !c1) c1 = w[i + 2]; @@ -2260,50 +2283,95 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui if (op == SPV_OP_DECORATE || op == OP_MEMBER_DECORATE_ || op == OP_DECORATION_GROUP || op == OP_GROUP_DECORATE || op == OP_GROUP_MEMBER_DECORATE || op == OP_DECORATE_ID || op == OP_DECORATE_STRING || op == OP_MEMBER_DECORATE_STRING) ann_end = i + l; + if (op >= 19 && op <= 39 && l >= 2 && w[i + 1] < bound) is_type[w[i + 1]] = 1; // OpTypeVoid .. OpTypeForwardPointer + else if (op == SPV_OP_CONSTANT && l >= 4 && w[i + 2] < bound) rtype[w[i + 2]] = w[i + 1]; i += l; } // pool texture variables: set 2, a binding below VK_POOL_KINDS (spirv_pool_rebind); one already at the marks // binding means the module was marked before - uint64_t sites = 0; + enum { RING = 8 }; + uint64_t sites = 0, deferred = 0, rings = 0, exits = 0; int marked = 0; for (uint32_t id = 0; id < bound; id++) if (set2[id] && bind[id] == VK_POOL_MARK_BINDING + 1) marked = 1; + site_def = fn_start && !marked ? calloc((n - fn_start) / 5 + 1, 1) : NULL; // an access chain is 5 words or more + if (fn_start && !marked && !site_def) return POOL_MARK_FREE(), *status = -1, *why = "no memory", NULL; + uint32_t loops[LOOP_DEPTH], fn = 0; + int depth = 0; for (uint64_t i = fn_start; fn_start && !marked && i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) break; + if (l >= 3 && w[i + 1] < bound && is_type[w[i + 1]] && w[i + 2] < bound) rtype[w[i + 2]] = w[i + 1]; + if (op == SPV_OP_FUNCTION && l >= 3) fn = w[i + 2], depth = 0; + else if (op == OP_LOOP_MERGE && l >= 3) depth < LOOP_DEPTH ? (void)(loops[depth++] = w[i + 1]) : (void)depth++; + else if (op == OP_LABEL_ && l >= 2 && depth > 0 && depth <= LOOP_DEPTH && loops[depth - 1] == w[i + 1]) depth--; if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && - bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) + bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) { + uint32_t ix = w[i + 4], t = ix < bound ? rtype[ix] : 0; + if (fragment_stage && entries == 1 && fs_entry && t && (t == uint_t || t == int_t)) { + int ring = fn != fs_entry || depth; + site_def[sites] = (uint8_t)(1 | (t == int_t ? 2 : 0) | (ring ? 4 : 0)); + ring ? rings++ : deferred++; + } sites++; - else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE || - (op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST)) + } else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE || + (op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST)) early = 0; + if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || + (fn == fs_entry && (op == OP_RETURN || op == OP_RETURN_VALUE))) + exits++; i += l; } - if (marked || !sites) return free(bind), free(set2), NULL; + if (marked || !sites) return POOL_MARK_FREE(), NULL; + uint64_t stores = deferred + rings; // what each way out stores: a variable, or a ring's entries in a loop + if (stores && (!exits || exits * (deferred + 3 * rings) > 8192)) // no way out to store at, or code that would grow + memset(site_def, 0, sites), deferred = rings = stores = 0; // too much *why = w[1] < 0x00010300 || w[1] > 0x00010600 ? "a SPIR-V version before 1.3 or after 1.6" : !ann_end || !fn_start ? "no decorations or no function" : vertex_stage && !marks_vs_ok ? "the GPU allows no stores in vertex-pipeline shaders" : fragment_stage && !marks_fs_ok ? "the GPU allows no stores in fragment shaders" : NULL; - if (*why) return free(bind), free(set2), *status = -1, NULL; + if (*why) return POOL_MARK_FREE(), *status = -1, NULL; early = early && fragment_stage && entries == 1 && em_end && em_end < ann_end; // From SPIR-V 1.4 on an entry point lists every global variable its functions use, not only the inputs and outputs // (the translator's mesh shaders are 1.4): each entry point gets the marks variable, a word more. int iface = w[1] >= 0x00010400; - // the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site - uint64_t on = n + 3 + (uint64_t)entries + 20 + 30 + 9 * sites; + // the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site; deferred: + // a word per entry point and variable, 80 words of types and constants, then per site its variables and the store of + // its slot (ring: 10 + 29), and per way out 29 words a variable and 8 * 29 a ring (its loop is 83) + uint64_t on = n + 3 + (uint64_t)entries * (1 + deferred + 2 * rings) + 20 + 30 + 9 * sites + 80 + deferred * 12 + + rings * 40 + (deferred * 29 + rings * RING * 29) * exits; uint32_t *o = malloc(on * 4); - if (!o) return free(bind), free(set2), *status = -1, *why = "no memory", NULL; + if (!o) return POOL_MARK_FREE(), *status = -1, *why = "no memory", NULL; uint32_t next = bound; uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++); uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++); + // deferred: the types, constants and variables (pv0: one per site; ra0, rc0: a ring and its count per ring site) + uint32_t new_bool = 0, ptr_pv = 0, c_none = 0, c_spare = 0, pv0 = 0, ring_t = 0, ptr_ring = 0, ring_none = 0, + cr[RING] = {0}, c_ring = 0, ra0 = 0, rc0 = 0; + if (stores) { + new_bool = bool_t ? 0 : (bool_t = next++); + ptr_pv = next++, c_none = next++, c_spare = next++, pv0 = next, next += (uint32_t)deferred; + if (rings) { + ring_t = next++, ptr_ring = next++, ring_none = next++, c_ring = next++; + for (int j = 0; j < RING; j++) cr[j] = next++; + ra0 = next, next += (uint32_t)rings, rc0 = next, next += (uint32_t)rings; + } + } uint64_t p = 5; memcpy(o, w, 5 * 4); for (uint64_t i = 5; i < ann_end;) { // the header up to the annotations' end: entry points widened, the mode added uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (early && i == em_end) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS; memcpy(o + p, w + i, l * 4); - if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) o[p] = (l + 1) << 16 | op, o[p + l] = var, p++; + if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) { + uint32_t nv = 0, *at = o + p + l + 1; // the marks buffer, then pv0.., ra0.. and rc0.. + o[p] = (uint32_t)(l + 1 + deferred + 2 * rings) << 16 | op, o[p + l] = var, p++; + for (uint32_t k = 0; k < deferred; k++) at[nv++] = pv0 + k; + for (uint32_t k = 0; k < rings; k++) at[nv++] = ra0 + k; + for (uint32_t k = 0; k < rings; k++) at[nv++] = rc0 + k; + p += nv; + } p += l; i += l; } @@ -2321,19 +2389,109 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_st, o[p++] = SC_STORAGE_BUFFER, o[p++] = st; if (new_ptr_u) o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_u, o[p++] = SC_STORAGE_BUFFER, o[p++] = uint_t; o[p++] = 4u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_st, o[p++] = var, o[p++] = SC_STORAGE_BUFFER; + if (stores) { + if (new_bool) o[p++] = 2u << 16 | OP_TYPE_BOOL, o[p++] = bool_t; + o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_pv, o[p++] = SC_PRIVATE, o[p++] = uint_t; + o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_none, o[p++] = 0xffffffffu; + o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_spare, o[p++] = VK_POOL_TEX; + if (rings) { + for (int j = 0; j < RING; j++) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = cr[j], o[p++] = (uint32_t)j; + o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_ring, o[p++] = RING; + o[p++] = 4u << 16 | OP_TYPE_ARRAY_, o[p++] = ring_t, o[p++] = uint_t, o[p++] = c_ring; + o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_ring, o[p++] = SC_PRIVATE, o[p++] = ring_t; + o[p++] = (3u + RING) << 16 | SPV_OP_CONSTANT_COMPOSITE, o[p++] = ring_t, o[p++] = ring_none; + for (int j = 0; j < RING; j++) o[p++] = c_none; + } + for (uint32_t k = 0; k < deferred; k++) + o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_pv, o[p++] = pv0 + k, o[p++] = SC_PRIVATE, o[p++] = c_none; + for (uint32_t k = 0; k < rings; k++) + o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_ring, o[p++] = ra0 + k, o[p++] = SC_PRIVATE, o[p++] = ring_none; + for (uint32_t k = 0; k < rings; k++) + o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_pv, o[p++] = rc0 + k, o[p++] = SC_PRIVATE, o[p++] = c0; + } + uint64_t site = 0; + uint32_t k_def = 0, k_ring = 0, cur = 0; // cur: the label of the block being copied + fn = 0; for (uint64_t i = fn_start; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; + if (op == SPV_OP_FUNCTION && l >= 3) fn = w[i + 2]; + if (op == OP_LABEL_ && l >= 2) cur = w[i + 1]; + if (stores && (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || + (fn == fs_entry && (op == OP_RETURN || op == OP_RETURN_VALUE)))) { + // marks[the slot, or the spare when none was noted] = 1 +#define POOL_MARK_STORE(from) \ + do { \ + uint32_t v_ = next++, ok_ = next++, ix_ = next++, at_ = next++; \ + o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = v_, o[p++] = (from); \ + o[p++] = 5u << 16 | OP_INOT_EQUAL, o[p++] = bool_t, o[p++] = ok_, o[p++] = v_, o[p++] = c_none; \ + o[p++] = 6u << 16 | OP_SELECT_, o[p++] = uint_t, o[p++] = ix_, o[p++] = ok_, o[p++] = v_, o[p++] = c_spare; \ + o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at_, o[p++] = var, o[p++] = c0, o[p++] = ix_; \ + o[p++] = 3u << 16 | OP_STORE_, o[p++] = at_, o[p++] = c1; \ + } while (0) + for (uint32_t k = 0; k < deferred; k++) POOL_MARK_STORE(pv0 + k); + for (uint32_t r = 0; r < rings; r++) { + if (op == OP_DEMOTE_TO_HELPER) { // not the block's end: every entry, without a loop + for (int j = 0; j < RING; j++) { + uint32_t from = next++; + o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = from, o[p++] = ra0 + r, o[p++] = cr[j]; + POOL_MARK_STORE(from); + } + continue; + } + // the block ends here (a return, kill or terminate), so it can end in a loop over the entries noted: + // for (j = 0; j < min(count, RING); j++) mark ring[j] + uint32_t cnt = next++, lt = next++, lim = next++, hd = next++, j = next++, mg = next++, ct = next++, + cond = next++, body = next++, from = next++, jn = next++; + o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = cnt, o[p++] = rc0 + r; + o[p++] = 5u << 16 | OP_ULESS_THAN, o[p++] = bool_t, o[p++] = lt, o[p++] = cnt, o[p++] = c_ring; + o[p++] = 6u << 16 | OP_SELECT_, o[p++] = uint_t, o[p++] = lim, o[p++] = lt, o[p++] = cnt, o[p++] = c_ring; + o[p++] = 2u << 16 | OP_BRANCH, o[p++] = hd; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = hd; + o[p++] = 7u << 16 | OP_PHI, o[p++] = uint_t, o[p++] = j, o[p++] = c0, o[p++] = cur, o[p++] = jn, o[p++] = ct; + o[p++] = 5u << 16 | OP_ULESS_THAN, o[p++] = bool_t, o[p++] = cond, o[p++] = j, o[p++] = lim; + o[p++] = 4u << 16 | OP_LOOP_MERGE, o[p++] = mg, o[p++] = ct, o[p++] = 0; + o[p++] = 4u << 16 | OP_BRANCH_CONDITIONAL, o[p++] = cond, o[p++] = body, o[p++] = mg; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = body; + o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = from, o[p++] = ra0 + r, o[p++] = j; + POOL_MARK_STORE(from); + o[p++] = 2u << 16 | OP_BRANCH, o[p++] = ct; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = ct; + o[p++] = 5u << 16 | OP_IADD, o[p++] = uint_t, o[p++] = jn, o[p++] = j, o[p++] = c1; + o[p++] = 2u << 16 | OP_BRANCH, o[p++] = hd; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = mg; + cur = mg; + } +#undef POOL_MARK_STORE + } memcpy(o + p, w + i, l * 4), p += l; if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) { - uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access - o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4]; - o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1; + uint8_t d = site_def[site++]; + if (d) { // the slot noted for the stores at the end + uint32_t ix = w[i + 4]; + if (d & 2) o[p++] = 4u << 16 | OP_BITCAST, o[p++] = uint_t, o[p++] = next, o[p++] = ix, ix = next++; + if (d & 4) { // ring[count % RING] = slot, count + 1 + uint32_t ra = ra0 + k_ring, rc = rc0 + k_ring++, cnt = next++, j = next++, at = next++, inc = next++; + o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = cnt, o[p++] = rc; + o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = uint_t, o[p++] = j, o[p++] = cnt, o[p++] = cr[RING - 1]; + o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = at, o[p++] = ra, o[p++] = j; + o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = ix; + o[p++] = 5u << 16 | OP_IADD, o[p++] = uint_t, o[p++] = inc, o[p++] = cnt, o[p++] = c1; + o[p++] = 3u << 16 | OP_STORE_, o[p++] = rc, o[p++] = inc; + } else { + o[p++] = 3u << 16 | OP_STORE_, o[p++] = pv0 + k_def++, o[p++] = ix; + } + } else { + uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access + o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4]; + o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1; + } } i += l; } o[3] = next; - free(bind), free(set2); + POOL_MARK_FREE(); +#undef POOL_MARK_FREE *out_n = p; *status = 1; return o; From 55ab90c66e7dbaa3a7a307b8f207be1e93363391 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:35 +0100 Subject: [PATCH 33/64] Mesh emulation: the stages of a pass's emulated draws run before the pass, in a command buffer of their own. Each emulated mesh draw ended the pass, ran its stages between four barriers and began the pass again: a pass of 20 emulated draws became 20 passes, and on an RX 580 6 to 7 ms of a frame's 30 ms of GPU time went to the gaps between passes (gpu_passes in Kokiri Forest: 8.7-10 ms per frame outside the game's passes and dispatches, 6.3-7.7 ms of it in 273-306 gaps with barriers only; 71 emulated draws a frame, 36 of them indirect). Now the first emulated draw of a pass ends the part there (vk_exec_split_now), so the pass begins a new part, and the stages of every emulated draw of that pass, the indirect ones' ring entries included, go into the part's pre-pass command buffer (vk_sync_pre), which the queue runs after the mirror prologue and before the part; the pass is not split again. NX2NAVI_MESH_EMU_BATCH=0 (Lama patches: Mesh emulation before the pass) brings the split back. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 ++ runtime/ui/menu.cpp | 5 +++ runtime/ui_text.c | 4 +++ runtime/ui_text.h | 1 + runtime/vk.h | 3 ++ runtime/vk_exec.c | 10 ++++-- runtime/vk_exec.h | 3 ++ runtime/vk_exec_draw.c | 80 +++++++++++++++++++++++++++++++++++------- runtime/vk_sync.c | 63 ++++++++++++++++++++++++++++----- 9 files changed, 147 insertions(+), 24 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index e546128b..6add4819 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -91,6 +91,7 @@ static const char *const known[] = { // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). static const struct { const char *key, *section; } sections[] = { // the menu's Lama patches tab (ui/menu.cpp tab_lama) + {"MESH_EMU_MB", "performance"}, {"MESH_EMU_TASKS", "performance"}, {"MESH_EMU_BATCH", "performance"}, {"VEC_INDEX_FIX", "performance"}, {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, {"FPS_LIMIT", "frame rate"}, {"FPS60", "frame rate"}, {"RTSS", "frame rate"}, {"REALTIME", "frame rate"}, {"CLOTH_REALTIME", "frame rate"}, {"HIGHFPS_FIX", "frame rate"}, @@ -302,6 +303,7 @@ static const CfgMeta metas[] = { {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit + {"MESH_EMU_TASKS", "16384", "256..262144", RESTART}, {"MESH_EMU_BATCH", "1", "0|1", RESTART}, {"VEC_INDEX_FIX", "0", "0|auto|1", RESTART}, {"PASS_TIMES", "0", "0|1", RESTART}, {"DUMP_KEY", "", NULL, RESTART}, // vk_crumbs.c, vk_dump.c // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index eabd4e64..49dd4c48 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1645,10 +1645,15 @@ static void tab_lama(float scale) { opt_toggle("NX2NAVI_ASTC_SCRATCH_KEEP", UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC); opt_group(UI_MENU_LAMA_RENDER); const bool native = vk_mesh_shader_native() != 0; // vk_exec_draw.c: the emulation only where the GPU lacks them + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_toggle("NX2NAVI_MESH_EMULATION", UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC); } + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_slider("NX2NAVI_MESH_EMU_MB", UI_MENU_LAMA_MESH_MB, 64, 2048, UI_MENU_LAMA_MESH_MB_DESC, 64, "%.0f MB"); + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_slider("NX2NAVI_MESH_EMU_TASKS", UI_MENU_LAMA_MESH_TASKS, 4096, 65536, UI_MENU_LAMA_MESH_TASKS_DESC, 4096, "%.0f"); + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); + opt_toggle("NX2NAVI_MESH_EMU_BATCH", UI_MENU_LAMA_MESH_BATCH, UI_MENU_LAMA_MESH_BATCH_DESC); // vk_exec_draw.c opt_toggle("NX2NAVI_TEX_CLOBBER", UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC); static const UiTextId vec[] = {UI_MENU_OFF, UI_MENU_LAMA_VEC_AUTO, UI_MENU_ON}; // gfx_shader.c patch_vec_index opt_choice("NX2NAVI_VEC_INDEX_FIX", UI_MENU_LAMA_VEC_INDEX, vec, 3, UI_MENU_LAMA_VEC_INDEX_DESC); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index ed89bbe4..a497bcda 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -123,6 +123,10 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_LAMA_MESH_NATIVE] = "this GPU has mesh shaders", [UI_MENU_LAMA_MESH_MB] = "Mesh emulation memory", [UI_MENU_LAMA_MESH_MB_DESC] = "Video memory for the emulated meshes' vertices and indices, in MB.", + [UI_MENU_LAMA_MESH_BATCH] = "Mesh emulation before the pass", + [UI_MENU_LAMA_MESH_BATCH_DESC] = "The emulated mesh stages of a render pass run before the pass in a command buffer " + "of their own, instead of splitting the pass at every emulated draw. Off: the " + "split, as before.", [UI_MENU_LAMA_MESH_TASKS] = "Mesh emulation task records", [UI_MENU_LAMA_MESH_TASKS_DESC] = "Task records per submission for the emulated meshes. Raise it when the log reports " "that a submission asked for more.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 7857fbf3..6ef06b5c 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -342,6 +342,7 @@ typedef enum { UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC, UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC, UI_MENU_LAMA_MESH_NATIVE, UI_MENU_LAMA_MESH_MB, UI_MENU_LAMA_MESH_MB_DESC, UI_MENU_LAMA_MESH_TASKS, UI_MENU_LAMA_MESH_TASKS_DESC, + UI_MENU_LAMA_MESH_BATCH, UI_MENU_LAMA_MESH_BATCH_DESC, UI_MENU_LAMA_VEC_INDEX, UI_MENU_LAMA_VEC_INDEX_DESC, UI_MENU_LAMA_VEC_AUTO, UI_MENU_LAMA_GUEST_OPT, UI_MENU_LAMA_GUEST_OPT_DESC, UI_MENU_LAMA_GUEST_OPT_NOTE, UI_MENU_LAMA_PASS_TIMES, UI_MENU_LAMA_PASS_TIMES_DESC, diff --git a/runtime/vk.h b/runtime/vk.h index 57fcd927..8564149c 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -750,6 +750,9 @@ VkCommandBuffer vk_sync_append_later(int n, int *at); void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n); VkCommandBuffer vk_sync_worker_begin(int w); void vk_sync_worker_end(VkCommandBuffer cb); +// The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun +// with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. +VkCommandBuffer vk_sync_pre(void); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 74a6f35a..6f7dd7f2 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1112,6 +1112,9 @@ static void set_rt(VkExec *ex, const p_set_rt *r) { static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_start; static uint64_t split_ns, part_t0; static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits; +int vk_exec_split_now(VkExec *ex) { + return 1; +} static int split_wanted(void) { if (split_draws >= split_at) return 1; @@ -2082,9 +2085,10 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end int begun = 0; const p_set_rt *rt = &ex->rt; *why = GE_LIMIT; - // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared) and while - // NX2NAVI_GPU_PASSES copies their workgroup counts (one list) - int mesh_serial = vk_ctx()->mesh_fallback || vk_gpu_passes_capturing(); + // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared), with the mesh emulation + // (no VK_EXT_mesh_shader: its arena, queue and the part's pre-pass command buffer are the submitting thread's) and + // while NX2NAVI_GPU_PASSES copies their workgroup counts (one list) + int mesh_serial = vk_ctx()->mesh_fallback || !vk_ctx()->has_mesh_shader || vk_gpu_passes_capturing(); for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it const void *a = nvn_op_payload(h); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 47fa8696..837153e0 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -125,6 +125,8 @@ typedef struct VkExec { const void *fb_fs; // feedback_scan's memo: fragment shader, texture bind serial, pass serial uint32_t fb_serial, fb_pass, fb_sampled; uint32_t tex_serial, pass_serial; // bumped by every texture or image bind and pass begin + uint32_t pre_pass; // the pass (pass_serial) the part's pre-pass command buffer serves (vk_sync_pre; + // vk_exec_draw.c mesh emulation); a pass begins after it with another serial } VkExec; // Begins dynamic rendering with the current targets unless a pass is open (a full memory barrier precedes it). Call @@ -140,6 +142,7 @@ void vk_exec_barrier(VkExec *ex); void vk_exec_barrier_dispatch(VkExec *ex); // NX2NAVI_TRACE=perf, every 600 frames: the barriers per frame by kind. void vk_exec_barrier_log(uint64_t frame); +int vk_exec_split_now(VkExec *ex); // vk_texture.c (NX2NAVI_TEX_ALIAS): commands that must run at the replay's current position. vk_exec_inline_begin returns // the command buffer being recorded after a full barrier (an open pass is ended; *reopen says so), or VK_NULL_HANDLE // when this thread is not recording a replay. vk_exec_inline_end begins the pass again when it was open. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 21093d83..6af2736b 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5575,6 +5575,7 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { ex->recorded++; } +static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args); // The primitive an emulated draw's mesh stage outputs: point meshes draw point lists, the rest triangles. static int mesh_emu_draw_prim(VkExec *ex) { static struct { const GfxShader *s; int prim; } memo[256]; @@ -5601,6 +5602,9 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); if (!fn && mesh_emu_on()) { if (d->first) warn_once(W_MESH, "an emulated mesh draw from workgroup %u: drawn from workgroup 0", d->first); + if (d->count && rt_knob("MESH", "1")) mesh_emu_draw(ex, d->count, NULL, NULL); + return; + } static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) if (!d->count || !on) return; @@ -5652,7 +5656,7 @@ uint32_t vk_exec_mesh_counts(const uint32_t **out) { *out = mesh_count_map; return mesh_count_map ? mesh_count_n : 0; } -static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { +static void mesh_count_note(VkExec *ex, VkCommandBuffer cb, const VkMemSpan *src) { uint32_t cap = (uint32_t)vk_gpu_passes_capturing(); if (!cap) { mesh_count_was = 0; return; } if (!mesh_count_was) mesh_count_was = 1, mesh_count_n = 0; @@ -5667,8 +5671,8 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { } if (mesh_count_n >= MESH_COUNTS) return; VkBufferCopy cp = {src->offset, 4ull * mesh_count_n++, 4}; - vkCmdCopyBuffer(ex->cb, src->buffer, mesh_count_buf, 1, &cp); - ex->recorded++; + vkCmdCopyBuffer(cb, src->buffer, mesh_count_buf, 1, &cp); + if (cb == ex->cb) ex->recorded++; } // NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record @@ -5705,7 +5709,11 @@ static int mesh_ahead_on(void) { return RT_KNOB_ON(mode); } -static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { +// A ring entry {count from src, 1, 1}, filled in cb; ex->cb: after a full barrier, which ends the pass (the next draw +// opens it again) and orders the culling's writes before the copy. Another cb (the pre-pass one of the mesh +// emulation) begins with one. d: the draw it is for, which may take an entry filled ahead (MESH_AHEAD, ex->cb only); +// NULL: none (the mesh emulation's). +static int mesh_ring_entry(VkExec *ex, VkCommandBuffer cb, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { DrawCtx *const cx = ex->dc; if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { char stops[160]; @@ -5714,7 +5722,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), (unsigned long long)mesh_ahead_taken, stops); } - if (cx->ma_i < cx->ma_n) { + if (d && cb == ex->cb && cx->ma_i < cx->ma_n) { if (cx->ma[cx->ma_i].d == d && cx->ma[cx->ma_i].ind == d->ind && cx->ma_brk == vk_exec_brk_now()) { *at = cx->ma[cx->ma_i++].at; __atomic_add_fetch(&mesh_ahead_taken, 1, __ATOMIC_RELAXED); @@ -5725,15 +5733,18 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan __atomic_add_fetch(&mesh_ahead_own, 1, __ATOMIC_RELAXED); if (!__atomic_load_n(&mesh_ring, __ATOMIC_ACQUIRE) && (vk_exec_in_chunk() || !mesh_ring_make())) return 0; *at = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); - vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy - vk_gpu_gap_note(VK_GAP_MESH_RING, 12); - vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1); + int own = cb == ex->cb; + if (own) { + vk_exec_barrier(ex); + cb = ex->cb; + vk_gpu_gap_note(VK_GAP_MESH_RING, 12); } + vkCmdFillBuffer(cb, mesh_ring, *at + 4, 8, 1); VkBufferCopy cp = {src->offset, *at, 4}; - vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); - mesh_count_note(ex, src); + vkCmdCopyBuffer(cb, src->buffer, mesh_ring, 1, &cp); + mesh_count_note(ex, cb, src); cx->ma_n = cx->ma_i = 0; - if (mesh_ahead_on()) { + if (d && own && mesh_ahead_on()) { const void *run[MESH_AHEAD]; int n = vk_exec_mesh_run(d, run, MESH_AHEAD); for (int i = 0; i < n; i++) { @@ -5746,7 +5757,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); VkBufferCopy fc = {ip.offset, fa, 4}; vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); - mesh_count_note(ex, &ip); + mesh_count_note(ex, ex->cb, &ip); ex->recorded += 2; cx->ma[cx->ma_n].d = f, cx->ma[cx->ma_n].ind = f->ind, cx->ma[cx->ma_n].at = fa; cx->ma_n++; @@ -5759,6 +5770,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan .dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); + if (own) ex->recorded += 3; return 1; } @@ -5777,6 +5789,11 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { (unsigned long long)d->ind); ex->dc->c0_binst = 0; VK_MEM_USE(ip); + mesh_emu_draw(ex, 0, NULL, &ip); + vk_mem_count(ip.cls, VK_MUSE_IND, 4); + VK_MEM_BOUND(ip); + return; + } if (fallback) { mfb_draw(ex, 0, &ip); vk_mem_count(ip.cls, VK_MUSE_IND, 4); @@ -5874,9 +5891,12 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); + } } } // count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from +// args: an indirect draw's arguments in game memory (count, 1, 1 copied into a ring entry, which ind then is) +static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args) { if (!me_arena()) return 0; ex->dc->c0_binst = 0; int prim = mesh_emu_draw_prim(ex); @@ -5895,13 +5915,41 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla : "its descriptors cannot be bound for compute"); ME.skipped++; return 0; + } VkPipelineLayout layout = info->layout; int push = info->push, pool = info->pool; VkDescriptorUpdateTemplate tpl_c = info->emu_tpl_c; VkDescriptorSet set0 = ex->dc->B.ds_bound[0]; + // NX2NAVI_MESH_EMU_BATCH (default on): the stages run in the part's pre-pass command buffer (vk_sync_pre), which + // the queue runs before the part, and the pass is not split for them. At the first emulated draw of a pass the part + // pass of 20 emulated draws was 20 passes, each emulated draw a pass end and begin with a full barrier, three more + // barriers and the targets stored and loaded again: on an RX 580 6 to 7 ms of a frame's GPU time went to the gaps + // between passes (gpu_passes, Kokiri Forest, 2026-11-10). + static int batch = -1; + if (batch < 0) { + batch = rt_knob("MESH_EMU_BATCH", "1"); + rt_log("vk: mesh emulation: the stages of a pass's emulated draws %s (NX2NAVI_MESH_EMU_BATCH=%d)", + batch ? "run before the pass, in a command buffer of their own" : "split the pass", batch); } + VkCommandBuffer pre = VK_NULL_HANDLE; + if (batch && ex->in_pass) { + if (ex->pre_pass != ex->pass_serial && vk_exec_split_now(ex)) { + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { // the pass again, in the new part + ex->pre_pass = ex->pass_serial; + set0 = ex->dc->B.ds_bound[0]; } + if (ex->pre_pass == ex->pass_serial) pre = vk_sync_pre(); } + if (!pre) vk_exec_barrier(ex); // ends the pass; what the game wrote before (its culling) is visible to the stages + VkCommandBuffer cb = pre ? pre : ex->cb; + VkMemSpan ring = {0}; + if (args) { + if (!mesh_ring_entry(ex, cb, NULL, args, &ring.offset)) { + warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + ME.skipped++; + return 0; + ring.buffer = mesh_ring, ind = ˚ + cb = pre ? pre : ex->cb; uint32_t ep = vk_texture_epoch(); if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over ME.epoch = ep; @@ -5925,6 +5973,14 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; } + if (!pre) { + // the draw: prepared again (the pass reopens, every binding is recorded again) + vk_exec_draw_new_cb(); + ex->dirty = VK_EXEC_DIRTY_ALL; + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { + ME.skipped++; + return 0; + } } cb = ex->cb; vkCmdBindIndexBuffer(cb, ME.buf, (VkDeviceSize)ME.iw * 4, VK_INDEX_TYPE_UINT32); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index f9109378..f0cbf4ff 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -764,6 +764,55 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { return &slots[cur_slot]; } +// ---- the part's pre-pass command buffer (vk.h vk_sync_pre; vk_exec_draw.c mesh emulation, NX2NAVI_MESH_EMU_BATCH) +// Begun at the first call of a part, submitted right before the part (after the mirror prologue) and ended there. Its +// own pool: with the recorder on, the recorder thread records the part (cmd_pool) while the replay records this. +static VkCommandPool pre_pool; +static CmdSlot *pre_slots; +static int npre, pre_pair = -1; +static VkCommandBuffer pre_cb; +VkCommandBuffer vk_sync_pre(void) { + if (pre_cb) return pre_cb; + if (!pre_pool) { + VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .queueFamilyIndex = c->gfx_family}; + if (vkCreateCommandPool(c->device, &pi, NULL, &pre_pool) != VK_SUCCESS) return VK_NULL_HANDLE; + } + uint64_t gpu = 0; + vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); + int i = 0; + while (i < npre && pre_slots[i].fence > gpu) i++; + if (i == npre) { + CmdSlot *n = realloc(pre_slots, sizeof *n * (size_t)(npre + 1)); + if (!n) return VK_NULL_HANDLE; + pre_slots = n; + VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pre_pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; + if (vkAllocateCommandBuffers(c->device, &ai, &pre_slots[npre].cb) != VK_SUCCESS) return VK_NULL_HANDLE; + npre++; + } + pre_slots[i].fence = cur_fence; // the submission's value: its parts are done by then + VkCommandBuffer cb = pre_slots[i].cb; + pre_pair = ts_open(cb); + // everything earlier in queue order (the parts before, the mirror prologue) before what it records + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); + pre_cb = cb; + return cb; +// The pre-pass command buffer of the part that goes now, ended; VK_NULL_HANDLE without one. +static VkCommandBuffer pre_end(uint64_t fence) { + VkCommandBuffer cb = pre_cb; + if (!cb) return VK_NULL_HANDLE; + pre_pair = -1; + vkEndCommandBuffer(cb); + pre_cb = VK_NULL_HANDLE; + return cb; +} + // The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is // submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the // part was recorded, sub_lock held; may move slots[]. @@ -878,10 +927,8 @@ static VkCommandBuffer split_part(int exec) { ts_end(cb, cur_fence); if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order - VkCommandBuffer pro = prologue(c, cur_fence); - VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = part_list(cbi, pro, cb, 1), - .pCommandBufferInfos = cbi}; + VkCommandBuffer pro = prologue(c, cur_fence), pre = pre_end(cur_fence); + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi}; VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; if (exec && exec_tl && exec_sig < cur_fence) @@ -954,7 +1001,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { if (cpu0) perf_replay_cpu(plat_thread_cpu_ns(me) - cpu0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); - VkCommandBuffer pro = prologue(c, fence); + VkCommandBuffer pro = prologue(c, fence), pre = pre_end(fence); s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; uint32_t ncbi = part_list(cbi, pro, s->cb, recorded); @@ -965,10 +1012,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; int both = exec_tl && exec_sig < fence; if (both) exec_sig = fence; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncbi, - .pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u, - .pSignalSemaphoreInfos = sig}; - ts_submitted(); + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi, + .signalSemaphoreInfoCount = both ? 2u : 1u, .pSignalSemaphoreInfos = sig}; queue_submit(c, &si, fence); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); s->fence = recorded ? fence : 0; From 38a3824b8fa14c86a763be0c5071c9cbe3fa8900 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:37 +0100 Subject: [PATCH 34/64] Mesh emulation: cut the part at a pass's first emulated draw instead of splitting it, so the texture pool set stays. A split hands the part to the queue and seals its texture pool set; the next part takes another set of the ring and refreshes its entries first. The splits for the mesh stages made 5 parts a submission and 6 set switches a frame at 2.3 ms of replay each (Kokiri Forest on 0.7.1, an RX 580: replay 32-38 ms a frame, the replay thread blocked on fences 26-35% of the time, 79 waits for a set the GPU still used). A cut (vk_sync_cut) ends the command buffer without submitting it: the part and its pre-pass command buffer wait in a chain and go to the queue, in order, with the next part that does, so the set is the same one throughout. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 3 ++ runtime/vk_exec.c | 17 ++++++++--- runtime/vk_exec.h | 2 ++ runtime/vk_exec_draw.c | 3 ++ runtime/vk_sync.c | 67 +++++++++++++++++++++++++++++++++++------- 5 files changed, 77 insertions(+), 15 deletions(-) diff --git a/runtime/vk.h b/runtime/vk.h index 8564149c..a56fe2bf 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -753,6 +753,9 @@ void vk_sync_worker_end(VkCommandBuffer cb); // The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun // with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. VkCommandBuffer vk_sync_pre(void); +// Ends the part being recorded without handing it to the queue (it goes with the next part that does, in order) and +// returns the command buffer the replay continues in; VK_NULL_HANDLE when no more parts can wait. Replay thread. +VkCommandBuffer vk_sync_cut(void); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 6f7dd7f2..5fcef4c4 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -1111,8 +1111,17 @@ static void set_rt(VkExec *ex, const p_set_rt *r) { // command, it was 4% of the replay in Kokiri Forest. static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_start; static uint64_t split_ns, part_t0; -static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits; +static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits, n_cuts; + int vk_exec_split_now(VkExec *ex) { + int in_pass = ex->in_pass; + if (in_pass) vk_exec_pass_end(ex); + VkCommandBuffer cb = vk_sync_cut(); // not handed to the queue: the texture pool set stays (vk_exec_pool_split) + if (!cb) return 0; + ex->cb = cb; + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + n_cuts++; return 1; } @@ -2583,10 +2592,10 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t if (n) n_subs++; if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) { rt_log("perf: replay command buffer parts per submission %.2f (%.2f first parts sent while the GPU was idle, %.2f " - "splits inside a pass); event order: held %llu, resumed %llu (%llu without their signal), switches %llu, " + "splits inside a pass, %.2f cuts for emulated mesh stages); event order: held %llu, resumed %llu (%llu without their signal), switches %llu, " "held now %d; waits a CPU signal met only after the submission's copies were taken: %llu", (double)(n_subs + n_parts) / (double)n_subs, (double)n_idle_parts / (double)n_subs, - (double)n_pass_splits / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed, + (double)n_pass_splits / (double)n_subs, (double)n_cuts / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed, (unsigned long long)evo_forced, (unsigned long long)evo_switches, evo_pending, (unsigned long long)vk_exec_event_late_take()); uint64_t xw, xe, xl, xm, le, lc, lo; @@ -2678,7 +2687,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; - n_subs = n_parts = n_idle_parts = n_pass_splits = 0; + n_subs = n_parts = n_idle_parts = n_pass_splits = n_cuts = 0; } rec_spent_ns += kern_now_ns() - in_ns; if (rec_spent_ns >= REC_CLAIM_NS) rec_claim(); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 837153e0..2b3a0a31 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -142,6 +142,8 @@ void vk_exec_barrier(VkExec *ex); void vk_exec_barrier_dispatch(VkExec *ex); // NX2NAVI_TRACE=perf, every 600 frames: the barriers per frame by kind. void vk_exec_barrier_log(uint64_t frame); +// Ends the open pass and the part without handing it to the queue (vk_sync_cut): the next draw begins the pass again +// at the start of a new command buffer, which the part's pre-pass command buffer precedes. 0 when no more can wait. int vk_exec_split_now(VkExec *ex); // vk_texture.c (NX2NAVI_TEX_ALIAS): commands that must run at the replay's current position. vk_exec_inline_begin returns // the command buffer being recorded after a full barrier (an open pass is ended; *reopen says so), or VK_NULL_HANDLE diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 6af2736b..80b75de4 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5922,6 +5922,9 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkDescriptorSet set0 = ex->dc->B.ds_bound[0]; // NX2NAVI_MESH_EMU_BATCH (default on): the stages run in the part's pre-pass command buffer (vk_sync_pre), which // the queue runs before the part, and the pass is not split for them. At the first emulated draw of a pass the part + // is cut there (vk_sync_cut: ended, but handed to the queue only with the next part, so the texture pool set stays; + // a split there took a new set, and each took 2.3 ms of the replay to refresh), so that the pass begins the next + // part and only what came before it runs before the stages. Split, a // pass of 20 emulated draws was 20 passes, each emulated draw a pass end and begin with a full barrier, three more // barriers and the targets stored and loaded again: on an RX 580 6 to 7 ms of a frame's GPU time went to the gaps // between passes (gpu_passes, Kokiri Forest, 2026-11-10). diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index f0cbf4ff..18685580 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -773,6 +773,7 @@ static int npre, pre_pair = -1; static VkCommandBuffer pre_cb; VkCommandBuffer vk_sync_pre(void) { if (pre_cb) return pre_cb; + VkCtx *c = vk_ctx(); if (!pre_pool) { VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, @@ -794,6 +795,9 @@ VkCommandBuffer vk_sync_pre(void) { } pre_slots[i].fence = cur_fence; // the submission's value: its parts are done by then VkCommandBuffer cb = pre_slots[i].cb; + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(cb, &bi); pre_pair = ts_open(cb); // everything earlier in queue order (the parts before, the mirror prologue) before what it records VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, @@ -803,6 +807,13 @@ VkCommandBuffer vk_sync_pre(void) { vkCmdPipelineBarrier2(cb, &di); pre_cb = cb; return cb; +} + +// Command buffers of the submission that a cut (vk_sync_cut) ended and that go to the queue with the next part, in +// order before it: a cut part's pre-pass command buffer, then the part. +static VkCommandBuffer chain[CHAIN_MAX]; +static int nchain; + // The pre-pass command buffer of the part that goes now, ended; VK_NULL_HANDLE without one. static VkCommandBuffer pre_end(uint64_t fence) { VkCommandBuffer cb = pre_cb; @@ -833,26 +844,59 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { return cb; } -// The command buffers of the part that ends now, in queue order, into cbi: the prologue (when there is one), the -// part's earlier pieces (vk_sync_append) and, with `with_last`, its last command buffer. Empties the list. -static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer last, int with_last) { +// What goes to the queue for a part: the mirror prologue, the parts cut before it with their pre-pass command buffers +// and pieces (the chain, emptied), the part's own pre-pass command buffer, the part's earlier pieces (vk_sync_append: +// the replay's and its workers', emptied), the part's last command buffer. Any of them may be missing. +static uint32_t chain_infos(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer pre, VkCommandBuffer cb) { uint32_t n = 0; - VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}; - if (pro) one.commandBuffer = pro, cbi[n++] = one; +#define CHAIN_ADD(x) \ + if (x) cbi[n++] = (VkCommandBufferSubmitInfo){.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = (x)} + CHAIN_ADD(pro); + for (int k = 0; k < nchain; k++) CHAIN_ADD(chain[k]); + CHAIN_ADD(pre); for (int i = 0; i < npart_cbs; i++) { if (!part_cbs[i]) rt_fatal("vk: a replay worker's command buffer is missing from a part"); - one.commandBuffer = part_cbs[i], cbi[n++] = one; + CHAIN_ADD(part_cbs[i]); } - if (with_last) one.commandBuffer = last, cbi[n++] = one; - npart_cbs = 0; + CHAIN_ADD(cb); +#undef CHAIN_ADD + nchain = 0, npart_cbs = 0; return n; } +// A cut: the part ends and the replay goes on in a new command buffer, as at a split, but nothing goes to the queue: +// the part (its pre-pass command buffer, its pieces and itself) waits in the chain for the next part that does. The +// texture pool set stays as it is, as nothing of the submission has been handed to the GPU. VK_NULL_HANDLE when the +// chain is full. +VkCommandBuffer vk_sync_cut(void) { + if (nchain + npart_cbs + 2 > CHAIN_MAX) return VK_NULL_HANDLE; + for (int i = 0; i < npart_cbs; i++) + if (!part_cbs[i]) return VK_NULL_HANDLE; // a replay worker's piece not filled yet: the part goes on + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = slots[cur_slot].cb; + vk_gpu_passes_part(cb, 1); + ts_end(cb, cur_fence); + if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); + VkCommandBuffer pre = pre_end(cur_fence); + if (pre) chain[nchain++] = pre; + for (int i = 0; i < npart_cbs; i++) chain[nchain++] = part_cbs[i]; + npart_cbs = 0; + chain[nchain++] = cb; + CmdSlot *s = take_slot(c, cur_fence); // the cut part keeps its slot until the value is reached + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(s->cb, &bi); + ts_begin(s->cb); + rec_begin(s->cb, cur_fence); + vk_gpu_passes_part(s->cb, 0); + return s->cb; +} + // The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the // part's list, then n places for the command buffers the workers record, in that order, and a new command buffer of // the same part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' // commands run between the replay's commands before and after. The places are filled (vk_sync_append_fill) before the -// part is submitted, which part_list checks. vk_sync_append_room: whether the list has room for n more. +// part is submitted, which chain_infos checks. vk_sync_append_room: whether the list has room for n more. int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; } VkCommandBuffer vk_sync_append_later(int n, int *at) { @@ -914,7 +958,6 @@ VkCommandBuffer vk_sync_worker_begin(int w) { } void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); } - static VkCommandBuffer split_part(int exec); VkCommandBuffer vk_sync_split(void) { return split_part(0); } // The part that ends here is the last with the submission's own commands: it signals the executed timeline. @@ -928,6 +971,7 @@ static VkCommandBuffer split_part(int exec) { if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order VkCommandBuffer pro = prologue(c, cur_fence), pre = pre_end(cur_fence); + uint32_t ncb = chain_infos(cbi, pro, pre, cb); // the mirror copies, the cut parts, the pre-pass work, the part VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi}; VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; @@ -1005,6 +1049,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { s = &slots[cur_slot]; VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; uint32_t ncbi = part_list(cbi, pro, s->cb, recorded); + uint32_t ncb = chain_infos(cbi, pro, pre, sent ? s->cb : VK_NULL_HANDLE); // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}, @@ -1016,7 +1061,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { .signalSemaphoreInfoCount = both ? 2u : 1u, .pSignalSemaphoreInfos = sig}; queue_submit(c, &si, fence); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); - s->fence = recorded ? fence : 0; + s->fence = sent ? fence : 0; vk_mem_note_submit(fence); plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); From f6199a461627518dd8d30c285d49479b81372107 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:39 +0100 Subject: [PATCH 35/64] Upload batches: their GPU time in the log (NX2NAVI_TRACE=perf), with how many go to the queue per frame. The replay's GPU time covers only its own parts. Upload batches are submissions of their own, the ASTC decoder's dispatches in them, so their time showed only as GPU idle "with it queued (untimed work or waits)": 10-16 ms a frame on an RX 580 in Kokiri Forest on 0.7.1 while the game streamed in about 7 textures a frame, against 0.6-1.2 ms where it streamed none. Each batch now writes a timestamp at its start and end, read when it retires; every 60 frames a perf line gives the batches per frame and their GPU time. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 26d5f4cb..9ac28a72 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -658,6 +658,7 @@ static int dead_pending(void); static void vram_log(uint64_t frame); static void outside_watch(uint64_t frame); +static void up_ts_log(uint64_t frame); static volatile uint64_t mirror_frame; // the frame vk_mem_census last saw, for mirror_sync's log lines @@ -4226,6 +4227,7 @@ typedef struct { int nchunk, cap_chunk; VkDeviceSize chunk_used; // bytes of the last chunk handed out uint64_t bytes; // staging bytes of its buffers (upload_cap) + int ts; // its pair of upload timestamps (up_ts_*), -1: not timed } Batch; enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; @@ -4239,6 +4241,58 @@ static int open_active; static Batch *inflight; static int ninflight, cap_inflight; static Batch *spare; // completed batches whose command buffer and fence are reused +// ---- upload GPU time (NX2NAVI_TRACE=perf): a timestamp at the start and the end of each upload batch, read when it +// retires. The replay's GPU time (vk_sync.c) times only its own parts; upload batches go to the queue as submissions of +// their own, with the ASTC decoder's dispatches in them, so their time showed only as GPU idle "with it queued". +// cmd_lock held throughout. +#define UP_TS_PAIRS 256 +static VkQueryPool up_ts_pool; +static int up_ts_state; // 0 not set up, 1 on, -1 off +static uint8_t up_ts_busy[UP_TS_PAIRS]; +static uint32_t up_ts_next; +static uint64_t up_ts_ns, up_ts_n, up_ts_max_ns, up_ts_missed, up_n_batches; // since the last log line + +static int up_ts_open(VkCtx *c, VkCommandBuffer cb) { + if (!(rt_trace & RT_TRACE_PERF) || up_ts_state < 0) return -1; + if (!up_ts_state) { + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * UP_TS_PAIRS}; + up_ts_state = c->props.limits.timestampComputeAndGraphics && + vkCreateQueryPool(c->device, &qi, NULL, &up_ts_pool) == VK_SUCCESS ? 1 : -1; + if (up_ts_state < 0) return -1; + } + uint32_t p = up_ts_next % UP_TS_PAIRS; + if (up_ts_busy[p]) { up_ts_missed++; return -1; } + up_ts_next++, up_ts_busy[p] = 1; + vkCmdResetQueryPool(cb, up_ts_pool, 2 * p, 2); + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, up_ts_pool, 2 * p); + return (int)p; +} + +static void up_ts_read(VkCtx *c, Batch *b) { // b retired: its fence signalled + if (b->ts < 0) return; + uint64_t t[2]; + if (vkGetQueryPoolResults(c->device, up_ts_pool, 2 * (uint32_t)b->ts, 2, sizeof t, t, sizeof t[0], + VK_QUERY_RESULT_64_BIT) == VK_SUCCESS && t[1] >= t[0]) { + uint64_t ns = (uint64_t)((double)(t[1] - t[0]) * c->props.limits.timestampPeriod); + up_ts_ns += ns, up_ts_n++; + if (ns > up_ts_max_ns) up_ts_max_ns = ns; + } + up_ts_busy[b->ts] = 0; + b->ts = -1; +} + +static void cmd_init(void *p); +static void up_ts_log(uint64_t frame) { // every 60 frames + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + rt_log("perf: frame %llu: upload batches %.1f per frame, on the GPU %.2f ms per frame (%llu timed, %.3f ms each, the " + "longest %.2f ms; %llu not timed)", (unsigned long long)frame, up_n_batches / 60.0, up_ts_ns / 60e6, + (unsigned long long)up_ts_n, up_ts_n ? up_ts_ns / 1e6 / (double)up_ts_n : 0.0, up_ts_max_ns / 1e6, + (unsigned long long)up_ts_missed); + up_ts_ns = up_ts_n = up_ts_max_ns = up_ts_missed = up_n_batches = 0; + plat_rmutex_unlock(&cmd_lock); +} static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; @@ -4325,6 +4379,7 @@ static void retire(VkCtx *c, int wait_all) { Batch *b = &inflight[k]; if (wait_all) vkWaitForFences(c->device, 1, &b->fence, VK_TRUE, UINT64_MAX); else if (vkGetFenceStatus(c->device, b->fence) != VK_SUCCESS) break; + up_ts_read(c, b); free_staging(c, b); done_seq = b->seq; if (nspare == cap_spare) grow((void **)&spare, &cap_spare, sizeof *spare); @@ -4462,6 +4517,7 @@ VkCommandBuffer vk_upload_begin(void) { VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(open_b.cb, &bi); + open_b.ts = up_ts_open(c, open_b.cb); open_b.seq = ++next_seq; vk_crumb(open_b.cb, VK_CRUMB_UPLOAD, open_b.seq); open_active = 1; @@ -4694,6 +4750,8 @@ uint64_t vk_upload_flush(int wait) { for (int i = 0; i < open_b.nchunk; i++) vmaFlushAllocation(c->vma, open_b.chunk[i].alloc, 0, i + 1 < open_b.nchunk ? VK_WHOLE_SIZE : open_b.chunk_used); vk_crumb(open_b.cb, VK_CRUMB_UPLOAD_END, open_b.seq); + if (open_b.ts >= 0) vkCmdWriteTimestamp(open_b.cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, up_ts_pool, 2 * (uint32_t)open_b.ts + 1); + up_n_batches++; vkEndCommandBuffer(open_b.cb); VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb}; vk_queue_lock(); From 78429f49334bc8d9bbd0d5aa5fef181cf31ee316 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:40 +0100 Subject: [PATCH 36/64] Held submissions keep every command buffer they carry: the hold buffer had room for two. queue_submit copies a held submission's command buffer infos into H.cbi[i], two entries each, the mirror prologue and the part. Since the mesh emulation's pre-pass command buffers and cuts (vk_sync_pre, vk_sync_cut) a submission carries up to three plus the cut parts, and the copy ran into the next held submission's entries and past the array: the next held submission's copy then overwrote them, so a submission could go to the queue without its own last command buffers and with another's twice. The buffer now has room for SUB_CBS (CHAIN_MAX + 3), and a larger count stops the run with a message instead of overrunning. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_sync.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 18685580..c661d6f3 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -71,10 +71,14 @@ static void results_run(uint64_t upto); // H is guarded by the queue lock. NX2NAVI_SUBMIT_BATCH=0 (or 1) hands each to the queue at once; =N holds up to N. #define HOLD_CAP 16 #define HOLD_NS 500000ull +// Command buffers of one submission: the mirror prologue, the parts cut before the part with their pre-pass command +// buffers (CHAIN_MAX), the part's pre-pass command buffer, the part (vk_sync_pre, vk_sync_cut, chain_infos). +#define CHAIN_MAX 64 +#define SUB_CBS (CHAIN_MAX + PART_CBS + 3) static struct { int n; VkSubmitInfo2 si[HOLD_CAP]; - VkCommandBufferSubmitInfo cbi[HOLD_CAP][PART_CBS + 2]; // the prologue and the part's command buffers + VkCommandBufferSubmitInfo cbi[HOLD_CAP][SUB_CBS]; VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried @@ -272,6 +276,7 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { if (H.n == HOLD_CAP) hold_flush_locked(c); int i = H.n++; H.si[i] = *si; + if (si->commandBufferInfoCount > SUB_CBS) rt_fatal("vk: a submission of %u command buffers", si->commandBufferInfoCount); if (si->commandBufferInfoCount) memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount); H.si[i].pCommandBufferInfos = H.cbi[i]; @@ -971,6 +976,7 @@ static VkCommandBuffer split_part(int exec) { if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order VkCommandBuffer pro = prologue(c, cur_fence), pre = pre_end(cur_fence); + VkCommandBufferSubmitInfo cbi[SUB_CBS]; uint32_t ncb = chain_infos(cbi, pro, pre, cb); // the mirror copies, the cut parts, the pre-pass work, the part VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi}; VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, @@ -1047,8 +1053,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_upload_flush(0); VkCommandBuffer pro = prologue(c, fence), pre = pre_end(fence); s = &slots[cur_slot]; - VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; - uint32_t ncbi = part_list(cbi, pro, s->cb, recorded); + VkCommandBufferSubmitInfo cbi[SUB_CBS]; + int sent = recorded || nchain || npart_cbs; // a cut part or a piece goes with the part after it, which goes too uint32_t ncb = chain_infos(cbi, pro, pre, sent ? s->cb : VK_NULL_HANDLE); // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, From 6de2be9ed5211a6a8b2e2954e120a3875b106fd1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:42 +0100 Subject: [PATCH 37/64] GPU timeline (NX2NAVI_TRACE=perf): every 60 frames, what the GPU's idle time was spent waiting for. The perf line's "idle with it queued" was the gap before each replay part handed over before the previous one ended: 13-17 ms a frame on an RX 580 in Kokiri Forest on 0.7.1, unexplained. Uploads and presents were not timed, a held submission (submit batching) counted as handed over while it still waited in vk_sync, and with cuts and pre-pass command buffers a submission closed more than the four timestamp pairs ts_unsent had room for: the rest went to the completion queue without a submit time and out of the GPU's order. Now every timed piece of GPU work (replay parts, mirror prologues, pre-pass command buffers, upload batches, presents) is noted with when it was handed to vk_sync, when it really went to the queue and its GPU start and end, and every 60 frames two lines split the idle time: starved (nothing in the queue, and how much of that a held submission waited in the batching) or with work in the queue, by the kind of piece that started after the gap, with the average delay from hand-over to start on an idle GPU per kind. ts_unsent has room for every command buffer of a submission, and the pairs are read in queue order. Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_present.c | 40 ++++++++++++ runtime/vk.h | 6 ++ runtime/vk_memory.c | 6 +- runtime/vk_sync.c | 137 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 178 insertions(+), 11 deletions(-) diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index b0d87802..b68fa709 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -140,6 +140,41 @@ static VkImageView g_sc_view[MAX_IMAGES]; // colour attachment views for the m static VkSemaphore g_render_done[MAX_IMAGES]; static VkCommandPool g_pool; static VkCommandBuffer g_cb[FRAMES]; + +// The GPU timeline (NX2NAVI_TRACE=perf, vk.h vk_gtl_note): a timestamp pair per present slot around the present's command +// buffer, read once the slot's fence is waited for again. Its gap before it shows a wait for the acquired image (the +// submission waits on the acquire semaphore) as idle time with work queued before presents. +static VkQueryPool pts_pool; +static int pts_state; // 0 not set up, 1 on, -1 off +static uint8_t pts_on[FRAMES]; // the slot's pair was written by its last present +static uint64_t pts_sub[FRAMES]; // when that present went to the queue + +static void pts_begin(VkCtx *c, VkCommandBuffer cb, unsigned slot) { + pts_on[slot] = 0; + if (!(rt_trace & RT_TRACE_PERF) || pts_state < 0) return; + if (!pts_state) { + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * FRAMES}; + pts_state = c->props.limits.timestampComputeAndGraphics && + vkCreateQueryPool(c->device, &qi, NULL, &pts_pool) == VK_SUCCESS ? 1 : -1; + if (pts_state < 0) return; + } + vkCmdResetQueryPool(cb, pts_pool, 2 * slot, 2); + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pts_pool, 2 * slot); + pts_on[slot] = 1; +} + +static void pts_end(VkCommandBuffer cb, unsigned slot) { + if (pts_on[slot]) vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, pts_pool, 2 * slot + 1); +} + +static void pts_read(VkCtx *c, unsigned slot) { // the slot's fence was waited for: its last present is done + if (!pts_on[slot]) return; + pts_on[slot] = 0; + uint64_t t[2]; + if (vkGetQueryPoolResults(c->device, pts_pool, 2 * slot, 2, sizeof t, t, sizeof t[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) + vk_gtl_note(GTL_PRESENT, pts_sub[slot], t[0], t[1]); +} static VkFence g_fence[FRAMES]; static VkSemaphore g_acquire[FRAMES]; static unsigned g_frame; @@ -1373,6 +1408,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w rt_log("gfx: present fence wait timed out"); goto out; } + pts_read(c, slot); // the GPU timeline (vk_gtl_note): this slot's last present is done uint32_t idx = 0; // NGX frame generation: the frame is composed into ng.bb and the presenter shows it (ng_show); no acquire here const int ngp = ng_ready(); @@ -1457,6 +1493,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(cb, &bi); + pts_begin(c, cb, slot); uint64_t crumb = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); vk_crumb(cb, VK_CRUMB_PRESENT, crumb); VkImage dst = ngp ? ng.bb.img : g_sc_img[idx]; @@ -1549,6 +1586,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w const int via_ng = ng_count >= 0; if (!via_ng) sync_barrier(cb, dst, last, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, last_stage, last_write, VK_PIPELINE_STAGE_2_NONE, 0); vk_crumb(cb, VK_CRUMB_PRESENT_END, crumb); + pts_end(cb, slot); vkEndCommandBuffer(cb); VkSemaphoreSubmitInfo wait = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = g_acquire[slot], @@ -1572,8 +1610,10 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w if (ng_n > 2) ng_wait_signaled(ng_n - 2); } vk_queue_lock(); + pts_sub[slot] = plat_now_ns(); r = vkQueueSubmit2(c->gfx_queue, 1, &si, g_fence[slot]); vk_queue_unlock(); + if (r != VK_SUCCESS) pts_on[slot] = 0; if (r == VK_SUCCESS && via_ng) { // the presenter shows the generated frames, then this one ng.n = ng_n; ng_hand(idx, ng_count, ng_set, ng_n); diff --git a/runtime/vk.h b/runtime/vk.h index a56fe2bf..781f5355 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -756,6 +756,12 @@ VkCommandBuffer vk_sync_pre(void); // Ends the part being recorded without handing it to the queue (it goes with the next part that does, in order) and // returns the command buffer the replay continues in; VK_NULL_HANDLE when no more parts can wait. Replay thread. VkCommandBuffer vk_sync_cut(void); +// The GPU timeline (NX2NAVI_TRACE=perf, vk_sync.c): a timed piece of GPU work of kind GTL_*, handed to the queue at +// sub_ns (plat_now_ns), its timestamps start and end (ticks of the graphics queue). vk_gtl_log: the gaps of the pieces +// noted since the last call, over frames frames, in the log. +enum { GTL_PART, GTL_MIRROR, GTL_PRE, GTL_UPLOAD, GTL_PRESENT, GTL_KINDS }; +void vk_gtl_note(int kind, uint64_t sub_ns, uint64_t start_tick, uint64_t end_tick); +void vk_gtl_log(uint64_t frame, int frames); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 9ac28a72..de6280f0 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -671,7 +671,7 @@ void vk_mem_census(uint64_t frame) { mirror_frame = frame; static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); - if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame); + if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line @@ -4228,6 +4228,7 @@ typedef struct { VkDeviceSize chunk_used; // bytes of the last chunk handed out uint64_t bytes; // staging bytes of its buffers (upload_cap) int ts; // its pair of upload timestamps (up_ts_*), -1: not timed + uint64_t sub_ns; // when it was handed to the queue (plat_now_ns), for the GPU timeline (vk_gtl_note) } Batch; enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; @@ -4241,6 +4242,7 @@ static int open_active; static Batch *inflight; static int ninflight, cap_inflight; static Batch *spare; // completed batches whose command buffer and fence are reused + // ---- upload GPU time (NX2NAVI_TRACE=perf): a timestamp at the start and the end of each upload batch, read when it // retires. The replay's GPU time (vk_sync.c) times only its own parts; upload batches go to the queue as submissions of // their own, with the ASTC decoder's dispatches in them, so their time showed only as GPU idle "with it queued". @@ -4276,6 +4278,7 @@ static void up_ts_read(VkCtx *c, Batch *b) { // b retired: its fence signalled VK_QUERY_RESULT_64_BIT) == VK_SUCCESS && t[1] >= t[0]) { uint64_t ns = (uint64_t)((double)(t[1] - t[0]) * c->props.limits.timestampPeriod); up_ts_ns += ns, up_ts_n++; + vk_gtl_note(GTL_UPLOAD, b->sub_ns, t[0], t[1]); if (ns > up_ts_max_ns) up_ts_max_ns = ns; } up_ts_busy[b->ts] = 0; @@ -4755,6 +4758,7 @@ uint64_t vk_upload_flush(int wait) { vkEndCommandBuffer(open_b.cb); VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb}; vk_queue_lock(); + open_b.sub_ns = plat_now_ns(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); if (r != VK_SUCCESS) { diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index c661d6f3..5208119c 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -84,6 +84,13 @@ static struct { uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried } H; static volatile int32_t held_n; // H.n, for readers without the lock +// The GPU timeline (vk_gtl_note): the timestamp pairs of the held submissions, stamped with the time they really go to +// the queue (ts_queue_ns), as a held part waits here and not in the queue. hold_pend: closed by ts_submitted for the +// submission queue_submit takes next (sub_lock); hold_pairs: those of the held ones (queue lock). +#define HOLD_PAIRS 512 +static uint32_t hold_pend[HOLD_PAIRS], hold_pairs[HOLD_PAIRS]; +static int nhold_pend, nhold_pairs; +static uint64_t ts_queue_ns[1024]; // per pair (TS_PAIRS): when its submission went to the queue static volatile uint64_t held_t0; // when the oldest held submission was held (kern_now_ns) static volatile uint64_t queued_v; // the newest timeline value handed to the queue static int hold_max = -1; @@ -248,6 +255,8 @@ static void hold_flush_locked(VkCtx *c) { (unsigned long long)H.last, (int)r); } if (H.last) __atomic_store_n(&queued_v, H.last, __ATOMIC_RELEASE); + for (int k = 0; k < nhold_pairs; k++) ts_queue_ns[hold_pairs[k]] = now; + nhold_pairs = 0; H.batches++, H.subs += (uint64_t)H.n; H.n = 0, H.last = 0; __atomic_store_n(&held_n, 0, __ATOMIC_RELEASE); @@ -274,6 +283,8 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { vk_queue_lock_raw(); perf_end(PERF_FENCE, p0); if (H.n == HOLD_CAP) hold_flush_locked(c); + for (int k = 0; k < nhold_pend && nhold_pairs < HOLD_PAIRS; k++) hold_pairs[nhold_pairs++] = hold_pend[k]; + nhold_pend = 0; int i = H.n++; H.si[i] = *si; if (si->commandBufferInfoCount > SUB_CBS) rt_fatal("vk: a submission of %u command buffers", si->commandBufferInfoCount); @@ -315,6 +326,7 @@ static void hold_release(uint64_t value, int aged, int try_only) { // part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue // family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed. #define TS_PAIRS 1024 +_Static_assert(TS_PAIRS == sizeof ts_queue_ns / sizeof ts_queue_ns[0], "ts_queue_ns has a slot per pair"); static VkQueryPool ts_pool; static int ts_state; // 0 not set up, 1 on, -1 off static double ts_period; // ns per tick @@ -331,13 +343,19 @@ static volatile int64_t ts_busy_all; // the previous one had ended (the GPU waited for the CPU: starved) or before (it waited for untimed work or a wait). static uint64_t ts_sub_ns[TS_PAIRS]; // CPU time the part was handed to vkQueueSubmit2 // A pair to read at its value. mirror: the copies of a mirror prologue (vk_sync.c prologue), counted apart as well. -typedef struct { uint32_t pair; int mirror; uint64_t fence; } TsRead; -static TsRead *ts_unsent[4]; // closed and not yet submitted (sub_lock) +// mirror: GTL_PART, GTL_MIRROR or GTL_PRE (the GPU timeline's kinds, below); cb: the command buffer it times. +typedef struct { uint32_t pair; int mirror; uint64_t fence; VkCommandBuffer cb; } TsRead; +// closed and not yet submitted (sub_lock): with cuts a submission carries up to SUB_CBS command buffers, each timed; +// with room for 4 the rest went to the completion queue at once, without a submit time and out of the GPU's order, and +// their gaps counted as idle "with it queued" +#define TS_UNSENT (2 * SUB_CBS) +static TsRead *ts_unsent[TS_UNSENT]; static int ts_n_unsent; static volatile int64_t ts_mirror_ns; // of ts_busy_ns, the mirror prologues' copies static uint64_t ts_last_end_ns; // completion thread: ts_last_end on the CPU clock static volatile int64_t ts_idle_ns, ts_starved_ns, ts_gaps, ts_max_gap_ns, ts_queued_ns; -static int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when it was taken +static volatile int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when + // it was taken; the GPU timeline's notes read them from other threads static PFN_vkGetCalibratedTimestampsKHR cal_fn; // On the first submission that wants timing (perf_gpu_timing: NX2NAVI_TRACE=perf or the full performance overlay). @@ -405,19 +423,114 @@ static void max64(volatile int64_t *m, int64_t v) { static void ts_read_cb(void *arg); +// ---- the GPU timeline (NX2NAVI_TRACE=perf, vk.h vk_gtl_note): every timed piece of GPU work, with the CPU time it was +// handed to the queue and its GPU start and end on the CPU clock (calibrated timestamps). The replay's parts, their +// mirror prologues and pre-pass command buffers come from here, the upload batches from vk_memory.c, the presents from +// gfx_present.c. Every 60 frames (vk_gtl_log) the pieces are put in GPU order and each gap between them is split: +// starved, nothing handed to the queue yet, or queued, the next piece handed over and not started; the queued part is +// summed by the kind of the piece that started after it. A gap with work queued is a wait inside the queue (a +// semaphore, the present's acquire) or the driver's and the system's delay to start work on an idle GPU. +#define GTL_N 16384 +typedef struct { int kind; int64_t call, sub, start, end; } GtlEv; // call: handed to vk_sync's submit; sub: to the queue +static GtlEv gtl[GTL_N]; +static int gtl_n; +static uint64_t gtl_dropped; +static PlatMutex gtl_lock = PLAT_MUTEX_INIT; +static const char *const gtl_names[GTL_KINDS] = {"replay parts", "mirror prologues", "pre-pass buffers", "upload batches", + "presents"}; + +static void vk_gtl_note_ns(int kind, int64_t call, int64_t sub, int64_t start, int64_t end) { + if (!(rt_trace & RT_TRACE_PERF) || kind < 0 || kind >= GTL_KINDS || !sub || end < start) return; + plat_mutex_lock(>l_lock); + if (gtl_n < GTL_N) gtl[gtl_n++] = (GtlEv){kind, call ? call : sub, sub, start, end}; + else gtl_dropped++; + plat_mutex_unlock(>l_lock); +} + +void vk_gtl_note(int kind, uint64_t sub_ns, uint64_t start_tick, uint64_t end_tick) { + if (!(rt_trace & RT_TRACE_PERF) || ts_state != 1 || !__atomic_load_n(&cal_at, __ATOMIC_ACQUIRE)) return; + uint64_t s = start_tick & ts_mask, e = end_tick & ts_mask; + if (e < s) return; + vk_gtl_note_ns(kind, (int64_t)sub_ns, (int64_t)sub_ns, tick_ns(s), tick_ns(e)); +} + + +} + +static int gtl_cmp(const void *a, const void *b) { + int64_t x = ((const GtlEv *)a)->start, y = ((const GtlEv *)b)->start; + return x < y ? -1 : x > y; +} + +void vk_gtl_log(uint64_t frame, int frames) { + static GtlEv ev[GTL_N]; + plat_mutex_lock(>l_lock); + int n = gtl_n; + memcpy(ev, gtl, sizeof ev[0] * (size_t)n); + gtl_n = 0; + uint64_t dropped = gtl_dropped; + gtl_dropped = 0; + plat_mutex_unlock(>l_lock); + if (n < 2 || frames < 1) return; + qsort(ev, (size_t)n, sizeof ev[0], gtl_cmp); + double busy = 0, starved = 0, queued = 0, q_by[GTL_KINDS] = {0}, lat[GTL_KINDS] = {0}, gpu[GTL_KINDS] = {0}; + double held = 0, held_idle = 0; // replay pieces: held in the submit batching; of it, while the GPU was idle + int big[GTL_KINDS] = {0}, nlat[GTL_KINDS] = {0}, cnt[GTL_KINDS] = {0}; + int64_t end = ev[0].start; + for (int i = 0; i < n; i++) { + const GtlEv *e = &ev[i]; + cnt[e->kind]++, gpu[e->kind] += (double)(e->end - e->start); + held += (double)(e->sub - e->call); + if (e->start > end) { // a gap before e + int64_t gap = e->start - end, s = e->sub > end ? e->sub - end : 0; + int64_t h0 = e->call > end ? e->call : end, h1 = e->sub < e->start ? e->sub : e->start; + if (h1 > h0) held_idle += (double)(h1 - h0); + if (s > gap) s = gap; + starved += (double)s, queued += (double)(gap - s), q_by[e->kind] += (double)(gap - s); + if (gap - s > 500000) big[e->kind]++; + if (e->sub >= end) lat[e->kind] += (double)(e->start - e->sub), nlat[e->kind]++; // handed to an idle GPU + } + int64_t from = e->start > end ? e->start : end; + if (e->end > from) busy += (double)(e->end - from), end = e->end; + } + double f = 1e6 * frames; + char by[512], la[512], gp[512]; + int a = 0, b = 0, g = 0; + for (int k = 0; k < GTL_KINDS; k++) { + if (!cnt[k]) continue; + a += snprintf(by + a, sizeof by - (size_t)a, "%s%s %.2f ms (%.1f gaps over 0.5 ms)", a ? ", " : "", gtl_names[k], + q_by[k] / f, (double)big[k] / frames); + b += snprintf(la + b, sizeof la - (size_t)b, "%s%s %.2f ms (%d)", b ? ", " : "", gtl_names[k], + nlat[k] ? lat[k] / nlat[k] / 1e6 : 0.0, nlat[k]); + g += snprintf(gp + g, sizeof gp - (size_t)g, "%s%s %.1f x %.2f ms", g ? ", " : "", gtl_names[k], + (double)cnt[k] / frames, gpu[k] / f); + } + rt_log("perf: frame %llu: GPU timeline per frame: busy %.2f ms; idle %.2f ms starved (nothing in the queue; %.2f ms of " + "it while the next piece was held in the submit batching) and %.2f ms with work in the queue, before %s; held " + "in the submit batching %.2f ms in all", (unsigned long long)frame, busy / f, starved / f, held_idle / f, + queued / f, by, held / f); + rt_log("perf: frame %llu: GPU timeline: pieces per frame and their GPU time: %s; from hand-over to start on an idle " + "GPU, on average: %s; %d pieces, %llu dropped", (unsigned long long)frame, gp, la, n, (unsigned long long)dropped); +} + // Just before vkQueueSubmit2 (sub_lock held): the parts closed since the last submit leave the CPU now. Their reads // are queued in the order the GPU runs them: a mirror prologue is closed after the part it was made for but runs // first. Read after its part, its copies counted neither as busy time nor as a gap of their own: they showed up as // idle time with work queued, 7-8 ms a frame over PCIe 3.0 in Kokiri Forest, and the frame as CPU-bound. -static void ts_submitted(void) { +// cbi: the submission's command buffers in queue order; their pairs are read in that order (a part's pre-pass command +// buffer and a cut part's are closed in another order than they run). +static void ts_submitted(const VkCommandBufferSubmitInfo *cbi, uint32_t ncb) { uint64_t now = plat_now_ns(); gfx_tl_note('S', now, cur_fence, 0); - for (int pass = 0; pass < 2; pass++) + for (uint32_t k = 0; k <= ncb; k++) // k == ncb: whatever no command buffer named, last for (int i = 0; i < ts_n_unsent; i++) { TsRead *t = ts_unsent[i]; - if (!t->mirror != pass) continue; // the prologues first + if (!t || (k < ncb ? t->cb != cbi[k].commandBuffer : 0)) continue; ts_sub_ns[t->pair] = now; + ts_queue_ns[t->pair] = 0; + if (nhold_pend < HOLD_PAIRS) hold_pend[nhold_pend++] = t->pair; vk_completion_add(t->fence, ts_read_cb, t); + ts_unsent[i] = NULL; } ts_n_unsent = 0; } @@ -441,12 +554,14 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order if (sub && sn > sub) plat_add64(&ts_queued_ns, sn - sub); gfx_tl_note('G', (uint64_t)sn, (uint64_t)en, (uint64_t)sub); if (en > ts_last_end_ns) ts_last_end_ns = en; + int64_t q = (int64_t)ts_queue_ns[t->pair]; + vk_gtl_note_ns(t->mirror, sub, q ? q : sub, sn, en); } uint64_t from = s > ts_last_end ? s : ts_last_end; if (e > from) { int64_t ns = (int64_t)((double)(e - from) * ts_period); plat_add64(&ts_busy_ns, ns); - if (t->mirror) plat_add64(&ts_mirror_ns, ns); + if (t->mirror == GTL_MIRROR) plat_add64(&ts_mirror_ns, ns); plat_add64(&ts_busy_all, ns); perf_gpu_busy((uint64_t)ns); } @@ -564,8 +679,8 @@ static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence, int mirror) { ts_sub_ns[pair] = 0; TsRead *t = malloc(sizeof *t); if (!t) rt_fatal("vk: out of memory"); - *t = (TsRead){(uint32_t)pair, mirror, fence}; - if (ts_n_unsent < 4) ts_unsent[ts_n_unsent++] = t; + *t = (TsRead){(uint32_t)pair, mirror, fence, cb}; + if (ts_n_unsent < TS_UNSENT) ts_unsent[ts_n_unsent++] = t; else vk_completion_add(fence, ts_read_cb, t); // more than a part and its prologue between two submits } @@ -823,6 +938,7 @@ static int nchain; static VkCommandBuffer pre_end(uint64_t fence) { VkCommandBuffer cb = pre_cb; if (!cb) return VK_NULL_HANDLE; + ts_close(cb, pre_pair, fence, GTL_PRE); pre_pair = -1; vkEndCommandBuffer(cb); pre_cb = VK_NULL_HANDLE; @@ -983,7 +1099,7 @@ static VkCommandBuffer split_part(int exec) { .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; if (exec && exec_tl && exec_sig < cur_fence) si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; - ts_submitted(); + ts_submitted(cbi, ncb); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted @@ -1065,6 +1181,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { if (both) exec_sig = fence; VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u, .pSignalSemaphoreInfos = sig}; + ts_submitted(cbi, ncb); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); s->fence = sent ? fence : 0; From 1dda8bc23c9afd6cab1b49c60fd42608cd403b0f Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:43 +0100 Subject: [PATCH 38/64] Lazy write-back: the armed list gets its own short lock; the replay's per-draw check no longer waits for freezes and thaws on other threads. Co-Authored-By: Claude Opus 5.5 --- runtime/memory.c | 95 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/runtime/memory.c b/runtime/memory.c index be2e0839..9802157d 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -168,9 +168,16 @@ static void unprobe_locked(uint64_t r) { // has the part's writes. Only a fault of the replay thread itself, a wait of 20 ms, or a release by a thaw goes ahead // inside the window (lz_gaps counts the faults): that part's writes to the range then reach guest memory with the // range's next write-back. +// The list of armed segments has a lock of its own, lz_lock, held only for a few steps over the list and never across a +// system call or a fetch: the replay asks it at every draw that writes a mirrored pool (mem_lazy_skip) and at the end of +// every part (mem_lazy_armed, mem_lazy_window). Under frz_lock it waited for freezes, thaws and arms on other threads, +// whose page queries and protection changes take that lock for tens to hundreds of microseconds: 8% of the replay +// thread's samples standing in Hyrule Field on an RX 580 (0.7.1, lazy write-back on by default). A change of the list +// holds both locks (frz_lock first); a read holds either. typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta -static LzSeg *lzs; // frz_lock -static int lz_n, lz_cap; // frz_lock +static PlatMutex lz_lock = PLAT_MUTEX_INIT; // inside frz_lock, never the other way round +static LzSeg *lzs; // changed under frz_lock and lz_lock, read under either +static int lz_n, lz_cap; // the same static volatile uint8_t lzr[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // a segment reaches into the region static uint32_t lz_hot[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 until which nothing is armed here static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: seconds of the region's last hold @@ -222,7 +229,7 @@ static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, static int lz_nnote; // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; -static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else +static volatile uint32_t lz_win_tid; // lz_lock: the replay thread's id while its part is between check and hand-over // For a fatal error's summary line (mem_lazy_note): written under frz_lock and read without it, since the thread that // fails may be the one inside a fetch. static volatile uint64_t lz_fetch_total; // fetches since the start @@ -239,11 +246,28 @@ static void lazy_flags_locked(uint64_t lo, uint64_t hi) { } } +// frz_lock and lz_lock held: segment i leaves the list, with the stale mark mem_lazy_skip may have set until now. A +// later mem_lazy_skip no longer finds it and writes its range back as ever. +static LzSeg lazy_take_locked(int i) { + LzSeg s = lzs[i]; + lzs[i] = lzs[--lz_n]; + return s; +} + +static void lazy_gone_locked(LzSeg s, int why); + // frz_lock held exclusively. Segment i goes: its bytes come down first when the mirror's are newer, then its pages // are read-write again. why: 0 a read fault, 1 a write fault (both hold the regions back), else a thaw or the mirror // going away. static void lazy_drop_locked(int i, int why) { - LzSeg s = lzs[i]; + plat_mutex_lock(&lz_lock); + LzSeg s = lazy_take_locked(i); + plat_mutex_unlock(&lz_lock); + lazy_gone_locked(s, why); +} + +// frz_lock held exclusively: the pages of s, out of the list already, get their bytes and their access back. +static void lazy_gone_locked(LzSeg s, int why) { if (s.stale && lz_fetch) { uint64_t t0 = plat_now_ns(); __atomic_store_n(&lz_fetch_now, s.hi - s.lo, __ATOMIC_RELAXED); @@ -258,7 +282,6 @@ static void lazy_drop_locked(int i, int why) { if (plat_mem_state(a, s.hi, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, PLAT_PROT_RW)) rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a)); - lzs[i] = lzs[--lz_n]; lazy_flags_locked(s.lo, s.hi); if (why > 1) { lz_other++; return; } lz_faults[why]++; @@ -428,6 +451,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t } uint64_t slo = x, shi = e; uint8_t stale = 0; + plat_mutex_lock(&lz_lock); for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer if (lzs[j].h0 == h0 && lzs[j].delta == delta && lzs[j].lo <= shi && lzs[j].hi >= slo) { if (lzs[j].lo < slo) slo = lzs[j].lo; @@ -437,6 +461,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t j = -1; } lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale}); + plat_mutex_unlock(&lz_lock); __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); armed += e - x; } @@ -455,13 +480,31 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o int any = 0, k = 0; for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r]; if (!any) return 0; - plat_mutex_lock(&frz_lock); - for (int i = lz_n; i-- > 0;) - if (lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta)) { - lzs[i].stale = 0; - lz_foreign++; - lazy_drop_locked(i, 4); + // A segment armed for another buffer goes back under frz_lock (lazy_gone_locked), with arms held off until the + // rest is marked, as before lz_lock; that is rare, and the replay's per-draw check holds lz_lock only. + int foreign = 0; + plat_mutex_lock(&lz_lock); + for (int i = 0; i < lz_n && !foreign; i++) + foreign = lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta); + if (foreign) { + plat_mutex_unlock(&lz_lock); + plat_mutex_lock(&frz_lock); + for (;;) { + plat_mutex_lock(&lz_lock); + int i = lz_n; + while (i-- > 0 && !(lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta))) {} + LzSeg s; + if (i >= 0) { + lzs[i].stale = 0; + lz_foreign++; + s = lazy_take_locked(i); + } + plat_mutex_unlock(&lz_lock); + if (i < 0) break; + lazy_gone_locked(s, 4); } + plat_mutex_lock(&lz_lock); + } for (int i = 0; i < lz_n && k < max; i++) { uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n; if (lo >= hi) continue; @@ -470,16 +513,19 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o for (; at > 0 && out[at - 1][0] > lo; at--) out[at][0] = out[at - 1][0], out[at][1] = out[at - 1][1]; out[at][0] = lo, out[at][1] = hi; } - plat_mutex_unlock(&frz_lock); + plat_mutex_unlock(&lz_lock); + if (foreign) plat_mutex_unlock(&frz_lock); return k; } + plat_mutex_lock(&lz_lock); + plat_mutex_unlock(&lz_lock); // [a, a + n) is armed as a whole still. int mem_lazy_armed(uint64_t a, uint64_t n) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; - plat_mutex_lock(&frz_lock); + plat_mutex_lock(&lz_lock); int ok = lazy_covered_locked(a, a + n); - plat_mutex_unlock(&frz_lock); + plat_mutex_unlock(&lz_lock); return ok; } @@ -530,16 +576,17 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64 return 1; } -// Opens (under frz_lock, so no release is half done) or closes the window of the replay's part. +// Opens (under lz_lock: a fault's segment is out of the list before it, or the fault sees it) or closes the window of +// the replay's part. void mem_lazy_window(int open) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; if (!open) { __atomic_store_n(&lz_win_tid, 0, __ATOMIC_RELEASE); return; } - plat_mutex_lock(&frz_lock); + plat_mutex_lock(&lz_lock); // a fault takes its segment out before this or sees the window __atomic_store_n(&lz_win_tid, plat_thread_id(), __ATOMIC_RELEASE); - plat_mutex_unlock(&frz_lock); + plat_mutex_unlock(&lz_lock); } static int lazy_fault(uint64_t a, int write) { @@ -547,14 +594,17 @@ static int lazy_fault(uint64_t a, int write) { uint32_t me = plat_thread_id(); for (int tries = 0;; tries++) { plat_mutex_lock(&frz_lock); + plat_mutex_lock(&lz_lock); // the window's test and the segment's leaving the list in one step int i = 0; while (i < lz_n && !(a >= lzs[i].lo && a < lzs[i].hi)) i++; if (i == lz_n) { + plat_mutex_unlock(&lz_lock); plat_mutex_unlock(&frz_lock); return 0; } uint32_t w = __atomic_load_n(&lz_win_tid, __ATOMIC_ACQUIRE); if (w && w != me && tries < 400) { // up to 20 ms: after the hand-over the fetch has that part's writes + plat_mutex_unlock(&lz_lock); plat_mutex_unlock(&frz_lock); plat_wait_ns(50000); continue; @@ -575,11 +625,14 @@ static int lazy_fault(uint64_t a, int write) { } } if (reader) lz_heavy_ms[r] = now | 1; - if (lz_chunk && !reader) { + if (lz_chunk && !reader) { // the piece leaves the list and the rest goes back into it under lz_lock out = lazy_piece_locked(i, a, write != 0); + plat_mutex_unlock(&lz_lock); if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back; - } else { - lazy_drop_locked(i, write != 0); + } else { // the segment leaves the list under lz_lock; its pages come back without it + LzSeg s = lazy_take_locked(i); + plat_mutex_unlock(&lz_lock); + lazy_gone_locked(s, write != 0); if (k >= 0) lz_note[k].hold = lz_back[r]; } if (k >= 0) lz_note[k].out = out; @@ -683,6 +736,8 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { // was read-write at the freeze, and mapping changes thaw or wait before they run, so these are all ours). strict: under // frz_lock, a failure ends the process; outside it (a thaw in flight, FRZ_THAWING) the pages are what the freeze left, // and a failure is only counted. +static void thaw_pages_vm(uint64_t r, int strict); + static void thaw_pages(uint64_t r, int strict) { uint64_t base = r << FRZ_SHIFT; const FrzRuns *k = frz_runs[r]; From 7d54ee73c74f21febefac72720769a063b1fd409 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:45 +0100 Subject: [PATCH 39/64] Waits for GPU work outside the replay (NX2NAVI_TRACE=perf): how much of each is the replay still recording and how much the GPU. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_sync.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 5208119c..dce3d334 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -16,6 +16,7 @@ #include "gfx.h" #include "host.h" #include "nvn_api.h" +#include "prof.h" #include "vk.h" #include "platform/platform.h" @@ -255,6 +256,7 @@ static void hold_flush_locked(VkCtx *c) { (unsigned long long)H.last, (int)r); } if (H.last) __atomic_store_n(&queued_v, H.last, __ATOMIC_RELEASE); + uint64_t now = plat_now_ns(); for (int k = 0; k < nhold_pairs; k++) ts_queue_ns[hold_pairs[k]] = now; nhold_pairs = 0; H.batches++, H.subs += (uint64_t)H.n; @@ -454,7 +456,74 @@ void vk_gtl_note(int kind, uint64_t sub_ns, uint64_t start_tick, uint64_t end_ti vk_gtl_note_ns(kind, (int64_t)sub_ns, (int64_t)sub_ns, tick_ns(s), tick_ns(e)); } +// ---- the game's waits for GPU work (NX2NAVI_TRACE=perf): for each submission value, when the game handed it to the +// replay (vk_sync_reserve; the submit itself without asynchronous replay), when the replay had recorded and submitted it +// (issued) and when it was done (GPU and completion). A fence wait of any thread but the replay's is split into its +// time until the waited submission was submitted (the replay still recording it or those before it) and its time after +// (the GPU, the submit batching's hold, the completion thread). Every 60 frames with the GPU timeline (vk_gtl_log). +#define GW_RING 4096 +typedef struct { uint64_t v, res, iss, done; } GwSub; +static GwSub gw[GW_RING]; +static PlatMutex gw_lock = PLAT_MUTEX_INIT; +static uint64_t gw_n, gw_ns, gw_rep_ns, gw_gpu_ns, gw_lead_ns, gw_age, gw_before, gw_lat_rep, gw_lat_gpu, gw_lat_n; +static GwSub *gw_at(uint64_t v) { // gw_lock held + GwSub *e = &gw[v & (GW_RING - 1)]; + if (e->v != v) *e = (GwSub){v, 0, 0, 0}; + return e; +} + +static void gw_mark(uint64_t v, int what) { + if (!(rt_trace & RT_TRACE_PERF) || !v) return; + uint64_t now = plat_now_ns(); + plat_mutex_lock(&gw_lock); + GwSub *e = gw_at(v); + if (what == 0) e->res = now; + else if (what == 1) e->iss = now; + else if (!e->done) e->done = now; + plat_mutex_unlock(&gw_lock); +} + +// the waits: t0 to t1 for value; newest: the newest value handed out at t0, sent: the newest submitted at t0 +static void gw_wait(uint64_t value, uint64_t t0, uint64_t t1, uint64_t newest, uint64_t sent) { + if (!(rt_trace & RT_TRACE_PERF) || t1 <= t0 || plat_thread_id() == prof_replay_tid) return; + uint64_t d = t1 - t0; + plat_mutex_lock(&gw_lock); + GwSub e = gw[value & (GW_RING - 1)]; + gw_n++, gw_ns += d; + if (d > gw_max_ns) gw_max_ns = d; + if (e.v != value || !e.iss) { + gw_unknown++, gw_gpu_ns += d; + } else { + uint64_t rep = e.iss > t0 ? (e.iss < t1 ? e.iss : t1) - t0 : 0; + gw_rep_ns += rep, gw_gpu_ns += d - rep; + if (e.res && e.res <= t0) gw_lead_ns += t0 - e.res; + if (e.res && e.done && e.iss >= e.res && e.done >= e.iss) + gw_lat_rep += e.iss - e.res, gw_lat_gpu += e.done - e.iss, gw_lat_n++; + } + if (newest > value) gw_age += newest - value; + if (value > sent + 1) gw_before += value - sent - 1; + plat_mutex_unlock(&gw_lock); +} + +static void gw_log(uint64_t frame, int frames) { + plat_mutex_lock(&gw_lock); + uint64_t n = gw_n, ns = gw_ns, rep = gw_rep_ns, gpu = gw_gpu_ns, lead = gw_lead_ns, age = gw_age, before = gw_before, + gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0; + plat_mutex_unlock(&gw_lock); + if (!n) { + rt_log("perf: frame %llu: waits for GPU work outside the replay: none", (unsigned long long)frame); + return; + } + double f = frames; + rt_log("perf: frame %llu: waits for GPU work outside the replay: %.1f per frame, %.2f ms per frame (longest %.2f ms): " + "%.2f ms until the replay had submitted the waited submission (%.2f submissions still before it in the replay " + "on average), %.2f ms after (GPU, submit batching hold, completion; %llu without a record); the waited one was " + "%.2f submissions behind the newest and handed to the replay %.2f ms before the wait; per waited submission " + "%.2f ms from hand-over to submit and %.2f ms from submit to done", + (unsigned long long)frame, n / f, ns / 1e6 / f, mx / 1e6, rep / 1e6 / f, (double)before / (double)n, + gpu / 1e6 / f, (unsigned long long)unk, (double)age / (double)n, lead / 1e6 / (double)n, + ln ? lr / 1e6 / (double)ln : 0.0, ln ? lg / 1e6 / (double)ln : 0.0); } static int gtl_cmp(const void *a, const void *b) { @@ -464,6 +533,7 @@ static int gtl_cmp(const void *a, const void *b) { void vk_gtl_log(uint64_t frame, int frames) { static GtlEv ev[GTL_N]; + gw_log(frame, frames); plat_mutex_lock(>l_lock); int n = gtl_n; memcpy(ev, gtl, sizeof ev[0] * (size_t)n); @@ -807,8 +877,10 @@ static void completion_main(void *arg) { // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each // checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload // path holds, so done waited behind uploads. + uint64_t was = done; __atomic_store_n(&done, reached, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_done); + for (uint64_t v = reached - was > GW_RING ? reached - GW_RING + 1 : was + 1; v <= reached; v++) gw_mark(v, 2); plat_mutex_unlock(&st_lock); vk_mem_collect(); plat_mutex_lock(&st_lock); @@ -1122,6 +1194,7 @@ uint64_t vk_sync_reserve(void) { if (reserved < issued) reserved = issued; uint64_t v = ++reserved; plat_mutex_unlock(&st_lock); + gw_mark(v, 0); return v; } @@ -1186,6 +1259,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_mem_lazy_handed(); s->fence = sent ? fence : 0; vk_mem_note_submit(fence); + gw_mark(fence, 1); plat_mutex_lock(&st_lock); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); @@ -1265,7 +1339,10 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { uint64_t p0 = perf_begin(); + uint64_t newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE), sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE); + uint64_t w0 = __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value ? 0 : plat_now_ns(); int ok = fence_wait_impl(value, timeout_ns); + if (w0 && ok) gw_wait(value, w0, plat_now_ns(), newest > sent ? newest : sent, sent); perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); return ok; From ad288e3bc1ce983618df4ec7a64f146fa6b551a3 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:47 +0100 Subject: [PATCH 40/64] Staged mirror uploads: the replay copies guest bytes for the VRAM mirrors while it records, and the first-submission wait ends once the replay has submitted; 8 texture pool sets. On 0.9.1 this is the release's own mirror capture (NX2NAVI_MIRROR_CAPTURE_KB) turned on: with NX2NAVI_MIRROR_STAGE=1 (the default) the pages the CPU wrote are copied out of guest memory, up to 64 MiB a part, when the part is handed to the queue. A submission whose prologue copied everything that way reads no guest memory late, so the game's wait after a frame's first submission ends once the replay has submitted it (vk_sync_recorded_wait). One with a range left to the GPU-time copy (a pool with a write-back in flight or a twin) or a draw that reads a pool without a mirror is noted (vk_mem_note_live), and its wait is the GPU's. Also 8 texture pool sets. Co-Authored-By: Claude Opus 5.5 --- runtime/host.h | 1 + runtime/memory.c | 14 +++++++++ runtime/nvn_api_core.c | 3 ++ runtime/vk.h | 8 ++++- runtime/vk_backend.c | 7 ++++- runtime/vk_memory.c | 69 +++++++++++++++++++++++++++++++++--------- runtime/vk_sync.c | 52 +++++++++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 16 deletions(-) diff --git a/runtime/host.h b/runtime/host.h index 3b5928db..7861672e 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -245,6 +245,7 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o int mem_lazy_armed(uint64_t a, uint64_t n); // Copies the ranges src[i] = {address, bytes} of guest memory whose pages read without a fault to dst[i], ok[i] set. int mem_copy_readable(uint8_t *const *dst, const uint64_t (*src)[2], int n, uint8_t *ok); +int mem_lazy_touches(uint64_t a, uint64_t n); // 0: the CPU can read [a, a + n) without a fault now void mem_lazy_window(int open); void mem_lazy_release(uint64_t a, uint64_t n); void mem_lazy_report(uint64_t frame, uint64_t skipped); diff --git a/runtime/memory.c b/runtime/memory.c index 9802157d..a1c9f7b4 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -518,8 +518,22 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o return k; } +// 0 when no page of [a, a + n) can be inaccessible right now (armed by the lazy write-back or probed), so the CPU can +// read it without a fault; 1 when one may be. +int mem_lazy_touches(uint64_t a, uint64_t n) { + if (!n) return 0; + if (a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT || __atomic_load_n(&prb_on, __ATOMIC_RELAXED)) return 1; + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; + int any = 0; + for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = __atomic_load_n(&lzr[r], __ATOMIC_ACQUIRE); + if (!any) return 0; + int hit = 0; plat_mutex_lock(&lz_lock); + for (int i = 0; i < lz_n && !hit; i++) hit = lzs[i].lo < a + n && lzs[i].hi > a; plat_mutex_unlock(&lz_lock); + return hit; +} + // [a, a + n) is armed as a whole still. int mem_lazy_armed(uint64_t a, uint64_t n) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 320d7160..c2a71790 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -741,6 +741,9 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. + // Since the staged mirror uploads (vk_memory.c) the copies of guest memory a submission reads are taken while the + // replay records it, and the wait ends once the replay has submitted it (vk_sync_recorded_wait); the GPU's wait + // below remains for a submission whose GPU reads guest memory itself. // The wait (exec_wait, vk_sync.c vk_sync_exec_wait) ends when the GPU has run the submission's passes and the // results the game reads once it goes on are in its memory, the last frame's occlusion counts among them; the // write-backs and the completion thread's other side effects are not waited for. NX2NAVI_WB_SPLIT=0 makes it diff --git a/runtime/vk.h b/runtime/vk.h index 781f5355..522df9b2 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -347,6 +347,10 @@ void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); void vk_mem_mirror_prologue(VkCommandBuffer cb); +// Staged mirror uploads (vk_memory.c): the GPU of the submission being recorded reads guest memory itself (noted by +// vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note. +void vk_mem_note_live(void); +int vk_mem_take_live(void); void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); void vk_mem_lazy_decide(void); // once the driver is known (after vk_caps_driver): on or off, with one log line @@ -606,7 +610,8 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: s // set and brings it up to date. vk_pool_takes counts the takes whose sync left no slot behind (vk_exec_draw.c pool_sync); // once it has moved on by more than VK_POOL_SETS since a pool entry changed, every set holds the change (vk_texture.c // retire_sweep). It may count fewer takes than there were, which only delays that. -#define VK_POOL_SETS 4 +#define VK_POOL_SETS 8 // 4 before: with the game no longer held for the GPU after a frame's first submission, the + // replay waited for a set's GPU work at 15% of the switches uint64_t vk_pool_takes(void); // Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after. // Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used. @@ -715,6 +720,7 @@ uint64_t vk_sync_reserve(void); // asynchronous replay: the value the int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted // The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); +int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)); int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own // NX2NAVI_TRACE=perf (vk_sync.c "the executed timeline"): what a wait on that timeline left behind since the last call, // whether the completion thread's callback running now belongs to a value whose waiter went on already, and the diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 1893f6fc..1bc66186 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -302,9 +302,14 @@ int vk_replay_async(void) { } // With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's. -static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { +static int vk_be_gpu_wait(uint64_t value, uint64_t timeout_ns) { return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns); } +// The submit wait (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): the GPU's only where the submission reads guest memory on the +// GPU (vk_sync_recorded_wait). +static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { + return vk_sync_recorded_wait(value, timeout_ns, vk_be_gpu_wait); +} // The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index de6280f0..34b62d60 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -2622,7 +2622,10 @@ void vk_mem_use(VkMemSpan *sp) { return; } // a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors) - if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return; + if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) { + vk_mem_note_live(); // the GPU reads this pool's guest memory as it runs (vk_mem_take_live) + return; + } int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); mem_use_locked(sp); @@ -2638,7 +2641,10 @@ static void mem_use_locked(VkMemSpan *sp) { r->mstate = MS_NONE, r->retry_sub = 0; // refuse sets it again } if (r->mstate == MS_NONE) mirror_create(r); - if (r->mstate != MS_ON) return; + if (r->mstate != MS_ON) { // the GPU reads this pool's guest memory as it runs (staged mirror uploads) + vk_mem_note_live(); + return; + } if (r->acc_sub != msub) mirror_sync(r); r->vd_use_sub = msub; sp->buffer = r->mbuf; @@ -2646,7 +2652,11 @@ static void mem_use_locked(VkMemSpan *sp) { void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; - if (!r || r->mstate != MS_ON) return; + if (!r) return; + if (r->mstate != MS_ON) { // guest memory itself, read by the GPU when it runs (vk_mem_take_live) + vk_mem_note_live(); + return; + } if (r->acc_sub != msub) { int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); @@ -3424,11 +3434,13 @@ static void hot_check(void) { if (mirror_hot < 0) { mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p; if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot); + } if (!mirror_hot) return; uint64_t fr = mirror_frame; if (!hot_frame0 || fr < hot_frame0) { hot_frame0 = fr ? fr : 1; return; + } if (fr - hot_frame0 < HOT_FRAMES || npend || nwbl) return; // nsnl: the previous submission's, recorded already double frames = (double)(fr - hot_frame0); hot_frame0 = fr; @@ -3442,6 +3454,7 @@ static void hot_check(void) { r->hot_bytes = (uint64_t)per; // kept for the log line below r->hot_bytes = 0, i++; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict) continue; + } mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first // as mirror_evict, under the exclusive lock (vk_pool_fini takes pools off the list under it), but for good: // MS_REFUSED with no retry (retry_sub 0), so neither vk_mem_use nor the budget retries mirror it again @@ -3456,6 +3469,7 @@ static void hot_check(void) { dead[nout] = r->mbuf, dead_a[nout] = r->malloc_; r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL; out[nout++] = r; + } plat_mutex_unlock(&pool_lock); for (int k = 0; k < nout; k++) { PoolRec *r = out[k]; @@ -3467,6 +3481,9 @@ static void hot_check(void) { if (++n_demoted <= 32) rt_log("mem: pool %#llx (%.1f MiB, %s) copied %.1f MiB a frame through its mirror: system memory from now on", (unsigned long long)r->lo, n / 1048576.0, f, per / 1048576.0); + } +} + void vk_mem_mirror_begin(void) { if (vk_mirror_on) hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions @@ -3516,28 +3533,51 @@ void vk_mem_mirror_begin(void) { int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; } -// NX2NAVI_MIRROR_CAPTURE_KB (default 0, off): the pages the CPU wrote since a VRAM mirror's last sync are copied out of -// guest memory by the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a -// host buffer the submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them -// from there into the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or -// more after the game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, -// values meant for a later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest -// memory is still on the GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over -// the mirror's newer ones), and when its device address table writes reach guest memory as well (a twin: what they -// write is not recorded); so does a range with a page that cannot be read without a fault (mem_copy_readable: a -// lazily armed range). The whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU. +// NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by +// the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the +// submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them from there into +// the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or more after the +// game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, values meant for a +// later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest memory is still on the +// GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over the mirror's newer ones), +// and when its device address table writes reach guest memory as well (a twin: what they write is not recorded); so +// does a range with a page that cannot be read without a fault (mem_copy_readable: a lazily armed range). The +// whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU. +// Unset or 0 with NX2NAVI_MIRROR_STAGE=1 (the default here): CAPTURE_STAGE_KB. A submission whose prologue copied +// everything this way reads nothing of guest memory late, so its submit wait ends once the replay has submitted it +// (vk_sync_recorded_wait); one with a range left to the GPU-time copy, or a draw that reads a pool without a mirror, +// is noted (vk_mem_note_live) and its submit wait is the GPU's. The first form of this (0.8.0) copied while the replay +// recorded and left out neither the write-backs in flight nor the twins. +#define CAPTURE_STAGE_KB (64 << 10) +static volatile int32_t mirror_live; // the submission being recorded had the GPU read guest memory + +static int stage_on(void) { + static int on = -1; + if (on < 0) on = rt_knob("MIRROR_STAGE", "1") != 0; + return on; +} + static uint64_t capture_cap(void) { static int64_t cap = -1; if (cap < 0) { int kb = cfg_int("NX2NAVI_MIRROR_CAPTURE_KB"); + if (kb <= 0 && stage_on()) kb = CAPTURE_STAGE_KB; cap = kb > 0 ? (int64_t)kb << 10 : 0; if (cap) rt_log("mem: mirror pages the CPU wrote are copied out of guest memory when their part is handed over, up to " - "%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB)", kb); + "%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB, NX2NAVI_MIRROR_STAGE)", kb); } return (uint64_t)cap; } +void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); } + +int vk_mem_take_live(void) { + int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED); + if (!vk_mirror_on || shadow_on || !capture_cap()) live = 1; + return live; +} + static MRange *capm; // replay: mirror_capture's merged ranges static uint64_t *caps; // replay: the chunk offset of each, UINT64_MAX for one left to the GPU-time copy static uint8_t **capd, *capok; // replay: mem_copy_readable's destinations and results @@ -3619,6 +3659,7 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); mirror_capture(cb); + if (npend) vk_mem_note_live(); // left to the GPU-time copy: the GPU reads guest memory late copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index dce3d334..abe93c1f 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -48,6 +48,10 @@ static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop static PlatCond cv_done = PLAT_COND_INIT; // done advanced static uint64_t issued, done; // newest submitted value; newest value whose side effects ran static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued) +// Staged mirror uploads (vk_memory.c "staged mirror uploads"): per submission value, whether its GPU still reads guest +// memory itself (value << 1 | live), under st_lock. A value not found counts as live. +#define LIVE_RING 4096 +static uint64_t live_of[LIVE_RING]; static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -509,6 +513,7 @@ static void gw_wait(uint64_t value, uint64_t t0, uint64_t t1, uint64_t newest, u static void gw_log(uint64_t frame, int frames) { plat_mutex_lock(&gw_lock); uint64_t n = gw_n, ns = gw_ns, rep = gw_rep_ns, gpu = gw_gpu_ns, lead = gw_lead_ns, age = gw_age, before = gw_before, + lr = gw_lat_rep, lg = gw_lat_gpu, ln = gw_lat_n, mx = gw_max_ns, unk = gw_unknown, rn = gw_rec_n, gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0; plat_mutex_unlock(&gw_lock); if (!n) { @@ -963,6 +968,7 @@ static VkCommandPool pre_pool; static CmdSlot *pre_slots; static int npre, pre_pair = -1; static VkCommandBuffer pre_cb; + VkCommandBuffer vk_sync_pre(void) { if (pre_cb) return pre_cb; VkCtx *c = vk_ctx(); @@ -1260,9 +1266,15 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { s->fence = sent ? fence : 0; vk_mem_note_submit(fence); gw_mark(fence, 1); + // after the last part's prologue: every staged copy of the submission is taken; a probe's submit leaves the note to + // the replay's submission it may have come in the middle of + int live = own ? 1 : vk_mem_take_live(); + int res = own ? 0x1f : vk_exec_take_results(); plat_mutex_lock(&st_lock); + live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0); __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); + plat_cond_broadcast(&cv_done); // vk_sync_recorded_wait plat_mutex_unlock(&st_lock); static unsigned subs; if (++subs % 4096 == 0) { @@ -1337,6 +1349,46 @@ static void results_run(uint64_t upto) { static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); + +// The game's wait after a frame's first submission (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): until the replay has +// submitted value, which took every copy of guest memory the submission reads; then also until the GPU has run it when +// its GPU reads guest memory itself (vk_mem_take_live) or NX2NAVI_SUBMIT_FINISH_GPU=1. 1 when reached. +int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)) { + vk_sync_start(); + if (gpu < 0) { + gpu = rt_knob("SUBMIT_FINISH_GPU", "0") != 0; + rt_log("vk: the wait after a frame's first submission ends %s (NX2NAVI_SUBMIT_FINISH_GPU=%d)", + gpu ? "when the GPU has run it" : "when the replay has submitted it, or the GPU has run it where it reads guest " + "memory itself", gpu); + } + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + if (gpu) return gpu_wait(value, timeout_ns); + uint64_t w0 = plat_now_ns(), newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE); + uint64_t sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE); + int live = 1, ok = 1; + plat_mutex_lock(&st_lock); + if (value > issued && value > reserved) ok = 0; // never handed over + while (ok && issued < value && done < value) { + if (timeout_ns < (1ull << 62)) { + uint64_t el = plat_now_ns() - w0; + if (el >= timeout_ns) { ok = 0; break; } + plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); + } else { + plat_cond_wait(&cv_done, &st_lock); + } + } + uint64_t w1 = plat_now_ns(); + if (w1 > w0) { + if (!live) gw_wait(value, w0, w1, newest > sent ? newest : sent, sent); // a GPU wait counts there + plat_mutex_lock(&gw_lock); + gw_rec_n++; + if (!live) gw_rec_done++; + plat_mutex_unlock(&gw_lock); + } + if (!ok) return 0; + return live ? gpu_wait(value, timeout_ns) : 1; +} + int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { uint64_t p0 = perf_begin(); uint64_t newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE), sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE); From d25c3a59d75358ca995da5c627ec066095ac3668 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:48 +0100 Subject: [PATCH 41/64] Submit wait: also until the newest submission up to it that hands the CPU results at its completion (events, counter reports, write-backs). Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 2 ++ runtime/vk_exec_xfer.c | 6 ++++++ runtime/vk_sync.c | 27 +++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/runtime/vk.h b/runtime/vk.h index 522df9b2..398737ab 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -351,6 +351,8 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb); // vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note. void vk_mem_note_live(void); int vk_mem_take_live(void); +// The submission being recorded hands results to the CPU at its completion (vk_exec_xfer.c): vk_sync_recorded_wait waits +// for the newest such submission up to the one it waits for. void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); void vk_mem_lazy_decide(void); // once the driver is known (after vk_caps_driver): on or off, with one log line diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 5e4c83f8..4b9cf391 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -62,6 +62,12 @@ static uint32_t max1(uint32_t v) { return v ? v : 1; } typedef struct { uint64_t addr, size; int back; } Range; // back: written into a VRAM mirror (vk_mem_mirrored) +// What the GPU work of the submission being recorded hands to the CPU at its completion without a fault to fetch it: +// event values, counter and occlusion reports, GPU writes of memory no VRAM mirror holds (vk_sync_recorded_wait waits +// for the newest such submission). Write-backs of the mirrors note it in vk_memory.c; the lazily written ones fetch on +// the CPU's first access. +int vk_exec_take_results(void) { return __atomic_exchange_n(&ex_results, 0, __ATOMIC_RELAXED); } + static void guest_written_cb(void *arg) { Range *r = arg; if (r->back) vk_gpu_written_back(r->addr, r->size); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index abe93c1f..cb209d30 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -52,6 +52,8 @@ static uint64_t reserved; // asynchronous replay: newest value handed ou // memory itself (value << 1 | live), under st_lock. A value not found counts as live. #define LIVE_RING 4096 static uint64_t live_of[LIVE_RING]; +// per value (index & LIVE_RING - 1): the newest value up to it whose completion hands results to the CPU (vk_exec_take_results) +static uint64_t res_of[LIVE_RING], res_last; static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -514,6 +516,7 @@ static void gw_log(uint64_t frame, int frames) { plat_mutex_lock(&gw_lock); uint64_t n = gw_n, ns = gw_ns, rep = gw_rep_ns, gpu = gw_gpu_ns, lead = gw_lead_ns, age = gw_age, before = gw_before, lr = gw_lat_rep, lg = gw_lat_gpu, ln = gw_lat_n, mx = gw_max_ns, unk = gw_unknown, rn = gw_rec_n, + rd = gw_rec_done, rs = gw_rec_self; gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0; plat_mutex_unlock(&gw_lock); if (!n) { @@ -529,6 +532,10 @@ static void gw_log(uint64_t frame, int frames) { (unsigned long long)frame, n / f, ns / 1e6 / f, mx / 1e6, rep / 1e6 / f, (double)before / (double)n, gpu / 1e6 / f, (unsigned long long)unk, (double)age / (double)n, lead / 1e6 / (double)n, ln ? lr / 1e6 / (double)ln : 0.0, ln ? lg / 1e6 / (double)ln : 0.0); + if (rn) rt_log("perf: frame %llu: submit waits: %llu, %llu of them without the GPU's run of the submission itself (%llu of " + "those waited for it anyway: it hands the CPU results), the rest went on to the GPU (the submission's GPU " + "read guest memory itself)", (unsigned long long)frame, (unsigned long long)rn, (unsigned long long)rd, + (unsigned long long)rs); } static int gtl_cmp(const void *a, const void *b) { @@ -1272,6 +1279,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { int res = own ? 0x1f : vk_exec_take_results(); plat_mutex_lock(&st_lock); live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0); + res_of[fence & (LIVE_RING - 1)] = res_last; __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); plat_cond_broadcast(&cv_done); // vk_sync_recorded_wait @@ -1355,7 +1363,12 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); // its GPU reads guest memory itself (vk_mem_take_live) or NX2NAVI_SUBMIT_FINISH_GPU=1. 1 when reached. int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)) { vk_sync_start(); + static int gpu = -1, res_mode = 1; if (gpu < 0) { + res_mode = rt_knob("SUBMIT_FINISH_RESULTS", "1"); + rt_log("vk: the wait after a frame's first submission also waits for %s (NX2NAVI_SUBMIT_FINISH_RESULTS=%d)", + res_mode == 2 ? "every submission before it" : res_mode ? "the newest submission up to it that hands the " + "CPU results at its completion (events, counter reports, write-backs)" : "nothing more", res_mode); gpu = rt_knob("SUBMIT_FINISH_GPU", "0") != 0; rt_log("vk: the wait after a frame's first submission ends %s (NX2NAVI_SUBMIT_FINISH_GPU=%d)", gpu ? "when the GPU has run it" : "when the replay has submitted it, or the GPU has run it where it reads guest " @@ -1377,12 +1390,26 @@ int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(u plat_cond_wait(&cv_done, &st_lock); } } + uint64_t need = 0; // the newest submission up to value whose completion hands the CPU results + if (ok) { + live = done < value && live_of[value & (LIVE_RING - 1)] != (value << 1); + if (res_mode == 2) need = value - 1; + uint64_t el = plat_now_ns() - w0; + if (el >= timeout_ns) { ok = 0; break; } + plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); + } else { + plat_cond_wait(&cv_done, &st_lock); + } + } + } + plat_mutex_unlock(&st_lock); uint64_t w1 = plat_now_ns(); if (w1 > w0) { if (!live) gw_wait(value, w0, w1, newest > sent ? newest : sent, sent); // a GPU wait counts there plat_mutex_lock(&gw_lock); gw_rec_n++; if (!live) gw_rec_done++; + if (!live && need == value) gw_rec_self++; plat_mutex_unlock(&gw_lock); } if (!ok) return 0; From 651073e4dbe95a6a6a91cf2f053236b8de1b592b Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:50 +0100 Subject: [PATCH 42/64] Submit wait: result kinds per submission, which ones the wait counts (NX2NAVI_SUBMIT_FINISH_KINDS) and which the waited submission had. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 3 +++ runtime/vk_exec_xfer.c | 5 +++++ runtime/vk_memory.c | 1 + runtime/vk_sync.c | 27 +++++++++++++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/runtime/vk.h b/runtime/vk.h index 398737ab..a4f25dc4 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -353,6 +353,9 @@ void vk_mem_note_live(void); int vk_mem_take_live(void); // The submission being recorded hands results to the CPU at its completion (vk_exec_xfer.c): vk_sync_recorded_wait waits // for the newest such submission up to the one it waits for. +enum { VK_RES_EVENT = 1, VK_RES_COUNTER = 2, VK_RES_OQ = 4, VK_RES_WRITE = 8, VK_RES_WRITEBACK = 16, VK_RES_KINDS = 5 }; +void vk_exec_note_results(int kind); // VK_RES_* +int vk_exec_take_results(void); // the kinds noted since the last take void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); void vk_mem_lazy_decide(void); // once the driver is known (after vk_caps_driver): on or off, with one log line diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 4b9cf391..292752f3 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -66,6 +66,7 @@ typedef struct { uint64_t addr, size; int back; } Range; // back: written into // event values, counter and occlusion reports, GPU writes of memory no VRAM mirror holds (vk_sync_recorded_wait waits // for the newest such submission). Write-backs of the mirrors note it in vk_memory.c; the lazily written ones fetch on // the CPU's first access. +static volatile int32_t ex_results; // VK_RES_* bits int vk_exec_take_results(void) { return __atomic_exchange_n(&ex_results, 0, __ATOMIC_RELAXED); } static void guest_written_cb(void *arg) { @@ -82,6 +83,7 @@ static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) { Range *r = malloc(sizeof *r); if (!r) rt_fatal("vk: out of memory"); *r = (Range){addr, size, vk_mem_mirrored(addr, size)}; + if (!r->back) vk_exec_note_results(VK_RES_WRITE); vk_mem_gpu_pending(addr, size); vk_completion_add(ex->fence, guest_written_cb, r); } @@ -520,6 +522,7 @@ static void oq_segment(VkExec *ex, uint64_t addr, int reset) { if (!s || (oq_nseg && !(s->q = malloc(sizeof *s->q * oq_nseg)))) rt_fatal("vk: out of memory"); s->cap = oq_nseg; s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; + if (addr) vk_exec_note_results(VK_RES_OQ); if (oq_nseg) memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); if (gs_on >= 0) { // the front pass: the workers add the queries of the group's ops before it __typeof__(GS[0]) *gs = &GS[gs_on]; @@ -734,6 +737,7 @@ static void signal_event(VkExec *ex, const p_signal_event *p) { ev_late++, ev_late_subs += vk_exec_submit_seq - ev_open[i].sub; ev_open[i] = ev_open[--ev_open_n]; } else i++; + vk_exec_note_results(VK_RES_EVENT); EventWrite *e = malloc(sizeof *e); if (!e) rt_fatal("vk: out of memory"); *e = (EventWrite){p->val, p->mode, p->value}; @@ -753,6 +757,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData)); if (samples) oq_report(ex, p->addr); else { + vk_exec_note_results(VK_RES_COUNTER); CounterWrite *a = malloc(sizeof *a); if (!a) rt_fatal("vk: out of memory"); *a = (CounterWrite){p->addr, report_seq}; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 34b62d60..ca75a463 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -3802,6 +3802,7 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { memcpy(f->c, chunks, sizeof(SnapChunk) * (size_t)nchunks); vk_completion_add(fence, snap_free, f); } + if (nsnl || nwbl) vk_exec_note_results(VK_RES_WRITEBACK); // write-backs: guest memory gets the GPU's bytes at the fence if (nwbl) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index cb209d30..bcd1d16c 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -54,6 +54,8 @@ static uint64_t reserved; // asynchronous replay: newest value handed ou static uint64_t live_of[LIVE_RING]; // per value (index & LIVE_RING - 1): the newest value up to it whose completion hands results to the CPU (vk_exec_take_results) static uint64_t res_of[LIVE_RING], res_last; +static uint8_t res_kinds[LIVE_RING]; // per value: its VK_RES_* kinds +static int res_mask = 0x1f; // NX2NAVI_SUBMIT_FINISH_KINDS: the kinds the wait counts static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -472,6 +474,8 @@ typedef struct { uint64_t v, res, iss, done; } GwSub; static GwSub gw[GW_RING]; static PlatMutex gw_lock = PLAT_MUTEX_INIT; static uint64_t gw_n, gw_ns, gw_rep_ns, gw_gpu_ns, gw_lead_ns, gw_age, gw_before, gw_lat_rep, gw_lat_gpu, gw_lat_n; +static uint64_t gw_max_ns, gw_unknown, gw_rec_n, gw_rec_done, gw_rec_self, gw_need_back, gw_need_n; +static uint64_t gw_need_kind[5], gw_skip_kind[5]; // VK_RES_KINDS // gw_rec: vk_sync_recorded_wait, those it ended static GwSub *gw_at(uint64_t v) { // gw_lock held GwSub *e = &gw[v & (GW_RING - 1)]; @@ -518,6 +522,10 @@ static void gw_log(uint64_t frame, int frames) { lr = gw_lat_rep, lg = gw_lat_gpu, ln = gw_lat_n, mx = gw_max_ns, unk = gw_unknown, rn = gw_rec_n, rd = gw_rec_done, rs = gw_rec_self; gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0; + uint64_t nb = gw_need_back, nn = gw_need_n, nk[5], sk[5]; + memcpy(nk, gw_need_kind, sizeof nk), memcpy(sk, gw_skip_kind, sizeof sk); + memset(gw_need_kind, 0, sizeof gw_need_kind), memset(gw_skip_kind, 0, sizeof gw_skip_kind); + gw_max_ns = gw_unknown = gw_rec_n = gw_rec_done = gw_rec_self = gw_need_back = gw_need_n = 0; plat_mutex_unlock(&gw_lock); if (!n) { rt_log("perf: frame %llu: waits for GPU work outside the replay: none", (unsigned long long)frame); @@ -536,6 +544,12 @@ static void gw_log(uint64_t frame, int frames) { "those waited for it anyway: it hands the CPU results), the rest went on to the GPU (the submission's GPU " "read guest memory itself)", (unsigned long long)frame, (unsigned long long)rn, (unsigned long long)rd, (unsigned long long)rs); + if (nn) rt_log("perf: frame %llu: the submission the submit wait waited for was %.2f submissions before the waited one; " + "its kinds: events %llu, counter reports %llu, occlusion reports %llu, unmirrored writes %llu, write-backs " + "%llu; the submissions after it (left out by the mask): %llu, %llu, %llu, %llu, %llu", + (unsigned long long)frame, (double)nb / (double)nn, (unsigned long long)nk[0], (unsigned long long)nk[1], + (unsigned long long)nk[2], (unsigned long long)nk[3], (unsigned long long)nk[4], (unsigned long long)sk[0], + (unsigned long long)sk[1], (unsigned long long)sk[2], (unsigned long long)sk[3], (unsigned long long)sk[4]); } static int gtl_cmp(const void *a, const void *b) { @@ -1279,6 +1293,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { int res = own ? 0x1f : vk_exec_take_results(); plat_mutex_lock(&st_lock); live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0); + res_kinds[fence & (LIVE_RING - 1)] = (uint8_t)res; + if (res & res_mask) res_last = fence; res_of[fence & (LIVE_RING - 1)] = res_last; __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); @@ -1366,6 +1382,8 @@ int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(u static int gpu = -1, res_mode = 1; if (gpu < 0) { res_mode = rt_knob("SUBMIT_FINISH_RESULTS", "1"); + rt_log("vk: result kinds the submit wait counts: %#x (NX2NAVI_SUBMIT_FINISH_KINDS: 1 events, 2 counter reports, " + "4 occlusion reports, 8 GPU writes of unmirrored memory, 16 mirror write-backs)", res_mask); rt_log("vk: the wait after a frame's first submission also waits for %s (NX2NAVI_SUBMIT_FINISH_RESULTS=%d)", res_mode == 2 ? "every submission before it" : res_mode ? "the newest submission up to it that hands the " "CPU results at its completion (events, counter reports, write-backs)" : "nothing more", res_mode); @@ -1410,6 +1428,15 @@ int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(u gw_rec_n++; if (!live) gw_rec_done++; if (!live && need == value) gw_rec_self++; + if (!live && need && need <= value) { + gw_need_back += value - need, gw_need_n++; + int k = res_kinds[need & (LIVE_RING - 1)]; + for (int b = 0; b < VK_RES_KINDS; b++) if (k >> b & 1) gw_need_kind[b]++; + if (need < value) { // what the submissions between need and value had that the mask leaves out + for (uint64_t v = need + 1; v <= value && v - need < 64; v++) + for (int b = 0; b < VK_RES_KINDS; b++) if (res_kinds[v & (LIVE_RING - 1)] >> b & 1) gw_skip_kind[b]++; + } + } plat_mutex_unlock(&gw_lock); } if (!ok) return 0; From 06be129b663b5dfdf0a86b875997467aa802539b Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:51 +0100 Subject: [PATCH 43/64] Submit wait: NX2NAVI_SUBMIT_FINISH_KINDS is read as a number. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_sync.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index bcd1d16c..80953490 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -1382,6 +1382,8 @@ int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(u static int gpu = -1, res_mode = 1; if (gpu < 0) { res_mode = rt_knob("SUBMIT_FINISH_RESULTS", "1"); + const char *km = cfg_get("NX2NAVI_SUBMIT_FINISH_KINDS"); // a number (rt_knob reads only 0, 1 and ab) + res_mask = (km && *km ? (int)strtol(km, NULL, 0) : 31) & 0x1f; rt_log("vk: result kinds the submit wait counts: %#x (NX2NAVI_SUBMIT_FINISH_KINDS: 1 events, 2 counter reports, " "4 occlusion reports, 8 GPU writes of unmirrored memory, 16 mirror write-backs)", res_mask); rt_log("vk: the wait after a frame's first submission also waits for %s (NX2NAVI_SUBMIT_FINISH_RESULTS=%d)", From e6ca9bc12933837231685f31674d1895a49d69c1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:53 +0100 Subject: [PATCH 44/64] Texture write-watch checks: a query widens to up to 8 regions on each side inside the texture's pool (NX2NAVI_SNAP_WIDEN). Co-Authored-By: Claude Opus 5.5 --- runtime/vk_memory.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ca75a463..ec28eadc 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -998,6 +998,43 @@ static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) { return calls; } +// NX2NAVI_SNAP_WIDEN= (default 8, 0 off): a texture check that has to query widens the query to up to this +// many 2 MiB regions on each side, inside the import pool that holds the texture, so the textures of one pool are +// snapped by one query a submission instead of one each. With freezing off (NX2NAVI_FREEZE_IDLE) an RX 580 in Hyrule +// Field made about 1,130 texture queries a frame, about 6 MiB each on average incl. repeats, 9.4 ms with the pools'. +static int snap_widen_n(void) { + static int n = -1; + if (n < 0) { + const char *e = cfg_get("NX2NAVI_SNAP_WIDEN"); + n = e && *e ? atoi(e) : 8; + if (n < 0) n = 0; + rt_log("mem: a texture check's write-watch query takes up to %d neighbouring 2 MiB regions of its pool on each " + "side (NX2NAVI_SNAP_WIDEN)", n); + } + return n; +} + +// r0..r1 widened inside the import pool that holds [lo, hi); pool_lock taken here (before gen_lock, as everywhere). +static void snap_widen(uint64_t lo, uint64_t hi, uint64_t *r0, uint64_t *r1) { + int w = snap_widen_n(); + if (!w) return; + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)lo, hi - lo); + const PoolRec *r = e ? e->buf : NULL; + if (r && r->kind == POOL_IMPORT && r->buf && r->import_hi > r->import_lo) { + uint64_t p0 = (r->import_lo + (1ull << SNAP_SHIFT) - 1) >> SNAP_SHIFT; // whole regions of the pool only + uint64_t p1 = r->import_hi >> SNAP_SHIFT; // [p0, p1) + if (p1 > p0) { + uint64_t a = *r0 > (uint64_t)w ? *r0 - (uint64_t)w : 0, b = *r1 + (uint64_t)w; + if (a < p0) a = p0; + if (b > p1 - 1) b = p1 - 1; + if (a < *r0) *r0 = a; + if (b > *r1 && b < SNAP_REGIONS) *r1 = b; + } + } + plat_mutex_unlock_shared(&pool_lock); +} + int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) { if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; @@ -1009,8 +1046,10 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e if (snap_at[r] != epoch) { excl = 1; break; } if (excl) { plat_mutex_unlock_shared(&gen_lock); + uint64_t w0 = r0, w1 = r1; + snap_widen(lo, hi, &w0, &w1); plat_mutex_lock(&gen_lock); - n_snap_calls += snap_regions(r0, r1, epoch); + n_snap_calls += snap_regions(w0, w1, epoch); } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; From 0383c392e8fb30f454282cbdea37ea345d2ce330 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:54 +0100 Subject: [PATCH 45/64] Thaws give back the runs their freeze recorded, and page calls take turns with the write-watch queries (vm_lock): a thaw waited about 16 ms per page call behind the queries. Co-Authored-By: Claude Opus 5.5 --- runtime/memory.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/runtime/memory.c b/runtime/memory.c index a1c9f7b4..9d868d4f 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -106,6 +106,12 @@ static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by m static int frz_async = 1; // the setting, read in mem_init typedef struct { uint8_t n; uint16_t run[16][2]; } FrzRuns; // pages of the region, [first, end) static FrzRuns *frz_runs[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock (written at a freeze): NULL or n 0 = not kept +// vm_lock, a leaf lock held around every write-watch query (written_pages) and around the page calls of a freeze and a +// thaw: on Windows each kind takes the process's address-space lock, and with both running at once a game thread's +// thaw waited about 16 ms per page call behind the replay's queries, 100-300 ms for a thaw of several runs (a frame of +// that length about every second standing in Hyrule Field on an RX 580). Taken in turns here, a thaw waits for one +// query at most. +static PlatMutex vm_lock = PLAT_MUTEX_INIT; static volatile int64_t frz_async_n; // thaws done outside the lock, since the start static volatile int64_t frz_protect_fails; // VirtualProtect failures of those thaws, since the start @@ -753,6 +759,12 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { static void thaw_pages_vm(uint64_t r, int strict); static void thaw_pages(uint64_t r, int strict) { + plat_mutex_lock(&vm_lock); + thaw_pages_vm(r, strict); + plat_mutex_unlock(&vm_lock); +} + +static void thaw_pages_vm(uint64_t r, int strict) { // vm_lock held uint64_t base = r << FRZ_SHIFT; const FrzRuns *k = frz_runs[r]; if (k && k->n) { @@ -948,19 +960,22 @@ void mem_thaw(uint64_t a, uint64_t n) { // reserved holes (heap pages given back) stay as they are, and thaw_locked finds the read-only runs again. static int freeze_locked(uint64_t r) { uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, run[16][2]; - int n = 0; - for (uint64_t x = a, y; x < end; x = y) { + int n = 0, ok = 1; + plat_mutex_lock(&vm_lock); + for (uint64_t x = a, y; x < end && ok; x = y) { int st = plat_mem_state(x, end, &y); if (st >= PLAT_PROT_NONE) { - if (st != PLAT_PROT_RW || n == 16) return 0; - run[n][0] = x, run[n][1] = y, n++; - } else if (st != PLAT_MEM_RESERVED) return 0; + if (st != PLAT_PROT_RW || n == 16) ok = 0; + else run[n][0] = x, run[n][1] = y, n++; + } else if (st != PLAT_MEM_RESERVED) ok = 0; } - for (int i = 0; i < n; i++) + for (int i = 0; ok && i < n; i++) if (!plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_READ)) { while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW); - return 0; + ok = 0; } + plat_mutex_unlock(&vm_lock); + if (!ok) return 0; // the runs, for a thaw without VirtualQuery (thaw_pages); kept per region from its first freeze on if (n && !frz_runs[r]) frz_runs[r] = calloc(1, sizeof *frz_runs[r]); if (frz_runs[r]) { @@ -1261,7 +1276,10 @@ int mem_start_note(int probe) { static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, int reset) { if (!max || !size) return 0; if (!watched(addr, size)) return UINT64_MAX; - return plat_mem_written(addr, size, pages, max, reset); + plat_mutex_lock(&vm_lock); + uint64_t n = plat_mem_written(addr, size, pages, max, reset); + plat_mutex_unlock(&vm_lock); + return n; } uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { From 8e5d555fbdf01a7a3ee005509224c5f5cf74ea64 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:56 +0100 Subject: [PATCH 46/64] NX2NAVI_SVC_WAITS: the guest main thread's time per kernel service and guest call chain, every 10 s. Co-Authored-By: Claude Opus 5.5 --- runtime/kernel.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/runtime/kernel.c b/runtime/kernel.c index 003e76e0..4243f01f 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -2372,7 +2372,62 @@ HOST_OVERRIDE(KERN_SYM_GET_CURRENT_FIBER, kern_fiber_current); // NX2NAVI_SVC_SLOW_MS (default 0, off): logs each kernel service that blocks longer than that, with the service number. // A stalled frame shows up as time in the replay; this says whether the guest was waiting in a service, and which one, // instead of leaving it to a sampler to find. +// NX2NAVI_SVC_WAITS=1 (diagnosis, default 0): the guest main thread's time in each kernel service, per service and the +// guest call chain that made it (the lr and up to three callers from the guest's frame records), logged every 10 s as +// milliseconds per second with the chain named (ldr_describe: the SDK's symbols). What the game's main thread waits for +// between frames: standing in Hyrule Field on an RX 580 it spent about a fifth of its time blocked in one service call. +#define SW_N 256 +#define SW_DEPTH 4 +static struct { uint64_t key, lr[SW_DEPTH], ns, n; uint32_t imm; } svw[SW_N]; +static uint64_t svw_t0; + +static void svc_wait_note(const Cpu *c, uint32_t imm, uint64_t ns) { + uint64_t lr[SW_DEPTH] = {c->x[30]}, fp = c->x[29], key = imm * 0x9E3779B97F4A7C15ull ^ c->x[30]; + for (int d = 1; d < SW_DEPTH && fp && !(fp & 7) && mem_readable(fp, 16); d++) { + uint64_t next = *(const volatile uint64_t *)(uintptr_t)fp, ret = *(const volatile uint64_t *)(uintptr_t)(fp + 8); + lr[d] = ret, key = (key ^ ret) * 0x100000001B3ull; + if (next <= fp) break; + fp = next; + } + if (!key) key = 1; + for (uint32_t k = 0; k < 32; k++) { + typeof(svw[0]) *e = &svw[(key + k) % SW_N]; + if (e->key && e->key != key) continue; + if (!e->key) e->key = key, e->imm = imm, memcpy(e->lr, lr, sizeof lr); + e->ns += ns, e->n++; + break; + } + uint64_t now = kern_now_ns(); + if (!svw_t0) svw_t0 = now; + if (now - svw_t0 < 10000000000ull) return; + double secs = (now - svw_t0) / 1e9; + svw_t0 = now; + for (int top = 0; top < 6; top++) { + int best = -1; + for (int i = 0; i < SW_N; i++) + if (svw[i].key && svw[i].ns && (best < 0 || svw[i].ns > svw[best].ns)) best = i; + if (best < 0) break; + char d[SW_DEPTH][96]; + for (int k = 0; k < SW_DEPTH; k++) { + if (svw[best].lr[k]) ldr_describe(svw[best].lr[k], d[k], sizeof d[k]); + else d[k][0] = 0; + } + rt_log("kernel: main thread in service %#x: %.1f ms/s over %llu calls (%.2f ms each) from %s < %s < %s < %s", + svw[best].imm, svw[best].ns / 1e6 / secs, (unsigned long long)svw[best].n, svw[best].ns / 1e6 / (double)svw[best].n, + d[0], d[1], d[2], d[3]); + svw[best].ns = 0; // listed: out of this round + } + memset(svw, 0, sizeof svw); +} + int kern_svc(Cpu *c, uint32_t imm) { + static int waits = -1; + if (waits && plat_thread_id() == prof_main_tid) { + uint64_t w0 = kern_now_ns(); + int r = kern_svc_inner(c, imm); + svc_wait_note(c, imm, kern_now_ns() - w0); + return r; + } static double lim = -1; if (lim < 0) lim = strtod(cfg_value("NX2NAVI_SVC_SLOW_MS"), NULL); if (lim <= 0) return kern_svc_inner(c, imm); From 0b4fdac38dc6a92c5322b4ed922cecbf453a261d Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:57 +0100 Subject: [PATCH 47/64] Defaults from the RX 580 runs: asynchronous replay, mesh emulation, mirror_hot 25, freeze_idle 1200, native zstd, game code at -O2; freeze and zstd and replay rows in the Lama patches tab. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 5 ++++- runtime/ui/menu.cpp | 7 +++++++ runtime/ui_text.c | 11 +++++++++++ runtime/ui_text.h | 2 ++ runtime/vk_exec_draw.c | 1 + runtime/vk_memory.c | 2 ++ runtime/vk_pipeline.c | 1 + tools/make_game.py | 1 + 8 files changed, 29 insertions(+), 1 deletion(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index 6add4819..bed3d8e0 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -91,6 +91,7 @@ static const char *const known[] = { // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). static const struct { const char *key, *section; } sections[] = { // the menu's Lama patches tab (ui/menu.cpp tab_lama) + {"ZSTD_NATIVE", "performance"}, {"MESH_EMU_MB", "performance"}, {"MESH_EMU_TASKS", "performance"}, {"MESH_EMU_BATCH", "performance"}, {"VEC_INDEX_FIX", "performance"}, {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, {"FPS_LIMIT", "frame rate"}, {"FPS60", "frame rate"}, {"RTSS", "frame rate"}, @@ -254,7 +255,7 @@ static const CfgMeta metas[] = { // offer's number, which this setting or the menu accepts), always (every room, the save still copied first), or a number. {"COOP_ACCEPT", "ask", NULL, LIVE}, // Advanced: scheduling - {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, + {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "1", "0|1", RESTART}, {"REPLAY_WORKERS", "3", "0..16", RESTART}, {"REPLAY_PIPE", "0", "0..4", RESTART}, {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds @@ -303,7 +304,9 @@ static const CfgMeta metas[] = { {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit + {"MESH_EMULATION", "1", "0|1", RESTART}, {"MESH_EMU_MB", "256", "32..2048", RESTART}, {"MESH_EMU_TASKS", "16384", "256..262144", RESTART}, {"MESH_EMU_BATCH", "1", "0|1", RESTART}, {"VEC_INDEX_FIX", "0", "0|auto|1", RESTART}, + {"GUEST_OPT", "2", "1|2", RESTART}, // tools/make_game.py: the next Rebuild the Game {"PASS_TIMES", "0", "0|1", RESTART}, {"DUMP_KEY", "", NULL, RESTART}, // vk_crumbs.c, vk_dump.c // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 49dd4c48..0341e6d7 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1633,6 +1633,7 @@ static void tab_lama(float scale) { static const char *const values[] = {"", "512", "1024", "2048", "4096", "8192"}; const char *names[] = {menu_text(UI_MENU_AUTO), "512", "1024", "2048", "4096", "8192"}; opt_choice_list("NX2NAVI_TEX_RELEASED_MAX", UI_MENU_LAMA_RELEASED, values, names, 6, UI_MENU_LAMA_RELEASED_DESC); + } opt_slider("NX2NAVI_MIRROR_HOT", UI_MENU_LAMA_MIRROR_HOT, 0, 100, UI_MENU_LAMA_MIRROR_HOT_DESC, 5, "%.0f %%"); { // vk_device.c vram_heap_limits static const char *const values[] = {"auto", "0"}; @@ -1643,7 +1644,13 @@ static void tab_lama(float scale) { const char *names[] = {"128 MB", "256 MB", "512 MB", "1024 MB", menu_text(UI_MENU_LAMA_NO_LIMIT)}; opt_choice_list("NX2NAVI_UPLOAD_BATCH_MB", UI_MENU_LAMA_UPLOAD_BATCH, values, names, 5, UI_MENU_LAMA_UPLOAD_BATCH_DESC); opt_toggle("NX2NAVI_ASTC_SCRATCH_KEEP", UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC); + { // vk_memory.c freeze_pass: submissions a region stays unwritten before it is frozen + static const char *const values[] = {"1800", "1200", "600", "60", "0"}; + const char *names[] = {"1800", "1200", "600", "60", menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_FREEZE_IDLE", UI_MENU_LAMA_FREEZE, values, names, 5, UI_MENU_LAMA_FREEZE_DESC); + opt_toggle("NX2NAVI_ZSTD_NATIVE", UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC); // ov_zstd.c trusted opt_group(UI_MENU_LAMA_RENDER); + opt_toggle("NX2NAVI_REPLAY_ASYNC", UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC); // vk_backend.c const bool native = vk_mesh_shader_native() != 0; // vk_exec_draw.c: the emulation only where the GPU lacks them if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_toggle("NX2NAVI_MESH_EMULATION", UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index a497bcda..c7729e2d 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -139,6 +139,17 @@ static const char *const text_en[UI_TEXT_COUNT] = { "of -O1: the game's own threads run faster on a slower CPU. The build takes " "several times longer.", [UI_MENU_LAMA_GUEST_OPT_NOTE] = "at the next rebuild", + [UI_MENU_LAMA_FREEZE] = "Freeze idle memory after", + [UI_MENU_LAMA_FREEZE_DESC] = "Submissions a memory region stays unwritten before it is made read-only, so the write " + "tracking can skip it. 1800 (about 15 s): few thaws and small tracking queries. 60: " + "the game thaws regions all the time and every thaw stalls a frame. Off: no stalls, but " + "the tracking scans all memory every frame.", + [UI_MENU_LAMA_ZSTD] = "Native data decompression", + [UI_MENU_LAMA_ZSTD_DESC] = "Decompresses the game's zstd data with the PC's libzstd instead of the game's own " + "decoder translated from the console. Lighter streaming while you move.", + [UI_MENU_LAMA_ASYNC] = "Asynchronous replay", + [UI_MENU_LAMA_ASYNC_DESC] = "Translates the game's GPU commands on a thread of their own instead of the game's, so " + "the game prepares the next frame meanwhile.", [UI_MENU_LAMA_PASS_TIMES] = "Pass times in the log", [UI_MENU_LAMA_PASS_TIMES_DESC] = "GPU time of every render pass and dispatch goes to the log, with the largest " "viewport each kind of pass draws with. Costs a little frame time.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 6ef06b5c..8875d9f7 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -335,6 +335,8 @@ typedef enum { // the Lama patches tab (ui/menu.cpp tab_lama): the settings of the RX 580 patches, English only UI_MENU_TAB_LAMA, UI_MENU_LAMA_INTRO, UI_MENU_LAMA_MEMORY, UI_MENU_LAMA_RENDER, UI_MENU_LAMA_BUILD, UI_MENU_LAMA_DIAG, + UI_MENU_LAMA_FREEZE, UI_MENU_LAMA_FREEZE_DESC, UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC, + UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC, UI_MENU_LAMA_RELEASED, UI_MENU_LAMA_RELEASED_DESC, UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC, UI_MENU_LAMA_MIRROR_HOT, UI_MENU_LAMA_MIRROR_HOT_DESC, diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 80b75de4..e5b6509f 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5593,6 +5593,7 @@ static int mesh_emu_draw_prim(VkExec *ex) { static int mesh_emu_on(void) { static int on = -1; if (on < 0) { + on = !vk_ctx()->has_mesh_shader && rt_knob("MESH_EMULATION", "1"); if (on) rt_log("vk: mesh shaders emulated with compute (NX2NAVI_MESH_EMULATION)"); } return on; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index ec28eadc..e278f085 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -3471,6 +3471,8 @@ static uint64_t hot_frame0, n_demoted; static void hot_check(void) { if (mirror_hot < 0) { + const char *v = cfg_value("NX2NAVI_MIRROR_HOT"); // the settings row's default (25) when unset + int p = v && *v ? atoi(v) : 25; mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p; if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot); } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 887a5134..92bb5445 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2839,6 +2839,7 @@ static Layout *layout_get(Bindings *B) { // set 0 gets a template per bind point. Caller holds P.lock. static int mesh_emu_on(void) { static int on = -1; + if (on < 0) on = rt_knob("MESH_EMULATION", "1"); return on; } diff --git a/tools/make_game.py b/tools/make_game.py index 0f5cbe26..8d95d4bb 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1448,6 +1448,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): and its command, which is part of its objects' keys, stays as it was. NX2NAVI_GUEST_OPT (an environment variable, or guest_opt in nx2navi.ini) 1 builds the lifted code at -O1; by default -O2: the game's own threads run it and bound the frame on a slower CPU; the build takes several times longer.""" + guest_opt = '-O1' if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '1' else '-O2' # -O2 unless guest_opt = 1 def cmd_for(src, dst): cpp = src.suffix in ('.cpp', '.cxx') # the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict From 7041b2fb6aa19477caab05cbb4c395916598116b Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:59 +0100 Subject: [PATCH 48/64] NX2NAVI_SVC_WAITS: read only when set; unset, cfg_bool ended the game (the key has no settings row). Co-Authored-By: Claude Opus 5.5 --- runtime/kernel.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/kernel.c b/runtime/kernel.c index 4243f01f..990813de 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -2422,6 +2422,10 @@ static void svc_wait_note(const Cpu *c, uint32_t imm, uint64_t ns) { int kern_svc(Cpu *c, uint32_t imm) { static int waits = -1; + if (waits < 0) { // no settings row: cfg_bool only on a set value (an unset key is fatal there) + const char *v = cfg_get("NX2NAVI_SVC_WAITS"); + waits = v && *v && cfg_bool("NX2NAVI_SVC_WAITS"); + } if (waits && plat_thread_id() == prof_main_tid) { uint64_t w0 = kern_now_ns(); int r = kern_svc_inner(c, imm); From f35e8d1eb2baa736d74f41d23e892b91d6c81a68 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:00 +0100 Subject: [PATCH 49/64] Recorder thread on for AMD GPUs without mesh shaders (GCN, RDNA 1): the reported device losses were all on RDNA 2-4; an RX 580 ran clean with NX2NAVI_REC_CHECK. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_rec.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 97098330..e9fea21e 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -166,10 +166,15 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence) { // past it). Keeping objects until their submission completed and ending a part from the recorder // (rec_end_cb) fixed the white scene on an RX 6700 XT, but not this: 0.2.8 turned the recorder on for every // GPU and the AMD reports came back the same day. NX2NAVI_VK_RECORDER=1 turns it on for an A/B run. + // On AMD GPUs without mesh shaders (GCN: RX 400/500, Vega; RDNA 1) it is on: every report above is from an + // RDNA 2-4 card, and those all have mesh shaders. An RX 580 ran with it and NX2NAVI_REC_CHECK + // (no hazards), the replay's part of the game's wait after a frame's first submission 6.9 ms instead of 12. const char *set = cfg_get("NX2NAVI_VK_RECORDER"); - mode = rt_knob("VK_RECORDER", vk_caps.amd ? "0" : "1"); - if (!mode && vk_caps.amd && !(set && *set)) - rt_log("vk_rec: no recorder thread on AMD GPUs (NX2NAVI_VK_RECORDER=1 turns it on)"); + int amd_off = vk_caps.amd && vk_ctx()->has_mesh_shader; + mode = rt_knob("VK_RECORDER", amd_off ? "0" : "1"); + if (!mode && amd_off && !(set && *set)) + rt_log("vk_rec: no recorder thread on AMD GPUs with mesh shaders (RDNA 2 and newer; NX2NAVI_VK_RECORDER=1 " + "turns it on)"); if (mode) { plat_event_init(&wake_ev, 0, 0); rec_prio = plat_thread_priority_get(); From ff1c1ed87a495d406279cf170e26468cfe8606b4 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:02 +0100 Subject: [PATCH 50/64] Mesh emulation: the stages of a pass's emulated draws recorded by stage, with one barrier between stages for all of them. Each draw's indices are now a range of its own and its task records too, reserved by an allocation shader (one invocation, mesh_emu_alloc_spirv) for the run of draw records queued: before the task stages the task records and the index ranges of draws without a task stage, after them the index ranges of the others (mesh workgroups with output, which the task stage now counts in the record, times OutputPrimitives x vertices per primitive). The mesh stage takes its indices from the draw's range with an atomic in its record; vertex slots stay the arena's, as the indices name them. So no draw's stages need another's, and the queue (MEQ) records the fills as the draws come, then at the pre-pass command buffer's end (vk_sync.c pre_end -> vk_exec_mesh_flush): allocation, every task stage, allocation, every mesh stage, a barrier between each. One by one each draw took three barriers and copied the arena's counters after the draw before it: an RX 580 spent 5.5 ms of a frame's GPU time on 69 draws' stages in Kokiri Forest. Co-Authored-By: Claude Opus 5.5 --- runtime/mesh_emu_spirv.c | 142 +++++++++++++++++++++++++++++++++- runtime/mesh_emu_spirv.h | 19 +++++ runtime/vk.h | 3 + runtime/vk_exec_draw.c | 160 +++++++++++++++++++++++++++++++++++++++ runtime/vk_sync.c | 3 + 5 files changed, 326 insertions(+), 1 deletion(-) diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c index 280b1f2e..114d8292 100644 --- a/runtime/mesh_emu_spirv.c +++ b/runtime/mesh_emu_spirv.c @@ -254,8 +254,13 @@ static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int uint32_t mz; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); { + // indices from the draw's own range (reserved by the allocation shader), vertex slots from the arena's + uint32_t drw = pc_u32(e, 1), vcap = pc_u32(e, 5); + uint32_t icap = word_load(e, base, add(e, drw, cu(e, MESH_EMU_R_ICAP))); + uint32_t ib = atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_ICUR)), isz); uint32_t vb = atomic(e, OpAtomicIAdd, base, cu(e, 0), vsz); uint32_t iok = E2(e, OpULessThanEqual, m->t_bool, add(e, ib, isz), icap); + ib = add(e, word_load(e, base, add(e, drw, cu(e, 2))), ib); // relative to the area, as firstIndex uint32_t vok = E2(e, OpULessThanEqual, m->t_bool, add(e, vb, vsz), vcap); uint32_t mi; if_begin(e, iok, &mi); @@ -358,6 +363,7 @@ static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint uint32_t m2; if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2); word_store(e, base, recw, nc); + atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_GROUPS)), nc); // for the index range if_end(e, m2); } if_end(e, m1); @@ -954,4 +960,138 @@ uint32_t *mesh_emu_vs(const MeshEmuInfo *mi, uint32_t need, const uint8_t types[ free(t.w), free(an.w), free(c.w); if (bad) { free(b.w); return NULL; } *out_n = b.n; - return b.w; \ No newline at end of file + return b.w; +#undef N +} + +// The allocation shader (mesh_emu_spirv.h): glslangValidator -V -g0 --target-env vulkan1.2, spirv-opt -O --strip-debug, +// spirv-val clean. Its source: +// #version 460 +// #extension GL_EXT_buffer_reference : require +// #extension GL_EXT_buffer_reference_uvec2 : require +// layout(local_size_x = 1) in; +// layout(buffer_reference, std430, buffer_reference_align = 4) buffer Words { uint w[]; }; +// layout(push_constant) uniform PC { uvec2 addr; uint first; uint n; uint mode; uint icap; uint trecs; uint pad; } pc; +// void main() { +// Words a = Words(pc.addr); +// for (uint d = 0u; d < pc.n; d++) { +// uint r = 64u + ((pc.first + d) % 4096u) * 16u; +// bool task = a.w[r + 14u] != 0u; +// if (pc.mode == 0u && task) { +// uint tb = a.w[2], k = a.w[r + 10u]; +// a.w[r + 8u] = tb; +// a.w[2] = k > pc.trecs ? pc.trecs + tb : tb + k; +// } else if ((pc.mode == 0u) != task) { +// uint g = pc.mode == 0u ? a.w[r + 10u] : a.w[r + 9u], per = a.w[r + 13u], ib = a.w[1]; +// uint ni = per != 0u && g > pc.icap / per ? pc.icap + 1u : g * per; +// a.w[r + 2u] = ib; +// a.w[r + 12u] = ib >= pc.icap ? 0u : min(ni, pc.icap - ib); +// a.w[1] = min(ib + ni, 0x7fffffffu); +// } +// } +// } +static const uint32_t alloc_words[] = { + 0x07230203, 0x00010500, 0x0008000b, 0x000000df, 0x00000000, 0x00020011, 0x00000001, 0x00020011, + 0x000014e3, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, + 0x000014e4, 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000f, + 0x00060010, 0x00000004, 0x00000011, 0x00000001, 0x00000001, 0x00000001, 0x00040047, 0x00000008, + 0x00000006, 0x00000004, 0x00030047, 0x00000009, 0x00000002, 0x00050048, 0x00000009, 0x00000000, + 0x00000023, 0x00000000, 0x00030047, 0x0000000d, 0x00000002, 0x00050048, 0x0000000d, 0x00000000, + 0x00000023, 0x00000000, 0x00050048, 0x0000000d, 0x00000001, 0x00000023, 0x00000008, 0x00050048, + 0x0000000d, 0x00000002, 0x00000023, 0x0000000c, 0x00050048, 0x0000000d, 0x00000003, 0x00000023, + 0x00000010, 0x00050048, 0x0000000d, 0x00000004, 0x00000023, 0x00000014, 0x00050048, 0x0000000d, + 0x00000005, 0x00000023, 0x00000018, 0x00050048, 0x0000000d, 0x00000006, 0x00000023, 0x0000001c, + 0x00040047, 0x000000cb, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, + 0x00000002, 0x00030027, 0x00000006, 0x000014e5, 0x00040015, 0x00000007, 0x00000020, 0x00000000, + 0x0003001d, 0x00000008, 0x00000007, 0x0003001e, 0x00000009, 0x00000008, 0x00040020, 0x00000006, + 0x000014e5, 0x00000009, 0x00040017, 0x0000000c, 0x00000007, 0x00000002, 0x0009001e, 0x0000000d, + 0x0000000c, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00040020, + 0x0000000e, 0x00000009, 0x0000000d, 0x0004003b, 0x0000000e, 0x0000000f, 0x00000009, 0x00040015, + 0x00000010, 0x00000020, 0x00000001, 0x0004002b, 0x00000010, 0x00000011, 0x00000000, 0x00040020, + 0x00000012, 0x00000009, 0x0000000c, 0x0004002b, 0x00000007, 0x00000018, 0x00000000, 0x0004002b, + 0x00000010, 0x0000001f, 0x00000002, 0x00040020, 0x00000020, 0x00000009, 0x00000007, 0x00020014, + 0x00000023, 0x0004002b, 0x00000010, 0x00000027, 0x00000001, 0x0004002b, 0x00000007, 0x0000002c, + 0x00001000, 0x0004002b, 0x00000007, 0x0000002e, 0x00000010, 0x00040020, 0x00000037, 0x000014e5, + 0x00000007, 0x0004002b, 0x00000010, 0x0000003b, 0x00000003, 0x0004002b, 0x00000010, 0x00000056, + 0x00000005, 0x0004002b, 0x00000010, 0x00000094, 0x00000004, 0x0004002b, 0x00000007, 0x000000a0, + 0x00000001, 0x0004002b, 0x00000007, 0x000000c5, 0x7fffffff, 0x00040017, 0x000000ca, 0x00000007, + 0x00000003, 0x0006002c, 0x000000ca, 0x000000cb, 0x000000a0, 0x000000a0, 0x000000a0, 0x0004002b, + 0x00000007, 0x000000d8, 0x0000004e, 0x0004002b, 0x00000007, 0x000000d9, 0x00000049, 0x0004002b, + 0x00000007, 0x000000da, 0x0000004a, 0x0004002b, 0x00000007, 0x000000db, 0x0000004d, 0x0004002b, + 0x00000007, 0x000000dc, 0x00000042, 0x0004002b, 0x00000007, 0x000000dd, 0x0000004c, 0x0004002b, + 0x00000007, 0x000000de, 0x00000048, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, + 0x000200f8, 0x00000005, 0x00050041, 0x00000012, 0x00000013, 0x0000000f, 0x00000011, 0x0004003d, + 0x0000000c, 0x00000014, 0x00000013, 0x0004007c, 0x00000006, 0x00000015, 0x00000014, 0x000200f9, + 0x00000019, 0x000200f8, 0x00000019, 0x000700f5, 0x00000007, 0x000000cc, 0x00000018, 0x00000005, + 0x000000c9, 0x0000001c, 0x00050041, 0x00000020, 0x00000021, 0x0000000f, 0x0000001f, 0x0004003d, + 0x00000007, 0x00000022, 0x00000021, 0x000500b0, 0x00000023, 0x00000024, 0x000000cc, 0x00000022, + 0x000400f6, 0x0000001b, 0x0000001c, 0x00000000, 0x000400fa, 0x00000024, 0x0000001a, 0x0000001b, + 0x000200f8, 0x0000001a, 0x00050041, 0x00000020, 0x00000028, 0x0000000f, 0x00000027, 0x0004003d, + 0x00000007, 0x00000029, 0x00000028, 0x00050080, 0x00000007, 0x0000002b, 0x00000029, 0x000000cc, + 0x00050089, 0x00000007, 0x0000002d, 0x0000002b, 0x0000002c, 0x00050084, 0x00000007, 0x0000002f, + 0x0000002d, 0x0000002e, 0x00050080, 0x00000007, 0x00000036, 0x0000002f, 0x000000d8, 0x00060041, + 0x00000037, 0x00000038, 0x00000015, 0x00000011, 0x00000036, 0x0006003d, 0x00000007, 0x00000039, + 0x00000038, 0x00000002, 0x00000004, 0x000500ab, 0x00000023, 0x0000003a, 0x00000039, 0x00000018, + 0x00050041, 0x00000020, 0x0000003c, 0x0000000f, 0x0000003b, 0x0004003d, 0x00000007, 0x0000003d, + 0x0000003c, 0x000500aa, 0x00000023, 0x0000003e, 0x0000003d, 0x00000018, 0x000500a7, 0x00000023, + 0x00000040, 0x0000003e, 0x0000003a, 0x000300f7, 0x00000042, 0x00000000, 0x000400fa, 0x00000040, + 0x00000041, 0x00000067, 0x000200f8, 0x00000041, 0x00060041, 0x00000037, 0x00000045, 0x00000015, + 0x00000011, 0x0000001f, 0x0006003d, 0x00000007, 0x00000046, 0x00000045, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x0000004b, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, 0x0000004c, + 0x00000015, 0x00000011, 0x0000004b, 0x0006003d, 0x00000007, 0x0000004d, 0x0000004c, 0x00000002, + 0x00000004, 0x00050080, 0x00000007, 0x00000051, 0x0000002f, 0x000000de, 0x00060041, 0x00000037, + 0x00000053, 0x00000015, 0x00000011, 0x00000051, 0x0005003e, 0x00000053, 0x00000046, 0x00000002, + 0x00000004, 0x00050041, 0x00000020, 0x00000057, 0x0000000f, 0x00000056, 0x0004003d, 0x00000007, + 0x00000058, 0x00000057, 0x000500ac, 0x00000023, 0x00000059, 0x0000004d, 0x00000058, 0x000300f7, + 0x0000005c, 0x00000000, 0x000400fa, 0x00000059, 0x0000005b, 0x00000061, 0x000200f8, 0x0000005b, + 0x00050080, 0x00000007, 0x00000060, 0x00000058, 0x00000046, 0x000200f9, 0x0000005c, 0x000200f8, + 0x00000061, 0x00050080, 0x00000007, 0x00000064, 0x00000046, 0x0000004d, 0x000200f9, 0x0000005c, + 0x000200f8, 0x0000005c, 0x000700f5, 0x00000007, 0x000000d0, 0x00000060, 0x0000005b, 0x00000064, + 0x00000061, 0x0005003e, 0x00000045, 0x000000d0, 0x00000002, 0x00000004, 0x000200f9, 0x00000042, + 0x000200f8, 0x00000067, 0x000500a5, 0x00000023, 0x0000006c, 0x0000003e, 0x0000003a, 0x000300f7, + 0x0000006e, 0x00000000, 0x000400fa, 0x0000006c, 0x0000006d, 0x0000006e, 0x000200f8, 0x0000006d, + 0x000300f7, 0x00000075, 0x00000000, 0x000400fa, 0x0000003e, 0x00000074, 0x0000007b, 0x000200f8, + 0x00000074, 0x00050080, 0x00000007, 0x00000078, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, + 0x00000079, 0x00000015, 0x00000011, 0x00000078, 0x0006003d, 0x00000007, 0x0000007a, 0x00000079, + 0x00000002, 0x00000004, 0x000200f9, 0x00000075, 0x000200f8, 0x0000007b, 0x00050080, 0x00000007, + 0x0000007f, 0x0000002f, 0x000000d9, 0x00060041, 0x00000037, 0x00000080, 0x00000015, 0x00000011, + 0x0000007f, 0x0006003d, 0x00000007, 0x00000081, 0x00000080, 0x00000002, 0x00000004, 0x000200f9, + 0x00000075, 0x000200f8, 0x00000075, 0x000700f5, 0x00000007, 0x000000cd, 0x0000007a, 0x00000074, + 0x00000081, 0x0000007b, 0x00050080, 0x00000007, 0x00000087, 0x0000002f, 0x000000db, 0x00060041, + 0x00000037, 0x00000088, 0x00000015, 0x00000011, 0x00000087, 0x0006003d, 0x00000007, 0x00000089, + 0x00000088, 0x00000002, 0x00000004, 0x00060041, 0x00000037, 0x0000008c, 0x00000015, 0x00000011, + 0x00000027, 0x0006003d, 0x00000007, 0x0000008d, 0x0000008c, 0x00000002, 0x00000004, 0x000500ab, + 0x00000023, 0x00000090, 0x00000089, 0x00000018, 0x000300f7, 0x00000092, 0x00000000, 0x000400fa, + 0x00000090, 0x00000091, 0x00000092, 0x000200f8, 0x00000091, 0x00050041, 0x00000020, 0x00000095, + 0x0000000f, 0x00000094, 0x0004003d, 0x00000007, 0x00000096, 0x00000095, 0x00050086, 0x00000007, + 0x00000098, 0x00000096, 0x00000089, 0x000500ac, 0x00000023, 0x00000099, 0x000000cd, 0x00000098, + 0x000200f9, 0x00000092, 0x000200f8, 0x00000092, 0x000700f5, 0x00000023, 0x0000009a, 0x00000090, + 0x00000075, 0x00000099, 0x00000091, 0x000300f7, 0x0000009d, 0x00000000, 0x000400fa, 0x0000009a, + 0x0000009c, 0x000000a2, 0x000200f8, 0x0000009c, 0x00050041, 0x00000020, 0x0000009e, 0x0000000f, + 0x00000094, 0x0004003d, 0x00000007, 0x0000009f, 0x0000009e, 0x00050080, 0x00000007, 0x000000a1, + 0x0000009f, 0x000000a0, 0x000200f9, 0x0000009d, 0x000200f8, 0x000000a2, 0x00050084, 0x00000007, + 0x000000a5, 0x000000cd, 0x00000089, 0x000200f9, 0x0000009d, 0x000200f8, 0x0000009d, 0x000700f5, + 0x00000007, 0x000000ce, 0x000000a1, 0x0000009c, 0x000000a5, 0x000000a2, 0x00050080, 0x00000007, + 0x000000aa, 0x0000002f, 0x000000dc, 0x00060041, 0x00000037, 0x000000ac, 0x00000015, 0x00000011, + 0x000000aa, 0x0005003e, 0x000000ac, 0x0000008d, 0x00000002, 0x00000004, 0x00050080, 0x00000007, + 0x000000b0, 0x0000002f, 0x000000dd, 0x00050041, 0x00000020, 0x000000b2, 0x0000000f, 0x00000094, + 0x0004003d, 0x00000007, 0x000000b3, 0x000000b2, 0x000500ae, 0x00000023, 0x000000b4, 0x0000008d, + 0x000000b3, 0x000300f7, 0x000000b7, 0x00000000, 0x000400fa, 0x000000b4, 0x000000b6, 0x000000b8, + 0x000200f8, 0x000000b6, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b8, 0x00050082, 0x00000007, + 0x000000bd, 0x000000b3, 0x0000008d, 0x0007000c, 0x00000007, 0x000000be, 0x00000001, 0x00000026, + 0x000000ce, 0x000000bd, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b7, 0x000700f5, 0x00000007, + 0x000000cf, 0x00000018, 0x000000b6, 0x000000be, 0x000000b8, 0x00060041, 0x00000037, 0x000000c0, + 0x00000015, 0x00000011, 0x000000b0, 0x0005003e, 0x000000c0, 0x000000cf, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x000000c4, 0x0000008d, 0x000000ce, 0x0007000c, 0x00000007, 0x000000c6, + 0x00000001, 0x00000026, 0x000000c4, 0x000000c5, 0x0005003e, 0x0000008c, 0x000000c6, 0x00000002, + 0x00000004, 0x000200f9, 0x0000006e, 0x000200f8, 0x0000006e, 0x000200f9, 0x00000042, 0x000200f8, + 0x00000042, 0x000200f9, 0x0000001c, 0x000200f8, 0x0000001c, 0x00050080, 0x00000007, 0x000000c9, + 0x000000cc, 0x00000027, 0x000200f9, 0x00000019, 0x000200f8, 0x0000001b, 0x000100fd, 0x00010038, +}; +const uint32_t *mesh_emu_alloc_spirv(size_t *n) { + *n = sizeof alloc_words / sizeof *alloc_words; + return alloc_words; +} +_Static_assert(MESH_EMU_DOFF / 4 == 64 && MESH_EMU_DRECS == 4096 && MESH_EMU_DWORDS == 16 && MESH_EMU_R_GROUPS == 9 && + MESH_EMU_R_FIRST == 10 && MESH_EMU_R_ICAP == 12 && MESH_EMU_R_PER == 13 && MESH_EMU_R_TASK == 14, + "the allocation shader's constants"); diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h index 430ca76a..0584e3ae 100644 --- a/runtime/mesh_emu_spirv.h +++ b/runtime/mesh_emu_spirv.h @@ -7,7 +7,13 @@ #include // The arena: one buffer, its address in the push constants of every emulation stage. +// words 0-3: vertex slots used, indices reserved, task records reserved, overflows (the replay zeroes them per +// submission) // MESH_EMU_DOFF: draw records, MESH_EMU_DWORDS words each: DrawIndexedIndirect {indexCount, instanceCount, +// firstIndex, vertexOffset, firstInstance}, mesh dispatch {x, y, z}, task base, then the MESH_EMU_R_* +// words; the replay fills them, the allocation shader (mesh_emu_alloc_spirv) reserves each draw's +// task records and index range, so that the stages of many draws can run together: a draw's +// indices are one range (its DrawIndexedIndirect) whatever the order its mesh workgroups run in // MESH_EMU_TOFF: task records, MESH_EMU_TWORDS words each: mesh workgroup count, then the payload // index area (uint32 indices), vertex area (16-byte slots: position, then each generic output) at offsets the // replay passes in the push constants. @@ -15,6 +21,12 @@ #define MESH_EMU_DWORDS 16u #define MESH_EMU_DRECS 4096u #define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +#define MESH_EMU_R_GROUPS 9u // mesh workgroups with output: the task stage adds the counts it emits +#define MESH_EMU_R_FIRST 10u // the first stage's workgroups (the replay: the count, or a copy of the indirect one) +#define MESH_EMU_R_ICUR 11u // indices taken from the draw's range (the mesh stage's atomics) +#define MESH_EMU_R_ICAP 12u // the draw's range: indices reserved (fit in the area) +#define MESH_EMU_R_PER 13u // indices one mesh workgroup may emit (OutputPrimitives x vertices per primitive) +#define MESH_EMU_R_TASK 14u // 1 with a task stage #define MESH_EMU_TWORDS 780u // 1 + payloads of up to 776 words (3072 bytes and the 32 the // translator adds for the payload header since 0.6.0), padded // task records: as many as the push constants say (the replay sizes them, NX2NAVI_MESH_EMU_TASKS); the index area @@ -26,6 +38,13 @@ // submission); word 3 counts mesh workgroups that did not fit. #define MESH_EMU_PC_BYTES 32u +// The allocation shader (GLCompute, one invocation): for the n draw records from ring index first, mode 0 reserves the +// task records of draws with a task stage (MESH_EMU_R_FIRST of them, from word 2) and the index ranges of draws +// without one (MESH_EMU_R_FIRST x MESH_EMU_R_PER, from word 1); mode 1, after the task stages, the index ranges of +// draws with one (MESH_EMU_R_GROUPS x MESH_EMU_R_PER). Push constants, 32 bytes: u64 arena address; uint first, n, +// mode, index capacity, task record capacity, unused. Static words, *n of them. +const uint32_t *mesh_emu_alloc_spirv(size_t *n); + typedef struct MeshEmuInfo { int task; // 1 TaskEXT, 0 MeshEXT uint32_t threads; // LocalSize x diff --git a/runtime/vk.h b/runtime/vk.h index a4f25dc4..fb7d9a4a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -764,6 +764,9 @@ void vk_sync_worker_end(VkCommandBuffer cb); // The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun // with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. VkCommandBuffer vk_sync_pre(void); +// vk_exec_draw.c: records into the pre-pass command buffer, as it ends, the stages of the emulated mesh draws queued +// for it (NX2NAVI_MESH_EMU_BATCH). A weak vk_sync.c version does nothing. +void vk_exec_mesh_flush(VkCommandBuffer cb); // Ends the part being recorded without handing it to the queue (it goes with the next part that does, in order) and // returns the command buffer the replay continues in; VK_NULL_HANDLE when no more parts can wait. Replay thread. VkCommandBuffer vk_sync_cut(void); diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index e5b6509f..e6423227 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5829,6 +5829,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { // NX2NAVI_MESH_EMU_TASKS its task records per submission (default 16384, 3 KiB each). Before the counters start over // they are copied to a readback ring; the perf log reports the most a submission asked for against the capacities. #define ME_RB 64 +static struct { VkBuffer buf, rb; VmaAllocation alloc, rb_alloc; uint32_t *rb_map; @@ -5837,8 +5838,29 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { int failed; uint64_t draws, skipped; uint32_t max[4]; + VkPipelineLayout alayout; // the allocation shader's (push constants only) + VkPipeline apipe; } ME; +// The allocation shader's pipeline (mesh_emu_spirv.h mesh_emu_alloc_spirv); 0 when it cannot be made. +static int me_alloc_pipe(void) { + VkDevice dev = vk_ctx()->device; + VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 32}; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .pushConstantRangeCount = 1, + .pPushConstantRanges = &pcr}; + if (vkCreatePipelineLayout(dev, &li, NULL, &ME.alayout) != VK_SUCCESS) return ME.alayout = VK_NULL_HANDLE, 0; + size_t n = 0; + const uint32_t *w = mesh_emu_alloc_spirv(&n); + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = n * 4, .pCode = w}; + VkShaderModule m; + if (vkCreateShaderModule(dev, &mi, NULL, &m) != VK_SUCCESS) return 0; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = ME.alayout}; + VkResult r = vkCreateComputePipelines(dev, VK_NULL_HANDLE, 1, &ci, NULL, &ME.apipe); + vkDestroyShaderModule(dev, m, NULL); + if (r != VK_SUCCESS) return ME.apipe = VK_NULL_HANDLE, 0; return 1; } @@ -5858,6 +5880,9 @@ static int me_arena(void) { VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!me_alloc_pipe()) { + ME.failed = 1; + rt_log("vk: mesh emulation: the allocation shader's pipeline could not be created: mesh draws skipped"); return 0; } if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) { @@ -5880,6 +5905,8 @@ static int me_arena(void) { if (vmaCreateBuffer(vk_ctx()->vma, &rbi, &rai, &ME.rb, &ME.rb_alloc, &rinfo) == VK_SUCCESS) ME.rb_map = rinfo.pMappedData; else ME.rb = VK_NULL_HANDLE; return 1; +} + // The access masks also carry the generic MEMORY_WRITE / MEMORY_READ bits. SHADER_STORAGE_READ/WRITE exist only in // synchronization2 (bits above 31), and AMD's driver for Polaris (23.10.01, RX 580) did not act on them: the mesh // dispatch that reads the {groups, task workgroups, 1} the task stage had just written read zeros and launched no @@ -5892,9 +5919,101 @@ static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFla .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; vkCmdPipelineBarrier2(cb, &di); +} + +// The stages of the emulated draws whose records were filled, waiting to be recorded (me_flush): the draws of a pass +// with NX2NAVI_MESH_EMU_BATCH, which run in the part's pre-pass command buffer when it ends (vk_exec_mesh_flush, from +// vk_sync.c), or the one draw that runs in the part. Recorded by stage, with one barrier between stages for all the +// draws: the fills, the allocation (task records; index ranges of draws without a task stage), every task stage, the +// allocation of the other index ranges, every mesh stage. Each draw's indices are a range of its own and its task +// records too (the allocation shader), so the draws' stages need nothing of each other's. One by one, as before +// 2026-11-11, each draw took three barriers, and its task base and first index were copies of the arena's counters +// after the draw before it: an RX 580 spent 5.5 ms of a frame's GPU time on 69 draws' stages (Kokiri Forest), +// most of it waiting at the barriers with the GPU nearly idle. +typedef struct { + VkPipeline tp, mp; + VkPipelineLayout layout; + VkDescriptorUpdateTemplate tpl_c; + VkDescriptorSet set0, pset; + int push; + uint32_t count, rec, pc[8]; + VkBuffer ib; // the first stage's indirect dispatch (VK_NULL_HANDLE: count) + VkDeviceSize io; + size_t doff; + uint32_t dbytes; +} MeQ; +#define MEQ_MAX 256 +static struct { + MeQ q[MEQ_MAX]; + int n, tasks; + uint8_t *d; // the push descriptors' data of the queued draws + size_t dn, dcap; +} MEQ; + +static void me_bind(VkCommandBuffer cb, const MeQ *q) { + if (q->push) rec_push_tpl(vk_pipeline_push_tpl_fn(), cb, q->tpl_c, q->layout, 0, MEQ.d + q->doff, q->dbytes); + else vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 0, 1, &q->set0, 0, NULL); + if (q->pset) vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 2, 1, &q->pset, 0, NULL); + vkCmdPushConstants(cb, q->layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, q->pc); +} + +static void me_first(VkCommandBuffer cb, const MeQ *q) { + if (q->ib) vkCmdDispatchIndirect(cb, q->ib, q->io); + else vkCmdDispatch(cb, q->count, 1, 1); +} + +static void me_alloc(VkCommandBuffer cb, uint32_t mode) { + uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), MEQ.q[0].rec, (uint32_t)MEQ.n, mode, ME.icap, ME.trecs, 0}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, ME.apipe); + vkCmdPushConstants(cb, ME.alayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, 1, 1, 1); +} + +// The queued draws' stages into cb (the pre-pass command buffer, or the part for a draw that runs there); returns the +// commands recorded. +static int me_flush(VkCommandBuffer cb) { + if (!MEQ.n || !cb) return 0; + const VkAccessFlags2 rw = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT; + const VkPipelineStageFlags2 cs = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT; + int k = 0; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 0), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + if (MEQ.tasks) { // the task stages wrote the mesh dispatch {max mesh groups, task workgroups, 1} into the draw record + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + if (!q->tp) continue; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->tp); + me_first(cb, q), k += 4; } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 1), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); } + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->mp); + if (q->tp) vkCmdDispatchIndirect(cb, ME.buf, MESH_EMU_DOFF + (VkDeviceSize)q->rec * MESH_EMU_DWORDS * 4 + 20); + else me_first(cb, q); + k += 4; } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT | VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | + cs | VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_2_INDEX_READ_BIT | rw | VK_ACCESS_2_TRANSFER_READ_BIT); + MEQ.n = MEQ.tasks = 0, MEQ.dn = 0; + return k + 4; +} + +// vk.h: the pre-pass command buffer ends (vk_sync.c pre_end): the stages of the draws queued for it go in first. +void vk_exec_mesh_flush(VkCommandBuffer cb) { me_flush(cb); } + // count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from // args: an indirect draw's arguments in game memory (count, 1, 1 copied into a ring entry, which ind then is) static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args) { @@ -5939,12 +6058,16 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const if (batch && ex->in_pass) { if (ex->pre_pass != ex->pass_serial && vk_exec_split_now(ex)) { if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { // the pass again, in the new part + ME.skipped++; + return 0; + } ex->pre_pass = ex->pass_serial; set0 = ex->dc->B.ds_bound[0]; } if (ex->pre_pass == ex->pass_serial) pre = vk_sync_pre(); } if (!pre) vk_exec_barrier(ex); // ends the pass; what the game wrote before (its culling) is visible to the stages + if (!pre && MEQ.n) me_flush(vk_sync_pre()); // the draws queued for the pre-pass command buffer go before this one VkCommandBuffer cb = pre ? pre : ex->cb; VkMemSpan ring = {0}; if (args) { @@ -5952,8 +6075,10 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); ME.skipped++; return 0; + } ring.buffer = mesh_ring, ind = ˚ cb = pre ? pre : ex->cb; + } uint32_t ep = vk_texture_epoch(); if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over ME.epoch = ep; @@ -5963,20 +6088,55 @@ static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const const uint32_t *h = ME.rb_map + 4 * ((ME.resets - 32) % ME_RB); for (int k = 0; k < 4; k++) if (h[k] > ME.max[k]) ME.max[k] = h[k]; + } VkBufferCopy rc = {0, 16 * (ME.resets % ME_RB), 16}; vkCmdCopyBuffer(cb, ME.buf, ME.rb, 1, &rc); ME.resets++; me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); + } vkCmdFillBuffer(cb, ME.buf, 0, 32, 0); // the counters, and the diagnosis words 4-7 (mesh_emu_spirv.c) me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + } uint32_t rec = ME.rec++ % MESH_EMU_DRECS; VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4; + if (MEQ.n && (MEQ.n == MEQ_MAX || (MEQ.q[0].rec + (uint32_t)MEQ.n) % MESH_EMU_DRECS != rec)) + me_flush(vk_sync_pre()); // full, or not the next record (the allocation shader takes a run of them) + // the record: DrawIndexedIndirect {0, 1, 0, 0, 0}, mesh dispatch, task base, then the words the allocation shader + // reads (mesh_emu_spirv.h MESH_EMU_R_*); an indirect first stage's count is copied from its arguments vkCmdFillBuffer(cb, ME.buf, ro, 4, 0); vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); + vkCmdFillBuffer(cb, ME.buf, ro + 8, (MESH_EMU_R_FIRST - 2) * 4, 0); + if (ind) { + const VkMemSpan *src = args ? args : ind; + VkBufferCopy cp = {src->offset, ro + MESH_EMU_R_FIRST * 4, 4}; + vkCmdCopyBuffer(cb, src->buffer, ME.buf, 1, &cp); + } else vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_FIRST * 4, 4, count); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_ICUR * 4, 8, 0); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_PER * 4, 4, mi.max_p * (mi.prim ? mi.prim : 3)); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_TASK * 4, 8, tp ? 1 : 0); // and the unused last word uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; + MeQ *q = &MEQ.q[MEQ.n]; + *q = (MeQ){.tp = tp, .mp = mp, .layout = layout, .tpl_c = tpl_c, .set0 = set0, .pset = pool ? PL.set : VK_NULL_HANDLE, + .push = push, .count = count, .rec = rec, .ib = ind ? ind->buffer : VK_NULL_HANDLE, .io = ind ? ind->offset : 0}; + memcpy(q->pc, pc, sizeof pc); + if (push) { + if (MEQ.dn + emu_d_bytes > MEQ.dcap) { + size_t c = MEQ.dcap ? MEQ.dcap * 2 : 65536; + while (c < MEQ.dn + emu_d_bytes) c *= 2; + uint8_t *d = realloc(MEQ.d, c); + if (!d) { + ME.skipped++; + return 0; + } + MEQ.d = d, MEQ.dcap = c; + } + memcpy(MEQ.d + MEQ.dn, emu_d, emu_d_bytes); + q->doff = MEQ.dn, q->dbytes = emu_d_bytes, MEQ.dn += emu_d_bytes; } + MEQ.n++, MEQ.tasks += !!tp; + if (!pre) ex->recorded += 3 + me_flush(cb); // the one draw's stages, here if (!pre) { // the draw: prepared again (the pass reopens, every binding is recorded again) vk_exec_draw_new_cb(); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 80953490..7c2c1e5e 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -812,6 +812,8 @@ static void gpu_stats(PerfGpu *g) { g->mirror_ns = (uint64_t)plat_xchg64(&ts_mirror_ns, 0); } +WEAK void vk_exec_mesh_flush(VkCommandBuffer cb) { (void)cb; } // without vk_exec_draw.c: nothing is queued + // Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted. WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { (void)cb, (void)fence; @@ -1037,6 +1039,7 @@ static int nchain; static VkCommandBuffer pre_end(uint64_t fence) { VkCommandBuffer cb = pre_cb; if (!cb) return VK_NULL_HANDLE; + vk_exec_mesh_flush(cb); // the stages of the pass's emulated draws, by stage ts_close(cb, pre_pair, fence, GTL_PRE); pre_pair = -1; vkEndCommandBuffer(cb); From 4718997313ae3359384fd7babf9623f99a925198 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:04 +0100 Subject: [PATCH 51/64] NX2NAVI_SUBMIT_FINISH_KINDS: a settings row (default 31) and a Lama patches row, all result kinds or all but events. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 1 + runtime/ui/menu.cpp | 6 ++++++ runtime/ui_text.c | 6 ++++++ runtime/ui_text.h | 1 + 4 files changed, 14 insertions(+) diff --git a/runtime/cfg.c b/runtime/cfg.c index bed3d8e0..c19af5d4 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -302,6 +302,7 @@ static const CfgMeta metas[] = { // the menu's Lama patches tab (ui/menu.cpp tab_lama): the settings of the RX 580 patches {"TEX_RELEASED_MAX", "", NULL, RESTART}, // vk_texture.c released_max: "" by the video memory budget, else 1..16384 {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, + {"SUBMIT_FINISH_KINDS", "31", "0..31", RESTART}, // vk_sync.c res_mask {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit {"MESH_EMULATION", "1", "0|1", RESTART}, {"MESH_EMU_MB", "256", "32..2048", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 0341e6d7..9cf0ddaa 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1639,6 +1639,8 @@ static void tab_lama(float scale) { static const char *const values[] = {"auto", "0"}; const char *names[] = {menu_text(UI_MENU_AUTO), menu_text(UI_MENU_OFF)}; opt_choice_list("NX2NAVI_VRAM_HEAP_LIMIT", UI_MENU_LAMA_HEAP_LIMIT, values, names, 2, UI_MENU_LAMA_HEAP_LIMIT_DESC); + } + } { // vk_memory.c upload_batch_cap static const char *const values[] = {"128", "256", "512", "1024", "0"}; const char *names[] = {"128 MB", "256 MB", "512 MB", "1024 MB", menu_text(UI_MENU_LAMA_NO_LIMIT)}; @@ -1648,6 +1650,10 @@ static void tab_lama(float scale) { static const char *const values[] = {"1800", "1200", "600", "60", "0"}; const char *names[] = {"1800", "1200", "600", "60", menu_text(UI_MENU_OFF)}; opt_choice_list("NX2NAVI_FREEZE_IDLE", UI_MENU_LAMA_FREEZE, values, names, 5, UI_MENU_LAMA_FREEZE_DESC); + { // vk_sync.c res_mask: the result kinds the submit wait counts (31 all, 30 all but events) + static const char *const values[] = {"31", "30"}; + const char *names[] = {menu_text(UI_MENU_LAMA_RESULTS_ALL), menu_text(UI_MENU_LAMA_RESULTS_NO_EVENTS)}; + opt_choice_list("NX2NAVI_SUBMIT_FINISH_KINDS", UI_MENU_LAMA_RESULTS, values, names, 2, UI_MENU_LAMA_RESULTS_DESC); opt_toggle("NX2NAVI_ZSTD_NATIVE", UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC); // ov_zstd.c trusted opt_group(UI_MENU_LAMA_RENDER); opt_toggle("NX2NAVI_REPLAY_ASYNC", UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC); // vk_backend.c diff --git a/runtime/ui_text.c b/runtime/ui_text.c index c7729e2d..5c35bb74 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -144,6 +144,12 @@ static const char *const text_en[UI_TEXT_COUNT] = { "tracking can skip it. 1800 (about 15 s): few thaws and small tracking queries. 60: " "the game thaws regions all the time and every thaw stalls a frame. Off: no stalls, but " "the tracking scans all memory every frame.", + [UI_MENU_LAMA_RESULTS] = "Wait for GPU results", + [UI_MENU_LAMA_RESULTS_DESC] = "After a frame's first submission the game waits until the GPU wrote the results it " + "reads back. All: events, counter and occlusion reports, write-backs. Without events: " + "a shorter wait; on an RX 580 nothing flickered.", + [UI_MENU_LAMA_RESULTS_ALL] = "All", + [UI_MENU_LAMA_RESULTS_NO_EVENTS] = "Without events", [UI_MENU_LAMA_ZSTD] = "Native data decompression", [UI_MENU_LAMA_ZSTD_DESC] = "Decompresses the game's zstd data with the PC's libzstd instead of the game's own " "decoder translated from the console. Lighter streaming while you move.", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 8875d9f7..37be47df 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -336,6 +336,7 @@ typedef enum { UI_MENU_TAB_LAMA, UI_MENU_LAMA_INTRO, UI_MENU_LAMA_MEMORY, UI_MENU_LAMA_RENDER, UI_MENU_LAMA_BUILD, UI_MENU_LAMA_DIAG, UI_MENU_LAMA_FREEZE, UI_MENU_LAMA_FREEZE_DESC, UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC, + UI_MENU_LAMA_RESULTS, UI_MENU_LAMA_RESULTS_DESC, UI_MENU_LAMA_RESULTS_ALL, UI_MENU_LAMA_RESULTS_NO_EVENTS, UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC, UI_MENU_LAMA_RELEASED, UI_MENU_LAMA_RELEASED_DESC, UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC, From 306de5058948735cdba42355d1cae37dc5a906c3 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:05 +0100 Subject: [PATCH 52/64] NX2NAVI_FOLIAGE (full, reduced, off): reduced skips the indirect mesh draws with a task stage (small grass and moss), off every indirect mesh draw (all foliage); a Lama patches row. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 ++ runtime/ui/menu.cpp | 7 +++++++ runtime/ui_text.c | 5 +++++ runtime/ui_text.h | 1 + runtime/vk_exec_draw.c | 28 +++++++++++++++++++++++++--- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/runtime/cfg.c b/runtime/cfg.c index c19af5d4..3c4f3531 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -91,6 +91,7 @@ static const char *const known[] = { // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). static const struct { const char *key, *section; } sections[] = { // the menu's Lama patches tab (ui/menu.cpp tab_lama) + {"TEX_RELEASED_MAX", "performance"}, {"MIRROR_HOT", "performance"}, {"FREEZE_IDLE", "performance"}, {"SUBMIT_FINISH_KINDS", "performance"}, {"FOLIAGE", "performance"}, {"ZSTD_NATIVE", "performance"}, {"MESH_EMU_MB", "performance"}, {"MESH_EMU_TASKS", "performance"}, {"MESH_EMU_BATCH", "performance"}, {"VEC_INDEX_FIX", "performance"}, {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, @@ -303,6 +304,7 @@ static const CfgMeta metas[] = { {"TEX_RELEASED_MAX", "", NULL, RESTART}, // vk_texture.c released_max: "" by the video memory budget, else 1..16384 {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, {"SUBMIT_FINISH_KINDS", "31", "0..31", RESTART}, // vk_sync.c res_mask + {"FOLIAGE", "full", "full|reduced|off", RESTART}, // vk_exec_draw.c foliage_skip {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit {"MESH_EMULATION", "1", "0|1", RESTART}, {"MESH_EMU_MB", "256", "32..2048", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 9cf0ddaa..55f5aaf9 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1645,21 +1645,28 @@ static void tab_lama(float scale) { static const char *const values[] = {"128", "256", "512", "1024", "0"}; const char *names[] = {"128 MB", "256 MB", "512 MB", "1024 MB", menu_text(UI_MENU_LAMA_NO_LIMIT)}; opt_choice_list("NX2NAVI_UPLOAD_BATCH_MB", UI_MENU_LAMA_UPLOAD_BATCH, values, names, 5, UI_MENU_LAMA_UPLOAD_BATCH_DESC); + } opt_toggle("NX2NAVI_ASTC_SCRATCH_KEEP", UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC); { // vk_memory.c freeze_pass: submissions a region stays unwritten before it is frozen static const char *const values[] = {"1800", "1200", "600", "60", "0"}; const char *names[] = {"1800", "1200", "600", "60", menu_text(UI_MENU_OFF)}; opt_choice_list("NX2NAVI_FREEZE_IDLE", UI_MENU_LAMA_FREEZE, values, names, 5, UI_MENU_LAMA_FREEZE_DESC); + } { // vk_sync.c res_mask: the result kinds the submit wait counts (31 all, 30 all but events) static const char *const values[] = {"31", "30"}; const char *names[] = {menu_text(UI_MENU_LAMA_RESULTS_ALL), menu_text(UI_MENU_LAMA_RESULTS_NO_EVENTS)}; opt_choice_list("NX2NAVI_SUBMIT_FINISH_KINDS", UI_MENU_LAMA_RESULTS, values, names, 2, UI_MENU_LAMA_RESULTS_DESC); + } opt_toggle("NX2NAVI_ZSTD_NATIVE", UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC); // ov_zstd.c trusted opt_group(UI_MENU_LAMA_RENDER); opt_toggle("NX2NAVI_REPLAY_ASYNC", UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC); // vk_backend.c const bool native = vk_mesh_shader_native() != 0; // vk_exec_draw.c: the emulation only where the GPU lacks them if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_toggle("NX2NAVI_MESH_EMULATION", UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC); + { // vk_exec_draw.c foliage_skip + static const char *const values[] = {"full", "reduced", "off"}; + const char *names[] = {menu_text(UI_MENU_LAMA_FOLIAGE_FULL), menu_text(UI_MENU_LAMA_FOLIAGE_REDUCED), menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_FOLIAGE", UI_MENU_LAMA_FOLIAGE, values, names, 3, UI_MENU_LAMA_FOLIAGE_DESC); } if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); opt_slider("NX2NAVI_MESH_EMU_MB", UI_MENU_LAMA_MESH_MB, 64, 2048, UI_MENU_LAMA_MESH_MB_DESC, 64, "%.0f MB"); diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 5c35bb74..f5327400 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -144,6 +144,11 @@ static const char *const text_en[UI_TEXT_COUNT] = { "tracking can skip it. 1800 (about 15 s): few thaws and small tracking queries. 60: " "the game thaws regions all the time and every thaw stalls a frame. Off: no stalls, but " "the tracking scans all memory every frame.", + [UI_MENU_LAMA_FOLIAGE] = "Foliage", + [UI_MENU_LAMA_FOLIAGE_DESC] = "Full: all of it. Reduced: no small grass or moss on the ground; ferns, leaves and " + "other plants stay. Off: no foliage. Terrain, rocks and buildings always stay.", + [UI_MENU_LAMA_FOLIAGE_FULL] = "Full", + [UI_MENU_LAMA_FOLIAGE_REDUCED] = "Reduced", [UI_MENU_LAMA_RESULTS] = "Wait for GPU results", [UI_MENU_LAMA_RESULTS_DESC] = "After a frame's first submission the game waits until the GPU wrote the results it " "reads back. All: events, counter and occlusion reports, write-backs. Without events: " diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 37be47df..4c7cb198 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -336,6 +336,7 @@ typedef enum { UI_MENU_TAB_LAMA, UI_MENU_LAMA_INTRO, UI_MENU_LAMA_MEMORY, UI_MENU_LAMA_RENDER, UI_MENU_LAMA_BUILD, UI_MENU_LAMA_DIAG, UI_MENU_LAMA_FREEZE, UI_MENU_LAMA_FREEZE_DESC, UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC, + UI_MENU_LAMA_FOLIAGE, UI_MENU_LAMA_FOLIAGE_DESC, UI_MENU_LAMA_FOLIAGE_FULL, UI_MENU_LAMA_FOLIAGE_REDUCED, UI_MENU_LAMA_RESULTS, UI_MENU_LAMA_RESULTS_DESC, UI_MENU_LAMA_RESULTS_ALL, UI_MENU_LAMA_RESULTS_NO_EVENTS, UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC, UI_MENU_LAMA_RELEASED, UI_MENU_LAMA_RELEASED_DESC, diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index e6423227..6e1bf0a1 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -5775,21 +5775,43 @@ static int mesh_ring_entry(VkExec *ex, VkCommandBuffer cb, const p_draw_mesh_ind return 1; } +// NX2NAVI_FOLIAGE (full, reduced, off; default full; the Lama patches tab): Ocarina draws its foliage with +// DrawMeshTasksIndirect, the instances its compute passes culled on the GPU. With a task stage: the small grass and moss +// that carpet the ground; without one: ferns, leaves, clover and the other plants. Its other mesh draws (DrawMeshTasks +// with a count: terrain, rocks, buildings, doors) are not foliage. reduced skips the indirect draws with a task stage, +// off every indirect one, in every pass (their shadows too). In a frame dump of Kokiri Forest on an RX 580 +// (f8118, 2026-11-10) the grass and moss were one draw with task stage 2ddf7b0bf74e5a05; the plants nine without. +static int foliage_skip(VkExec *ex) { + static int mode = -1; + if (mode < 0) { + const char *v = cfg_get("NX2NAVI_FOLIAGE"); + mode = v && !strcmp(v, "off") ? 2 : v && !strcmp(v, "reduced") ? 1 : 0; + rt_log("vk: foliage %s (NX2NAVI_FOLIAGE)", mode == 2 ? "off" : mode ? "reduced: no small grass or moss" : "full"); + } + if (!mode) return 0; + if (mode == 2) return 1; + const uint8_t *ctl, *code; + uint32_t len; + return ex->prog && find_stage(ex->dc, ex->prog, NVN_STAGE_TASK, &ctl, &code, &len); +} + // DrawMeshTasksIndirect: one draw from its ring entry (mesh_ring_entry); x2-x4 of the call vary like leftovers, so they // are not read as a count or stride. static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { PFN_vkCmdDrawMeshTasksIndirectEXT fn = draw_mesh_ind_fn(); - int fallback = vk_ctx()->mesh_fallback; - if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + int emu = !fn && mesh_emu_on(), fallback = !emu && vk_ctx()->mesh_fallback; + if (!fn && !emu && !fallback) + return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader (mesh_emulation = 1 emulates them): skipped"); static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); - if (!on) return; + if (!on || foliage_skip(ex)) return; VkMemSpan ip; if (!vk_mem_resolve(d->ind, 12, &ip) || ip.offset % 4) return warn_once(W_INDIRECT, "indirect mesh draw arguments at %#llx are not backed or not aligned: skipped", (unsigned long long)d->ind); ex->dc->c0_binst = 0; VK_MEM_USE(ip); + if (emu) { // emulated: a ring entry {count, 1, 1}, made where the stages run, is the first stage's dispatch mesh_emu_draw(ex, 0, NULL, &ip); vk_mem_count(ip.cls, VK_MUSE_IND, 4); VK_MEM_BOUND(ip); From aa1045d411720e15f7efde7abfc673ee754ba932 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:07 +0100 Subject: [PATCH 53/64] GPU passes: result points, where in the waited-for submissions the results the game's wait needs are done on the GPU, and how much earlier the wait could end. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 6 +++ runtime/vk_crumbs.c | 101 +++++++++++++++++++++++++++++++++++++++++ runtime/vk_exec_xfer.c | 4 ++ runtime/vk_sync.c | 4 ++ 4 files changed, 115 insertions(+) diff --git a/runtime/vk.h b/runtime/vk.h index fb7d9a4a..2e6c36c0 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -146,6 +146,12 @@ enum { VK_GAP_MESH_RING, VK_GAP_BC_BACK, VK_GAP_N }; void vk_gpu_passes_part(VkCommandBuffer cb, int end); +// Result points (NX2NAVI_GPU_PASSES): the submission whose parts follow (vk_sync.c), a result just recorded (VK_RES_* +// bits, vk_exec_note_results), a submission the game's wait after a frame's first submission waited for. +void vk_gpu_passes_fence(uint64_t fence); +void vk_gpu_passes_result(int kinds); +void vk_gpu_passes_waited(uint64_t need); +int vk_sync_result_mask(void); // NX2NAVI_SUBMIT_FINISH_KINDS as read void vk_gpu_gap_note(int kind, uint64_t units); // Pass marks wanted and possible (NX2NAVI_PASS_CRUMBS and the extensions): set by vk_crumbs_init. extern int vk_pass_marks_on; diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index 72bc56c5..110355aa 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -197,6 +197,40 @@ static uint64_t gp_shader_hash(const GfxShader *s) { static int kind_is_pass(const VkPassMark *m) { return m->kind == VK_MARK_PASS; } +// Result points (gpu_passes): where in a submission's GPU time the results the CPU waits for are done. The replay +// notes a result (vk_exec_note_results: an event, a counter or occlusion report, a GPU write of unmirrored memory, a +// mirror write-back) as it records it; the next timestamp the capture writes after it (a pass's or dispatch's start or +// end, a part's end) is when the GPU is done with it at the latest. The report compares that, for the submissions the +// game's wait after a frame's first submission waited for (vk_sync_recorded_wait), with the submission's end: how much +// earlier the wait could end if it ended at the results instead of at the whole submission's completion. +#define GP_RES 16384 +typedef struct { uint64_t fence; uint32_t q; uint8_t kind; } GpRes; // q: query in gp_pool, UINT32_MAX until written +static GpRes gp_res[GP_RES]; +static uint32_t gp_nres, gp_res_open; +static uint64_t gp_fence, gp_part_fence[GP_SLOTS]; +#define GP_WAITED 256 +static uint64_t gp_waited[GP_WAITED]; +static uint32_t gp_nwaited; // atomic: the game's threads add + +static void gp_resolve(uint32_t q) { + for (uint32_t i = gp_res_open; i < gp_nres; i++) gp_res[i].q = q; + gp_res_open = gp_nres; +} + +void vk_gpu_passes_fence(uint64_t fence) { gp_fence = fence; } + +void vk_gpu_passes_result(int kinds) { + if (!gp_capture || gp_part == UINT32_MAX) return; + for (int b = 0; b < VK_RES_KINDS && gp_nres < GP_RES; b++) + if (kinds >> b & 1) gp_res[gp_nres++] = (GpRes){gp_fence, UINT32_MAX, (uint8_t)b}; +} + +void vk_gpu_passes_waited(uint64_t need) { + if (!gp_capture) return; + uint32_t i = __atomic_fetch_add(&gp_nwaited, 1, __ATOMIC_RELAXED); + if (i < GP_WAITED) __atomic_store_n(&gp_waited[i], need, __ATOMIC_RELEASE); +} + int vk_gpu_passes_capturing(void) { return gp_capture; } static void gp_gap_close(uint32_t to) { @@ -216,12 +250,14 @@ void vk_gpu_passes_part(VkCommandBuffer cb, int end) { if (!gp_capture || gp_n >= GP_SLOTS) return; gp_labels[gp_n] = (GpLabel){.kind = GP_KIND_PART}; vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, gp_n * 2); + gp_part_fence[gp_n] = gp_fence; gp_part = gp_n++; if (!gp_in_mark) gp_gap_from = gp_part * 2; return; } if (gp_part == UINT32_MAX) return; vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, gp_part * 2 + 1); + gp_resolve(gp_part * 2 + 1); if (!gp_in_mark) gp_gap_close(gp_part * 2 + 1); gp_labels[gp_part].used = 1; gp_part = UINT32_MAX, gp_gap_from = UINT32_MAX; @@ -233,6 +269,7 @@ static void gp_begin(VkCommandBuffer cb, VkPassMark *m) { if (gp_part != UINT32_MAX) gp_gap_close(gp_n * 2); // the gap before this mark ends at its start gp_labels[gp_n].used = 0; vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, gp_n * 2); + gp_resolve(gp_n * 2); if (gp_prims && kind_is_pass(m)) vkCmdBeginQuery(cb, gp_prims, gp_n, 0); if (gp_stats && kind_is_pass(m)) vkCmdBeginQuery(cb, gp_stats, gp_n, 0); m->gp = ++gp_n; @@ -244,6 +281,7 @@ static void gp_end(VkCommandBuffer cb, const VkPassMark *m) { if (gp_prims && kind_is_pass(m)) vkCmdEndQuery(cb, gp_prims, i); if (gp_stats && kind_is_pass(m)) vkCmdEndQuery(cb, gp_stats, i); vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, i * 2 + 1); + gp_resolve(i * 2 + 1); GpLabel *l = &gp_labels[i]; l->kind = m->kind, l->w = m->w, l->h = m->h, l->d = m->d, l->draws = m->draws == UINT32_MAX ? 0 : m->draws; l->layer = m->layer; @@ -310,6 +348,66 @@ static void gp_report_gaps(double tick_ms, double parts_ms, uint32_t parts) { gp_kind_names[b], (double)gp_kind_n[b] / GP_FRAMES, gp_kind_units[b] / 1e6 / GP_FRAMES); } +// The result points of the timed frames (vk_gpu_passes_result), for the submissions the game's wait waited for. +static void gp_report_results(double tick_ms) { + VkCtx *c = vk_ctx(); + static const char *const names[VK_RES_KINDS] = {"events", "counter reports", "occlusion reports", + "unmirrored writes", "mirror write-backs"}; + uint32_t nw = __atomic_load_n(&gp_nwaited, __ATOMIC_ACQUIRE); + if (nw > GP_WAITED) nw = GP_WAITED; + uint64_t seen[GP_WAITED]; + uint32_t ns = 0, subs = 0; + double span = 0, last[VK_RES_KINDS] = {0}, cnt[VK_RES_KINDS] = {0}, mask_end = 0, all_end = 0; + int mask = vk_sync_result_mask(); + for (uint32_t w = 0; w < nw; w++) { + uint64_t f = __atomic_load_n(&gp_waited[w], __ATOMIC_ACQUIRE); + int dup = !f; + for (uint32_t k = 0; k < ns && !dup; k++) dup = seen[k] == f; + if (dup) continue; + seen[ns++] = f; + uint64_t t0 = UINT64_MAX, t1 = 0, t; + for (uint32_t i = 0; i < gp_n; i++) { // the submission's parts: its first start, its last end + if (gp_labels[i].kind != GP_KIND_PART || !gp_labels[i].used || gp_part_fence[i] != f) continue; + if (gp_tick(c, i * 2, &t) && t < t0) t0 = t; + if (gp_tick(c, i * 2 + 1, &t) && t > t1) t1 = t; + } + if (t0 == UINT64_MAX || t1 <= t0) continue; + uint64_t lk[VK_RES_KINDS] = {0}; + int any = 0; + for (uint32_t i = 0; i < gp_nres; i++) + if (gp_res[i].fence == f && gp_res[i].q != UINT32_MAX && gp_tick(c, gp_res[i].q, &t) && t >= t0) { + if (t > lk[gp_res[i].kind]) lk[gp_res[i].kind] = t; + any = 1; + } + if (!any) continue; + subs++; + span += (double)(t1 - t0) * tick_ms; + uint64_t me = t0, ae = t0; + for (int b = 0; b < VK_RES_KINDS; b++) { + if (!lk[b]) continue; + last[b] += (double)(lk[b] - t0) * tick_ms, cnt[b]++; + if (lk[b] > ae) ae = lk[b]; + if (mask >> b & 1 && lk[b] > me) me = lk[b]; + } + mask_end += (double)(me - t0) * tick_ms, all_end += (double)(ae - t0) * tick_ms; + } + if (!subs) { + rt_log("gpu passes: result points: none of the %u waited-for submissions was timed with its results", ns); + return; + } + char list[400]; + int o = 0; + for (int b = 0; b < VK_RES_KINDS && o < (int)sizeof list - 80; b++) + if (cnt[b]) + o += snprintf(list + o, sizeof list - (size_t)o, "%s%s %.2f ms (%.0f of %u)", o ? ", " : "", names[b], + last[b] / cnt[b], cnt[b], subs); + list[o] = 0; + rt_log("gpu passes: result points: %u submissions the game's wait waited for, %.2f ms of GPU time each from their " + "start to their end; their last results done at: %s", subs, span / subs, list); + rt_log("gpu passes: result points: the wait could end %.2f ms before the submission's end (the kinds it counts, " + "%#x), %.2f ms with every kind", (span - mask_end) / subs, mask, (span - all_end) / subs); +} + typedef struct { GpLabel l; double ms; uint32_t n; uint64_t draws, prims, frags; } GpSum; static int gp_sum_cmp(const void *a, const void *b) { @@ -382,6 +480,7 @@ static void gp_report(uint64_t frame) { (double)sum / GP_FRAMES, top, list); } gp_report_gaps(tick_ms, parts_ms, parts); + gp_report_results(tick_ms); for (int s = 0; s < n && s < GP_TOP; s++) { const GpSum *e = &sums[s]; if (e->l.kind == VK_MARK_DISPATCH) @@ -453,6 +552,8 @@ void vk_gpu_passes_frame(uint64_t frame) { if (gp_stats) vkResetQueryPool(c->device, gp_stats, 0, GP_SLOTS); gp_ngaps = 0, gp_gap_from = UINT32_MAX, gp_gap_kinds = 0, gp_part = UINT32_MAX; memset(gp_kind_n, 0, sizeof gp_kind_n), memset(gp_kind_units, 0, sizeof gp_kind_units); + gp_nres = gp_res_open = 0; + __atomic_store_n(&gp_nwaited, 0, __ATOMIC_RELEASE); gp_n = 0, gp_start = frame, gp_capture = 1, gp_phase = 1; } else if (gp_phase == 1) { if (frame < gp_start + GP_FRAMES) return; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 292752f3..ef379279 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -67,6 +67,10 @@ typedef struct { uint64_t addr, size; int back; } Range; // back: written into // for the newest such submission). Write-backs of the mirrors note it in vk_memory.c; the lazily written ones fetch on // the CPU's first access. static volatile int32_t ex_results; // VK_RES_* bits +void vk_exec_note_results(int kind) { + __atomic_fetch_or(&ex_results, kind, __ATOMIC_RELAXED); + vk_gpu_passes_result(kind); // NX2NAVI_GPU_PASSES: where in the submission it is done +} int vk_exec_take_results(void) { return __atomic_exchange_n(&ex_results, 0, __ATOMIC_RELAXED); } static void guest_written_cb(void *arg) { diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 7c2c1e5e..7f0b37e3 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -1256,6 +1256,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence); ts_begin(s->cb); rec_begin(s->cb, fence); + vk_gpu_passes_fence(fence); // the submission the parts and results that follow belong to vk_gpu_passes_part(s->cb, 0); int recorded = 1; if (own) own->record(s->cb, own->arg); @@ -1380,6 +1381,8 @@ static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); // The game's wait after a frame's first submission (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): until the replay has // submitted value, which took every copy of guest memory the submission reads; then also until the GPU has run it when // its GPU reads guest memory itself (vk_mem_take_live) or NX2NAVI_SUBMIT_FINISH_GPU=1. 1 when reached. +int vk_sync_result_mask(void) { return res_mask; } + int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)) { vk_sync_start(); static int gpu = -1, res_mode = 1; @@ -1433,6 +1436,7 @@ int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(u gw_rec_n++; if (!live) gw_rec_done++; if (!live && need == value) gw_rec_self++; + if (!live && need) vk_gpu_passes_waited(need); if (!live && need && need <= value) { gw_need_back += value - need, gw_need_n++; int k = res_kinds[need & (LIVE_RING - 1)]; From 15744c688adeecee1ee12451b9909a240a2c86cf Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:09 +0100 Subject: [PATCH 54/64] Texture pool refresh: the slots a refresh left are kept in a list, and the next refresh of the submission looks up those and the changed slots instead of walking the whole table again. While an area loads, every refresh left slots, and with any left the next one walked the table: about 67,000 slot reads and 1.65 ms of the replay per submission in Kokiri Forest on an RX 580, six submissions a frame (profile 2026-11-11: pool_fill and the slot lookups under the first pool draws, mesh_emu_draw's among them). Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 56 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 6e1bf0a1..a561763e 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -2218,6 +2218,22 @@ static uint64_t pool_create_ns(void) { return (uint64_t)(pool_loading && ns ? load_ns : ns); } static uint32_t pool_defer[VK_POOL_TEX]; // per slot: 1 + the low bits of the frame a refresh left it in +// The slots a refresh left (pool_refill failed: the creation budget or the frame's time was spent), so that the next +// refresh of the same submission looks up these and the changed slots (vk_texture_dirty_slots_locked) instead of +// walking the whole table again. While an area loads, slots are left at every refresh, and each refresh walked the +// table: about 67,000 slot reads and 1.65 ms of the replay per submission in Kokiri Forest on an RX 580 (2026-11-11), +// with six submissions a frame. pool_left_all: the list overflowed (or nothing tracks it yet): walk the table. +static uint32_t pool_left[VK_POOL_TEX], pool_nleft; +static uint8_t pool_is_left[VK_POOL_TEX]; +static int pool_left_all = 1; +static void pool_left_add(uint32_t id) { + if (id >= VK_POOL_TEX || pool_is_left[id]) return; + pool_is_left[id] = 1, pool_left[pool_nleft++] = id; +} +static void pool_left_clear(void) { + for (uint32_t i = 0; i < pool_nleft; i++) pool_is_left[pool_left[i]] = 0; + pool_nleft = 0; +} static VkWriteDescriptorSet pool_w[POOL_WBATCH]; static VkDescriptorImageInfo pool_ii[POOL_WBATCH]; static uint32_t pool_nw; @@ -2603,7 +2619,7 @@ static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_loc if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++; else PL.n_by_addr++; if (pool_refill(d->ex, (uint32_t)id, d->used, t, d->spent)) PL.tok[id] = t; - else d->retry++; + else d->retry++, pool_left_add((uint32_t)id); } // Every entry as the slots, kinds and samplers are now (pool_sync_entries has nothing to write). @@ -2705,7 +2721,7 @@ static void pool_sync_entries(VkExec *ex) { if (fresh || gen != PL.gen || ep != PL.epoch || PL.retry) { vk_texture_create_budget_locked(spent ? 0 : pool_create_budget()); PL.n_frame_stops += (uint64_t)spent; - // Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that + // Only a slot change since the last sync (no new kinds, no new submission): the slots that // changed (vk_texture_dirty_slots_locked), not the whole table. With NX2NAVI_POOL_FAST (default on) a new // submission too: its freshness check goes through the entries the slots resolved to and the slots that // resolved to none (vk_texture_pool_fresh_locked, pool_unres), not every slot. @@ -2715,8 +2731,32 @@ static void pool_sync_entries(VkExec *ex) { int grp = pool_group(); if (grp && !PL.gmode) vk_texture_groups_unstamp_locked(); // members were refilled one by one meanwhile PL.gmode = grp; - int walked = !fresh && (!newep || pool_fast()) && !PL.retry && PL.dirty_pos && - vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd); + // With slots left by an earlier refresh (the creation budget or the frame's time spent): those (pool_left) + // first, then the changed slots, instead of the walk over every slot. While an area loads, slots are left at + // every refresh, and each one walked the table: about 67,000 slot reads and 1.65 ms of the replay per + // submission in Kokiri Forest on an RX 580 (0.7.1, 2026-11-11), six submissions a frame. pool_left_all: no + // list kept yet (until the first walk): walk. + int quick = !fresh && (!newep || pool_fast()) && (!PL.retry || !pool_left_all) && PL.dirty_pos; + if (quick && PL.retry) { + uint32_t k = 0; + for (uint32_t i = 0; i < pool_nleft; i++) { + uint32_t id = pool_left[i]; + uint64_t t = (int)id < n ? vk_texture_slot_token((int)id) : 0; + int left = 0; + if (t != PL.tok[id]) { + PL.n_refill++; + if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++; + else PL.n_by_addr++; + if (pool_refill(ex, id, used, t, spent)) PL.tok[id] = t; + else left = 1; + } + if (left) pool_left[k++] = id; + else pool_is_left[id] = 0; + } + pool_nleft = k; + PL.n_slots += (uint64_t)k; + } + int walked = quick && vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd); if (walked) { int at = newep ? 0 : 1; PL.n_walk[at]++, PL.walk_visit[at] += PL.n_visit - v0, PL.walk_refill[at] += PL.n_refill - f0; @@ -2750,7 +2790,7 @@ static void pool_sync_entries(VkExec *ex) { PL.unres_ns += kern_now_ns() - tu; PL.n_ep++, PL.ep_ns += kern_now_ns() - te; } - if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(); + if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(), pool_left_clear(), pool_left_all = 0; uint64_t tw = walked ? 0 : kern_now_ns(); for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync uint64_t t = vk_texture_slot_token(id); @@ -2759,7 +2799,7 @@ static void pool_sync_entries(VkExec *ex) { if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++; else PL.n_by_addr++; if (pool_refill(ex, (uint32_t)id, used, t, spent)) PL.tok[id] = t; - else retry++; + else retry++, pool_left_add((uint32_t)id); } else if (t && fresh) { pool_fill(ex, (uint32_t)id, fresh, t); } else if (t && ep != PL.epoch) { @@ -2771,10 +2811,11 @@ static void pool_sync_entries(VkExec *ex) { PL.n_slots += (uint64_t)n, PL.n_full++, PL.full_ns += kern_now_ns() - tw; vk_texture_groups_unstamp_locked(); // the walk over every slot refilled members one by one } + retry = (int)pool_nleft; // every slot left, by this refresh or an earlier one not filled since vk_texture_create_budget_locked(-1); } PL.retry = retry != 0; - // retry counts the slots this sync left; a sync with slots left walks the whole table, so that is all of them + // retry counts every slot left (pool_left: this sync's and the earlier ones' not filled since) int backlog = pool_load_backlog(); if (!pool_loading && backlog > 0 && retry >= backlog) { pool_load_made = (uint64_t)retry, pool_load_frame0 = PL.frame; @@ -3429,6 +3470,7 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded. // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. +static _Thread_local const VkDescSlot *emu_d; // set 0's descriptors of the last bind_resources build (mesh emulation) static uint32_t emu_d_bytes; static void bind_resources(VkExec *ex, const VkPipeInfo *info) { DrawCtx *const cx = ex->dc; From 40272299305a574d7749062389a2a9a82b98501f Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:10 +0100 Subject: [PATCH 55/64] Mesh emulation: its compute pipelines are built outside the lock, and looked up without one A draw of an emulated mesh pipeline looked its entry up under the lock the build held across the driver's compile of both stages: on a first run every such draw waited for whatever entry was being built, 2.5 s of the replay thread in Kokiri Forest on an RX 580. Entries are now published complete at the head of one of 256 buckets and read without a lock. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_pipeline.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 92bb5445..ef816fec 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -2896,8 +2896,23 @@ typedef struct MeshEmuEnt { VkPipeline mesh, task; MeshEmuInfo mi, ti; } MeshEmuEnt; +// Each draw of an emulated mesh pipeline looks its entry up (vk_pipeline_mesh_emu), so the lookup takes no lock: an +// entry is complete before it is published at its bucket's head (release) and is never removed. emu_lock only orders +// the publishing. The build (two compute pipelines, up to seconds each in the driver on a first run) is done outside +// it: held across the build, as before, it stopped every draw of the replay that looked up any entry meanwhile, 2.5 s +// of the replay thread's samples in a first 0.8.0 run in Kokiri Forest on an RX 580 (2026-11-11). +#define EMU_BUCKETS 256 +static MeshEmuEnt *emu_ents[EMU_BUCKETS]; static PlatMutex emu_lock = PLAT_MUTEX_INIT; +static uint32_t emu_bucket(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout) { + uint64_t h = (uint64_t)(uintptr_t)ms * 0x9e3779b97f4a7c15ull ^ (uint64_t)(uintptr_t)ts * 0xc2b2ae3d27d4eb4full ^ + (uint64_t)(uintptr_t)layout * 0x165667b19e3779f9ull; + return (uint32_t)(h >> 40) % EMU_BUCKETS; +} + +static MeshEmuEnt *emu_find(MeshEmuEnt *e, const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout) { + while (e && !(e->ms == ms && e->ts == ts && e->layout == layout)) e = __atomic_load_n(&e->next, __ATOMIC_ACQUIRE); return e; } @@ -2915,7 +2930,11 @@ static VkPipeline emu_compute(const uint32_t *w, size_t n, VkPipelineLayout layo } static const MeshEmuEnt *emu_get(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, uint32_t toff, int create) { + MeshEmuEnt **head = &emu_ents[emu_bucket(ms, ts, layout)]; + MeshEmuEnt *e = emu_find(__atomic_load_n(head, __ATOMIC_ACQUIRE), ms, ts, layout); + if (e || !create) return e; e = calloc(1, sizeof *e); + if (!e) return NULL; e->ms = ms, e->ts = ts, e->layout = layout; const char *why = NULL; uint64_t n = 0; @@ -2938,7 +2957,21 @@ static const MeshEmuEnt *emu_get(const GfxShader *ms, const GfxShader *ts, VkPip } free(r), free(c); } + plat_mutex_lock(&emu_lock); + MeshEmuEnt *had = emu_find(*head, ms, ts, layout); // another build of the same stages published first: keep that + if (!had) { + static int logged; + if (!e->ok && logged++ < 16) rt_log("vk_pipeline: mesh emulation: a mesh program is not emulated: %s", why ? why : "?"); + e->next = *head; + __atomic_store_n(head, e, __ATOMIC_RELEASE); + } plat_mutex_unlock(&emu_lock); + if (had) { + if (e->mesh) vkDestroyPipeline(P.device, e->mesh, NULL); + if (e->task) vkDestroyPipeline(P.device, e->task, NULL); + free(e); + return had; + } return e; } From 293731d083a4a262535c1571af57603a9276dc64 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:12 +0100 Subject: [PATCH 56/64] Frame generation: present and acquire queues from another family where the graphics family has one queue AMD's Windows driver gives the graphics family a single queue. vk_device.c then made no frame generation queues, and vk_fg_create_swapchain returned without a log line: AMD FSR 3.1 frame generation set in the menu did nothing on an RX 580. The present queue (which nothing else may submit to) and the image acquire queue now come from a family that can present, compute families first, with their families passed to AMD's swapchain; the log names them, or says that there are none, and the menu offers FSR 3.1 frame generation only where they exist. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_device.c | 68 +++++++++++++++++++++++++++++++++++---------- runtime/vk_ffx.cpp | 4 ++- runtime/vk_fg.cpp | 19 +++++++++---- runtime/vk_fg.h | 7 +++-- runtime/vk_fg4.cpp | 11 +++++--- 5 files changed, 83 insertions(+), 26 deletions(-) diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 9b547287..ac68eb3a 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -21,6 +21,10 @@ PFN_vkGetDeviceProcAddr vk_dpa = vkGetDeviceProcAddr; static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_device) +#define QFAM_MAX 16 +static uint32_t g_nqfam; // the device's queue families (pick_device): their flags and queue counts +static VkQueueFlags g_qfam_flags[QFAM_MAX]; +static uint32_t g_qfam_count[QFAM_MAX]; static VkCtx g_vk; static int g_ready; @@ -271,6 +275,8 @@ retry: vkGetPhysicalDeviceQueueFamilyProperties(g_vk.phys, &nq, qp); g_vk.gfx_family = find_gfx_family(g_vk.phys, qp, nq); g_gfx_queues = qp[g_vk.gfx_family].queueCount; + g_nqfam = nq < QFAM_MAX ? nq : QFAM_MAX; + for (uint32_t i = 0; i < g_nqfam; i++) g_qfam_flags[i] = qp[i].queueFlags, g_qfam_count[i] = qp[i].queueCount; g_vk.xfer_family = g_vk.gfx_family; for (uint32_t i = 0; i < nq; i++) // a family that does transfer only (async DMA engine) if ((qp[i].queueFlags & VK_QUEUE_TRANSFER_BIT) && !(qp[i].queueFlags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT))) { @@ -541,18 +547,44 @@ static int create_device(void) { has_fbl && c->fbl.attachmentFeedbackLoopLayout, has_lr && c->lr.dynamicRenderingLocalRead); } - // Frame generation (vk_fg.cpp) presents on a second queue of the graphics family and acquires on a third - static const float prio[3] = {1.0f, 1.0f, 1.0f}; + // Frame generation (vk_fg.cpp) presents on a second queue of the graphics family and acquires on a third. AMD's + // swapchain needs a present queue nothing else submits to (FidelityFX: the engine must not use it, or it can + // deadlock); the image acquire queue needs no capability. Where the graphics family has one queue (AMD's Windows + // driver, an RX 580), both come from another family: a queue of a family that can present for the present, and + // a second queue of it, or of any other family, for the acquire (the present queue itself when there is none). + // Without them vk_fg_create_swapchain gave up without a word: frame generation set in the menu did nothing. + static const float prio[4] = {1.0f, 1.0f, 1.0f, 1.0f}; uint32_t nq_gfx = vk_fg_queue_count() < g_gfx_queues ? vk_fg_queue_count() : g_gfx_queues; - VkDeviceQueueCreateInfo qi[2] = { - {.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, .queueFamilyIndex = g_vk.gfx_family, .queueCount = nq_gfx, - .pQueuePriorities = prio}}; - uint32_t nqi = 1; - if (g_vk.xfer_family != g_vk.gfx_family) { - qi[1] = (VkDeviceQueueCreateInfo){.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, - .queueFamilyIndex = g_vk.xfer_family, .queueCount = 1, .pQueuePriorities = prio}; - nqi = 2; + uint32_t used[QFAM_MAX] = {0}; + if (g_vk.gfx_family < QFAM_MAX) used[g_vk.gfx_family] = nq_gfx; + if (g_vk.xfer_family != g_vk.gfx_family && g_vk.xfer_family < QFAM_MAX) used[g_vk.xfer_family] = 1; + uint32_t fg_pf = UINT32_MAX, fg_pi = 0, fg_af = UINT32_MAX, fg_ai = 0; + if (nq_gfx >= 2) { + fg_pf = fg_af = g_vk.gfx_family, fg_pi = 1, fg_ai = nq_gfx - 1; + } else { + for (int pass = 0; pass < 2 && fg_pf == UINT32_MAX; pass++) // compute families first, then the others + for (uint32_t f = 0; f < g_nqfam && fg_pf == UINT32_MAX; f++) { + if (f == g_vk.gfx_family || used[f] >= g_qfam_count[f]) continue; + if (!(g_qfam_flags[f] & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT))) continue; + if (!pass != !!(g_qfam_flags[f] & VK_QUEUE_COMPUTE_BIT)) continue; + if (!gfx_window_vk_presentable(g_vk.instance, g_vk.phys, f)) continue; + fg_pf = f, fg_pi = used[f]++; + } + if (fg_pf != UINT32_MAX) { + if (used[fg_pf] < g_qfam_count[fg_pf] && used[fg_pf] < 4) fg_af = fg_pf, fg_ai = used[fg_pf]++; + for (uint32_t f = 0; f < g_nqfam && fg_af == UINT32_MAX; f++) + if (f != g_vk.gfx_family && used[f] < g_qfam_count[f] && used[f] < 4) fg_af = f, fg_ai = used[f]++; + if (fg_af == UINT32_MAX) fg_af = fg_pf, fg_ai = fg_pi; + } } + VkDeviceQueueCreateInfo qi[QFAM_MAX]; + uint32_t nqi = 0; + qi[nqi++] = (VkDeviceQueueCreateInfo){.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, + .queueFamilyIndex = g_vk.gfx_family, .queueCount = nq_gfx, .pQueuePriorities = prio}; + for (uint32_t f = 0; f < g_nqfam; f++) + if (f != g_vk.gfx_family && used[f]) + qi[nqi++] = (VkDeviceQueueCreateInfo){.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, + .queueFamilyIndex = f, .queueCount = used[f], .pQueuePriorities = prio}; VkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, .pNext = &c->f2, .queueCreateInfoCount = nqi, .pQueueCreateInfos = qi, .enabledExtensionCount = n_ext, .ppEnabledExtensionNames = ext}; VkResult r = vk_dlssg_create_device(g_vk.instance, g_vk.phys, &di, &g_vk.device); @@ -564,11 +596,19 @@ static int create_device(void) { rt_log("vk: device entry points come through Streamline's interposer (DLSS Frame Generation)"); } vkGetDeviceQueue(g_vk.device, g_vk.gfx_family, 0, &g_vk.gfx_queue); - if (nq_gfx >= 2) { + if (fg_pf != UINT32_MAX) { VkQueue present, acquire; - vkGetDeviceQueue(g_vk.device, g_vk.gfx_family, 1, &present); - vkGetDeviceQueue(g_vk.device, g_vk.gfx_family, nq_gfx - 1, &acquire); - vk_fg_queues(present, acquire); + vkGetDeviceQueue(g_vk.device, fg_pf, fg_pi, &present); + vkGetDeviceQueue(g_vk.device, fg_af, fg_ai, &acquire); + vk_fg_queues(present, fg_pf, acquire, fg_af); + if (fg_pf != g_vk.gfx_family) + rt_log("vk: frame generation queues: the graphics family %u has %u queue%s; presents on family %u queue %u, " + "image acquires on family %u queue %u", g_vk.gfx_family, g_gfx_queues, g_gfx_queues == 1 ? "" : "s", + fg_pf, fg_pi, fg_af, fg_ai); + } else { + rt_log("vk: frame generation queues: none (the graphics family %u has %u queue%s and no other family with a free " + "queue can present): AMD FSR 3.1 frame generation cannot run", g_vk.gfx_family, g_gfx_queues, + g_gfx_queues == 1 ? "" : "s"); } if (g_vk.xfer_family != g_vk.gfx_family) vkGetDeviceQueue(g_vk.device, g_vk.xfer_family, 0, &g_vk.xfer_queue); else { g_vk.xfer_queue = g_vk.gfx_queue; g_vk.xfer_shared = 1; } diff --git a/runtime/vk_ffx.cpp b/runtime/vk_ffx.cpp index 41272894..eca5062e 100644 --- a/runtime/vk_ffx.cpp +++ b/runtime/vk_ffx.cpp @@ -28,6 +28,7 @@ extern "C" { #include "platform/platform.h" #include "vk.h" #include "vk_ffx.h" +#include "vk_fg.h" } namespace { @@ -260,7 +261,8 @@ extern "C" int vk_ffx_available(void) { #endif } -extern "C" int vk_ffx_framegen_available(void) { return vk_ffx_available(); } // the same library, its API calls +// the same library, its API calls; and a queue for the frame generation swapchain's presents (vk_fg_has_queues) +extern "C" int vk_ffx_framegen_available(void) { return vk_ffx_available() && vk_fg_has_queues(); } // NX2NAVI_FSR4 (default 0, live): FSR 4 where it runs (vk_ffx4.cpp) and has not failed in a row, else FSR 3.1. extern "C" int vk_ffx_fsr4(void) { return cfg_bool("NX2NAVI_FSR4") && vk_ffx4_available() && !vk_ffx4_failed(); } diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 3ed69bef..290564ee 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -35,6 +35,7 @@ namespace { struct { VkQueue present = VK_NULL_HANDLE, acquire = VK_NULL_HANDLE; + uint32_t pfam = 0, afam = 0; // their queue families ffxContext sc = nullptr; // the swapchain context ffxContext fg = nullptr; // the frame generation context VkExtent2D fg_ext = {0, 0}; @@ -102,6 +103,12 @@ extern "C" int vk_fg_wanted(void) { if (F.sc) return !F.failed && !four; if (four) return 1; if (F.failed) return 0; + if (!F.present) { // no queue to present on besides the game's (vk_device.c says why): the plain swapchain stays + static int told_q; + if (!__atomic_exchange_n(&told_q, 1, __ATOMIC_RELAXED)) + rt_log("fg: AMD FSR 3.1 frame generation is set, but the device got no queue for its presents: off"); + return 0; + } if (vk_ffx_framegen_available()) return 1; static int told; if (!__atomic_exchange_n(&told, 1, __ATOMIC_RELAXED)) @@ -112,11 +119,13 @@ extern "C" int vk_fg_wanted(void) { // Always the three queues: the setting can be switched on during a run, and idle queues cost nothing. extern "C" uint32_t vk_fg_queue_count(void) { return 3; } -extern "C" void vk_fg_queues(VkQueue present, VkQueue acquire) { - F.present = present, F.acquire = acquire; - vk_fg4_queues(present, acquire); +extern "C" void vk_fg_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family) { + F.present = present, F.acquire = acquire, F.pfam = present_family, F.afam = acquire_family; + vk_fg4_queues(present, present_family, acquire, acquire_family); } +extern "C" int vk_fg_has_queues(void) { return F.present != VK_NULL_HANDLE; } + // AMD's swapchain is in place: the window's swapchain handle is AMD's object, and every swapchain call has to go // through AMD's functions. That holds after a failure too, until the present code has replaced the swapchain // (vk_fg_wanted): the plain Vulkan calls must never get that handle. @@ -147,8 +156,8 @@ extern "C" VkResult vk_fg_create_swapchain(const VkSwapchainCreateInfoKHR *ci, V d.createInfo.oldSwapchain = VK_NULL_HANDLE; d.gameQueue = {c->gfx_queue, c->gfx_family, submit_game}; d.asyncComputeQueue = {VK_NULL_HANDLE, c->gfx_family, nullptr}; - d.presentQueue = {F.present, c->gfx_family, nullptr}; - d.imageAcquireQueue = {F.acquire, c->gfx_family, nullptr}; + d.presentQueue = {F.present, F.pfam, nullptr}; + d.imageAcquireQueue = {F.acquire, F.afam, nullptr}; ffxReturnCode_t r = vk_ffx_create(&F.sc, &d.header); if (r != FFX_API_RETURN_OK) { F.sc = nullptr; diff --git a/runtime/vk_fg.h b/runtime/vk_fg.h index 31a97cb7..54771529 100644 --- a/runtime/vk_fg.h +++ b/runtime/vk_fg.h @@ -5,7 +5,10 @@ int vk_fg_wanted(void); // NX2NAVI_FRAMEGEN = fsr3 with DLSS or FSR 3.1 upscaling on (live), where FSR 3.1 loads uint32_t vk_fg_queue_count(void); // graphics-family queues the device creates -void vk_fg_queues(VkQueue present, VkQueue acquire); // vk_device.c, after the device exists +// vk_device.c, after the device exists: the queues AMD's swapchain presents and acquires on, and their families (the +// graphics family's own where it has three queues, else queues of another family: AMD's Windows driver gives one) +void vk_fg_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family); +int vk_fg_has_queues(void); // vk_fg_queues got a present queue: frame generation can run here int vk_fg_active(void); // the frame generation swapchain exists int vk_fg_generating(void); // ... and the last frame had generation on (not paused for the menu) // gfx_present.c's swapchain calls while vk_fg_wanted: the first create makes AMD's swapchain context, later ones @@ -34,7 +37,7 @@ int vk_fg4_failed(void); // it failed in this run: FSR 3.1's const char *vk_fg4_name(void); // the frame generation version the loader lists first for the device ("3.1.6"), "" unknown int vk_fg4_active(void); int vk_fg4_generating(void); -void vk_fg4_queues(VkQueue present, VkQueue acquire); +void vk_fg4_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family); VkResult vk_fg4_create_swapchain(const VkSwapchainCreateInfoKHR *ci, VkSwapchainKHR *sc); void vk_fg4_destroy_swapchain(VkSwapchainKHR sc); VkResult vk_fg4_get_images(VkSwapchainKHR sc, uint32_t *n, VkImage *img); diff --git a/runtime/vk_fg4.cpp b/runtime/vk_fg4.cpp index c380e156..6ca06b23 100644 --- a/runtime/vk_fg4.cpp +++ b/runtime/vk_fg4.cpp @@ -28,7 +28,7 @@ extern "C" int vk_fg4_failed(void) { return 0; } extern "C" const char *vk_fg4_name(void) { return ""; } extern "C" int vk_fg4_active(void) { return 0; } extern "C" int vk_fg4_generating(void) { return 0; } -extern "C" void vk_fg4_queues(VkQueue, VkQueue) {} +extern "C" void vk_fg4_queues(VkQueue, uint32_t, VkQueue, uint32_t) {} extern "C" VkResult vk_fg4_create_swapchain(const VkSwapchainCreateInfoKHR *, VkSwapchainKHR *) { return VK_ERROR_FEATURE_NOT_PRESENT; } @@ -68,6 +68,7 @@ namespace { struct { VkQueue present = VK_NULL_HANDLE, acquire = VK_NULL_HANDLE; + uint32_t pfam = 0, afam = 0; // their queue families ffxContext sc = nullptr; // the swapchain context ffxContext fg = nullptr; // the frame generation context VkSwapchainKHR handle = VK_NULL_HANDLE; // the swapchain the context made: remade in place, destroyed after the context @@ -200,7 +201,9 @@ extern "C" int vk_fg4_failed(void) { return F.failed; } extern "C" const char *vk_fg4_name(void) { return g_fg_name; } -extern "C" void vk_fg4_queues(VkQueue present, VkQueue acquire) { F.present = present, F.acquire = acquire; } +extern "C" void vk_fg4_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family) { + F.present = present, F.acquire = acquire, F.pfam = present_family, F.afam = acquire_family; +} // The SDK's swapchain is in place: the window's swapchain handle is its object, and every swapchain call has to go // through its functions, after a failure too, until the present code has taken it down. @@ -230,8 +233,8 @@ extern "C" VkResult vk_fg4_create_swapchain(const VkSwapchainCreateInfoKHR *ci, F.sc_desc.createInfo.oldSwapchain = VK_NULL_HANDLE; F.sc_desc.gameQueue = {c->gfx_queue, c->gfx_family, submit_game}; F.sc_desc.asyncComputeQueue = {VK_NULL_HANDLE, c->gfx_family, nullptr}; - F.sc_desc.presentQueue = {F.present, c->gfx_family, nullptr}; - F.sc_desc.imageAcquireQueue = {F.acquire, c->gfx_family, nullptr}; + F.sc_desc.presentQueue = {F.present, F.pfam, nullptr}; + F.sc_desc.imageAcquireQueue = {F.acquire, F.afam, nullptr}; // the version this code was built against, which the SDK asks for with every context, and a test's version F.sc_ver = {}; F.sc_ver.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_FRAMEGENERATIONSWAPCHAIN_VERSION_VK; From 046c2f6812d7d2743f23bf92e0aafa17732caf6e Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:13 +0100 Subject: [PATCH 57/64] Frame generation: its counters every 600 frames, whether the frame generates or not The line was logged only from a generating frame (the second, then every 3600th): a run whose frames never generated said nothing about why. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_fg.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 290564ee..7eb466f0 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -276,6 +276,15 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, else if (!hudless) F.no_hudless++; else if (!on) F.paused++; F.frame++; + // Every 600 frames, whether this one generates or not: what kept frames from being generated, and how many frames + // AMD's swapchain presented (generated ones included). Logged only from a generating frame, as before, a run whose + // frames never generated (the menu open at the swapchain's second frame, no inputs, no HUD-less picture) said nothing. + if (F.frame == 120 || F.frame % 600 == 0) + rt_log("fg: %llu frames, %llu generated (%llu with the HUD kept out), %llu presents by the swapchain; not generated: " + "%llu menu open, %llu without depth and motion, %llu without the HUD-less picture", + (unsigned long long)F.frame, (unsigned long long)F.generated, (unsigned long long)F.hudless, + (unsigned long long)vk_fg_presents(sc), (unsigned long long)F.paused, (unsigned long long)F.no_inputs, + (unsigned long long)F.no_hudless); ffxConfigureDescFrameGeneration cfg = {}; cfg.header.type = FFX_API_CONFIGURE_DESC_TYPE_FRAMEGENERATION; cfg.swapChain = (void *)sc; @@ -319,11 +328,6 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, p.motionVectors = image(&mv); r = vk_ffx_dispatch(&F.fg, &p.header); if (r != FFX_API_RETURN_OK) return fail("frame generation preparation", r); - if (F.frame == 2 || F.frame % 3600 == 0) - rt_log("fg: %llu frames presented, %llu generated (%llu with the HUD kept out); paused %llu (menu), %llu without depth " - "and motion, %llu without the HUD-less picture", (unsigned long long)F.frame, (unsigned long long)F.generated, - (unsigned long long)F.hudless, (unsigned long long)F.paused, (unsigned long long)F.no_inputs, - (unsigned long long)F.no_hudless); } extern "C" void vk_fg_release(void) { From 81cdd6708637dbacae56830deb950fdbe34d50ba Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:15 +0100 Subject: [PATCH 58/64] Frame generation: the HUD-less picture also where the upscaler's output is smaller than the presented picture In handheld mode the scene is 75% of the picture (1440x810 in 1920x1080) and the game's last passes scale it up with the HUD. hudless_record took the upscaler's output only at the presented picture's size: no HUD-less picture, and no frame generated (5 of 3600 on an RX 580). The crop is now scaled to the smaller picture. Co-Authored-By: Claude Opus 5.5 --- runtime/gfx_present.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index b68fa709..86aa44b8 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -942,8 +942,18 @@ static VkImage hudless_record(VkCommandBuffer cb, unsigned slot, int w, int h, c VkImageView pv; uint32_t pw, ph; VkFormat pf; - if (!(vk_fg_active() || ng_running()) || !vk_dlaa_fg_hudless(&pic, &pv, &pw, &ph, &pf) || (int)pw != w || (int)ph != h) + if (!(vk_fg_active() || ng_running()) || !vk_dlaa_fg_hudless(&pic, &pv, &pw, &ph, &pf) || !pw || !ph || w <= 0 || h <= 0) return VK_NULL_HANDLE; + // The upscaler's output can be smaller than the presented picture: the scene is 75% of it in handheld mode + // (widescreen.c, 1440x810 in a 1920x1080 picture) and the game's last passes scale it up with the HUD. Taken only + // at the same size, as before, that frame had no HUD-less picture and was not generated: 5 of 3600 frames generated + // in handheld mode on an RX 580 (2026-11-11). The crop is taken at the same place of the smaller picture. + int prect[4] = {rect[0], rect[1], rect[2], rect[3]}; + if ((int)pw != w || (int)ph != h) { + prect[0] = (int)((int64_t)rect[0] * pw / w), prect[1] = (int)((int64_t)rect[1] * ph / h); + prect[2] = (int)((int64_t)rect[2] * pw / w), prect[3] = (int)((int64_t)rect[3] * ph / h); + if (prect[2] <= 0 || prect[3] <= 0) return VK_NULL_HANDLE; + } VkCtx *c = vk_ctx(); if (g_hl && (g_hl_w != g_sc_ext.width || g_hl_h != g_sc_ext.height || g_hl_fmt != g_sc_fmt)) { vk_queue_lock(), vkDeviceWaitIdle(c->device), vk_queue_unlock(); @@ -980,7 +990,7 @@ static VkImage hudless_record(VkCommandBuffer cb, unsigned slot, int w, int h, c } // the steps' per-slot views of their source: a slot of its own (the presented picture's views stay valid for this // command buffer); with HDR the PQ encode as for the presented float picture - put_picture(cb, slot + FRAMES, pic, pf, w, h, rect, g_hl, to, general); + put_picture(cb, slot + FRAMES, pic, pf, (int)pw, (int)ph, prect, g_hl, to, general); sync_barrier(cb, g_hl, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); From 39db2ae2a12fac384f0b778820ca50d0fad3521a Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:16 +0100 Subject: [PATCH 59/64] Video memory: every VMA allocation by the heaps' live budgets, with room kept for FidelityFX The heap limits were fixed at device creation, and only texture images looked at the budget. FidelityFX allocates outside VMA (FSR 3.1 contexts, frame generation): 882 MiB of it with 4644 MiB of VMA's filled the 5526 MiB budget of heap 0 while heap 2 had 1757 MiB free, and an upload submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. vmaCreateImage, vmaCreateBuffer and vmaAllocateMemory now go through vk_vma_budget.c, which leaves out a device-local heap whose driver-reported use is within NX2NAVI_VRAM_RESERVE_MB (default 512) of its budget; a Lama patches row sets it. Co-Authored-By: Claude Opus 5.5 --- runtime/cfg.c | 2 + runtime/ui/menu.cpp | 4 ++ runtime/ui_text.c | 4 ++ runtime/ui_text.h | 1 + runtime/vk.h | 14 +++++ runtime/vk_vma_budget.c | 124 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 149 insertions(+) create mode 100644 runtime/vk_vma_budget.c diff --git a/runtime/cfg.c b/runtime/cfg.c index 3c4f3531..30d170a5 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -93,6 +93,7 @@ static const struct { const char *key, *section; } sections[] = { // the menu's Lama patches tab (ui/menu.cpp tab_lama) {"TEX_RELEASED_MAX", "performance"}, {"MIRROR_HOT", "performance"}, {"FREEZE_IDLE", "performance"}, {"SUBMIT_FINISH_KINDS", "performance"}, {"FOLIAGE", "performance"}, {"ZSTD_NATIVE", "performance"}, + {"VRAM_HEAP_LIMIT", "performance"}, {"VRAM_RESERVE_MB", "performance"}, {"UPLOAD_BATCH_MB", "performance"}, {"MESH_EMULATION", "performance"}, {"MESH_EMU_MB", "performance"}, {"MESH_EMU_TASKS", "performance"}, {"MESH_EMU_BATCH", "performance"}, {"VEC_INDEX_FIX", "performance"}, {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, {"FPS_LIMIT", "frame rate"}, {"FPS60", "frame rate"}, {"RTSS", "frame rate"}, @@ -306,6 +307,7 @@ static const CfgMeta metas[] = { {"SUBMIT_FINISH_KINDS", "31", "0..31", RESTART}, // vk_sync.c res_mask {"FOLIAGE", "full", "full|reduced|off", RESTART}, // vk_exec_draw.c foliage_skip {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB + {"VRAM_RESERVE_MB", "512", "0..4096", RESTART}, // vk_vma_budget.c: video memory kept free of VMA's allocations {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit {"MESH_EMULATION", "1", "0|1", RESTART}, {"MESH_EMU_MB", "256", "32..2048", RESTART}, {"MESH_EMU_TASKS", "16384", "256..262144", RESTART}, {"MESH_EMU_BATCH", "1", "0|1", RESTART}, {"VEC_INDEX_FIX", "0", "0|auto|1", RESTART}, diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index 55f5aaf9..7b40d8d7 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -1640,6 +1640,10 @@ static void tab_lama(float scale) { const char *names[] = {menu_text(UI_MENU_AUTO), menu_text(UI_MENU_OFF)}; opt_choice_list("NX2NAVI_VRAM_HEAP_LIMIT", UI_MENU_LAMA_HEAP_LIMIT, values, names, 2, UI_MENU_LAMA_HEAP_LIMIT_DESC); } + { // vk_vma_budget.c reserve_bytes + static const char *const values[] = {"512", "256", "768", "1024", "0"}; + const char *names[] = {"512 MB", "256 MB", "768 MB", "1024 MB", menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_VRAM_RESERVE_MB", UI_MENU_LAMA_RESERVE, values, names, 5, UI_MENU_LAMA_RESERVE_DESC); } { // vk_memory.c upload_batch_cap static const char *const values[] = {"128", "256", "512", "1024", "0"}; diff --git a/runtime/ui_text.c b/runtime/ui_text.c index f5327400..125b0913 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -105,6 +105,10 @@ static const char *const text_en[UI_TEXT_COUNT] = { "its own. Fixes the teal splotches in Kokiri Forest's fog.", [UI_MENU_LAMA_MIRROR_HOT] = "Pools copied every frame stay in system memory", [UI_MENU_LAMA_MIRROR_HOT_DESC] = "A game memory pool whose copies to and from video memory each frame reach this " + [UI_MENU_LAMA_RESERVE] = "Video memory kept free", + [UI_MENU_LAMA_RESERVE_DESC] = "Room kept free in each video memory heap for what FidelityFX (FSR 3.1, frame generation) and the " + "driver allocate on their own: a heap with less than this left takes no new allocation while " + "another heap can. Off: video memory can fill up with frame generation on, and the game ends.", [UI_MENU_LAMA_HEAP_LIMIT] = "Video memory heap limit", [UI_MENU_LAMA_HEAP_LIMIT_DESC] = "Auto: on GPUs with a second video memory heap (an RX 580 shows 6 GB and 2 GB), " "each heap is filled only to its budget before the next one is used, so the " diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 4c7cb198..6793d7be 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -342,6 +342,7 @@ typedef enum { UI_MENU_LAMA_RELEASED, UI_MENU_LAMA_RELEASED_DESC, UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC, UI_MENU_LAMA_MIRROR_HOT, UI_MENU_LAMA_MIRROR_HOT_DESC, + UI_MENU_LAMA_HEAP_LIMIT, UI_MENU_LAMA_HEAP_LIMIT_DESC, UI_MENU_LAMA_RESERVE, UI_MENU_LAMA_RESERVE_DESC, UI_MENU_LAMA_UPLOAD_BATCH, UI_MENU_LAMA_UPLOAD_BATCH_DESC, UI_MENU_LAMA_NO_LIMIT, UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC, UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC, UI_MENU_LAMA_MESH_NATIVE, diff --git a/runtime/vk.h b/runtime/vk.h index 2e6c36c0..6bf6cef8 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -6,6 +6,20 @@ #pragma clang diagnostic ignored "-Wnullability-completeness" #include "vk_mem_alloc.h" #pragma clang diagnostic pop +// Every VMA allocation goes by the device-local heaps' live budgets (vk_vma_budget.c): a heap within the reserve of its +// budget takes nothing new while another heap can. +VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, + VmaAllocation *a, VmaAllocationInfo *info); +VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info); +VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, + VmaAllocation *a, VmaAllocationInfo *info); +uint64_t vk_vma_moved(void); // allocations placed outside a full heap so far +#ifndef VK_VMA_BUDGET_IMPL +#define vmaCreateImage vk_vma_create_image +#define vmaCreateBuffer vk_vma_create_buffer +#define vmaAllocateMemory vk_vma_allocate_memory +#endif #include "nvn_obj.h" // pool_t, tex_t, view_t, samp_par, NVNcopyRegion (memory and textures block) #include "gfx.h" // GfxShader, GfxShaderRes, GfxShaderAnswers (pipelines block) diff --git a/runtime/vk_vma_budget.c b/runtime/vk_vma_budget.c new file mode 100644 index 00000000..a0b7f971 --- /dev/null +++ b/runtime/vk_vma_budget.c @@ -0,0 +1,124 @@ +// Video memory by the heaps' live budgets, for every allocation through VMA (vk.h maps vmaCreateImage, vmaCreateBuffer +// and vmaAllocateMemory here). +// +// An RX 580 (8 GiB, AMD 23.10) has a 6 GiB device-local heap and a 2 GiB device-local, host-visible one. The heap limits +// of vk_device.c (vram_heap_limits) are fixed when the device is made: each heap's budget then, less a tenth for the +// memory VMA does not see. That memory grows later: AMD's FidelityFX (FSR 3.1 upscaling contexts, up to six, and frame +// generation with its swapchain and context) allocates its own, outside VMA. With FSR 3.1 and frame generation on at +// 1080p it reached 882 MiB, and heap 0 held 4644 MiB of VMA's beside it: 5526 of a 5526 MiB budget, while heap 2 had +// 1757 MiB free. The next upload submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY and the game ended (2026-11-11). +// Only texture images looked at the budget (vk_texture.c image_alloc); render targets, pool memory, mirrors and the +// rest were placed in heap 0 up to VMA's fixed limit. +// +// Here each allocation leaves out the memory types of a device-local heap whose use, as the driver reports it +// (VK_EXT_memory_budget: VMA's budget, which counts the memory VMA does not see), is within the reserve of its budget, +// and VMA places it in the other device-local heap, or system memory, as it does past the fixed limits. The reserve +// (NX2NAVI_VRAM_RESERVE_MB, default 512, 0 off) is room left for allocations VMA does not make, which can come at +// once: AMD's contexts are made when the upscaler or the frame generation setting changes. An allocation that finds +// no room that way is made as asked, as before. Allocations from a VMA pool and those for one memory type alone (the +// headroom's anchors, vk_headroom.c) are made as asked. +#define VK_VMA_BUDGET_IMPL +#include +#include "vk.h" +#include "cfg.h" +#include "host.h" +#include "platform/platform.h" + +static VkDeviceSize reserve_bytes(void) { + static int64_t r = -1; + if (r < 0) { + const char *e = cfg_get("NX2NAVI_VRAM_RESERVE_MB"); + long mb = e && *e ? atol(e) : 512; + r = (int64_t)(mb < 0 ? 0 : mb) << 20; + } + return (VkDeviceSize)r; +} + +static volatile int64_t n_moved; // allocations placed outside a full heap +static volatile uint32_t full_seen; // heaps already logged as full (bit per heap) + +// The memory types of bits that may take a new allocation: those of device-local heaps within reserve of their budget +// are left out. Returns bits itself when nothing is left out or nothing would be left. +static uint32_t budget_bits(VmaAllocator vma, uint32_t bits) { + VkDeviceSize reserve = reserve_bytes(); + if (!reserve) return bits; + const VkPhysicalDeviceMemoryProperties *mp; + vmaGetMemoryProperties(vma, &mp); + VmaBudget b[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(vma, b); + uint32_t full = 0; + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if ((mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && b[h].budget && + b[h].usage + reserve > b[h].budget) + full |= 1u << h; + if (!full) return bits; + uint32_t out = 0; + for (uint32_t t = 0; t < mp->memoryTypeCount; t++) + if ((bits >> t & 1) && !(full >> mp->memoryTypes[t].heapIndex & 1)) out |= 1u << t; + if (!out || out == bits) return bits; + uint32_t told = __atomic_fetch_or(&full_seen, full, __ATOMIC_RELAXED); + for (uint32_t h = 0; h < VK_MAX_MEMORY_HEAPS; h++) + if ((full >> h & 1) && !(told >> h & 1)) + rt_log("vk: video memory heap %u full: %llu of its %llu MiB budget in use (%llu MiB of it VMA's), less %llu MiB " + "kept free (NX2NAVI_VRAM_RESERVE_MB): new allocations go to the other heaps while it is", h, + (unsigned long long)(b[h].usage >> 20), (unsigned long long)(b[h].budget >> 20), + (unsigned long long)(b[h].statistics.blockBytes >> 20), (unsigned long long)(reserve >> 20)); + return out; +} + +static void note_moved(void) { + int64_t n = __atomic_add_fetch(&n_moved, 1, __ATOMIC_RELAXED); + if (!(n & (n - 1)) && n >= 16) + rt_log("vk: video memory: %lld allocations placed outside a full heap so far", (long long)n); +} + +uint64_t vk_vma_moved(void) { return (uint64_t)__atomic_load_n(&n_moved, __ATOMIC_RELAXED); } + +// Device-local types of bits (0: all), from the allocator's properties: whether the request could land in a heap +// budget_bits leaves out +static uint32_t all_bits(VmaAllocator vma, uint32_t bits) { + if (bits) return bits; + const VkPhysicalDeviceMemoryProperties *mp; + vmaGetMemoryProperties(vma, &mp); + return mp->memoryTypeCount >= 32 ? ~0u : (1u << mp->memoryTypeCount) - 1; +} + +VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, + VmaAllocation *a, VmaAllocationInfo *info) { + if (!ai->pool) { + uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); + if (nb != bits) { + VmaAllocationCreateInfo b = *ai; + b.memoryTypeBits = nb; + if (vmaCreateImage(vma, ici, &b, img, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + } + } + return vmaCreateImage(vma, ici, ai, img, a, info); +} + +VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info) { + if (!ai->pool) { + uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); + if (nb != bits) { + VmaAllocationCreateInfo b = *ai; + b.memoryTypeBits = nb; + if (vmaCreateBuffer(vma, bci, &b, buf, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + } + } + return vmaCreateBuffer(vma, bci, ai, buf, a, info); +} + +VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, + VmaAllocation *a, VmaAllocationInfo *info) { + uint32_t bits = req->memoryTypeBits & all_bits(vma, ai->memoryTypeBits); + if (!ai->pool && (bits & (bits - 1))) { // more than one type: one alone is asked for on purpose + uint32_t nb = budget_bits(vma, bits); + if (nb != bits) { + VkMemoryRequirements r = *req; + r.memoryTypeBits = nb; + if (vmaAllocateMemory(vma, &r, ai, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + } + } + return vmaAllocateMemory(vma, req, ai, a, info); +} From ccd7fb14c2be49c0a4d1b68eed718ff6c38eb4f2 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:18 +0100 Subject: [PATCH 60/64] mirror_hot: only pools the CPU does not write go back to system memory A pool read in place gives the GPU the game's bytes as they are when it reads them. The demoted pools were ones the CPU refills every frame, which the game rewrote for the next frame while the replay and the GPU still drew the last: small grass and moss flickered and slid over the ground, and with 0.8.0's mesh fallback every mesh flashed. Pools with more than 64 KiB of CPU writes a frame (written pages and flushes) now stay mirrored. Co-Authored-By: Claude Opus 5.5 --- runtime/ui_text.c | 4 ++++ runtime/vk_memory.c | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 125b0913..30f95186 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -105,6 +105,10 @@ static const char *const text_en[UI_TEXT_COUNT] = { "its own. Fixes the teal splotches in Kokiri Forest's fog.", [UI_MENU_LAMA_MIRROR_HOT] = "Pools copied every frame stay in system memory", [UI_MENU_LAMA_MIRROR_HOT_DESC] = "A game memory pool whose copies to and from video memory each frame reach this " + "percentage of its size is read by the GPU from system memory instead, if the " + "CPU does not write it (a pool it refills each frame stays in video memory: read " + "in place, the GPU saw the next frame's data). For GPUs and buses short of " + "bandwidth; 25 suits an RX 580. 0: off.", [UI_MENU_LAMA_RESERVE] = "Video memory kept free", [UI_MENU_LAMA_RESERVE_DESC] = "Room kept free in each video memory heap for what FidelityFX (FSR 3.1, frame generation) and the " "driver allocate on their own: a heap with less than this left takes no new allocation while " diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index e278f085..50824654 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -83,6 +83,7 @@ typedef struct PoolRec { int fl_n; // flush_lock: ranges in fl; read without the lock to skip an empty list uint64_t fl_calls, fl_sum; // flush_lock: the calls and bytes behind fl, for mirror_sync's statistics uint64_t hot_bytes; // replay: uploads, write-backs and merges since the last hot check (mirror_hot) + uint64_t hot_up; // of them the uploads: pages the CPU wrote (mirror_hot leaves such pools mirrored) // Shadow storage (POOL_IMPORT in shadow mode, "shadow storage" below): buf is a host-visible copy of the guest pages. VmaAllocation salloc; uint8_t *smap; // its mapping; offsets as in buf (from import_lo) @@ -2532,7 +2533,7 @@ static void mirror_sync(PoolRec *r) { for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) { range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT; - r->st_pages += 1ull << GEN_PAGE_SHIFT; + r->st_pages += 1ull << GEN_PAGE_SHIFT, r->hot_bytes += 1ull << GEN_PAGE_SHIFT, r->hot_up += 1ull << GEN_PAGE_SHIFT; } } r->mgen = gen_now; @@ -2547,6 +2548,7 @@ static void mirror_sync(PoolRec *r) { r->fl_n = 0, r->fl_calls = r->fl_sum = 0; plat_mutex_unlock(&flush_lock); mrep.flush_calls += calls, mrep.flush_sum += sum; + r->hot_up += sum; // the CPU wrote them (mirror_hot) for (int i = 0; i < n; i++) { uint64_t a = fl[i].lo > lo ? fl[i].lo : lo, b = fl[i].hi < hi ? fl[i].hi : hi; if (a >= b) continue; @@ -3464,6 +3466,13 @@ static void freeze_pass(void) { // previous one has queued its write-backs and nothing is pending for the next: the pool leaves the mirrored list, the // device address table points at its imported buffer again, and the mirror is freed once the submissions that may still // use it are done (vk_defer_buffer, as mirror_evict). Pools under HOT_MIN a frame stay mirrored. +// Only pools the CPU does not write go: a pool read in place gives the GPU the game's bytes as they are when it reads +// them, not as they were at the submit (the mirror's copy). The game refills its per-frame pools for the next frame +// while the replay (asynchronous, up to a frame behind) and the GPU still draw the last one from them: the small grass +// and moss, drawn from instances in such pools, flickered and slid over the ground as the camera turned, and with +// 0.8.0's mesh fallback every mesh flashed (RX 580, Kokiri Forest, 2026-11-11; 18 pools of 2-15 MiB had gone, each +// refilled by the CPU every frame). A pool with more than HOT_UP_MAX of CPU writes a frame stays mirrored. +#define HOT_UP_MAX (64ull << 10) #define HOT_FRAMES 120 #define HOT_MIN (1ull << 20) static int mirror_hot = -1; @@ -3492,7 +3501,11 @@ static void hot_check(void) { plat_mutex_lock(&pool_lock); for (int i = 0; i < nm;) { PoolRec *r = mlist[i]; + uint64_t n = r->import_hi - r->import_lo, per = (uint64_t)(r->hot_bytes / frames), up = (uint64_t)(r->hot_up / frames); r->hot_bytes = (uint64_t)per; // kept for the log line below + r->hot_up = 0; + if (nout == 8 || per < HOT_MIN || per * 100 < n * (uint64_t)mirror_hot || up > HOT_UP_MAX || !r->mbuf || + (!r->twin && r->gpa_rec)) { r->hot_bytes = 0, i++; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict) continue; } From f23f41320263ab1859c8dc233001452cd41a84f3 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:20 +0100 Subject: [PATCH 61/64] Staged mirror uploads: a submission that reads through the device address table waits for the GPU The table names pools without a mirror, guest memory the GPU reads as it runs, and nothing marked such a submission live: with NX2NAVI_MIRROR_STAGE the submit wait ended once the replay had submitted, and the game wrote the next frame's foliage instances while the GPU still drew from them (small grass and moss flickering and sliding as the camera turned). Co-Authored-By: Claude Opus 5.5 --- runtime/vk_exec_draw.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a561763e..596c978e 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1545,6 +1545,12 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * // submission as their draws: either way this ran for the guest submission. if (r->cbuf_slot == VK_GPA_CB_SLOT) { vk_mem_shadow_table_read(); + // What the table names is mostly guest memory itself (pools without a mirror), which the GPU reads as it runs: + // the submission's submit wait is the GPU's (vk_mem_take_live). Without this the staged mirror uploads + // (NX2NAVI_MIRROR_STAGE) ended the wait once the replay had submitted, and the game wrote the next frame's + // instances while the GPU still drew from them: the small grass and moss, which reach their instances through + // the table, flickered and slid over the ground as the camera turned (RX 580, Kokiri Forest, 2026-11-11). + vk_mem_note_live(); VkDescriptorBufferInfo t = vk_gpa_info(); return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE}; } From 922e364f50439cf1a77786530f1eff03c52990a5 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:21 +0100 Subject: [PATCH 62/64] Staged mirror uploads: device address table readers noted at every bind gpl noted them in ssbo_info, which the bind skip and the kept descriptors leave out once they have warmed up: the submissions were taken as reading no guest memory again, and the grass and moss flickered again after a minute of play (every submit wait of a 2100-frame run ended without the GPU). gpa_reader looks once per pipeline for a table binding and notes the submission at each bind_resources and mo_desc. Co-Authored-By: Claude Opus 5.5 --- runtime/vk.h | 2 ++ runtime/vk_exec_draw.c | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/runtime/vk.h b/runtime/vk.h index 6bf6cef8..ddbb5b61 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -880,6 +880,8 @@ typedef struct { // mesh stages are compute pipelines of the same layout, vk_pipeline_mesh_emu), and set 0's compute-side template. int emu; VkDescriptorUpdateTemplate emu_tpl_c; + int8_t gpa; // vk_exec_draw.c gpa_reader: 0 not known yet, 1 no binding of the device + // address table, 2 one (its draws read guest memory as the GPU runs) } VkPipeInfo; // One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the // bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 596c978e..84a9c7ae 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1550,6 +1550,7 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * // (NX2NAVI_MIRROR_STAGE) ended the wait once the replay had submitted, and the game wrote the next frame's // instances while the GPU still drew from them: the small grass and moss, which reach their instances through // the table, flickered and slid over the ground as the camera turned (RX 580, Kokiri Forest, 2026-11-11). + // Not every bind comes here (the bind skip, the kept descriptors): gpa_reader notes it at each bind. vk_mem_note_live(); VkDescriptorBufferInfo t = vk_gpa_info(); return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE}; @@ -3478,7 +3479,29 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. static _Thread_local const VkDescSlot *emu_d; // set 0's descriptors of the last bind_resources build (mesh emulation) static uint32_t emu_d_bytes; +// A pipeline that binds the device address table reads what it names, mostly guest memory itself (pools without a +// mirror), as the GPU runs: the submission's submit wait is the GPU's (vk_mem_take_live). Without this the staged mirror +// uploads (NX2NAVI_MIRROR_STAGE) ended the wait once the replay had submitted, and the game wrote the next frame's +// instances while the GPU still drew from them: the small grass and moss, which reach their instances through the +// table, flickered and slid over the ground as the camera turned (RX 580, Kokiri Forest, 2026-11-11). Noted at every +// bind, before the bind skip and the kept descriptors (bind_sig, pb_take), which leave ssbo_info out: noted only there, +// it was missed once those had warmed up, and the flicker came back after a minute of play. +// the pipeline's binding list holds a table binding +static void gpa_reader(const VkPipeInfo *info) { + int8_t g = __atomic_load_n(&info->gpa, __ATOMIC_RELAXED); // the replay workers bind too + if (!g) { + g = 1; + for (uint32_t i = 0; i < info->n_bindings && g == 1; i++) { + const VkPipeBinding *b = &info->bindings[i]; + if (b->kind == GFX_RES_SSBO && b->res && b->res->cbuf_slot == VK_GPA_CB_SLOT) g = 2; + } + __atomic_store_n(&((VkPipeInfo *)info)->gpa, g, __ATOMIC_RELAXED); + } + if (g == 2) vk_mem_note_live(); +} + static void bind_resources(VkExec *ex, const VkPipeInfo *info) { + gpa_reader(info); DrawCtx *const cx = ex->dc; VkWriteDescriptorSet *w = cx->bind_w; VkDescSlot *d = cx->bind_d; @@ -4657,6 +4680,7 @@ static VkWriteDescriptorSet mo_write(const VkPipeBinding *b, const VkDescSlot *d // would hold the same); a pushed set is pushed by mo_emit. 0 for a pipeline with storage buffers, images or texel // buffers (not drawn again). static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) { + gpa_reader(info); if (info->n_bindings > MAX_BINDINGS || info->pool) return 0; uint32_t total = 0; for (uint32_t i = 0; i < info->n_bindings; i++) total += info->bindings[i].count ? info->bindings[i].count : 1; From 045217bc735df1e112222af44687604f4a4674d1 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:23 +0100 Subject: [PATCH 63/64] Submit wait: also until the newest earlier submission that reads guest memory itself is done The wait is for the frame's first submission; with staged mirror uploads it ended at that one's hand-over unless that one itself read guest memory. The previous frame's submissions that read the grass and moss instances through the device address table were often still on the GPU, and the game wrote the next frame's instances under them: the flicker came back with every submit wait ending without the GPU (2220-frame run, RX 580). Co-Authored-By: Claude Opus 5.5 --- runtime/vk_sync.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 7f0b37e3..ae107252 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -52,6 +52,13 @@ static uint64_t reserved; // asynchronous replay: newest value handed ou // memory itself (value << 1 | live), under st_lock. A value not found counts as live. #define LIVE_RING 4096 static uint64_t live_of[LIVE_RING]; +// Per submission value: the newest submission up to it whose GPU reads guest memory itself (live_of), 0 none. The +// submit wait is for the frame's first submission, but the game's writes after it race with every submission before it +// that still reads guest memory: those of the frame before (the instances of its grass and moss, read through the +// device address table) were often still on the GPU when the wait ended at the first one's hand-over, and the game +// wrote the next frame's instances under them (flicker and sliding foliage, RX 580, 2026-11-11). The wait goes on until +// that one is done. +static uint64_t live_last_of[LIVE_RING], live_last; // per value (index & LIVE_RING - 1): the newest value up to it whose completion hands results to the CPU (vk_exec_take_results) static uint64_t res_of[LIVE_RING], res_last; static uint8_t res_kinds[LIVE_RING]; // per value: its VK_RES_* kinds @@ -1297,6 +1304,8 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { int res = own ? 0x1f : vk_exec_take_results(); plat_mutex_lock(&st_lock); live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0); + if (live) live_last = fence; + live_last_of[fence & (LIVE_RING - 1)] = live_last; res_kinds[fence & (LIVE_RING - 1)] = (uint8_t)res; if (res & res_mask) res_last = fence; res_of[fence & (LIVE_RING - 1)] = res_last; @@ -1419,7 +1428,14 @@ int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(u uint64_t need = 0; // the newest submission up to value whose completion hands the CPU results if (ok) { live = done < value && live_of[value & (LIVE_RING - 1)] != (value << 1); + int known = live_of[value & (LIVE_RING - 1)] >> 1 == value; + need = known ? res_of[value & (LIVE_RING - 1)] : value; if (res_mode == 2) need = value - 1; + if (!res_mode) need = 0; + uint64_t lneed = known ? live_last_of[value & (LIVE_RING - 1)] : value; // the newest reading guest memory + if (lneed > need) need = lneed; + while (!live && need && done < need) { + if (timeout_ns < (1ull << 62)) { uint64_t el = plat_now_ns() - w0; if (el >= timeout_ns) { ok = 0; break; } plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); From 83eade69d150132d71a1dc27f7da6d9aeb301068 Mon Sep 17 00:00:00 2001 From: Tobim6 <102646588+Tobim6@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:24 +0100 Subject: [PATCH 64/64] Prewarm: emulated mesh pipelines are built ahead of play On a GPU without mesh shaders the boot prewarm dropped every recorded mesh pipeline, also with the mesh emulation, which builds them as a vertex pipeline plus compute stages (pipe_miss: layout_get_emu). Each was then built at its first draw, 113-830 ms on an RX 580 (22 of the 48 pipelines missed entering Kokiri Forest on 0.9.1), and the draws waiting for it were skipped: frames held back or shown without them. With NX2NAVI_MESH_EMULATION on they are built by the prewarm like the others. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_pipeline.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index ef816fec..5e206d9f 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -5325,8 +5325,12 @@ static void prewarm_run(void) { VkPipeState s = r[i].s; s.vs = vs, s.fs = fs, s.ts = ts; // a mesh pipeline of the shared list (keys_open_seed) on a GPU without mesh shaders, which draws them through - // the mesh fallback: none to build, as the preparation skips it (seed_pipelines) - if (is_mesh(&s) && !vk_ctx()->has_mesh_shader) { + // the mesh fallback: none to build, as the preparation skips it (seed_pipelines). With the mesh emulation + // (NX2NAVI_MESH_EMULATION) it is a vertex pipeline and its compute stages (pipe_miss: layout_get_emu), built + // here like the others: left to the first draw, each took 113-830 ms on an RX 580 and the draws waiting for it + // were skipped (22 of the 48 pipelines missed entering Kokiri Forest, 0.9.1), frames held back or shown without + // them. + if (is_mesh(&s) && !vk_ctx()->has_mesh_shader && !mesh_emu_on()) { dropped++; continue; }