diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index f2c21f5e..68a2a25f 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1097,6 +1097,11 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; // vk_c0_fill reads the storage slots, and the uniform slots and dispatch size for compute or the base instance for // graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed. + // A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record + // is zero): those two are compared without building and comparing the whole 1.8 KiB record on every draw. + if (k < 2 && B.c0_valid[k] && B.c0_in[k].binst == c0_binst && + !memcmp(B.c0_in[k].ssbo, ex->ssbo[st], sizeof B.c0_in[k].ssbo)) + return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; typeof(B.c0_in[0]) in; memset(&in, 0, sizeof in); memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo); @@ -2944,9 +2949,10 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } } else { kind = 1; - vk_texture_lock(); // tex_info looks up with the texture lock held + int locked = 0; // tex_info looks up with the texture lock held; kept descriptors (pb_take) need none for (uint32_t k = 0; k < cnt; k++) { if (pb_take(ex, b, nd + k, k, &o[k])) continue; + if (!locked) vk_texture_lock(), locked = 1; int slot = r->slot + (int)k; uint64_t h = 0; if (img) h = slot >= 0 && slot < 32 ? ex->img[st][slot] : 0; @@ -2957,7 +2963,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { o[k].w[2] = (uint32_t)o[k].image.imageLayout; // zero the tail padding pb_keep(nd + k, &o[k]); } - vk_texture_unlock(); + if (locked) vk_texture_unlock(); } } else { rt_fatal("vk: binding %u has an unknown kind %d", (unsigned)b->binding, (int)b->kind); @@ -3144,6 +3150,13 @@ static void bind_vertex_buffers(VkExec *ex) { // (vk_mem_gen, read before any resolve), resolves to that buffer again: most draws change one stream or none. uint64_t gen = vk_mem_gen(); int same_gen = B.vb_valid && B.vgen == gen; + // Most draws change no stream at all: with every range as it was bound and no pool or mirror change, the loop + // below would take each buffer from B again and return at the compare after it. + if (same_gen && !memcmp(B.vin, ex->vbuf, sizeof B.vin)) { + int same = 1; + for (int b = 0; !dyn && b < 16 && same; b++) same = B.vstr[b] == ex->pipe.vstream[b].stride; + if (same) return; + } for (int b = 0; b < 17; b++) { vb[b] = Z.zbuf, vo[b] = 0, vsz[b] = ZERO_BYTES, vstr[b] = b < 16 ? ex->pipe.vstream[b].stride : 0; if (b == VK_PIPE_ZERO_BINDING) continue; diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index b7fe82b6..a06f0e97 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -3,7 +3,11 @@ // call's arguments; its pointers point into the packet itself, so the recorder passes the payload to the driver as // it is. A packet never wraps: one that does not fit before the end is preceded by a pad. The producer publishes head // after each packet; the recorder makes the calls and advances tail, which the producer waits on when the ring is -// full and in rec_drain. While a part is recorded the recorder spins on head; idle, it sleeps on an event. +// full and in rec_drain. While a part is recorded the recorder spins on head; idle, it sleeps on an event, and a sleep +// that ends with nothing new goes on without spinning first. The producer wakes a sleeping recorder once: it takes the +// sleeping flag back itself, so the packets it makes while the recorder is still waking cost no further system call. +// head, tail and the producer's own words sit on cache lines of their own: the producer writes head and its counters +// per packet while the recorder reads head in its spin, and the recorder writes tail. #define VK_REC_IMPL #include #include @@ -16,10 +20,12 @@ #define RING_WORDS (1u << 21) // 16 MiB: a part of a submission is ~1-2 MiB of packets in the town static uint64_t ring[RING_WORDS]; -static volatile uint64_t head, tail; // words published by the producer / consumed by the recorder +static _Alignas(64) volatile uint64_t head; // words published by the producer +static _Alignas(64) volatile uint64_t tail; // words consumed by the recorder static volatile int32_t sleeping; // the recorder waits for wake_ev static PlatEvent wake_ev; // auto-reset -static uint64_t put_h; // producer: words written (published up to head) +static _Alignas(64) uint64_t put_h; // producer: words written (published up to head) +static uint64_t tail_seen; // producer: tail as it last read it (wait_space); tail only grows static VkCommandBuffer rec_cb; // the part being recorded (set while the recorder is idle) static int rec_live; // producer: rec_begin started the recorder for rec_cb static PlatThread *rec_thread; @@ -56,18 +62,21 @@ static void rec_main(void *arg) { (void)arg; plat_thread_priority(rec_prio); uint64_t t = 0; + int slept = 0; // the last thing it did was sleep: nothing new then means still idle, so no spin before the next sleep for (;;) { uint64_t h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); if (t == h) { - for (int i = 0; i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); + for (int i = 0; !slept && i < 4096 && t == h; i++) _mm_pause(), h = __atomic_load_n(&head, __ATOMIC_ACQUIRE); if (t == h) { __atomic_store_n(&sleeping, 1, __ATOMIC_SEQ_CST); h = __atomic_load_n(&head, __ATOMIC_SEQ_CST); if (t == h) plat_event_wait_ns(&wake_ev, 2000000); // a missed wake costs at most 2 ms __atomic_store_n(&sleeping, 0, __ATOMIC_RELAXED); + slept = 1; continue; } } + slept = 0; // NX2NAVI_REC_DELAY_US (debug): the recorder falls this far behind each batch it finds, as on a slow machine, // so objects the replay lets go of meet packets it has not made yet (NX2NAVI_REC_CHECK). if (rec_delay_ns) plat_sleep_ns(rec_delay_ns); @@ -85,18 +94,27 @@ static void rec_main(void *arg) { // ---- producer +// Wakes the recorder when it sleeps or is about to: whoever takes the flag back sets the event, once per sleep. Setting +// the event for a recorder that has not reached its wait yet makes that wait return at once. +static void wake_if_sleeping(void) { + if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED) && __atomic_exchange_n(&sleeping, 0, __ATOMIC_SEQ_CST)) + plat_event_set(&wake_ev); +} + static void wake(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); - if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + wake_if_sleeping(); } static void wait_space(uint64_t need) { - if (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) <= RING_WORDS) return; + if (put_h + need - tail_seen <= RING_WORDS) return; // room even by the older tail + tail_seen = __atomic_load_n(&tail, __ATOMIC_ACQUIRE); + if (put_h + need - tail_seen <= RING_WORDS) return; RS_.full_waits++; uint64_t t0 = now_ns(); __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); wake(); - while (put_h + need - __atomic_load_n(&tail, __ATOMIC_ACQUIRE) > RING_WORDS) _mm_pause(); + while (put_h + need - (tail_seen = __atomic_load_n(&tail, __ATOMIC_ACQUIRE)) > RING_WORDS) _mm_pause(); full_ns_all += now_ns() - t0; } @@ -122,7 +140,7 @@ static void *put(uint32_t op, size_t bytes) { static void pub(void) { __atomic_store_n(&head, put_h, __ATOMIC_RELEASE); - if (__atomic_load_n(&sleeping, __ATOMIC_RELAXED)) plat_event_set(&wake_ev); + wake_if_sleeping(); } void rec_drain(void) { diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index 62f07ef4..b83544c8 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -313,7 +313,11 @@ static int first_above(uint64_t a) { int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out) { if (!U.on || !n) return 0; - int i = first_above(addr); + // Consecutive bindings mostly name the region the last one did: that one is tried before the binary search. The + // regions are sorted and disjoint, so the region that holds addr is the one first_above finds. + static int hint; + int i = hint; + if (i >= U.nreg || addr < U.reg[i].lo || addr >= U.reg[i].hi) i = first_above(addr); if (i >= U.nreg) return 0; const Reg *r = &U.reg[i]; if (r->dropped || addr < r->lo || addr + n > r->hi) return 0; @@ -321,6 +325,7 @@ int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferI if (off % align) return 0; *out = (VkDescriptorBufferInfo){U.arena->buf, off, n}; U.hits++; + hint = i; return 1; }