diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index 57cad6b2..a4cb4b47 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -12,8 +12,11 @@ // // The decode is recorded into the upload batch (vk_memory.c) in chunks of at most 32 MiB of texels: a chunk's raw // blocks go from staging to a device-local scratch buffer, its dispatches write RGBA8 texels behind them, and a -// buffer-to-image copy puts them into the image; the next chunk reuses the scratch after a barrier. The scratch -// buffer is freed once the batch completes (vk_defer_buffer). +// buffer-to-image copy puts them into the image; the next chunk reuses the scratch after a barrier. The next texture +// reuses it too, behind the same barrier, while it records into the same command buffer (scratch_get): a buffer per +// texture, each kept until its batch completed, made a scene load (2641 textures in one frame in Kokiri Forest) hold +// gigabytes of scratch at once and allocate 256 MiB device blocks inline, about half a second of the load frame. A +// larger scratch replaces it; the old one is freed once the batches recorded so far completed (vk_defer_buffer). #include #include #include @@ -418,6 +421,42 @@ static void mem_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 src, VkAccessF // Scratch per vk_astc_record: texels of at most this many bytes at a time (or one region, when larger), so a large // texture array (169 M texels, 677 MB as RGBA8, on monsters2) does not need its whole RGBA8 copy as scratch. #define CHUNK_OUT (32ull << 20) +#define SCRATCH_MIN (64ull << 20) // the shared scratch's least size: one chunk's texels plus its blocks and BC3 + +// The shared scratch and the command buffer it was last recorded into. An upload batch's command buffer is used again +// only after that batch completed (vk_memory.c: completed batches go to the spare list), so the same handle means the +// open batch, ordered by a barrier, or a finished one. g_lock held. +static struct { VkBuffer buf; VmaAllocation alloc; VkDeviceSize size; VkCommandBuffer cb; VkDevice dev; } S; +static uint64_t n_scratch_new, n_scratch_reuse; + +// A scratch of at least need bytes for recording into cb; *reused: earlier work in cb may still use it. 0 on failure. +static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int *reused) { + plat_mutex_lock(&g_lock); + *reused = S.buf && S.dev == c->device && S.cb == cb && S.size >= need; + if (!*reused) { + VkDeviceSize size = need < SCRATCH_MIN ? SCRATCH_MIN : need; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | + VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VkBuffer b; + VmaAllocation a; + if (vmaCreateBuffer(c->vma, &bi, &aci, &b, &a, NULL) != VK_SUCCESS) { + plat_mutex_unlock(&g_lock); + return VK_NULL_HANDLE; + } + // the old one may still be read by its command buffer: freed once the batches recorded so far completed + if (S.buf && S.dev == c->device) vk_defer_buffer(S.buf, S.alloc); + S.buf = b, S.alloc = a, S.size = size, S.dev = c->device; + n_scratch_new++; + } else { + n_scratch_reuse++; + } + S.cb = cb; + VkBuffer out = S.buf; + plat_mutex_unlock(&g_lock); + return out; +} static VkDeviceSize region_out(const VkAstcRegion *r) { return (VkDeviceSize)r->w * r->h * r->slices * 4; } static VkDeviceSize region_in(const VkAstcRegion *r, uint32_t bw, uint32_t bh) { @@ -463,16 +502,12 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons cap_in = (cap_in + al - 1) / al * al; cap_out = (cap_out + al - 1) / al * al; if (!bc) cap_bc = 0; - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = cap_in + cap_out + cap_bc, - .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | - VK_BUFFER_USAGE_TRANSFER_DST_BIT}; - VmaAllocationCreateInfo aci = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; - VkBuffer sb; - VmaAllocation sa; - if (vmaCreateBuffer(c->vma, &bi, &aci, &sb, &sa, NULL) != VK_SUCCESS) { + int reused; + VkBuffer sb = scratch_get(c, cb, cap_in + cap_out + cap_bc, &reused); + if (!sb) { if (plat_inc32(&n_fail) <= 16) rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded", - (unsigned long long)(cap_in + cap_out)); + (unsigned long long)(cap_in + cap_out + cap_bc)); return 0; } plat_mutex_lock(&g_lock); @@ -522,7 +557,7 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons in += region_in(&rg[i1], bw, bh), out += region_out(&rg[i1]); } vk_crumb(cb, VK_CRUMB_ASTC, ++n_chunks); - if (chunks) // the previous chunk's dispatches and image copy are done with the scratch + if (chunks || reused) // the previous chunk's (or texture's) dispatches and copies are done with the scratch mem_barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_READ_BIT, VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, @@ -601,13 +636,14 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons plat_mutex_unlock(&t_lock); } if ((n_dispatch + (uint64_t)n) / 1024 != n_dispatch / 1024 || !n_dispatch) { - rt_log("vk: ASTC: %llu regions, %.1f M texels decoded on the GPU so far", - (unsigned long long)(n_dispatch + (uint64_t)n), (double)n_texels / 1e6); + rt_log("vk: ASTC: %llu regions, %.1f M texels decoded on the GPU so far; shared scratch %llu MiB, made %llu " + "times, reused %llu", (unsigned long long)(n_dispatch + (uint64_t)n), (double)n_texels / 1e6, + (unsigned long long)(S.size >> 20), (unsigned long long)n_scratch_new, + (unsigned long long)n_scratch_reuse); vk_astc_time_summary(); } n_dispatch += (uint64_t)n; free(copies), free(bcp); - vk_defer_buffer(sb, sa); return 1; } @@ -626,6 +662,8 @@ void vk_astc_release(void) { if (G.dsl) vkDestroyDescriptorSetLayout(c->device, G.dsl, NULL); if (G.tab) vmaDestroyBuffer(c->vma, G.tab, G.tab_alloc); } + if (c && S.buf && S.dev == c->device) vmaDestroyBuffer(c->vma, S.buf, S.alloc); // the device is idle here + memset(&S, 0, sizeof S); memset(&G.dsl, 0, sizeof G - offsetof(__typeof__(G), dsl)); G.dev = VK_NULL_HANDLE; G.mode = -1; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a8d0893c..4bad9fbf 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1329,6 +1329,24 @@ static uint32_t attached_as(const VkExec *ex, const SubRange *r) { return m; } +// Knobs the draw and dispatch paths test on every call, read once: each cfg lookup walks the settings table +// (cfg.c raw_get), about 0.2 ms a frame of replay in Kokiri Forest when they ran per draw. +static int knob_log_mesh(void) { // NX2NAVI_LOG_MESH=1: the first mesh draws' state + static int on = -1; + if (on < 0) on = rt_knob("LOG_MESH", "0"); + return on; +} +static int knob_tm_dispatch(void) { // NX2NAVI_TM_DISPATCH: the scene upscaler stand-in's mode + static int on = -1; + if (on < 0) on = rt_knob("TM_DISPATCH", "1"); + return on; +} +static int knob_log_cs_binds(void) { // NX2NAVI_LOG_CS_BINDS set: the compute bindings of each program's first dispatch + static int on = -1; + if (on < 0) on = cfg_get("NX2NAVI_LOG_CS_BINDS") != NULL; + return on; +} + // NX2NAVI_TEX_MEMO (tex_info, feedback_scan): 0 off, 1 on, 2 (=check) computes every hit again and stops on a difference. static int td_mode(void) { static int on = -1; @@ -1360,10 +1378,16 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { int n = gfx_shader_resources(fs, GFX_RES_TEX, &res); uint32_t m = 0, gen = vk_texture_slots_gen(); // before any lookup int memo = td_mode(), locked = 0; - for (int r = 0; r < n; r++) - for (int k = 0; k < (res[r].count ? res[r].count : 1); k++) { + for (int r = 0; r < n; r++) { + // Only bound slots (0..127) are looked at. The texture and sampler pool arrays (res.pool: 32768 and 4096 + // descriptors, bindless handles) have slot -1: walking them ran up to about 100,000 empty iterations per scan, + // about 3.6 ms of replay per frame in Kokiri Forest, and read tex[FS][0..127] for them by mistake. + if (res[r].pool || res[r].slot < 0) continue; + int count = res[r].count ? res[r].count : 1; + if (count > 128 - res[r].slot) count = 128 - res[r].slot; + for (int k = 0; k < count; k++) { int slot = res[r].slot + k; - uint64_t h = slot >= 0 && slot < 128 ? ex->tex[NVN_STAGE_FS][slot] : 0; + uint64_t h = ex->tex[NVN_STAGE_FS][slot]; if (!h) continue; typeof(FM[0]) *f = &FM[(h * 0x9E3779B97F4A7C15ull) >> 51]; if (memo && f->h == h && slot_still(&f->gen, f->tok, h, gen)) { @@ -1383,6 +1407,7 @@ static uint32_t feedback_scan(VkExec *ex, const GfxShader *fs) { if (tok) *f = (typeof(*f)){.h = h, .tok = tok, .gen = gen, .r = sr}; else f->h = 0; } + } if (locked) vk_texture_unlock(); ex->fb_fs = fs, ex->fb_serial = ex->tex_serial, ex->fb_pass = ex->pass_serial, ex->fb_sampled = m; return m; @@ -3688,7 +3713,7 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); vk_exec_frame.draws++; static uint32_t n_logged; - if (n_logged < 8 && rt_knob("LOG_MESH", "0")) { + if (n_logged < 8 && knob_log_mesh()) { n_logged++; rt_log("vk: mesh draw: %u workgroups, program %llu", d->count, (unsigned long long)ex->prog->serial); } @@ -3720,7 +3745,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; vk_exec_frame.draws++; static uint32_t n_logged; - if (n_logged < 8 && rt_knob("LOG_MESH", "0")) { + if (n_logged < 8 && knob_log_mesh()) { n_logged++; rt_log("vk: indirect mesh draw: arguments at %#llx, program %llu", (unsigned long long)d->ind, (unsigned long long)ex->prog->serial); @@ -3878,7 +3903,7 @@ static VkImage fallback_scratch(VkExec *ex, uint32_t w, uint32_t h, VkFormat f) .extent = {w, h, 1}, .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | - (f == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 && rt_knob("TM_DISPATCH", "1") != 2 ? VK_IMAGE_USAGE_SAMPLED_BIT : 0), + (f == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 && knob_tm_dispatch() != 2 ? VK_IMAGE_USAGE_SAMPLED_BIT : 0), .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; VmaAllocationCreateInfo ia = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; VkImage img; VmaAllocation a; @@ -3960,7 +3985,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) { src = t; } if (dst && src) vk_texture_storage_bound_locked(dst); - if (dst && src && cfg_get("NX2NAVI_LOG_CS_BINDS")) { // each distinct (program, source, target) once, 256 at most + if (dst && src && knob_log_cs_binds()) { // each distinct (program, source, target) once, 256 at most static struct { uint64_t prog, s, d; } seen[256]; static int n; uint64_t pr = ex->prog ? ex->prog->serial : 0; @@ -3993,7 +4018,7 @@ static void untranslated_dispatch_fallback(VkExec *ex) { VkImage si = src ? src->image : VK_NULL_HANDLE, di = dst ? dst->image : VK_NULL_HANDLE; VkImageView dst_view = VK_NULL_HANDLE; // for the tonemap pass, while the lock keeps the image if (src && dst && (dst->usage & VK_IMAGE_USAGE_STORAGE_BIT) && raw_9e5(src->image, src->format) && - dst->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && rt_knob("TM_DISPATCH", "1") != 2) { + dst->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && knob_tm_dispatch() != 2) { view_t nv; memset(&nv, 0, sizeof nv); dst_view = get_view(ex, dst, &nv, 0, 3, 1); // 3: Ryujinx Texture2D @@ -4094,7 +4119,7 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { dummies_init(); int present; const GfxShader *cs = shader_for(ex, NVN_STAGE_CS, &present, NULL, 0); // synchronous: a skipped dispatch changes results - if (cfg_get("NX2NAVI_LOG_CS_BINDS")) { // diagnosis: the compute bindings of the first dispatch of each program + if (knob_log_cs_binds()) { // diagnosis: the compute bindings of the first dispatch of each program static uint64_t seen[64]; static int nseen; int known = 0; for (int i = 0; i < nseen; i++) known |= seen[i] == ex->prog->serial; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index e6dce958..f5437d8a 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -690,11 +690,17 @@ static void clear_tex(VkExec *ex, const p_clear_tex *p) { if (img->aspect & VK_IMAGE_ASPECT_COLOR_BIT) { rng.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; VkClearColorValue cv; - if (int_format(img->format) != (p->is_uint != 0)) { - warn_once(W_CLEAR, "texture clear value kind (uint %d) does not match format %d: skipped", p->is_uint, (int)img->format); - return; - } memcpy(&cv, &p->c, sizeof cv); + if (int_format(img->format) != (p->is_uint != 0)) { + // A value of the other kind that is all zero bits (0.0f is) is the same clear either way: Ocarina clears its + // rg16ui edge map (480x270) with a float 0 each frame, and skipped, the post chain's edge classification + // read the previous frames' edges. Other values stay skipped. + if (cv.uint32[0] | cv.uint32[1] | cv.uint32[2] | cv.uint32[3]) { + warn_once(W_CLEAR, "texture clear value kind (uint %d) does not match format %d: skipped", p->is_uint, + (int)img->format); + return; + } + } vk_exec_pass_end(ex); vk_exec_barrier(ex); vkCmdClearColorImage(ex->cb, img->image, VK_IMAGE_LAYOUT_GENERAL, &cv, 1, &rng); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index ffdf2435..1ae8dc2a 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -1970,14 +1970,20 @@ static struct { // enters each Pipe as building under the key a draw would compute, queues it to the low queue and builds the queue on // every core. Records are deduplicated by a hash of their bytes; they are collected under P.lock by the worker that // built the pipeline and appended by the present thread (vk_pipeline_autosave) or at exit (vk_pipeline_save). A file -// with another magic, version or record size is ignored and rewritten. Bump KEYS_VERSION when VkPipeState or the -// record changes meaning without changing size. -#define KEYS_VERSION 3 // 3: shaders named by input id (gfx_shader.c) +// with another magic, version or record size is ignored and rewritten; a version 3 file (no task stage) is converted +// and rewritten. Bump KEYS_VERSION when VkPipeState or the record changes meaning without changing size. +#define KEYS_VERSION 4 // 3: shaders named by input id (gfx_shader.c); 4: mesh pipelines, with their task stage typedef struct { char magic[8]; uint32_t version, rec_size; } KeysHeader; typedef struct { - uint8_t vs[32], fs[32]; // gfx_shader_input_id of each stage; fs all zero with has_fs 0 + uint8_t vs[32], fs[32]; uint32_t has_fs, pad; - VkPipeState s; // vs, fs zero + VkPipeState s; +} KeyRecV3; +typedef struct { + uint8_t vs[32], fs[32], ts[32]; // gfx_shader_input_id of each stage (vs: the mesh stage of a mesh pipeline); fs all + // zero with has_fs 0, ts with has_ts 0 + uint32_t has_fs, has_ts; + VkPipeState s; // vs, fs, ts zero } KeyRec; static struct { char path[PLAT_PATH_MAX * 2]; @@ -2031,8 +2037,9 @@ static void rec_fill(KeyRec *r, const VkPipeState *s) { memset(r, 0, sizeof *r); memcpy(r->vs, gfx_shader_input_id(s->vs), 32); if (s->fs) memcpy(r->fs, gfx_shader_input_id(s->fs), 32), r->has_fs = 1; + if (s->ts) memcpy(r->ts, gfx_shader_input_id(s->ts), 32), r->has_ts = 1; r->s = *s; - r->s.vs = r->s.fs = NULL; + r->s.vs = r->s.fs = r->s.ts = NULL; r->s.prim_restart = 0; // dynamic state: the same pipeline either way } @@ -2041,6 +2048,7 @@ static void keys_note(const VkPipeState *s) { if (!A.on || !K.path[0]) return; static const uint8_t none[32]; if (s->fs && !memcmp(gfx_shader_input_id(s->fs), none, 32)) return; // the runtime's own shader (gfx_shader_own) + if (s->ts && !memcmp(gfx_shader_input_id(s->ts), none, 32)) return; KeyRec r; rec_fill(&r, s); if (!seen_add(rec_hash(&r))) return; @@ -2093,8 +2101,8 @@ static void keys_open(void) { fseek(f, 0, SEEK_END); long len = ftell(f); fseek(f, 0, SEEK_SET); - if (len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8) && - h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) { + int head = len >= (long)sizeof h && fread(&h, sizeof h, 1, f) == 1 && !memcmp(h.magic, "NX2NPKEY", 8); + if (head && h.version == KEYS_VERSION && h.rec_size == sizeof(KeyRec)) { K.header_ok = 1; size_t n = ((size_t)len - sizeof h) / sizeof(KeyRec); if (n && (K.file = malloc(n * sizeof(KeyRec))) && fread(K.file, sizeof(KeyRec), n, f) == n) K.file_n = n; @@ -2102,6 +2110,22 @@ static void keys_open(void) { for (size_t i = 0; i < K.file_n; i++) seen_add(rec_hash((const KeyRec *)K.file + i)); if (len != (long)(sizeof h + n * sizeof(KeyRec))) // a torn append: rewrite the file from what was read K.header_ok = 0; + } else if (head && h.version == 3 && h.rec_size == sizeof(KeyRecV3)) { + // the vertex pipelines of a version 3 file keep their prewarm: each record without a task stage, and the + // file rewritten in this version below (header_ok 0) + size_t n = ((size_t)len - sizeof h) / sizeof(KeyRecV3); + KeyRecV3 *old = n ? malloc(n * sizeof *old) : NULL; + if (old && fread(old, sizeof *old, n, f) == n && (K.file = calloc(n, sizeof(KeyRec)))) { + KeyRec *r = (KeyRec *)K.file; + for (size_t i = 0; i < n; i++) { + memcpy(r[i].vs, old[i].vs, 32), memcpy(r[i].fs, old[i].fs, 32); + r[i].has_fs = old[i].has_fs, r[i].s = old[i].s; + seen_add(rec_hash(&r[i])); + } + K.file_n = n; + rt_log("vk_pipeline: %s: %zu version 3 records converted; the file is rewritten", K.path, n); + } + free(old); } else { rt_log("vk_pipeline: %s has another format or version; it is rewritten", K.path); } @@ -2160,7 +2184,8 @@ static void pipe_worker(void *arg) { } // Step 1 for a job a draw waits for: its stage libraries and the fast link. Step 2, the optimised build, goes to // the head of the low queue, so a burst of new pipelines gets every fast link before any optimised build. - if (P.gpl && (!j->prebuild || j->demanded) && !j->p->info.pipeline && !j->p->fast_tried && j->p->lay) { + if (P.gpl && (!j->prebuild || j->demanded) && !j->p->info.pipeline && !j->p->fast_tried && j->p->lay && + !is_mesh(&j->s)) { int final; VkPipeline f = gpl_link(&j->s, &j->k, j->p->lay, 1, &final); // releases P.lock while building j->p->fast_tried = 1; @@ -2285,9 +2310,10 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode int mesh = is_mesh(s); if (mesh) { // Mesh draws (VK_EXT_mesh_shader): one set holding the mesh stage's bindings, the fragment stage's after them and - // the task stage's after those (create_graphics renumbers both modules); built here, without libraries or workers. + // the task stage's after those (create_graphics renumbers both modules); no libraries (the GPL fast link is for + // vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its + // whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note). uint32_t fspan = s->fs ? binding_span(s->fs) : 1; - mode = 0; if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders"; else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch"; else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) || @@ -2317,7 +2343,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode rt_log("vk_pipeline: graphics pipeline not built: %s", why); } else if (!j) { P.stats.graphics++; - if (mode != 2 && !mesh) keys_note(s); // the key file holds vertex pipelines only + if (mode != 2) keys_note(s); } Pipe *p = pipe_add(k, sizeof *k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_GRAPHICS, gfx_shader_inputs(s->vs, NULL)); if (!j || !p) { @@ -2442,7 +2468,7 @@ static VkPipeline graphics(const VkPipeState *s, const VkPipeInfo **info, int as : "builds its libraries itself (NX2NAVI_PIPE_WAIT=draw)"); } double end = wait_ms ? now_ms() + wait_ms : 0; - if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && p->lay) { + if (p && p->building && P.gpl && !p->info.pipeline && !p->fast_tried && p->lay && !is_mesh(s)) { int final = 0, build = wait_ms != 0 && draw_build; uint64_t p0 = perf_begin(), t0 = qpc(); if (build && U.on) { @@ -2557,8 +2583,8 @@ void vk_pipeline_prewarm(void) { } const KeyRec *r = (const KeyRec *)K.file; size_t n = K.file_n, m = 0; - uint8_t (*ids)[32] = malloc(2 * n * 32); - const GfxShader **sh = calloc(2 * n, sizeof *sh); + uint8_t (*ids)[32] = malloc(3 * n * 32); + const GfxShader **sh = calloc(3 * n, sizeof *sh); if (!ids || !sh) { free(ids), free(sh); return; @@ -2566,6 +2592,7 @@ void vk_pipeline_prewarm(void) { for (size_t i = 0; i < n; i++) { memcpy(ids[m++], r[i].vs, 32); if (r[i].has_fs) memcpy(ids[m++], r[i].fs, 32); + if (r[i].has_ts) memcpy(ids[m++], r[i].ts, 32); } qsort(ids, m, 32, id_cmp); size_t u = 0; @@ -2582,7 +2609,7 @@ void vk_pipeline_prewarm(void) { for (size_t i = 0; i < n; i++) { const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp); const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL; - if (!v || !f || !sh[v - ids] || !sh[f - ids]) continue; + if (!v || !f || !sh[v - ids] || !sh[f - ids] || gfx_shader_stage(sh[v - ids]) != GFX_STAGE_VS) continue; GfxShaderIo io[32]; vs_outs(sh[v - ids], sh[f - ids], io); pair += (uint64_t)__builtin_popcount(gfx_shader_inputs(sh[f - ids], NULL)), pairs++; @@ -2598,17 +2625,20 @@ void vk_pipeline_prewarm(void) { for (size_t i = 0; i < n; i++) { const uint8_t (*v)[32] = bsearch(r[i].vs, ids, u, 32, id_cmp); const uint8_t (*f)[32] = r[i].has_fs ? bsearch(r[i].fs, ids, u, 32, id_cmp) : NULL; - const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = f ? sh[f - ids] : NULL; - if (!vs || (r[i].has_fs && !fs)) { + const uint8_t (*t)[32] = r[i].has_ts ? bsearch(r[i].ts, ids, u, 32, id_cmp) : NULL; + const GfxShader *vs = v ? sh[v - ids] : NULL, *fs = f ? sh[f - ids] : NULL, *ts = t ? sh[t - ids] : NULL; + if (!vs || (r[i].has_fs && !fs) || (r[i].has_ts && !ts)) { dropped++; continue; } VkPipeState s = r[i].s; - s.vs = vs, s.fs = fs; + s.vs = vs, s.fs = fs, s.ts = ts; GKey k; gkey_fill(&s, &k); - pr_hist[(k.clamp & 1) | (k.poly_mode & 3) << 1 | (k.minus_w & 1) << 3]++; - fs_hist[k.samples >= 8 ? 3 : k.samples >= 4 ? 2 : k.samples >= 2 ? 1 : 0]++; + if (!is_mesh(&s)) { // the stage libraries lb_start prebuilds are for vertex pipelines + pr_hist[(k.clamp & 1) | (k.poly_mode & 3) << 1 | (k.minus_w & 1) << 3]++; + fs_hist[k.samples >= 8 ? 3 : k.samples >= 4 ? 2 : k.samples >= 2 ? 1 : 0]++; + } uint64_t h = key_hash(&k, sizeof k); int q = 0; if (pipe_find(&k, sizeof k, h)) known++; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 119960c6..8bd88337 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -115,8 +115,11 @@ static uint64_t n_checks; // draw-binding storage checks (make_fresh_snap) si // tex_gen per storage address (hashed): a slot resolves through the entries at its own address only (slot_resolve), so // a slot's resolution stays valid while its address's generation does. NX2navi finalizes and initializes transient // textures every frame: about 90 tex_gen and 160 slots_gen bumps per frame in the monsters fight, one every 16 draws, -// which emptied every cache keyed on the global generations. -#define AGEN_N 4096 +// which emptied every cache keyed on the global generations. 65536 buckets, the most the 16-bit index of sinfo, +// slot_bucket and the dirty ring's bucket entries holds: with 4096, about 8 slots shared each bucket, and every +// transient texture made or released at one address made the texture pool refresh (vk_exec_draw.c pool_sync_entries) +// refill its neighbours too, most of its 3-3.7 ms per frame in Kokiri Forest. +#define AGEN_N 65536 static uint32_t agen[AGEN_N]; // Per slot id, for vk_texture_slot_token without the lock: the registration serial (bits 16-47; 0: never registered) // and the address generation index (bits 0-15), one word so a reader gets a consistent pair. Fixed-size (slot ids are @@ -124,7 +127,7 @@ static uint32_t agen[AGEN_N]; static uint64_t *sinfo; static uint32_t reg_serial; -static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); } +static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 48); } // Slots that may resolve differently since a position, for the texture pool descriptors (vk_exec_draw.c // pool_sync_entries), which otherwise looked at every registered slot after each change: Ocarina registers slots all @@ -484,6 +487,35 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { (unsigned long long)k->addr, w, h, d->format); } } + // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: + // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load + // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that + // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom + // thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference. + enum { SUBALLOC_MAX = 32 << 20 }; + if (!img && tex_mirrored(t) && samples == 1) { + if (vkCreateImage(c->device, &ci, NULL, &img) == VK_SUCCESS) { + // an image the driver requires (not only prefers) dedicated memory for keeps it: vmaCreateImage below + VkMemoryDedicatedRequirements dr = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 mr2 = {.sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, .pNext = &dr}; + VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; + vkGetImageMemoryRequirements2(c->device, &ri, &mr2); + VkMemoryRequirements mr = mr2.memoryRequirements; + VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && + vmaAllocateMemory(c->vma, &mr, &ai, &e->alloc, NULL) == VK_SUCCESS; + if (ok && vmaBindImageMemory(c->vma, e->alloc, img) != VK_SUCCESS) { + vmaFreeMemory(c->vma, e->alloc); + ok = 0; + } + if (!ok) { + vkDestroyImage(c->device, img, NULL); + img = VK_NULL_HANDLE, e->alloc = NULL; + } + } else { + img = VK_NULL_HANDLE; + } + } if (!img) { VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; VkResult r = vmaCreateImage(c->vma, &ci, &ai, &img, &e->alloc, NULL); @@ -1201,6 +1233,13 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { // 2250x1080 map target through an (R,G,B,1) descriptor and followed a 960x60 target at the same address. Tex *own = e, *any = NULL; e = NULL; + // A slot whose content the CPU writes (tex_mirrored) follows another image only for content its own bytes do + // not hold, the GPU's: Ocarina rasterizes each UI caption on the CPU into a small r8 glyph atlas and reuses the + // memory of a freed one for the next, of another width. Following the older 256x39 atlas at the address, the + // replay uploaded the new 128-wide bytes with a 256-texel row, and the Basic Controls screen's ZL caption + // ("Target") sampled a mix of its glyphs and stale ones. Without such an image the slot gets its own below + // (entry_of), uploaded from its storage with its own geometry. + int cpu_slot = tex_mirrored(&s->t); for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) { if (p->released) continue; // Only an entry of the slot's target: Ocarina's volumetric fog registers a 3D two-channel volume (32x24x64) @@ -1208,10 +1247,11 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { // that 2D image, no 3D view can be made of it (vk_exec_draw.c get_view) and they read and wrote stand-ins. // With the target kept they get the volume itself. if (p->key.d.target != k.d.target) continue; - if (!any) any = p; if (same_but_swizzle(&p->key, &k)) { e = p; break; } + if (!any && (!cpu_slot || alias_has_gpu_content(p))) any = p; } - if (!e) e = any ? any : own; + // own is missing or released here; for a CPU slot entry_of below finds or revives it by key + if (!e) e = any ? any : cpu_slot ? NULL : own; } if (!e && s->t.mem) { if (create_budget < 0) e = entry_of(&s->t);