diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 47fbaf20..06850775 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -247,6 +247,12 @@ # A render target that needs an image of its own takes the image of a released one of the same size and format # instead of making a new one (0 = always a new image). # tex_reuse = 1 +# A texture the game does not fill from the CPU itself, made over memory the CPU writes (a second description of a +# texture's bytes), starts with those bytes and takes the game's later writes to them until something is drawn into +# it (1, default). 0: it starts empty, as before, and what is drawn with it can come out flat and pale. 2: it gets +# the bytes once and no later write. Costs an upload for each such texture; a render target the game makes over +# CPU memory starts with the last picture left there instead of an empty one. +# tex_split_fill = 1 # 1: shaders mark the textures they sample, so that while video memory or the system's memory runs low, textures no # shader sampled for a while go, even those the game keeps registered (the areas it has loaded); one sampled again is # made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay. diff --git a/runtime/cfg.c b/runtime/cfg.c index d8ee9355..a60af1e1 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -67,7 +67,7 @@ static const char *const known[] = { "SVC_SLOW_MS", "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", - "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", + "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_SPLIT_FILL", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", @@ -135,7 +135,7 @@ static const struct { const char *key, *section; } sections[] = { {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, - {"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, + {"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"FFX_PIPE_CACHE", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, @@ -292,7 +292,7 @@ static const CfgMeta metas[] = { {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, - {"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_SPLIT_FILL", "1", "0|1|2", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 06e465f8..ba42af7b 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -13,6 +13,7 @@ #include "nvn_api.h" #include "tex.h" #include "vk.h" +#include "vk_exec.h" #include "vk_gpa.h" #include "gpa_read_spv.h" #include "platform/platform.h" @@ -1068,6 +1069,165 @@ static int test_textures(pool_t *tpool, pool_t *phys) { return 0; } +// Level 0 of image (w x h RGBA8) read into out through a staging buffer after everything recorded so far has run. +static void image_bytes(VkImage image, uint32_t w, uint32_t h, uint8_t *out) { + VkCtx *c = vk_ctx(); + VkBuffer rb; + VmaAllocation ra; + size_t n = (size_t)w * h * 4; + uint8_t *r = vk_staging(n, &rb, &ra); + VkCommandBuffer cb = vk_upload_begin(); + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); + VkBufferImageCopy rg = {.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .imageExtent = {w, h, 1}}; + vkCmdCopyImageToBuffer(cb, image, VK_IMAGE_LAYOUT_GENERAL, rb, 1, &rg); + mb = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, + .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT}; + vkCmdPipelineBarrier2(cb, &di); + vk_upload_end(); + vk_upload_flush(1); + vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE); + memcpy(out, r, n); + vmaDestroyBuffer(c->vma, rb, ra); +} + +// ---- two descriptors on one CPU-written storage (vk_texture.c "split fill", start_at) +// D1 is a texture the CPU writes. D2 names the same storage with a texture flag for which the cache does not take the +// content from the CPU (COMPRESSIBLE here), so a slot of D2 shows D1's image for as long as D1 has a live entry. +// What the slot shows once D1's entry is gone, in the two orders a game can let go of them: +// - keep: D2's slot stays registered while D1 is released and registered again (a texture made again in place); +// - both: D1 and D2 are released, the slot of D2 is looked up all the same (the texture pool refresh looks at every +// registered slot), then the storage gets new bytes and both are registered again (the next texture in that memory). +// recorded: the lookup that makes D2's entry runs while a submission is recorded, as a draw of a replay makes it, so a +// copy from D1's image can be recorded there; otherwise outside any recording, where only the storage can fill it. +// *what: 0 the slot's image is all zero, 1 it holds the storage's bytes as they are at the end, 2 the bytes the storage +// held when D2's entry was made (the same as 1 in "keep", where the storage is not rewritten), 3 none of these. +enum { SPLIT_W = 32, SPLIT_H = 32, SPLIT_N = SPLIT_W * SPLIT_H * 4 }; + +static int split_what(const uint8_t *got, const uint8_t *first, const uint8_t *last) { + static const uint8_t zero[SPLIT_N]; + return !memcmp(got, zero, SPLIT_N) ? 0 : !memcmp(got, last, SPLIT_N) ? 1 : !memcmp(got, first, SPLIT_N) ? 2 : 3; +} + +typedef struct { int id; const VkTexImage *got; } SplitRec; + +static void split_record(VkCommandBuffer cb, void *arg) { // as gpa_read_record: the frame of a recorded submission + SplitRec *r = arg; + vk_texture_new_submit(); + vk_mem_mirror_begin(); + vk_exec_probe_inline(cb, vk_sync_issued() + 1); + r->got = vk_texture_by_id(r->id, NULL, NULL); + vk_exec_probe_inline(VK_NULL_HANDLE, 0); + vk_mem_mirror_end(cb, vk_sync_issued() + 1); +} + +static int split_case(pool_t *pool, uint64_t off, uint64_t serial, int both, int recorded, int *what) { + uint8_t *lin[1] = {0}, first[SPLIT_N], got[SPLIT_N]; + tex_t d1 = make_tex(pool, off, 0x25, SPLIT_W, SPLIT_H, 1, serial); + tex_t d2 = d1; + d2.d.flags |= NVN_TEXTURE_FLAGS_COMPRESSIBLE, d2.serial = serial + 1; + GCHECK(off + d1.L.size <= pool->size, "the two-descriptor case's storage does not fit the probe's pool"); + if (fill_storage(&d1, lin)) return 1; + memcpy(first, lin[0], SPLIT_N); + const VkTexImage *e1 = vk_texture_get(&d1); + GCHECK(e1 && e1->mirrored, "the CPU-written texture of the two-descriptor case"); + VkImage img1 = e1->image; + vk_texture_register(6, &d2, NULL); + const VkTexImage *s2 = vk_texture_by_id(6, NULL, NULL); + GCHECK(s2 && s2->image == img1, "the slot of the second descriptor does not show the CPU-written texture's image"); + vk_texture_release(&d1); + if (both) vk_texture_release(&d2); + if (recorded) { + SplitRec r = {6, NULL}; + uint64_t fence = vk_sync_probe_submit(split_record, &r); + GCHECK(vk_sync_fence_wait(fence, 5000000000ull), "the submission that looks the second descriptor's slot up timed out"); + s2 = r.got; + } else { + vk_texture_new_submit(); + s2 = vk_texture_by_id(6, NULL, NULL); // the texture pool refresh looks the slot up + } + GCHECK(s2 && s2->image != img1 && !s2->mirrored, "after the release of the CPU-written texture the slot of the second " + "descriptor has no image of its own (%s)", s2 ? "it still shows the released one" : "none at all"); + VkImage img2 = s2->image; + if (both) { // the next texture in that memory + if (fill_storage(&d1, lin)) return 1; + vk_texture_register(6, &d2, NULL); + } + vk_texture_register(7, &d1, NULL); + vk_texture_new_submit(); + const VkTexImage *s1 = vk_texture_by_id(7, NULL, NULL); + GCHECK(s1 && s1->mirrored && s1->image != img2, "the CPU-written texture registered again has no image of its own"); + image_bytes(s1->image, SPLIT_W, SPLIT_H, got); + GCHECK(!memcmp(got, lin[0], SPLIT_N), "the CPU-written texture registered again does not hold its storage's bytes"); + s2 = vk_texture_by_id(6, NULL, NULL); + GCHECK(s2 && s2->image == img2, "the slot of the second descriptor left its own image when the first came back"); + image_bytes(s2->image, SPLIT_W, SPLIT_H, got); + *what = split_what(got, first, lin[0]); + // a shader sampled the slot: with NX2NAVI_TRACE=perf the log names an entry that never had content + uint32_t marks[8] = {0}; + marks[6] = 1; + vk_pool_marks_read(marks, 8); + g_gpu_checks += 6; + vk_texture_release(&d1), vk_texture_release(&d2); + free(lin[0]); + return 0; +} + +// The three cases with split fill off (tex_split_fill = 0), on (1) and filling once (2). Off, the slot shows an image +// nothing filled, which is what the setting is for; on, the storage's bytes, copied from D1's image where a submission +// was being recorded and uploaded from the storage elsewhere; filling once, the bytes of the texture that was in the +// memory before when both descriptors were let go of. +static int test_split(pool_t *pool) { + static const char *const names[4] = {"zeros", "the storage's bytes", "the bytes the storage held before", "other bytes"}; + int what[3][3]; + uint64_t made[3][3][2]; + for (int mode = 0; mode < 3; mode++) { + vk_texture_split_fill_set(mode); + for (int k = 0; k < 3; k++) { // keep, both, keep with the lookup recorded + uint64_t c0, f0, c1, f1; + vk_texture_split_counts(&c0, &f0); + if (split_case(pool, 0x80000 + (uint64_t)(mode * 3 + k) * 0x4000, 30 + (uint64_t)(mode * 3 + k) * 2, k == 1, k == 2, + &what[mode][k])) + return 1; + vk_texture_split_counts(&c1, &f1); + made[mode][k][0] = c1 - c0, made[mode][k][1] = f1 - f0; + } + } + vk_texture_split_fill_set(-1); // the setting decides again + vk_sync_finish(); + printf("probe-vkmem: a second descriptor on a CPU-written storage, not CPU-written itself, once the first one's entry " + "is gone. Its slot shows, with tex_split_fill 0 / 1 / 2: when it stayed registered %s / %s / %s; when both were " + "released and registered again over new bytes %s / %s / %s; when its entry was made in a recorded submission " + "%s / %s / %s (copies from the first one's image %llu / %llu / %llu, fills from storage %llu / %llu / %llu)\n", + names[what[0][0]], names[what[1][0]], names[what[2][0]], names[what[0][1]], names[what[1][1]], names[what[2][1]], + names[what[0][2]], names[what[1][2]], names[what[2][2]], (unsigned long long)made[0][2][0], + (unsigned long long)made[1][2][0], (unsigned long long)made[2][2][0], (unsigned long long)made[0][2][1], + (unsigned long long)made[1][2][1], (unsigned long long)made[2][2][1]); + for (int k = 0; k < 3; k++) { + // off: an image nothing filled (zeros with NX2NAVI_FIX_ZERO, the default; else whatever the memory held) + GCHECK(vk_fix_zero() ? what[0][k] == 0 : what[0][k] != 1, "with split fill off the slot shows %s in case %d: the " + "sequence no longer leaves the second descriptor an empty image", names[what[0][k]], k); + GCHECK(!made[0][k][0] && !made[0][k][1], "with split fill off an entry was filled in case %d", k); + GCHECK(what[1][k] == 1, "with split fill on the slot shows %s in case %d, not the storage's bytes", names[what[1][k]], k); + GCHECK(what[2][k] == (k == 1 ? 2 : 1), "filling once, the slot shows %s in case %d", names[what[2][k]], k); + for (int mode = 1; mode < 3; mode++) + GCHECK(made[mode][k][0] + made[mode][k][1] == 1, "split fill %d made %llu copies and %llu fills in case %d, not " + "one of them", mode, (unsigned long long)made[mode][k][0], (unsigned long long)made[mode][k][1], k); + } + for (int mode = 1; mode < 3; mode++) { + GCHECK(made[mode][0][1] == 1 && made[mode][1][1] == 1, "outside a recording split fill %d did not fill from storage", mode); +#ifdef _WIN32 // elsewhere an import's pinning can mark pages written without a CPU write, and the storage is uploaded + GCHECK(made[mode][2][0] == 1, "in a recorded submission split fill %d did not copy from the first one's image", mode); +#endif + } + g_gpu_checks += 9; + return 0; +} + static int test_gpu(void) { uint64_t heap = 0; GCHECK(mem_set_heap_size(16ull << 20, &heap) == 0 && heap, "SetHeapSize 16 MiB"); @@ -1086,6 +1246,7 @@ static int test_gpu(void) { int detail_bad = test_detail(&dpool); vk_pool_fini(&dpool); if (detail_bad) return 1; + if (test_split(&tpool)) return 1; VkMemStats st; vk_mem_stats(&st); GCHECK(st.imported_pools == 2 && st.physical_pools == 1 && st.virtual_pools == 1 && !st.import_failures, diff --git a/runtime/vk.h b/runtime/vk.h index 0ab3966a..aad3ca73 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -466,6 +466,10 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback); void vk_astc_scratch_fail(int n); // a test's hook: the next n scratch buffer requests fail (tex_probe.c) uint64_t vk_texture_upload_retries(void); // uploads left to be tried again: their decode had no memory (vk_texture.c upload) +// vk_texture.c "split fill", a test's hooks (tex_probe.c): the mode from here on (tex_split_fill: 0, 1 or 2), and how +// many entries started with a copy of the released entry they replaced and how many were filled from their storage. +void vk_texture_split_fill_set(int mode); +void vk_texture_split_counts(uint64_t *copies, uint64_t *fills); VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh); // source bytes the region reads void vk_astc_stats(uint64_t *dispatches, uint64_t *texels); // since start uint64_t vk_astc_scratch_bytes(void); // the decoder's shared scratch now diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 8ef76626..48c703e2 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -785,6 +785,11 @@ void vk_exec_inline_end(int reopen) { if (reopen) vk_exec_pass_begin(&X); } +void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence) { + X.cb = cb, X.fence = fence; + x_here = cb != VK_NULL_HANDLE; +} + typedef struct { VkBuffer b; VmaAllocation a; } InlineBuf; static void free_inline_buf(void *arg) { // completion thread InlineBuf *t = arg; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 4e4e533a..b58560bf 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -143,6 +143,10 @@ void vk_exec_barrier_log(uint64_t frame); // vk_exec_inline_buffer: a device buffer for transfers, destroyed when the submission completes. VkCommandBuffer vk_exec_inline_begin(int *reopen); void vk_exec_inline_end(int reopen); +// A test's hook (tex_probe.c): between a call with cb and one with VK_NULL_HANDLE, the calling thread counts as +// recording a replay into cb for submission `fence`, so vk_exec_inline_begin hands cb out. Called from the record +// callback of vk_sync_probe_submit, where a replay would be recorded. +void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence); // vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex // buffers): the replay continues in a new one (vk_sync_split). void vk_exec_draw_new_cb(void); diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index fa3d7897..1b910638 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -734,16 +734,19 @@ static VkResult pipe_create(int kind, void *ci, VkPipeline *pipe) { // read, so a file that is dropped never takes its size in memory. Sizes are 64-bit all the way // (vk_cache_file.h cache_file_read): a file past 2 GB is read with the limit raised or off, and refused by // its real size otherwise. +static int cache_over_cap; // the last file cache_data_read was asked for is over the limit static void *cache_data_read(VkCtx *c, const char *path, size_t *size) { *size = 0; + cache_over_cap = 0; if (!path[0]) return NULL; uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_MAX_MB"), NULL, 10) << 20, len = 0; int why = CACHE_FILE_NONE; void *data = cache_file_read(path, sizeof(VkPipelineCacheHeaderVersionOne), cap, &len, &why); if (!data) { if (why == CACHE_FILE_OVER_CAP) - rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; this run " - "rewrites it", path, len / 1048576.0, (unsigned long long)(cap >> 20)); + cache_over_cap = 1, + rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; it is " + "replaced when the play file is next folded in, at the next start that finds one", path, len / 1048576.0, (unsigned long long)(cap >> 20)); else if (why != CACHE_FILE_NONE) rt_log("vk_pipeline: %s (%llu bytes, %.1f MB) is not read: %s; starting empty", path, (unsigned long long)len, len / 1048576.0, cache_file_why(why)); @@ -787,6 +790,7 @@ static void cache_load(VkCtx *c) { } size_t size = 0, kept = 0; void *data = cache_data_read(c, P.cache_path, &size); + int base_over = cache_over_cap; // an oversized file is never read again: the play file takes its place at once P.cache = cache_make(c, data, size, &kept); free(data); P.cache_loaded = kept; @@ -808,7 +812,7 @@ static void cache_load(VkCtx *c) { VkPipelineCache play = psize || on ? cache_make(c, pdata, psize, &pkept) : VK_NULL_HANDLE; free(pdata); uint64_t fold_at = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_PLAY_MB"), NULL, 10) << 20; - int fold = play && psize && (!on || psize >= fold_at); + int fold = play && psize && (!on || psize >= fold_at || base_over); if (fold) { // Into P.cache now; the file is written by the first save, which then removes the play file. A run that ends // before that save finds both files as they were and merges again. diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 7a49de3d..d0ce9593 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -93,6 +93,18 @@ typedef struct Tex { uint8_t in_gwin; // in gwin[] int pidx; // 1 + its index in pent[] (vk_texture_pool_note_locked); 0: not listed int ph; // 1 + its index in pref[] (pool_ref); 0: none + // Where the entry and its content came from ("origins", below drop_note), for the lines of NX2NAVI_TRACE=perf + uint8_t had; // HAD_*: what put content into the image of an entry the CPU does not write + uint8_t og_start; // START_*: how its content started (start_at) + uint8_t og_drop; // DROP_*: the drop of this descriptor's last entry, when the ring still knew it + uint8_t og_reimg; // made with the image of a released entry (create_tex, reuse) + uint8_t said; // marks_check wrote its line for this entry + uint8_t sfill; // split fill: the CPU does not write it, and its image is the storage's bytes: 1 as a + // CPU-written one's until the GPU writes it, 2 once (from_storage) + uint32_t og_backs; // times it was taken back after a release (entry_of) + uint64_t og_frame; // gfx_counters.frames when it was made + uint64_t og_drop_frame; // gfx_counters.frames of the drop og_drop names + uint64_t og_back_frame; // gfx_counters.frames when it was last taken back } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the @@ -483,18 +495,24 @@ static int same_but_swizzle(const TexKey *a, const TexKey *b) { // Textures whose content the CPU provides in pool memory. Display, compressible, zcull, // depth, multisample and GPU-only storage stays GPU-owned. -static int tex_mirrored(const tex_t *t) { - if (!t->mem || nvn_gpu_only((uintptr_t)t->mem)) return 0; - if (t->has_layout) return 1; - if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0; +// mirror_refused: NULL for such a texture, else the rule that leaves it out, in words for a log line. +static const char *mirror_refused(const tex_t *t) { + if (!t->mem) return "no storage"; + if (nvn_gpu_only((uintptr_t)t->mem)) return "storage in the GPU-only address window"; + if (t->has_layout) return NULL; + if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) + return "a pool the CPU has no access to"; if (t->d.flags & (NVN_TEXTURE_FLAGS_DISPLAY | NVN_TEXTURE_FLAGS_COMPRESSIBLE | NVN_TEXTURE_FLAGS_ADAPTIVE_ZCULL | NVN_TEXTURE_FLAGS_DEPTH16_PREFER_FAST_CLEAR | NVN_TEXTURE_FLAGS_ZCULL_SUPPORT_STENCIL)) - return 0; - if (t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0; - if (t->d.format >= 0x31 && t->d.format <= 0x36) return 0; // stencil / depth formats - return 1; + return "texture flags (one of 0x1, 0x8, 0x80, 0x100, 0x400)"; + if (t->d.samples > 1) return "multisample"; + if (t->d.target == NVN_TEXTURE_TARGET_BUFFER) return "a buffer texture"; + if (t->d.format >= 0x31 && t->d.format <= 0x36) return "a depth or stencil format"; + return NULL; } +static int tex_mirrored(const tex_t *t) { return !mirror_refused(t); } + static void log_limited(volatile int32_t *n, const char *fmt, ...) __attribute__((format(printf, 2, 3))); static void log_limited(volatile int32_t *n, const char *fmt, ...) { if (plat_inc32(n) > 16) return; @@ -627,6 +645,85 @@ static int tex_reuse_on(void) { static void destroy_tex(Tex *e); +// Why an entry was destroyed (drop_tex), in the order of drop_names. +enum { DROP_NONE, DROP_MAX, DROP_CAP, DROP_COMMIT, DROP_VRAM, DROP_IDLE_VRAM, DROP_IDLE_COMMIT, DROP_DISUSE, + DROP_OVERWRITTEN, DROP_REPLACED, DROP_REUSED, DROP_RANGE, DROP_EXACT, DROP_SHUTDOWN, DROP_N }; +static void drop_tex(Tex *e, int why); +// How an entry's content started (start_at; Tex.og_start), in the order of start_names. +enum { START_NEW, START_UPLOAD_LIVE, START_COPY_LIVE, START_COPY_RELEASED, START_EMPTY_LIVE, START_EMPTY_RELEASED, + START_COPY_GONE, START_FILL, START_N }; +// What put content into the image of an entry the CPU does not write (Tex.had); a GPU write is pub.gpu_owned. +enum { HAD_COPY = 1, HAD_TEXELS = 2, HAD_FILL = 4 }; + +// ---- split fill (NX2NAVI_TEX_SPLIT_FILL, tex_split_fill in nx2navi.ini: 1 the default, 0 off, 2 filled once) +// Two descriptors can name one storage the CPU writes: a texture whose content the cache takes from the CPU +// (tex_mirrored), and a second one with a texture flag for which tex_mirrored refuses. The second has no upload of its +// own: its slot shows the first one's image for as long as that entry is live (slot_resolve). Once the first is +// released, a lookup of the second's slot makes the second an entry of its own (entry_of), start_at destroys the +// released first one, and the new image has nothing to start from: cleared, never uploaded, and from then on what +// the slot shows, also after the first descriptor is registered again and gets a new entry beside it. On the console +// both descriptors read the same bytes. tex_probe.c test_split makes the sequence and reads the image back: zeros. +// +// With split fill, a new entry the CPU does not write, at storage the CPU can see, starts with those bytes: +// - by a copy on the GPU from the released CPU-written entry start_at is about to destroy, when that image holds the +// storage as it is now (uploaded, and no write since its last check) or content the GPU wrote, and +// alias_compatible allows; +// - else from the storage itself, uploaded as a CPU-written texture's is (upload). +// With 1 the image then follows the storage as a CPU-written one's does (from_storage: make_fresh, the pool's list) +// until the GPU or WriteTexels writes it. The texture pool refresh makes such an entry also for a slot whose descriptor +// the game has let go of, long before the next texture's bytes arrive in that memory, and when both descriptors are +// registered again the slot finds that entry: filled once (2), it shows the texture that was there before. +// Left out: depth, stencil and multisample images, buffer textures, pools the CPU has no access to, storage over +// SPLIT_FILL_MAX, and an entry made with a released entry's image, whose first transition is recorded at the replay's +// position, after the upload batch (init_new); so create_tex gives an entry that is to be filled a new image. Nothing +// is filled where a live entry at the address holds content the GPU wrote (that is what the new one should show, and +// start_at copies it when it can), nor with NX2NAVI_TEX_ALIAS off. +// +// What follows the storage also takes what a stale descriptor's entry does not need: an entry made for a slot the game +// let go of is uploaded again whenever its pages are written, by the next textures loaded into that memory, read in +// its own format. The perf line counts those ("filled again after a write"); 2 does without them. +// +// What it costs: an upload (a copy out of the storage, a decode for ASTC) for each such entry that finds no image to +// copy from, where the image was only cleared before. A render target in a CPU pool that the game makes anew often +// pays that every time and no longer takes a released entry's image; the perf line counts the fills, their bytes and +// their time. Until the GPU first writes it, its storage is also checked for writes once a submission. +// What it can show: the bytes in the storage, whatever they are. For a render target that is the last picture the +// game or a write-back left there, where the image was zero before: a pass that reads a new target before anything +// drew into it sees that older picture instead of nothing. +#define SPLIT_FILL_MAX ((uint64_t)32 << 20) +static int split_mode = -1; +// since the last perf line: entries started by a copy from the released entry they replace, filled from storage (with +// the bytes and the time), filled again after a write; left empty for a reused image, for their size +static uint64_t n_split_copy, n_split_fill, n_split_bytes, n_split_ns, n_split_refill, n_split_reused, n_split_big; +static uint64_t split_total[2]; // copies and fills so far (vk_texture_split_counts) + +static int split_fill_mode(void) { + if (split_mode < 0) { + int v = cfg_int("NX2NAVI_TEX_SPLIT_FILL"); + split_mode = v == 0 ? 0 : v == 2 ? 2 : 1; + rt_log("vk: a texture the CPU does not write, made at storage the CPU writes, %s (NX2NAVI_TEX_SPLIT_FILL=%d)", + !split_mode ? "starts empty" : split_mode == 2 ? "starts with the storage's bytes, once" + : "starts with the storage's bytes and follows them", split_mode); + } + return split_mode; +} + +// A test's hook (tex_probe.c): the mode from here on, and the copies and fills made so far. +void vk_texture_split_fill_set(int mode) { split_mode = mode; } +void vk_texture_split_counts(uint64_t *copies, uint64_t *fills) { *copies = split_total[0], *fills = split_total[1]; } + +// Whether split fill gives a new entry for t its storage's bytes, apart from the size (SPLIT_FILL_MAX). +static int split_wants(const tex_t *t, const TexVkFormat *f) { + if (!split_fill_mode() || !t->mem || nvn_gpu_only((uintptr_t)t->mem) || tex_mirrored(t)) return 0; + if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0; + if (f->depth || f->stencil || t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0; + int bw, bh; + return nvn_fmt_bpb(t->d.format, &bw, &bh) != 0; +} + +// Whether e's image takes its content from the storage: a texture the CPU writes, or one split fill fills. +static int from_storage(const Tex *e) { return e->pub.mirrored || e->sfill == 1 || (e->sfill == 2 && !e->uploaded); } + // Whether an image with these requirements can lie in the pool memory at e's address (*sp: that memory). static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, uint64_t guest_size, VkMemSpan *sp) { if ((e->f.depth || e->f.stencil) && req->size > guest_size) return 0; @@ -1094,7 +1191,11 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { } } int gpu_only = !tex_mirrored(t); - if (!img && gpu_only && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above) + // an entry split fill is to fill gets a new image: start_at uploads into it in the upload batch (see "split fill") + int to_fill = gpu_only && split_wants(t, &f) && L.size <= SPLIT_FILL_MAX; + for (const Tex *p = to_fill ? nvn_hm_get(&by_addr, &k->addr) : NULL; p; p = p->next_at) + if (alias_has_gpu_content(p)) { to_fill = 0; break; } // start_at copies that in instead + if (!img && gpu_only && !to_fill && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above) // Not one released in this frame or the one before: Ocarina releases its render targets and registers them // again every frame, and DLSS's scene target (2560x1440 at a fixed address in Kokiri Forest) lost its image // to a moving target of the same size in between, and DLSS its history (vk_dlaa.c evaluate), several times a @@ -1117,8 +1218,8 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { img = a->pub.image, e->alloc = a->alloc; vk_exec_image_gone(img); // the views and records of the texture it was a->pub.image = VK_NULL_HANDLE, a->alloc = NULL; - destroy_tex(a); - e->reused = 1; + drop_tex(a, DROP_REUSED); + e->reused = 1, e->og_reimg = 1; n_reused++; } } @@ -1361,7 +1462,7 @@ int vk_texture_pool_note_locked(int id, uint32_t *ent) { if (tslots[id].ce && tslots[id].cgen != agen[sinfo[id] & 0xffff]) vk_texture_by_id_locked(id, NULL, NULL); Tex *e = tslots[id].ce; if (!e) return 0; - if (!e->pidx && e->pub.mirrored && !e->pub.gpu_owned) { + if (!e->pidx && from_storage(e) && !e->pub.gpu_owned) { if (npent == cpent) { int n = cpent ? cpent * 2 : 1024; PEnt *p = realloc(pent, sizeof *p * (size_t)n); @@ -1474,6 +1575,132 @@ static void destroy_tex(Tex *e) { vk_defer_free(e); } +// ---- origins (NX2NAVI_TRACE=perf; a run without it tests the flag and counts the drops, nothing more) +// A texture drawn without its content leaves nothing in the log: no call fails and the image exists. What can still be +// told afterwards is where the entry came from, so each entry keeps how it was made and how its content started +// (og_*, had), and the last DROPS_N destroyed entries keep their descriptor and the reason they went. An entry made +// for a descriptor that is still in that ring says which drop it follows (origin_made), and the lines about one +// address list what was dropped there (others_at). The users are marks_check, which names an entry a shader sampled +// while nothing had ever put content into its image, and the lines of start_at. +#define DROPS_N 4096 +typedef struct { uint64_t kh, addr, frame; uint32_t format, flags, w, h; uint8_t why, mirrored, released; } DropRec; +static DropRec drops[DROPS_N]; +static uint64_t ndrops; +static uint64_t n_drop_by[DROP_N]; // entries destroyed since the last perf line, by reason +static const char *const drop_names[DROP_N] = { + "", "past RELEASED_MAX released entries", "past the released entries' size cap", "under commit pressure", + "under VRAM pressure", "idle, under VRAM pressure", "idle, under commit pressure", "unsampled (disuse)", + "overwritten at another address", "replaced by another descriptor at its address", "its image went to another texture", + "its storage went", "made whole again (texture detail)", "shutdown"}; +static const char *const start_names[START_N] = { + "started on its own", "filled from storage over a live entry", "content copied from a live entry", + "content copied from a released entry", "started empty over a live entry", "started empty over a released entry", + "content copied from the released CPU-written entry it replaced (split fill)", "filled from its storage (split fill)"}; + +static uint64_t frames_now(void) { return __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); } + +static uint64_t key_hash(const TexKey *k) { // FNV-1a; make_key zeroes the key's padding + uint64_t h = 0xcbf29ce484222325ull; + const uint8_t *b = (const uint8_t *)k; + for (size_t i = 0; i < sizeof *k; i++) h = (h ^ b[i]) * 0x100000001b3ull; + return h; +} + +// destroy_tex with the reason, which every caller gives. +static void drop_tex(Tex *e, int why) { + n_drop_by[why]++; + if (rt_trace & RT_TRACE_PERF) + drops[ndrops++ % DROPS_N] = (DropRec){key_hash(&e->key), e->key.addr, frames_now(), (uint32_t)e->t.d.format, + (uint32_t)e->t.d.flags, e->t.d.width, e->t.d.height, (uint8_t)why, + e->pub.mirrored, e->released != 0}; + destroy_tex(e); +} + +// A new entry: the frame, and the drop of its descriptor's last entry when the ring still holds it. +static void origin_made(Tex *e) { + e->og_frame = frames_now(); + if (!(rt_trace & RT_TRACE_PERF)) return; + uint64_t kh = key_hash(&e->key), n = ndrops < DROPS_N ? ndrops : DROPS_N; + for (uint64_t i = 1; i <= n; i++) { // newest first + const DropRec *r = &drops[(ndrops - i) % DROPS_N]; + if (r->kh != kh || r->addr != e->key.addr) continue; + e->og_drop = r->why, e->og_drop_frame = r->frame; + return; + } +} + +// "fmt 0x25 (vk 37) 256x256x1, 9 levels, 1 layers, target 1, samples 0, flags 0x1, pool flags 0x22" +static int desc_text(char *out, size_t n, const Tex *e) { + const nvn_texdesc *d = &e->t.d; + return snprintf(out, n, "fmt %#x (vk %d) %ux%ux%u, %u levels, %u layers, target %d, samples %d, flags %#x, pool flags %#x", + d->format, (int)e->f.vk, d->width, d->height, d->depth, e->pub.levels + e->pub.skip, e->pub.layers, + d->target, d->samples, (unsigned)d->flags, (unsigned)e->t.pool_flags); +} + +// "made at frame 100 again after a drop at frame 90 (under commit pressure); started empty over a released entry; ..." +static int origin_text(char *out, size_t n, const Tex *e) { + int k = snprintf(out, n, "made at frame %llu", (unsigned long long)e->og_frame); + if (e->og_drop && k < (int)n) + k += snprintf(out + k, n - (size_t)k, " again after a drop at frame %llu (%s)", (unsigned long long)e->og_drop_frame, + drop_names[e->og_drop]); + if (k < (int)n) k += snprintf(out + k, n - (size_t)k, "; %s", start_names[e->og_start]); + if (e->og_reimg && k < (int)n) k += snprintf(out + k, n - (size_t)k, "; with a released entry's image"); + if (e->og_backs && k < (int)n) + k += snprintf(out + k, n - (size_t)k, "; taken back %u times, last at frame %llu", e->og_backs, + (unsigned long long)e->og_back_frame); + return k < (int)n ? k : (int)n - 1; +} + +// Whether nothing ever put content into e's image: an entry the CPU writes that was not uploaded, or one it does not +// write, at an address the CPU can see, that the GPU never wrote and that got no copy, no texels and no split fill. +static int no_content(const Tex *e) { + if (e->pub.gpu_owned) return 0; + if (e->pub.mirrored) return !e->uploaded; + return !e->had && !(e->sfill && e->uploaded) && e->key.addr && !nvn_gpu_only(e->key.addr); +} + +// Lines per address for one kind of event: up to `most` each, in a table of the first SAID_N addresses that had one +// (a later address always gets its line), so a session's limit is by address, not a count of lines. +#define SAID_N 4096 +typedef struct { uint64_t addr[SAID_N]; uint16_t n[SAID_N]; } SaidAt; +static int said_at(SaidAt *s, uint64_t addr, unsigned most) { + uint32_t i = (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); + for (int probe = 0; probe < 64; probe++, i = (i + 1) % SAID_N) { + if (s->addr[i] && s->addr[i] != addr) continue; + s->addr[i] = addr; + return s->n[i] < most ? (s->n[i]++, 1) : 0; + } + return 1; +} + +// The other descriptors at e's address: every entry in the cache with its state, and what the ring knows was dropped +// there, newest first. skip: an entry the caller describes itself (NULL: none). +static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { + int k = 0, any = 0; + out[0] = 0; + for (const Tex *p = nvn_hm_get(&by_addr, &e->key.addr); p && k < (int)n - 1; p = p->next_at) { + if (p == e || p == skip) continue; + k += snprintf(out + k, n - (size_t)k, "%s[%s%s%s%s: fmt %#x flags %#x %ux%u, %s, %s]", any++ ? " " : "", + p->released ? "released" : "live", p->superseded ? ", superseded" : "", p->retiring ? ", retiring" : "", + p->pub.gpu_owned ? ", GPU-written" : "", p->t.d.format, (unsigned)p->t.d.flags, p->t.d.width, + p->t.d.height, p->pub.mirrored ? (p->uploaded ? "CPU-written, uploaded" : "CPU-written, not uploaded") + : (no_content(p) ? "not CPU-written, no content" : "not CPU-written"), + start_names[p->og_start]); + } + uint64_t have = ndrops < DROPS_N ? ndrops : DROPS_N; + int listed = 0; + for (uint64_t i = 1; i <= have && listed < 4 && k < (int)n - 1; i++) { + const DropRec *r = &drops[(ndrops - i) % DROPS_N]; + if (r->addr != e->key.addr) continue; + k += snprintf(out + k, n - (size_t)k, "%s[dropped at frame %llu (%s): fmt %#x flags %#x %ux%u, %s%s]", any++ ? " " : "", + (unsigned long long)r->frame, drop_names[r->why], r->format, r->flags, r->w, r->h, + r->mirrored ? "CPU-written" : "not CPU-written", r->released ? ", released then" : ", live then"); + listed++; + } + if (!any) k = snprintf(out, n, "none"); + return k < (int)n ? k : (int)n - 1; +} + // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries the // oldest one goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). NX2navi // finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames. @@ -1481,7 +1708,7 @@ static void destroy_tex(Tex *e) { // frames every 40-50 s on a GTX 1050 Ti, then the textures still in use were created and uploaded again. #define RELEASED_MAX 2048 static void sweep_released(void) { - while (nreleased > RELEASED_MAX && rel_head) destroy_tex(rel_head); + while (nreleased > RELEASED_MAX && rel_head) drop_tex(rel_head, DROP_MAX); } // NX2NAVI_TEX_VRAM_PRESSURE (percent of the device-local budget, default VRAM_PRESSURE_DEFAULT; 0 = off): above it, each @@ -1700,7 +1927,7 @@ static uint64_t stale_after(const Tex *e) { return gap > GPU_STALE_FRAMES ? gap : GPU_STALE_FRAMES; } -static int drop_stale(uint64_t keep, uint64_t *count) { // returns the entries dropped: PRESSURE_STEP, more may follow +static int drop_stale(uint64_t keep, uint64_t *count, int why) { // returns the entries dropped: PRESSURE_STEP, more may follow uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); int dropped = 0; const uint64_t stale = STALE_FRAMES; @@ -1710,7 +1937,7 @@ static int drop_stale(uint64_t keep, uint64_t *count) { // returns the entries next = e->rel_next; if (!e->bytes || (!any && !comes_back_intact(e) && !e->overwritten)) continue; if (now - e->rel_frame < stale_after(e) || (!e->overwritten && !idle(e, now))) continue; - destroy_tex(e); + drop_tex(e, why); dropped++, (*count)++; } return dropped; @@ -1724,7 +1951,7 @@ static void drop_overwritten(void) { for (Tex *e = rel_head, *next; e && now - e->rel_frame >= OVERWRITTEN_KEEP && dropped < PRESSURE_STEP; e = next) { next = e->rel_next; if (!e->overwritten || now - e->ow_frame < OVERWRITTEN_KEEP) continue; - destroy_tex(e); + drop_tex(e, DROP_OVERWRITTEN); dropped++, n_over_drops++; } } @@ -1758,7 +1985,7 @@ void vk_texture_want_room(uint64_t bytes) { __ATOMIC_RELAXED)) {} } -static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // tex_lock held; returns the bytes freed +static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget, int why) { // tex_lock held; returns the bytes freed if (vk_sync_fence_done() < evict_wait) return 0; Tex *v[EVICT_STEP]; int nv = 0; @@ -1776,7 +2003,7 @@ static uint64_t evict_idle(uint64_t want, uint64_t used, uint64_t budget) { // } uint64_t freed = 0; int n = 0; - for (; n < nv && freed < want; n++) freed += v[n]->bytes, destroy_tex(v[n]); + for (; n < nv && freed < want; n++) freed += v[n]->bytes, drop_tex(v[n], why); if (!n) return 0; n_evict += (uint64_t)n, n_evict_bytes += freed; evict_wait = vk_sync_issued() + 2; @@ -1884,7 +2111,7 @@ static void retire_sweep(void) { e->retiring = 0, retiring_bytes -= e->bytes; nvn_hm_put(&holds, &e->key, (void *)(uintptr_t)1); n_retire_freed++; - destroy_tex(e); + drop_tex(e, DROP_DISUSE); continue; } retiring[k++] = e; @@ -2028,9 +2255,79 @@ static uint64_t evict_disused(uint64_t want) { return retiring_bytes; } +// The registered slots at e's address, for marks_check's line: how many, those of e's own descriptor ("f": finalized), +// those whose last lookup gave e (a slot of another descriptor among them follows e), and the other descriptors' with +// format and flags; up to 8 ids of each. +static void slot_note(int32_t id, const Tex *e, int *n, int *n_own, int *n_show, int *n_other, char *own, char *show, + char *other, size_t own_cap, size_t other_cap) { + const TexSlot *s = &tslots[id]; + if (!s->used || (uintptr_t)s->t.mem != e->key.addr) return; + ++*n; + TexKey sk; + make_key(&s->t, &sk); + size_t at; + if (!memcmp(&sk, &e->key, sizeof sk)) { + if (++*n_own <= 8 && (at = strlen(own)) < own_cap - 1) + snprintf(own + at, own_cap - at, " %d%s", (int)id, s->finalized ? "f" : ""); + } else if (++*n_other <= 8 && (at = strlen(other)) < other_cap - 1) { + snprintf(other + at, other_cap - at, " %d (fmt %#x flags %#x%s)", (int)id, s->t.d.format, (unsigned)s->t.d.flags, + s->finalized ? ", finalized" : ""); + } + if (s->ce == e && s->cgen == agen[sinfo[id] & 0xffff] && ++*n_show <= 8 && (at = strlen(show)) < own_cap - 1) + snprintf(show + at, own_cap - at, " %d", (int)id); +} + +static void slots_text(char *out, size_t n, const Tex *e) { + char own[112] = "", show[112] = "", other[360] = ""; + int total = 0, n_own = 0, n_show = 0, n_other = 0; + uint32_t b = agen_idx(e->key.addr); + for (int32_t id = bucket_head[b]; id >= 0; id = slot_next[id]) + slot_note(id, e, &total, &n_own, &n_show, &n_other, own, show, other, sizeof own, sizeof other); + for (int32_t g = bucket_ghead[b]; g >= 0; g = grp[g].gnext) + for (int32_t id = grp[g].pub.head; id >= 0; id = slot_next[id]) + slot_note(id, e, &total, &n_own, &n_show, &n_other, own, show, other, sizeof own, sizeof other); + snprintf(out, n, "%d registered; %d of this descriptor:%s; %d showing this entry:%s; %d of other descriptors:%s", total, + n_own, n_own ? own : " none", n_show, n_show ? show : " none", n_other, n_other ? other : " none"); +} + +// NX2NAVI_TRACE=perf, from vk_pool_marks_read: a shader sampled slot id, and nothing has ever put content into e, the +// entry the slot shows or one credited with the mark (no_content): the picture drawn through it is an empty or +// undefined image, which nothing else reports. One line per entry and up to MARK_LINES per address (said_at); the +// marked slots that met such an entry are counted in the texture origins line whether or not a line was written. +static uint64_t n_mark_empty, n_mark_none; // since the last perf line: marked slots that met such an entry; that show none +#define MARK_LINES 6 +#define MARK_LINES_ALL 20000 +static void marks_check(Tex *e, uint32_t id, uint64_t now, int followed) { + if (!no_content(e)) return; + // an entry the CPU writes is uploaded by its first check, which one just made (again) has not had yet + if (e->pub.mirrored && now - e->og_frame < 4) return; + n_mark_empty++; + if (e->said) return; + e->said = 1; + static SaidAt said; + static uint32_t total; + if (total >= MARK_LINES_ALL || !said_at(&said, e->key.addr, MARK_LINES)) return; + total++; + char d[224], og[320], state[160], slots[640], others[640]; + desc_text(d, sizeof d, e); + origin_text(og, sizeof og, e); + slots_text(slots, sizeof slots, e); + others_at(others, sizeof others, e, NULL); + if (e->pub.mirrored) + snprintf(state, sizeof state, "CPU-written, not uploaded%s", + e->up_retry ? " (an upload that had no memory waits for its next try)" : ""); + else + snprintf(state, sizeof state, "not CPU-written (%s), never drawn into, copied into or filled", mirror_refused(&e->t)); + rt_log("perf: frame %llu: sampled without content: texture %#llx (%llu bytes) [%s]: %s; %s; marked slot %u%s; slots " + "at the address: %s; other descriptors there: %s", (unsigned long long)now, (unsigned long long)e->key.addr, + (unsigned long long)e->t.L.size, d, state, og, id, followed ? " (of another descriptor, credited to this entry)" : "", + slots, others); +} + void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { lock(); uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + int trace = (rt_trace & RT_TRACE_PERF) != 0; if (n > (uint32_t)ntslots) n = (uint32_t)ntslots; uint32_t marked = 0; for (uint32_t id = 0; id < n; id++) { @@ -2040,14 +2337,17 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { if (s->ce && s->cgen == agen[sinfo[id] & 0xffff]) { // what the slot resolves to now if (s->ce->retiring) retire_cancel(s->ce); // not expected: retiring bumps the slot's generation s->ce->samp_frame = now; + if (trace) marks_check(s->ce, id, now, 0); continue; } + if (trace && !s->ce && s->cgen == agen[sinfo[id] & 0xffff]) n_mark_none++; // its last lookup gave no entry TexKey k; make_key(&s->t, &k); Tex *e = nvn_hm_get(&by_key, &k); if (e && !e->released) { if (e->retiring) retire_cancel(e); // retired, and its stand-in was sampled: back as it is e->samp_frame = now; + if (trace) marks_check(e, id, now, 0); continue; } // The slot has no entry of its own: it samples another descriptor's image at its address (slot_resolve follows @@ -2064,6 +2364,7 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n) { if (p->retiring) retire_cancel(p); p->samp_frame = now; followed = 1; + if (trace) marks_check(p, id, now, 1); } // The slot's own key may be held although an entry at its address took the mark: the slot shows the stand-in // then (slot_resolve follows fewer entries than the loop above credits), and only this drop makes its texture @@ -2109,7 +2410,7 @@ static void commit_evict(void) { // tex_lock held, the commit is low and the r } uint64_t used = 0, budget = 0; vk_mem_vram_live(&used, &budget); - uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget); + uint64_t freed = evict_idle(COMMIT_EVICT_STEP, used, budget, DROP_IDLE_COMMIT); if (freed < COMMIT_EVICT_STEP) evict_disused(COMMIT_EVICT_STEP - freed); } @@ -2154,18 +2455,18 @@ static void sweep_pressure(void) { // tex_lock held next = e->rel_next; if (!comes_back_intact(e)) continue; freed += e->bytes; - destroy_tex(e); + drop_tex(e, DROP_VRAM); dropped++, n_pressure_drops++; } - if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget); + if (tex_evict && freed < over) freed += evict_idle(over - freed, used, budget, DROP_IDLE_VRAM); if (tex_evict && freed < over) evict_disused(over - freed); } int low = commit_is_low(), more = 0; // more: the released entries have stale ones left for the next check if (rel_head) { drop_overwritten(); uint64_t cap = released_cap_bytes(); - if (rel_bytes > cap) drop_stale(cap, &n_cap_drops); - if (rel_bytes > COMMIT_LOW_KEEP && low) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops) == PRESSURE_STEP; + if (rel_bytes > cap) drop_stale(cap, &n_cap_drops, DROP_CAP); + if (rel_bytes > COMMIT_LOW_KEEP && low) more = drop_stale(COMMIT_LOW_KEEP, &n_commit_drops, DROP_COMMIT) == PRESSURE_STEP; } if (low && !more) commit_evict(); } @@ -2571,6 +2872,7 @@ static void upload(Tex *e) { uint64_t dt = kern_now_ns() - u0; if (e->uploaded) { n_reuploads++, perf_count(PERF_CNT_TEX_REUP); + n_split_refill += e->sfill != 0; ru_n++, ru_bytes += src_bytes, ru_ns += dt; if (e->wt_n) { ru_wt_n++, ru_wt_bytes += src_bytes, ru_wt_ns += dt; @@ -2645,9 +2947,11 @@ static int same_content(const Tex *e) { return 1; } -// A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation. +// A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation. So is +// one split fill fills (from_storage), while it is not released: the game may be unmapping a released one's storage, +// and nothing keeps a slot from resolving to it as the stale rule does for a CPU-written one (slot_resolve). static void make_fresh(Tex *e) { - if (!e->pub.mirrored || e->pub.gpu_owned) return; + if (!from_storage(e) || e->pub.gpu_owned || (e->sfill && e->released)) return; uint64_t size = e->t.L.size; if (!e->uploaded) { if (up_wait(e)) return; @@ -2666,7 +2970,7 @@ static void make_fresh(Tex *e) { // vk_mem_changed_snap) instead of a system call per texture. A write the snapshot of the texture's region missed (made // after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot. static void make_fresh_snap(Tex *e) { - if (!e->pub.mirrored || e->pub.gpu_owned) return; + if (!from_storage(e) || e->pub.gpu_owned || (e->sfill && e->released)) return; if (!e->uploaded && up_wait(e)) return; uint32_t now; n_checks++; @@ -2800,6 +3104,61 @@ static void create_failed(const TexKey *k) { failed_hold[i].k = *k, failed_hold[i].until_ns = plat_now_ns() + 2000000000ull; } +// NX2NAVI_TRACE=perf: a line for each event of the two kinds that can leave a descriptor showing an image without +// content, up to START_LINES per address and kind (said_at). +// - over: e, which the CPU writes, was made over the live entry `over` and is filled from its storage. `over` stays +// as it is, and a slot that names its descriptor goes on showing it, whatever it holds. +// - gone: e, which the CPU does not write, was made at an address the CPU can see, and the released CPU-written entry +// there was destroyed (gone: its description, gone_rel: the frame of its release). Nothing uploads e; a slot of e's +// descriptor that showed the destroyed entry's image (slot_resolve follows it while it is live) shows e's from now on. +// flag: e was taken back (over); how e started (gone): 0 empty, 1 with a copy of another entry's content, 2 with a +// copy of the destroyed entry's, 3 filled from its storage (the last two: split fill). +static uint64_t n_cpu_empty[3]; // since the last perf line: entries the CPU does not write, at an address it can see, + // that started without content: on their own, over a released entry, over a live one +#define START_LINES 8 +#define START_LINES_ALL 20000 +static void start_line(const Tex *e, const Tex *over, const char *gone, uint64_t gone_rel, int flag) { + static SaidAt said[2]; + static uint32_t total[2]; + int kind = over ? 0 : 1; + if (total[kind] >= START_LINES_ALL || !said_at(&said[kind], e->key.addr, START_LINES)) return; + total[kind]++; + char d[224], o[224], og[320]; + desc_text(d, sizeof d, e); + if (over) { + const char *why = mirror_refused(&over->t); + desc_text(o, sizeof o, over); + origin_text(og, sizeof og, over); + rt_log("perf: frame %llu: texture %#llx filled from storage over a live entry%s: new [%s]; live [%s], %s%s%s, %s", + (unsigned long long)frames_now(), (unsigned long long)e->key.addr, flag ? " (taken back)" : "", d, o, + why ? "not CPU-written: " : "CPU-written", why ? why : "", no_content(over) ? ", no content" : "", og); + } else { + static const char *const how[4] = {"started empty", "got another entry's content", "got the destroyed entry's content", + "was filled from its storage"}; + rt_log("perf: frame %llu: texture %#llx, not CPU-written (%s), %s after the released CPU-written entry at its " + "address was destroyed: new [%s]; destroyed [%s], released at frame %llu", + (unsigned long long)frames_now(), (unsigned long long)e->key.addr, mirror_refused(&e->t), how[flag & 3], d, + gone, (unsigned long long)gone_rel); + } +} + +// Split fill from the storage: e, new and through its first transition, gets its storage's bytes in the upload batch +// (make_fresh, as a CPU-written texture's first use) and is marked to follow them or not (Tex.sfill). 0 when e is left +// as it is: storage over SPLIT_FILL_MAX, an image taken from a released entry (see "split fill"), storage not mapped, +// or an upload that had no memory, which make_fresh tries again later. +static int split_fill(Tex *e) { + if (e->t.L.size > SPLIT_FILL_MAX) return n_split_big++, 0; + if (e->reused) return n_split_reused++, 0; + if (!mem_readable(e->pub.addr, e->t.L.size)) return 0; + uint64_t t0 = kern_now_ns(); + e->sfill = (uint8_t)split_mode; + make_fresh(e); + if (!e->uploaded) return 0; + n_split_fill++, split_total[1]++, n_split_bytes += e->t.L.size, n_split_ns += kern_now_ns() - t0; + e->had |= HAD_FILL; + return 1; +} + static int depth_alias_log(void) { // NX2NAVI_DEPTH_ALIAS_LOG=1: start_at logs each new depth entry over a live texture static int on = -1; if (on < 0) on = rt_knob("DEPTH_ALIAS_LOG", "0"); @@ -2821,7 +3180,16 @@ static int depth_alias_log(void) { // NX2NAVI_DEPTH_ALIAS_LOG=1: start_at logs // it there (a black frame with FSR 3.1 and its frame generation, two presents long). static void start_at(Tex *e, const TexKey *k, int taken_back) { int alias = 0; + int trace = (rt_trace & RT_TRACE_PERF) != 0; + int gone = 0, rel_other = 0; // released entries at the address: CPU-written ones destroyed here, others superseded + char gone_desc[224] = ""; // trace: the last destroyed one, described while it existed + uint64_t gone_rel = 0; Tex *src = NULL, *rsrc = NULL; + // split fill: e is to start with its storage's bytes; msrc: the released CPU-written entry whose image holds them + // (destroyed after the copy), msrc_gpu: it holds what the GPU wrote instead, msrc_gen: the storage's generation + int fill = !taken_back && tex_alias && split_wants(&e->t, &e->f), msrc_gpu = 0; + uint32_t msrc_gen = 0; + Tex *msrc = NULL; const char *why = "no live texture with GPU content"; // for the log of an alias left empty const Tex *other = NULL; int newer = 0; // taken back: a texture at this address holds GPU content written since e's release @@ -2832,10 +3200,27 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { if (taken_back && p->use_frame >= e->rel_frame && alias_has_gpu_content(p)) newer = 1; if (p->released) { if (p->pub.mirrored) { - destroy_tex(p); + if (trace) { + int n = desc_text(gone_desc, sizeof gone_desc, p); + if (n > 0 && n < (int)sizeof gone_desc) + snprintf(gone_desc + n, sizeof gone_desc - (size_t)n, ", %s", p->pub.gpu_owned ? "GPU-written" + : p->uploaded ? "uploaded" : "not uploaded"); + gone_rel = p->rel_frame; + } + gone++; + const char *no; + uint32_t gen = 0; + int gpu = alias_has_gpu_content(p); + if (fill && !msrc && (gpu || (p->uploaded && !p->pub.gpu_owned && + !vk_mem_changed(p->pub.addr, p->t.L.size, p->wgen, &gen))) && + alias_compatible(e, p, &no)) { + msrc = p, msrc_gpu = gpu, msrc_gen = gen; + continue; + } + drop_tex(p, DROP_REPLACED); continue; } - p->superseded = 1; + p->superseded = 1, rel_other = 1; const char *unused; if (tex_alias && alias_has_gpu_content(p) && alias_compatible(e, p, &unused) && (!rsrc || p->use_frame > rsrc->use_frame)) @@ -2876,9 +3261,26 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { if (taken_back) init_new(e); return; } - if (src && alias_copy(e, src)) { + // split fill only where nothing live holds content the storage lacks: that content is what e should show + if (src || live_content) fill = 0; + int copied = src && alias_copy(e, src), filled = 0; + if (fill && msrc && alias_copy(e, msrc)) { // from the released entry this one replaces + copied = filled = 1; + n_split_copy++, split_total[0]++; + e->og_start = START_COPY_GONE; + e->had |= HAD_COPY; + if (!msrc_gpu && e->t.L.size <= SPLIT_FILL_MAX) { // the storage's bytes as of msrc_gen: followed from there + e->sfill = (uint8_t)split_mode, e->uploaded = 1, e->wgen = msrc_gen; + e->chash_ok = 0; + } + } + if (msrc) drop_tex(msrc, DROP_REPLACED); + if (filled) { // nothing more: e is not CPU-written + } else if (copied) { n_alias_copied++; if (from_released) n_alias_released++; + e->og_start = from_released ? START_COPY_RELEASED : START_COPY_LIVE; + e->had |= HAD_COPY; if (e->pub.mirrored) { // the content is src's GPU render, newer than the storage bytes vk_mem_changed(e->pub.addr, e->t.L.size, 0, &e->wgen); e->uploaded = 1; @@ -2888,9 +3290,16 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { e->chash_ok = 0; } else { init_new(e); + if (fill) filled = split_fill(e); + e->og_start = filled ? START_FILL : alias ? (e->pub.mirrored ? START_UPLOAD_LIVE : START_EMPTY_LIVE) + : (gone || rel_other) && !e->pub.mirrored ? START_EMPTY_RELEASED : START_NEW; + // an entry the CPU does not write, at an address the CPU can see, that starts without content + if (!filled && !e->pub.mirrored && !nvn_gpu_only(k->addr)) n_cpu_empty[alias ? 2 : gone || rel_other ? 1 : 0]++; if (alias) { - if (e->pub.mirrored) n_alias_upload++; - else { + if (e->pub.mirrored) { + n_alias_upload++; + if (trace) start_line(e, other, NULL, 0, taken_back); + } else if (!filled) { n_alias_uninit++; static int logged; // what an empty start over live memory was, to tell a black frame's cause; depth buffers start empty @@ -2907,6 +3316,8 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) { } } } + if (trace && gone && !e->pub.mirrored && !nvn_gpu_only(k->addr)) + start_line(e, NULL, gone_desc, gone_rel, e->og_start == START_COPY_GONE ? 2 : filled ? 3 : copied); } static Tex *entry_of(const tex_t *t) { @@ -2925,6 +3336,7 @@ static Tex *entry_of(const tex_t *t) { rt_log("vk: texture aliases: %s", tex_alias ? "a new image takes the live image's content" : "a new image starts empty"); } if (!(e = create_tex(t, &k, tex_alias))) return create_failed(&k), NULL; + origin_made(e); perf_count(PERF_CNT_TEX_NEW); start_at(e, &k, 0); e->next_at = nvn_hm_get(&by_addr, &k.addr); @@ -2941,6 +3353,7 @@ static Tex *entry_of(const tex_t *t) { if (gap > e->back_gap) e->back_gap = gap > UINT32_MAX ? UINT32_MAX : (uint32_t)gap; e->released = 0, nreleased--, rel_unlink(e), n_site[2]++, tex_gen_flip(e, e->superseded || e->overwritten); e->overwritten = 0; + e->og_backs++, e->og_back_frame = e->samp_frame; if (e->superseded) { // its image may still be in use: the first barrier waits for all earlier work e->superseded = 0, e->reused = 1; n_restarts++; @@ -3106,7 +3519,7 @@ void vk_texture_range_gone(uint64_t lo, uint64_t size, VmaAllocation alloc) { Tex *e = by_key.vals[i]; if (e && ((e->key.addr >= lo && e->key.addr - lo < size) || (alloc && e->bound_to == alloc))) list[n++] = e; } - for (int i = 0; i < n; i++) destroy_tex(list[i]); + for (int i = 0; i < n; i++) drop_tex(list[i], DROP_RANGE); free(list); unlock(); } @@ -3407,7 +3820,7 @@ const VkTexImage *vk_texture_exact_by_id_locked(int id, view_t *view, int *has_v log_limited(&n_logged, "texture detail: texture %#llx (fmt %#x %ux%u, %u levels) is read by texel position or its " "size is asked for: made whole again (%llu so far)", (unsigned long long)e->pub.addr, e->t.d.format, e->t.d.width, e->t.d.height, e->pub.levels + e->pub.skip, (unsigned long long)n_exact); - destroy_tex(e); + drop_tex(e, DROP_EXACT); } return vk_texture_by_id_locked(id, view, has_view); } @@ -3483,6 +3896,24 @@ void vk_texture_new_submit(void) { n_cap_drops = n_pressure_drops = n_commit_drops = n_commit_past = n_evict = n_evict_bytes = 0; n_disuse = n_disuse_bytes = n_disuse_back = n_retire_freed = n_retire_cancel = 0; n_reused = n_reuse_new = n_bind_skips = n_restarts = n_overwritten = n_over_drops = 0; + rt_log("perf: frame %llu: texture origins: released entries dropped past RELEASED_MAX %llu, replaced by another " + "descriptor at their address %llu; entries the CPU does not write made without content at an address it " + "can see: on their own %llu, over a released entry %llu, over a live one %llu; marked slots that met an " + "entry without content %llu, that show no entry %llu", + (unsigned long long)f * 60, (unsigned long long)n_drop_by[DROP_MAX], (unsigned long long)n_drop_by[DROP_REPLACED], + (unsigned long long)n_cpu_empty[0], (unsigned long long)n_cpu_empty[1], (unsigned long long)n_cpu_empty[2], + (unsigned long long)n_mark_empty, (unsigned long long)n_mark_none); + memset(n_drop_by, 0, sizeof n_drop_by), memset(n_cpu_empty, 0, sizeof n_cpu_empty); + n_mark_empty = n_mark_none = 0; + if (split_mode > 0) + rt_log("perf: frame %llu: split fill: entries the CPU does not write started with their storage's bytes: " + "copied from the released entry they replaced %llu, filled from storage %llu (%.1f MiB, %.2f ms), " + "filled again after a write %llu; left empty: image taken from a released entry %llu, storage over " + "%llu MiB %llu", (unsigned long long)f * 60, (unsigned long long)n_split_copy, + (unsigned long long)n_split_fill, n_split_bytes / 1048576.0, n_split_ns / 1e6, + (unsigned long long)n_split_refill, (unsigned long long)n_split_reused, + (unsigned long long)(SPLIT_FILL_MAX >> 20), (unsigned long long)n_split_big); + n_split_copy = n_split_fill = n_split_bytes = n_split_ns = n_split_refill = n_split_reused = n_split_big = 0; } unlock(); } @@ -3612,6 +4043,7 @@ static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t if (g->level < e->pub.skip) return 1; uint32_t il = g->level - e->pub.skip; if (il >= e->pub.levels) return 0; + e->had |= HAD_TEXELS, e->sfill = 0; // its content is no longer the storage's (split fill) uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height; int rgb = e->f.expand_rgb, astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu(); size_t texels = (size_t)w * h, ib = agpu ? 0 : astc ? 4 : rgb ? (size_t)g->bpb / 3 * 4 : 0; @@ -4405,7 +4837,7 @@ void vk_texture_shutdown(void) { int n = 0; for (uint32_t i = 0; i < by_key.cap; i++) if (by_key.vals[i]) list[n++] = by_key.vals[i]; - for (int i = 0; i < n; i++) destroy_tex(list[i]); + for (int i = 0; i < n; i++) drop_tex(list[i], DROP_SHUTDOWN); free(list); for (uint32_t i = 0; i < samplers.cap; i++) if (samplers.vals[i] && c) vkDestroySampler(c->device, (VkSampler)samplers.vals[i], NULL);