Start a texture the CPU does not write, made over storage the CPU writes, with that storage's bytes.

Two descriptors can name one CPU-written storage: one whose content the cache takes from the CPU, and a second with a texture flag for which it does not. The second has no upload of its own and its slot shows the first one's image while that entry is live. Once the first is released, a lookup of the second's slot makes it an entry of its own, start_at destroys the released first one, and the new image starts cleared and is never uploaded; the slot shows it from then on, also after the first descriptor is registered again. The same happens when both are released and the texture pool refresh looks the second's slot up before the next texture arrives in that memory.

With tex_split_fill = 1 (the default) start_at gives such an entry the storage's bytes: by a copy from the released CPU-written entry it is about to destroy when that image is current and alias_compatible allows, else by an upload from the storage. The image then follows the storage as a CPU-written texture's does until the GPU or WriteTexels writes it; 2 fills it once, 0 is the behaviour before. Depth, stencil and multisample images, pools without CPU access and storage over 32 MiB are left out, and an entry that is to be filled gets a new image instead of a released entry's, since a reused image goes through its first transition after the upload batch. The perf trace counts the copies, the fills with their bytes and time, the fills made again after a write and the entries left empty.

--probe-vkmem makes both sequences with the three values and reads the second descriptor's image back: zeros with 0, the storage's bytes with 1, and with 2 the bytes of the texture that was in the memory before when both descriptors were let go of. One case makes the entry inside a recorded submission, through a hook that lets the probe stand in for a replay, and there the content comes by the copy.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 05:57:23 -05:00
parent 15236bf5c2
commit 6c991d628d
7 changed files with 341 additions and 23 deletions

View File

@@ -243,6 +243,12 @@
# A render target that needs an image of its own takes the image of a released one of the same size and format
# instead of making a new one (0 = always a new image).
# tex_reuse = 1
# A texture the game does not fill from the CPU itself, made over memory the CPU writes (a second description of a
# texture's bytes), starts with those bytes and takes the game's later writes to them until something is drawn into
# it (1, default). 0: it starts empty, as before, and what is drawn with it can come out flat and pale. 2: it gets
# the bytes once and no later write. Costs an upload for each such texture; a render target the game makes over
# CPU memory starts with the last picture left there instead of an empty one.
# tex_split_fill = 1
# 1: shaders mark the textures they sample, so that while video memory or the system's memory runs low, textures no
# shader sampled for a while go, even those the game keeps registered (the areas it has loaded); one sampled again is
# made again within a few frames. The first start after changing it builds the pipelines again. 0: such textures stay.

View File

@@ -67,7 +67,7 @@ static const char *const known[] = {
"SVC_SLOW_MS",
"SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_IDLE", "SPLIT_IN_PASS", "SPLIT_US", "SPV_MARK_TEST", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS",
"SWKBD_ANSWER",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_REUSE", "TEX_SPLIT_FILL", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64",
"VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS",
"WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS",
"ZSTD_CHECK", "ZSTD_NATIVE",
@@ -135,7 +135,7 @@ static const struct { const char *key, *section; } sections[] = {
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"PIPE_LINK_DRAW_MS", "performance"}, {"PIPE_AHEAD", "performance"}, {"PIPE_FAST_WORKERS", "performance"}, {"GPL_FEEDBACK", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"OQ_BATCH", "performance"}, {"GATE_WAIT_MS", "performance"}, {"PIPE_PREBUILD", "performance"},
{"COMPUTE_KEEP", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
{"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
{"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"},
{"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"},
{"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"},
@@ -291,7 +291,7 @@ static const CfgMeta metas[] = {
{"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again
// textures and memory
{"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART},
{"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},
{"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_SPLIT_FILL", "1", "0|1|2", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART},
{"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART},
{"ASTC_SCRATCH_KEEP", "0", "0|1", RESTART}, {"ASTC_ARENA", "0", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena
// render fixes

View File

@@ -13,6 +13,7 @@
#include "nvn_api.h"
#include "tex.h"
#include "vk.h"
#include "vk_exec.h"
#include "vk_gpa.h"
#include "gpa_read_spv.h"
#include "platform/platform.h"
@@ -1068,6 +1069,165 @@ static int test_textures(pool_t *tpool, pool_t *phys) {
return 0;
}
// Level 0 of image (w x h RGBA8) read into out through a staging buffer after everything recorded so far has run.
static void image_bytes(VkImage image, uint32_t w, uint32_t h, uint8_t *out) {
VkCtx *c = vk_ctx();
VkBuffer rb;
VmaAllocation ra;
size_t n = (size_t)w * h * 4;
uint8_t *r = vk_staging(n, &rb, &ra);
VkCommandBuffer cb = vk_upload_begin();
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
.dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
VkBufferImageCopy rg = {.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .imageExtent = {w, h, 1}};
vkCmdCopyImageToBuffer(cb, image, VK_IMAGE_LAYOUT_GENERAL, rb, 1, &rg);
mb = (VkMemoryBarrier2){.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COPY_BIT,
.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT,
.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT};
vkCmdPipelineBarrier2(cb, &di);
vk_upload_end();
vk_upload_flush(1);
vmaInvalidateAllocation(c->vma, ra, 0, VK_WHOLE_SIZE);
memcpy(out, r, n);
vmaDestroyBuffer(c->vma, rb, ra);
}
// ---- two descriptors on one CPU-written storage (vk_texture.c "split fill", start_at)
// D1 is a texture the CPU writes. D2 names the same storage with a texture flag for which the cache does not take the
// content from the CPU (COMPRESSIBLE here), so a slot of D2 shows D1's image for as long as D1 has a live entry.
// What the slot shows once D1's entry is gone, in the two orders a game can let go of them:
// - keep: D2's slot stays registered while D1 is released and registered again (a texture made again in place);
// - both: D1 and D2 are released, the slot of D2 is looked up all the same (the texture pool refresh looks at every
// registered slot), then the storage gets new bytes and both are registered again (the next texture in that memory).
// recorded: the lookup that makes D2's entry runs while a submission is recorded, as a draw of a replay makes it, so a
// copy from D1's image can be recorded there; otherwise outside any recording, where only the storage can fill it.
// *what: 0 the slot's image is all zero, 1 it holds the storage's bytes as they are at the end, 2 the bytes the storage
// held when D2's entry was made (the same as 1 in "keep", where the storage is not rewritten), 3 none of these.
enum { SPLIT_W = 32, SPLIT_H = 32, SPLIT_N = SPLIT_W * SPLIT_H * 4 };
static int split_what(const uint8_t *got, const uint8_t *first, const uint8_t *last) {
static const uint8_t zero[SPLIT_N];
return !memcmp(got, zero, SPLIT_N) ? 0 : !memcmp(got, last, SPLIT_N) ? 1 : !memcmp(got, first, SPLIT_N) ? 2 : 3;
}
typedef struct { int id; const VkTexImage *got; } SplitRec;
static void split_record(VkCommandBuffer cb, void *arg) { // as gpa_read_record: the frame of a recorded submission
SplitRec *r = arg;
vk_texture_new_submit();
vk_mem_mirror_begin();
vk_exec_probe_inline(cb, vk_sync_issued() + 1);
r->got = vk_texture_by_id(r->id, NULL, NULL);
vk_exec_probe_inline(VK_NULL_HANDLE, 0);
vk_mem_mirror_end(cb, vk_sync_issued() + 1);
}
static int split_case(pool_t *pool, uint64_t off, uint64_t serial, int both, int recorded, int *what) {
uint8_t *lin[1] = {0}, first[SPLIT_N], got[SPLIT_N];
tex_t d1 = make_tex(pool, off, 0x25, SPLIT_W, SPLIT_H, 1, serial);
tex_t d2 = d1;
d2.d.flags |= NVN_TEXTURE_FLAGS_COMPRESSIBLE, d2.serial = serial + 1;
GCHECK(off + d1.L.size <= pool->size, "the two-descriptor case's storage does not fit the probe's pool");
if (fill_storage(&d1, lin)) return 1;
memcpy(first, lin[0], SPLIT_N);
const VkTexImage *e1 = vk_texture_get(&d1);
GCHECK(e1 && e1->mirrored, "the CPU-written texture of the two-descriptor case");
VkImage img1 = e1->image;
vk_texture_register(6, &d2, NULL);
const VkTexImage *s2 = vk_texture_by_id(6, NULL, NULL);
GCHECK(s2 && s2->image == img1, "the slot of the second descriptor does not show the CPU-written texture's image");
vk_texture_release(&d1);
if (both) vk_texture_release(&d2);
if (recorded) {
SplitRec r = {6, NULL};
uint64_t fence = vk_sync_probe_submit(split_record, &r);
GCHECK(vk_sync_fence_wait(fence, 5000000000ull), "the submission that looks the second descriptor's slot up timed out");
s2 = r.got;
} else {
vk_texture_new_submit();
s2 = vk_texture_by_id(6, NULL, NULL); // the texture pool refresh looks the slot up
}
GCHECK(s2 && s2->image != img1 && !s2->mirrored, "after the release of the CPU-written texture the slot of the second "
"descriptor has no image of its own (%s)", s2 ? "it still shows the released one" : "none at all");
VkImage img2 = s2->image;
if (both) { // the next texture in that memory
if (fill_storage(&d1, lin)) return 1;
vk_texture_register(6, &d2, NULL);
}
vk_texture_register(7, &d1, NULL);
vk_texture_new_submit();
const VkTexImage *s1 = vk_texture_by_id(7, NULL, NULL);
GCHECK(s1 && s1->mirrored && s1->image != img2, "the CPU-written texture registered again has no image of its own");
image_bytes(s1->image, SPLIT_W, SPLIT_H, got);
GCHECK(!memcmp(got, lin[0], SPLIT_N), "the CPU-written texture registered again does not hold its storage's bytes");
s2 = vk_texture_by_id(6, NULL, NULL);
GCHECK(s2 && s2->image == img2, "the slot of the second descriptor left its own image when the first came back");
image_bytes(s2->image, SPLIT_W, SPLIT_H, got);
*what = split_what(got, first, lin[0]);
// a shader sampled the slot: with NX2NAVI_TRACE=perf the log names an entry that never had content
uint32_t marks[8] = {0};
marks[6] = 1;
vk_pool_marks_read(marks, 8);
g_gpu_checks += 6;
vk_texture_release(&d1), vk_texture_release(&d2);
free(lin[0]);
return 0;
}
// The three cases with split fill off (tex_split_fill = 0), on (1) and filling once (2). Off, the slot shows an image
// nothing filled, which is what the setting is for; on, the storage's bytes, copied from D1's image where a submission
// was being recorded and uploaded from the storage elsewhere; filling once, the bytes of the texture that was in the
// memory before when both descriptors were let go of.
static int test_split(pool_t *pool) {
static const char *const names[4] = {"zeros", "the storage's bytes", "the bytes the storage held before", "other bytes"};
int what[3][3];
uint64_t made[3][3][2];
for (int mode = 0; mode < 3; mode++) {
vk_texture_split_fill_set(mode);
for (int k = 0; k < 3; k++) { // keep, both, keep with the lookup recorded
uint64_t c0, f0, c1, f1;
vk_texture_split_counts(&c0, &f0);
if (split_case(pool, 0x80000 + (uint64_t)(mode * 3 + k) * 0x4000, 30 + (uint64_t)(mode * 3 + k) * 2, k == 1, k == 2,
&what[mode][k]))
return 1;
vk_texture_split_counts(&c1, &f1);
made[mode][k][0] = c1 - c0, made[mode][k][1] = f1 - f0;
}
}
vk_texture_split_fill_set(-1); // the setting decides again
vk_sync_finish();
printf("probe-vkmem: a second descriptor on a CPU-written storage, not CPU-written itself, once the first one's entry "
"is gone. Its slot shows, with tex_split_fill 0 / 1 / 2: when it stayed registered %s / %s / %s; when both were "
"released and registered again over new bytes %s / %s / %s; when its entry was made in a recorded submission "
"%s / %s / %s (copies from the first one's image %llu / %llu / %llu, fills from storage %llu / %llu / %llu)\n",
names[what[0][0]], names[what[1][0]], names[what[2][0]], names[what[0][1]], names[what[1][1]], names[what[2][1]],
names[what[0][2]], names[what[1][2]], names[what[2][2]], (unsigned long long)made[0][2][0],
(unsigned long long)made[1][2][0], (unsigned long long)made[2][2][0], (unsigned long long)made[0][2][1],
(unsigned long long)made[1][2][1], (unsigned long long)made[2][2][1]);
for (int k = 0; k < 3; k++) {
// off: an image nothing filled (zeros with NX2NAVI_FIX_ZERO, the default; else whatever the memory held)
GCHECK(vk_fix_zero() ? what[0][k] == 0 : what[0][k] != 1, "with split fill off the slot shows %s in case %d: the "
"sequence no longer leaves the second descriptor an empty image", names[what[0][k]], k);
GCHECK(!made[0][k][0] && !made[0][k][1], "with split fill off an entry was filled in case %d", k);
GCHECK(what[1][k] == 1, "with split fill on the slot shows %s in case %d, not the storage's bytes", names[what[1][k]], k);
GCHECK(what[2][k] == (k == 1 ? 2 : 1), "filling once, the slot shows %s in case %d", names[what[2][k]], k);
for (int mode = 1; mode < 3; mode++)
GCHECK(made[mode][k][0] + made[mode][k][1] == 1, "split fill %d made %llu copies and %llu fills in case %d, not "
"one of them", mode, (unsigned long long)made[mode][k][0], (unsigned long long)made[mode][k][1], k);
}
for (int mode = 1; mode < 3; mode++) {
GCHECK(made[mode][0][1] == 1 && made[mode][1][1] == 1, "outside a recording split fill %d did not fill from storage", mode);
#ifdef _WIN32 // elsewhere an import's pinning can mark pages written without a CPU write, and the storage is uploaded
GCHECK(made[mode][2][0] == 1, "in a recorded submission split fill %d did not copy from the first one's image", mode);
#endif
}
g_gpu_checks += 9;
return 0;
}
static int test_gpu(void) {
uint64_t heap = 0;
GCHECK(mem_set_heap_size(16ull << 20, &heap) == 0 && heap, "SetHeapSize 16 MiB");
@@ -1086,6 +1246,7 @@ static int test_gpu(void) {
int detail_bad = test_detail(&dpool);
vk_pool_fini(&dpool);
if (detail_bad) return 1;
if (test_split(&tpool)) return 1;
VkMemStats st;
vk_mem_stats(&st);
GCHECK(st.imported_pools == 2 && st.physical_pools == 1 && st.virtual_pools == 1 && !st.import_failures,

View File

@@ -466,6 +466,10 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons
uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback);
void vk_astc_scratch_fail(int n); // a test's hook: the next n scratch buffer requests fail (tex_probe.c)
uint64_t vk_texture_upload_retries(void); // uploads left to be tried again: their decode had no memory (vk_texture.c upload)
// vk_texture.c "split fill", a test's hooks (tex_probe.c): the mode from here on (tex_split_fill: 0, 1 or 2), and how
// many entries started with a copy of the released entry they replaced and how many were filled from their storage.
void vk_texture_split_fill_set(int mode);
void vk_texture_split_counts(uint64_t *copies, uint64_t *fills);
VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh); // source bytes the region reads
void vk_astc_stats(uint64_t *dispatches, uint64_t *texels); // since start
uint64_t vk_astc_scratch_bytes(void); // the decoder's shared scratch now

View File

@@ -785,6 +785,11 @@ void vk_exec_inline_end(int reopen) {
if (reopen) vk_exec_pass_begin(&X);
}
void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence) {
X.cb = cb, X.fence = fence;
x_here = cb != VK_NULL_HANDLE;
}
typedef struct { VkBuffer b; VmaAllocation a; } InlineBuf;
static void free_inline_buf(void *arg) { // completion thread
InlineBuf *t = arg;

View File

@@ -143,6 +143,10 @@ void vk_exec_barrier_log(uint64_t frame);
// vk_exec_inline_buffer: a device buffer for transfers, destroyed when the submission completes.
VkCommandBuffer vk_exec_inline_begin(int *reopen);
void vk_exec_inline_end(int reopen);
// A test's hook (tex_probe.c): between a call with cb and one with VK_NULL_HANDLE, the calling thread counts as
// recording a replay into cb for submission `fence`, so vk_exec_inline_begin hands cb out. Called from the record
// callback of vk_sync_probe_submit, where a replay would be recorded.
void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence);
// vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex
// buffers): the replay continues in a new one (vk_sync_split).
void vk_exec_draw_new_cb(void);

View File

@@ -99,6 +99,8 @@ typedef struct Tex {
uint8_t og_drop; // DROP_*: the drop of this descriptor's last entry, when the ring still knew it
uint8_t og_reimg; // made with the image of a released entry (create_tex, reuse)
uint8_t said; // marks_check wrote its line for this entry
uint8_t sfill; // split fill: the CPU does not write it, and its image is the storage's bytes: 1 as a
// CPU-written one's until the GPU writes it, 2 once (from_storage)
uint32_t og_backs; // times it was taken back after a release (entry_of)
uint64_t og_frame; // gfx_counters.frames when it was made
uint64_t og_drop_frame; // gfx_counters.frames of the drop og_drop names
@@ -649,9 +651,78 @@ enum { DROP_NONE, DROP_MAX, DROP_CAP, DROP_COMMIT, DROP_VRAM, DROP_IDLE_VRAM, DR
static void drop_tex(Tex *e, int why);
// How an entry's content started (start_at; Tex.og_start), in the order of start_names.
enum { START_NEW, START_UPLOAD_LIVE, START_COPY_LIVE, START_COPY_RELEASED, START_EMPTY_LIVE, START_EMPTY_RELEASED,
START_N };
START_COPY_GONE, START_FILL, START_N };
// What put content into the image of an entry the CPU does not write (Tex.had); a GPU write is pub.gpu_owned.
enum { HAD_COPY = 1, HAD_TEXELS = 2 };
enum { HAD_COPY = 1, HAD_TEXELS = 2, HAD_FILL = 4 };
// ---- split fill (NX2NAVI_TEX_SPLIT_FILL, tex_split_fill in nx2navi.ini: 1 the default, 0 off, 2 filled once)
// Two descriptors can name one storage the CPU writes: a texture whose content the cache takes from the CPU
// (tex_mirrored), and a second one with a texture flag for which tex_mirrored refuses. The second has no upload of its
// own: its slot shows the first one's image for as long as that entry is live (slot_resolve). Once the first is
// released, a lookup of the second's slot makes the second an entry of its own (entry_of), start_at destroys the
// released first one, and the new image has nothing to start from: cleared, never uploaded, and from then on what
// the slot shows, also after the first descriptor is registered again and gets a new entry beside it. On the console
// both descriptors read the same bytes. tex_probe.c test_split makes the sequence and reads the image back: zeros.
//
// With split fill, a new entry the CPU does not write, at storage the CPU can see, starts with those bytes:
// - by a copy on the GPU from the released CPU-written entry start_at is about to destroy, when that image holds the
// storage as it is now (uploaded, and no write since its last check) or content the GPU wrote, and
// alias_compatible allows;
// - else from the storage itself, uploaded as a CPU-written texture's is (upload).
// With 1 the image then follows the storage as a CPU-written one's does (from_storage: make_fresh, the pool's list)
// until the GPU or WriteTexels writes it. The texture pool refresh makes such an entry also for a slot whose descriptor
// the game has let go of, long before the next texture's bytes arrive in that memory, and when both descriptors are
// registered again the slot finds that entry: filled once (2), it shows the texture that was there before.
// Left out: depth, stencil and multisample images, buffer textures, pools the CPU has no access to, storage over
// SPLIT_FILL_MAX, and an entry made with a released entry's image, whose first transition is recorded at the replay's
// position, after the upload batch (init_new); so create_tex gives an entry that is to be filled a new image. Nothing
// is filled where a live entry at the address holds content the GPU wrote (that is what the new one should show, and
// start_at copies it when it can), nor with NX2NAVI_TEX_ALIAS off.
//
// What follows the storage also takes what a stale descriptor's entry does not need: an entry made for a slot the game
// let go of is uploaded again whenever its pages are written, by the next textures loaded into that memory, read in
// its own format. The perf line counts those ("filled again after a write"); 2 does without them.
//
// What it costs: an upload (a copy out of the storage, a decode for ASTC) for each such entry that finds no image to
// copy from, where the image was only cleared before. A render target in a CPU pool that the game makes anew often
// pays that every time and no longer takes a released entry's image; the perf line counts the fills, their bytes and
// their time. Until the GPU first writes it, its storage is also checked for writes once a submission.
// What it can show: the bytes in the storage, whatever they are. For a render target that is the last picture the
// game or a write-back left there, where the image was zero before: a pass that reads a new target before anything
// drew into it sees that older picture instead of nothing.
#define SPLIT_FILL_MAX ((uint64_t)32 << 20)
static int split_mode = -1;
// since the last perf line: entries started by a copy from the released entry they replace, filled from storage (with
// the bytes and the time), filled again after a write; left empty for a reused image, for their size
static uint64_t n_split_copy, n_split_fill, n_split_bytes, n_split_ns, n_split_refill, n_split_reused, n_split_big;
static uint64_t split_total[2]; // copies and fills so far (vk_texture_split_counts)
static int split_fill_mode(void) {
if (split_mode < 0) {
int v = cfg_int("NX2NAVI_TEX_SPLIT_FILL");
split_mode = v == 0 ? 0 : v == 2 ? 2 : 1;
rt_log("vk: a texture the CPU does not write, made at storage the CPU writes, %s (NX2NAVI_TEX_SPLIT_FILL=%d)",
!split_mode ? "starts empty" : split_mode == 2 ? "starts with the storage's bytes, once"
: "starts with the storage's bytes and follows them", split_mode);
}
return split_mode;
}
// A test's hook (tex_probe.c): the mode from here on, and the copies and fills made so far.
void vk_texture_split_fill_set(int mode) { split_mode = mode; }
void vk_texture_split_counts(uint64_t *copies, uint64_t *fills) { *copies = split_total[0], *fills = split_total[1]; }
// Whether split fill gives a new entry for t its storage's bytes, apart from the size (SPLIT_FILL_MAX).
static int split_wants(const tex_t *t, const TexVkFormat *f) {
if (!split_fill_mode() || !t->mem || nvn_gpu_only((uintptr_t)t->mem) || tex_mirrored(t)) return 0;
if (!(t->pool_flags & (NVN_MEMORY_POOL_FLAGS_CPU_UNCACHED | NVN_MEMORY_POOL_FLAGS_CPU_CACHED))) return 0;
if (f->depth || f->stencil || t->d.samples > 1 || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return 0;
int bw, bh;
return nvn_fmt_bpb(t->d.format, &bw, &bh) != 0;
}
// Whether e's image takes its content from the storage: a texture the CPU writes, or one split fill fills.
static int from_storage(const Tex *e) { return e->pub.mirrored || e->sfill == 1 || (e->sfill == 2 && !e->uploaded); }
// Whether an image with these requirements can lie in the pool memory at e's address (*sp: that memory).
static int pool_fits(VkCtx *c, const Tex *e, const VkMemoryRequirements *req, uint64_t guest_size, VkMemSpan *sp) {
@@ -1120,7 +1191,11 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) {
}
}
int gpu_only = !tex_mirrored(t);
if (!img && gpu_only && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above)
// an entry split fill is to fill gets a new image: start_at uploads into it in the upload batch (see "split fill")
int to_fill = gpu_only && split_wants(t, &f) && L.size <= SPLIT_FILL_MAX;
for (const Tex *p = to_fill ? nvn_hm_get(&by_addr, &k->addr) : NULL; p; p = p->next_at)
if (alias_has_gpu_content(p)) { to_fill = 0; break; } // start_at copies that in instead
if (!img && gpu_only && !to_fill && tex_reuse_on()) { // the image of a released entry with these parameters (reuse, above)
// Not one released in this frame or the one before: Ocarina releases its render targets and registers them
// again every frame, and DLSS's scene target (2560x1440 at a fixed address in Kokiri Forest) lost its image
// to a moving target of the same size in between, and DLSS its history (vk_dlaa.c evaluate), several times a
@@ -1387,7 +1462,7 @@ int vk_texture_pool_note_locked(int id, uint32_t *ent) {
if (tslots[id].ce && tslots[id].cgen != agen[sinfo[id] & 0xffff]) vk_texture_by_id_locked(id, NULL, NULL);
Tex *e = tslots[id].ce;
if (!e) return 0;
if (!e->pidx && e->pub.mirrored && !e->pub.gpu_owned) {
if (!e->pidx && from_storage(e) && !e->pub.gpu_owned) {
if (npent == cpent) {
int n = cpent ? cpent * 2 : 1024;
PEnt *p = realloc(pent, sizeof *p * (size_t)n);
@@ -1519,7 +1594,8 @@ static const char *const drop_names[DROP_N] = {
"its storage went", "made whole again (texture detail)", "shutdown"};
static const char *const start_names[START_N] = {
"started on its own", "filled from storage over a live entry", "content copied from a live entry",
"content copied from a released entry", "started empty over a live entry", "started empty over a released entry"};
"content copied from a released entry", "started empty over a live entry", "started empty over a released entry",
"content copied from the released CPU-written entry it replaced (split fill)", "filled from its storage (split fill)"};
static uint64_t frames_now(void) { return __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); }
@@ -1576,11 +1652,11 @@ static int origin_text(char *out, size_t n, const Tex *e) {
}
// Whether nothing ever put content into e's image: an entry the CPU writes that was not uploaded, or one it does not
// write, at an address the CPU can see, that the GPU never wrote and that got no copy and no texels.
// write, at an address the CPU can see, that the GPU never wrote and that got no copy, no texels and no split fill.
static int no_content(const Tex *e) {
if (e->pub.gpu_owned) return 0;
if (e->pub.mirrored) return !e->uploaded;
return !e->had && e->key.addr && !nvn_gpu_only(e->key.addr);
return !e->had && !(e->sfill && e->uploaded) && e->key.addr && !nvn_gpu_only(e->key.addr);
}
// Lines per address for one kind of event: up to `most` each, in a table of the first SAID_N addresses that had one
@@ -2796,6 +2872,7 @@ static void upload(Tex *e) {
uint64_t dt = kern_now_ns() - u0;
if (e->uploaded) {
n_reuploads++, perf_count(PERF_CNT_TEX_REUP);
n_split_refill += e->sfill != 0;
ru_n++, ru_bytes += src_bytes, ru_ns += dt;
if (e->wt_n) {
ru_wt_n++, ru_wt_bytes += src_bytes, ru_wt_ns += dt;
@@ -2870,9 +2947,11 @@ static int same_content(const Tex *e) {
return 1;
}
// A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation.
// A mirrored texture is uploaded on first use and again when its storage pages carry a newer write generation. So is
// one split fill fills (from_storage), while it is not released: the game may be unmapping a released one's storage,
// and nothing keeps a slot from resolving to it as the stale rule does for a CPU-written one (slot_resolve).
static void make_fresh(Tex *e) {
if (!e->pub.mirrored || e->pub.gpu_owned) return;
if (!from_storage(e) || e->pub.gpu_owned || (e->sfill && e->released)) return;
uint64_t size = e->t.L.size;
if (!e->uploaded) {
if (up_wait(e)) return;
@@ -2891,7 +2970,7 @@ static void make_fresh(Tex *e) {
// vk_mem_changed_snap) instead of a system call per texture. A write the snapshot of the texture's region missed (made
// after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot.
static void make_fresh_snap(Tex *e) {
if (!e->pub.mirrored || e->pub.gpu_owned) return;
if (!from_storage(e) || e->pub.gpu_owned || (e->sfill && e->released)) return;
if (!e->uploaded && up_wait(e)) return;
uint32_t now;
n_checks++;
@@ -3032,7 +3111,8 @@ static void create_failed(const TexKey *k) {
// - gone: e, which the CPU does not write, was made at an address the CPU can see, and the released CPU-written entry
// there was destroyed (gone: its description, gone_rel: the frame of its release). Nothing uploads e; a slot of e's
// descriptor that showed the destroyed entry's image (slot_resolve follows it while it is live) shows e's from now on.
// flag: e was taken back (over), e got a copy of another entry's content (gone).
// flag: e was taken back (over); how e started (gone): 0 empty, 1 with a copy of another entry's content, 2 with a
// copy of the destroyed entry's, 3 filled from its storage (the last two: split fill).
static uint64_t n_cpu_empty[3]; // since the last perf line: entries the CPU does not write, at an address it can see,
// that started without content: on their own, over a released entry, over a live one
#define START_LINES 8
@@ -3053,13 +3133,32 @@ static void start_line(const Tex *e, const Tex *over, const char *gone, uint64_t
(unsigned long long)frames_now(), (unsigned long long)e->key.addr, flag ? " (taken back)" : "", d, o,
why ? "not CPU-written: " : "CPU-written", why ? why : "", no_content(over) ? ", no content" : "", og);
} else {
static const char *const how[4] = {"started empty", "got another entry's content", "got the destroyed entry's content",
"was filled from its storage"};
rt_log("perf: frame %llu: texture %#llx, not CPU-written (%s), %s after the released CPU-written entry at its "
"address was destroyed: new [%s]; destroyed [%s], released at frame %llu",
(unsigned long long)frames_now(), (unsigned long long)e->key.addr, mirror_refused(&e->t),
flag ? "got another entry's content" : "started empty", d, gone, (unsigned long long)gone_rel);
(unsigned long long)frames_now(), (unsigned long long)e->key.addr, mirror_refused(&e->t), how[flag & 3], d,
gone, (unsigned long long)gone_rel);
}
}
// Split fill from the storage: e, new and through its first transition, gets its storage's bytes in the upload batch
// (make_fresh, as a CPU-written texture's first use) and is marked to follow them or not (Tex.sfill). 0 when e is left
// as it is: storage over SPLIT_FILL_MAX, an image taken from a released entry (see "split fill"), storage not mapped,
// or an upload that had no memory, which make_fresh tries again later.
static int split_fill(Tex *e) {
if (e->t.L.size > SPLIT_FILL_MAX) return n_split_big++, 0;
if (e->reused) return n_split_reused++, 0;
if (!mem_readable(e->pub.addr, e->t.L.size)) return 0;
uint64_t t0 = kern_now_ns();
e->sfill = (uint8_t)split_mode;
make_fresh(e);
if (!e->uploaded) return 0;
n_split_fill++, split_total[1]++, n_split_bytes += e->t.L.size, n_split_ns += kern_now_ns() - t0;
e->had |= HAD_FILL;
return 1;
}
static int depth_alias_log(void) { // NX2NAVI_DEPTH_ALIAS_LOG=1: start_at logs each new depth entry over a live texture
static int on = -1;
if (on < 0) on = rt_knob("DEPTH_ALIAS_LOG", "0");
@@ -3086,6 +3185,11 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
char gone_desc[224] = ""; // trace: the last destroyed one, described while it existed
uint64_t gone_rel = 0;
Tex *src = NULL, *rsrc = NULL;
// split fill: e is to start with its storage's bytes; msrc: the released CPU-written entry whose image holds them
// (destroyed after the copy), msrc_gpu: it holds what the GPU wrote instead, msrc_gen: the storage's generation
int fill = !taken_back && tex_alias && split_wants(&e->t, &e->f), msrc_gpu = 0;
uint32_t msrc_gen = 0;
Tex *msrc = NULL;
const char *why = "no live texture with GPU content"; // for the log of an alias left empty
const Tex *other = NULL;
int newer = 0; // taken back: a texture at this address holds GPU content written since e's release
@@ -3104,6 +3208,15 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
gone_rel = p->rel_frame;
}
gone++;
const char *no;
uint32_t gen = 0;
int gpu = alias_has_gpu_content(p);
if (fill && !msrc && (gpu || (p->uploaded && !p->pub.gpu_owned &&
!vk_mem_changed(p->pub.addr, p->t.L.size, p->wgen, &gen))) &&
alias_compatible(e, p, &no)) {
msrc = p, msrc_gpu = gpu, msrc_gen = gen;
continue;
}
drop_tex(p, DROP_REPLACED);
continue;
}
@@ -3148,8 +3261,22 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
if (taken_back) init_new(e);
return;
}
int copied = src && alias_copy(e, src);
if (copied) {
// split fill only where nothing live holds content the storage lacks: that content is what e should show
if (src || live_content) fill = 0;
int copied = src && alias_copy(e, src), filled = 0;
if (fill && msrc && alias_copy(e, msrc)) { // from the released entry this one replaces
copied = filled = 1;
n_split_copy++, split_total[0]++;
e->og_start = START_COPY_GONE;
e->had |= HAD_COPY;
if (!msrc_gpu && e->t.L.size <= SPLIT_FILL_MAX) { // the storage's bytes as of msrc_gen: followed from there
e->sfill = (uint8_t)split_mode, e->uploaded = 1, e->wgen = msrc_gen;
e->chash_ok = 0;
}
}
if (msrc) drop_tex(msrc, DROP_REPLACED);
if (filled) { // nothing more: e is not CPU-written
} else if (copied) {
n_alias_copied++;
if (from_released) n_alias_released++;
e->og_start = from_released ? START_COPY_RELEASED : START_COPY_LIVE;
@@ -3163,15 +3290,16 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
e->chash_ok = 0;
} else {
init_new(e);
e->og_start = alias ? (e->pub.mirrored ? START_UPLOAD_LIVE : START_EMPTY_LIVE)
if (fill) filled = split_fill(e);
e->og_start = filled ? START_FILL : alias ? (e->pub.mirrored ? START_UPLOAD_LIVE : START_EMPTY_LIVE)
: (gone || rel_other) && !e->pub.mirrored ? START_EMPTY_RELEASED : START_NEW;
// an entry the CPU does not write, at an address the CPU can see, that starts without content
if (!e->pub.mirrored && !nvn_gpu_only(k->addr)) n_cpu_empty[alias ? 2 : gone || rel_other ? 1 : 0]++;
if (!filled && !e->pub.mirrored && !nvn_gpu_only(k->addr)) n_cpu_empty[alias ? 2 : gone || rel_other ? 1 : 0]++;
if (alias) {
if (e->pub.mirrored) {
n_alias_upload++;
if (trace) start_line(e, other, NULL, 0, taken_back);
} else {
} else if (!filled) {
n_alias_uninit++;
static int logged;
// what an empty start over live memory was, to tell a black frame's cause; depth buffers start empty
@@ -3188,7 +3316,8 @@ static void start_at(Tex *e, const TexKey *k, int taken_back) {
}
}
}
if (trace && gone && !e->pub.mirrored && !nvn_gpu_only(k->addr)) start_line(e, NULL, gone_desc, gone_rel, copied);
if (trace && gone && !e->pub.mirrored && !nvn_gpu_only(k->addr))
start_line(e, NULL, gone_desc, gone_rel, e->og_start == START_COPY_GONE ? 2 : filled ? 3 : copied);
}
static Tex *entry_of(const tex_t *t) {
@@ -3776,6 +3905,15 @@ void vk_texture_new_submit(void) {
(unsigned long long)n_mark_empty, (unsigned long long)n_mark_none);
memset(n_drop_by, 0, sizeof n_drop_by), memset(n_cpu_empty, 0, sizeof n_cpu_empty);
n_mark_empty = n_mark_none = 0;
if (split_mode > 0)
rt_log("perf: frame %llu: split fill: entries the CPU does not write started with their storage's bytes: "
"copied from the released entry they replaced %llu, filled from storage %llu (%.1f MiB, %.2f ms), "
"filled again after a write %llu; left empty: image taken from a released entry %llu, storage over "
"%llu MiB %llu", (unsigned long long)f * 60, (unsigned long long)n_split_copy,
(unsigned long long)n_split_fill, n_split_bytes / 1048576.0, n_split_ns / 1e6,
(unsigned long long)n_split_refill, (unsigned long long)n_split_reused,
(unsigned long long)(SPLIT_FILL_MAX >> 20), (unsigned long long)n_split_big);
n_split_copy = n_split_fill = n_split_bytes = n_split_ns = n_split_refill = n_split_reused = n_split_big = 0;
}
unlock();
}
@@ -3905,7 +4043,7 @@ static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t
if (g->level < e->pub.skip) return 1;
uint32_t il = g->level - e->pub.skip;
if (il >= e->pub.levels) return 0;
e->had |= HAD_TEXELS;
e->had |= HAD_TEXELS, e->sfill = 0; // its content is no longer the storage's (split fill)
uint32_t w = (uint32_t)r->width, h = (uint32_t)r->height;
int rgb = e->f.expand_rgb, astc = e->f.astc_bw != 0, agpu = astc && vk_astc_gpu();
size_t texels = (size_t)w * h, ib = agpu ? 0 : astc ? 4 : rgb ? (size_t)g->bpb / 3 * 4 : 0;