Merge the shadow sync once per guest submission that binds the device address table.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 06:54:21 -05:00
10 changed files with 218 additions and 375 deletions

View File

@@ -286,10 +286,11 @@ well: the shadow copies are chosen for the measured cost above, not for a missin
The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest
pointers into either. A shader that reads through the table resolves nothing, so the pool it reaches is not queued for
the copy-in: a part whose pipelines declare the table is marked at its draws and dispatches, and its prologue brings
the shadow of every pool in the table up to date with the pages the CPU wrote and queues the same pages for those
pools' VRAM mirrors (`vk_memory.c` `vk_mem_gpa_sync`). That copies the written pages of every CPU-visible pool in
each such part, which the cost above does not include. What a shader stores through the table lands in the shadow copy
the copy-in: a guest submission whose record binds the table (`vk_exec_draw.c` `ssbo_info`) brings the shadow of
every pool in the table up to date with the pages the CPU wrote, once, before the first part recorded after the
binding is submitted, and queues the same pages for those pools' VRAM mirrors (`vk_memory.c` `vk_mem_shadow_sync`).
That copies the written pages of every CPU-visible pool once in each such guest submission, which the cost above does
not include. What a shader stores through the table lands in the shadow copy
and in the pool's VRAM mirror, and no binding names it, so nothing notes it as a GPU write: it does not come back to
guest memory, and the next whole copy-in of its page (the CPU rewrites other words of the same records) puts guest
memory's stale bytes back over it in both copies. In the runs measured so far that copy-in always came before the
@@ -299,8 +300,9 @@ then on like a page with a noted GPU write, and has its stores written back to g
submission that uses the table. It is off by default (`runtime/vk_memory.c`, "Pages a shader stores into", says why).
`NX2NAVI_SHADOW_CHECK=1` logs every 60 frames how often a copy-in overwrote such a store and for which pools, how many
write-backs landed on pages the CPU had written since their copy-in, and what the copy-ins cost and where they went.
`--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is never bound, and must read
`0x13579BDF` and, after a plain CPU write, `0x2468ACE0`; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU
`--probe-vkmem` checks the sync: the shadow of a pool that is never bound must hold `0x13579BDF`, keep it through a
plain CPU write and a guest submission that does not bind the table, and hold the written `0x2468ACE0` after one that
does; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU
write into a shadow page that nothing notes must survive the CPU's writes of that page and reach guest memory. On
Linux, run it for each backing:

View File

@@ -1,106 +0,0 @@
; --probe-vkmem (runtime/tex_probe.c test_gpa_read): one invocation reads a 32-bit word of guest memory through the
; device address table as the game's translated shaders do, by the table's layout (runtime/vk_gpa.c): L1 at word 16,
; one entry per 16 MiB; L2 blocks of 4096 page entries from word 65552; word 2 holds the word index of the records,
; four words each, the first two the buffer's device address minus the guest address of its byte 0. The pool that
; holds the word is never a descriptor of the shader: only the table (binding 0) and a result word (binding 1) are.
; An address no page maps gives 0. Written in SPIR-V assembly (spirv-as; no GLSL compiler in the build). As GLSL:
; #extension GL_EXT_buffer_reference : require
; layout(local_size_x = 1) in;
; layout(std430, binding = 0) readonly buffer Table { uint t[]; };
; layout(std430, binding = 1) buffer Result { uint r[]; };
; layout(push_constant) uniform Push { uint64_t guest; } pc;
; layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer Word { uint v; };
; void main() {
; uint rec = t[65552u + (t[16u + uint(pc.guest >> 24)] << 12) + (uint(pc.guest >> 12) & 4095u)];
; if (rec == 0u) { r[0] = 0u; return; }
; uint at = t[2] + (rec << 2);
; r[0] = Word(pc.guest + (uint64_t(t[at]) | uint64_t(t[at + 1u]) << 32)).v;
; }
OpCapability Shader
OpCapability Int64
OpCapability PhysicalStorageBufferAddresses
OpExtension "SPV_KHR_physical_storage_buffer"
OpMemoryModel PhysicalStorageBuffer64 GLSL450
OpEntryPoint GLCompute %main "main" %table %result %push
OpExecutionMode %main LocalSize 1 1 1
OpDecorate %_runtimearr_uint ArrayStride 4
OpDecorate %Words Block
OpMemberDecorate %Words 0 Offset 0
OpDecorate %table DescriptorSet 0
OpDecorate %table Binding 0
OpDecorate %result DescriptorSet 0
OpDecorate %result Binding 1
OpDecorate %Push Block
OpMemberDecorate %Push 0 Offset 0
%void = OpTypeVoid
%uint = OpTypeInt 32 0
%ulong = OpTypeInt 64 0
%bool = OpTypeBool
%fn = OpTypeFunction %void
%_runtimearr_uint = OpTypeRuntimeArray %uint
%Words = OpTypeStruct %_runtimearr_uint
%ptr_words = OpTypePointer StorageBuffer %Words
%_ptr_StorageBuffer_uint = OpTypePointer StorageBuffer %uint
%table = OpVariable %ptr_words StorageBuffer
%result = OpVariable %ptr_words StorageBuffer
%Push = OpTypeStruct %ulong
%ptr_push = OpTypePointer PushConstant %Push
%push = OpVariable %ptr_push PushConstant
%_ptr_PushConstant_ulong = OpTypePointer PushConstant %ulong
%_ptr_PhysicalStorageBuffer_uint = OpTypePointer PhysicalStorageBuffer %uint
%uint_0 = OpConstant %uint 0
%uint_1 = OpConstant %uint 1
%uint_2 = OpConstant %uint 2
%uint_4 = OpConstant %uint 4
%uint_12 = OpConstant %uint 12
%uint_16 = OpConstant %uint 16
%uint_24 = OpConstant %uint 24
%uint_32 = OpConstant %uint 32
%uint_4095 = OpConstant %uint 4095
%uint_65552 = OpConstant %uint 65552
%main = OpFunction %void None %fn
%entry = OpLabel
%28 = OpAccessChain %_ptr_PushConstant_ulong %push %uint_0
%guest = OpLoad %ulong %28
%30 = OpShiftRightLogical %ulong %guest %uint_24
%31 = OpUConvert %uint %30
%59 = OpIAdd %uint %31 %uint_16
%32 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %59
%block = OpLoad %uint %32
%34 = OpShiftRightLogical %ulong %guest %uint_12
%35 = OpUConvert %uint %34
%page = OpBitwiseAnd %uint %35 %uint_4095
%62 = OpShiftLeftLogical %uint %block %uint_12
%63 = OpIAdd %uint %uint_65552 %62
%42 = OpIAdd %uint %63 %page
%37 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %42
%rec = OpLoad %uint %37
%out = OpAccessChain %_ptr_StorageBuffer_uint %result %uint_0 %uint_0
%mapped = OpINotEqual %bool %rec %uint_0
OpSelectionMerge %done None
OpBranchConditional %mapped %is_mapped %not_mapped
%is_mapped = OpLabel
%43 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %uint_2
%recs = OpLoad %uint %43
%45 = OpShiftLeftLogical %uint %rec %uint_2
%at = OpIAdd %uint %recs %45
%47 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %at
%48 = OpLoad %uint %47
%49 = OpIAdd %uint %at %uint_1
%50 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %49
%51 = OpLoad %uint %50
%52 = OpUConvert %ulong %48
%53 = OpUConvert %ulong %51
%54 = OpShiftLeftLogical %ulong %53 %uint_32
%delta = OpBitwiseOr %ulong %52 %54
%56 = OpIAdd %ulong %guest %delta
%ptr = OpConvertUToPtr %_ptr_PhysicalStorageBuffer_uint %56
%value = OpLoad %uint %ptr Aligned 4
OpStore %out %value
OpBranch %done
%not_mapped = OpLabel
OpStore %out %uint_0
OpBranch %done
%done = OpLabel
OpReturn
OpFunctionEnd

View File

@@ -15,7 +15,6 @@
#include "vk.h"
#include "vk_exec.h"
#include "vk_gpa.h"
#include "gpa_read_spv.h"
#include "platform/platform.h"
static uint64_t g_rng = 0x9E3779B97F4A7C15ull;
@@ -372,7 +371,7 @@ static int test_import(pool_t *pool) {
// second sync in the same epoch would not see the CPU writes made since the first), then the part's CPU inputs
// go in before its GPU work.
vk_texture_new_submit();
vk_mem_shadow_sync();
vk_mem_shadow_sync(0);
copy_wait(sb, 0, sp.buffer, sp.offset, 0x20000);
vk_gpu_written(a + 0x1000, 0x20000);
GCHECK(!memcmp(m + 0x1000, s, 0x20000), "GPU copy into the imported heap range differs from the source");
@@ -391,7 +390,7 @@ static int test_import(pool_t *pool) {
// add to and the CPU resets (the game's lighting accumulators) depends on the reset reaching the GPU first.
vk_mem_gpu_pending(a + 0x30000, 0x4000);
vk_texture_new_submit();
vk_mem_shadow_sync();
vk_mem_shadow_sync(0);
copy_wait(sp.buffer, sp.offset, sb, 0, 0x4000);
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
GCHECK(!memcmp(s, m + 0x30000, 0x4000), "GPU read of CPU-written heap differs");
@@ -439,145 +438,83 @@ static int test_remade(uint8_t *guest) {
return 0;
}
// test_gpa_read's shader (runtime/shaders/gpa_read.spvasm): one invocation follows the device address table from a
// guest address (the push constant) to the buffer that holds it and stores the 32-bit word found there in the result
// buffer. Storage bindings: 0 the table itself, 1 the result. The pool that holds the word is never a descriptor, a
// transfer source or a resolved span of the submission.
typedef struct {
VkPipeline pipe;
VkPipelineLayout layout;
VkDescriptorSet set;
uint64_t source;
int setup_mirror, mirrored;
} GpaRead;
static void gpa_read_record(VkCommandBuffer cb, void *arg) {
GpaRead *r = arg;
// Match vk_backend.c submit_now and vk_exec.c vk_exec_record, including epochs on descriptor reuse.
vk_texture_new_submit();
vk_mem_mirror_begin();
// The only explicit use of the source pool is making its mirror, before the first read: that submission takes its
// shadow touch and its first upload. After the CPU's write nothing resolves, uses or flushes the pool.
if (r->setup_mirror) r->mirrored = vk_mem_mirror_probe(r->source), r->setup_mirror = 0;
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, r->pipe);
vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, r->layout, 0, 1, &r->set, 0, NULL);
vkCmdPushConstants(cb, r->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof r->source, &r->source);
vk_mem_gpa_use(); // the shader reads the table, in the second submission through the same descriptor set
vkCmdDispatch(cb, 1, 1, 1);
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT};
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
vkCmdPipelineBarrier2(cb, &di);
vk_mem_mirror_end(cb, vk_sync_issued() + 1); // called under the submission lock, before the value is issued
}
static int test_gpa_read(uint8_t *guest) {
VkCtx *c = vk_ctx();
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xA00000, .size = 0x10000,
.game_mem = guest + 0xA00000};
if (!vk_gpa_wanted()) { // no buffer device addresses, or NX2NAVI_GPA=0: shaders read zeros through it by design
printf("probe-vkmem: no device address table: the read through it is skipped\n");
return 0;
}
volatile uint32_t *source = (volatile uint32_t *)pool.mem;
*source = 0x13579BDFu; // before vk_pool_init: a shadow copy starts with the bytes an import would show
GCHECK(vk_gpa_on() && vk_pool_init(&pool), "the device address table and a CPU-visible pool for the read through it");
VkBuffer out = VK_NULL_HANDLE;
VmaAllocation alloc = NULL;
VkShaderModule module = VK_NULL_HANDLE;
VkDescriptorSetLayout dsl = VK_NULL_HANDLE;
VkDescriptorPool dp = VK_NULL_HANDLE;
// With shadow storage the pool also gets a VRAM mirror, which the table then names: the prologue of a part that
// uses the table keeps both up to date (vk_mem_gpa_sync). Without it the pool stays an import, guest memory itself:
// a mirror is not synced for the table's readers there (vk_exec_draw.c ssbo_info).
int want_mirror = vk_mirror_on && vk_mem_shadow_storage();
GpaRead r = {.source = (uintptr_t)pool.mem, .setup_mirror = want_mirror};
int bad = 1;
#define GPA_CHECK(cond, what) do { if (!(cond)) { printf("probe-vkmem: FAIL GPA " what "\n"); goto done; } } while (0)
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 4, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT};
VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT,
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT};
VmaAllocationInfo info;
GPA_CHECK(vmaCreateBuffer(c->vma, &bi, &ai, &out, &alloc, &info) == VK_SUCCESS && info.pMappedData, "result buffer");
VkDescriptorSetLayoutBinding bindings[2] = {
{.binding = 0, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT},
{.binding = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}};
VkDescriptorSetLayoutCreateInfo dl = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .bindingCount = 2,
.pBindings = bindings};
GPA_CHECK(vkCreateDescriptorSetLayout(c->device, &dl, NULL, &dsl) == VK_SUCCESS, "descriptor layout");
VkPushConstantRange push = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof r.source};
VkPipelineLayoutCreateInfo pl = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
.pSetLayouts = &dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &push};
GPA_CHECK(vkCreatePipelineLayout(c->device, &pl, NULL, &r.layout) == VK_SUCCESS, "pipeline layout");
VkShaderModuleCreateInfo sm = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof gpa_read_spv,
.pCode = gpa_read_spv};
GPA_CHECK(vkCreateShaderModule(c->device, &sm, NULL, &module) == VK_SUCCESS, "shader module");
VkComputePipelineCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, .layout = r.layout,
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT,
.module = module, .pName = "main"}};
GPA_CHECK(vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &cp, NULL, &r.pipe) == VK_SUCCESS, "compute pipeline");
VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2};
VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .maxSets = 1,
.poolSizeCount = 1, .pPoolSizes = &ps};
GPA_CHECK(vkCreateDescriptorPool(c->device, &pi, NULL, &dp) == VK_SUCCESS, "descriptor pool");
VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = dp,
.descriptorSetCount = 1, .pSetLayouts = &dsl};
GPA_CHECK(vkAllocateDescriptorSets(c->device, &da, &r.set) == VK_SUCCESS, "descriptor set");
VkDescriptorBufferInfo buffers[2] = {vk_gpa_info(), {out, 0, 4}};
VkWriteDescriptorSet writes[2];
for (unsigned i = 0; i < 2; i++)
writes[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = r.set, .dstBinding = i,
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
.pBufferInfo = &buffers[i]};
vkUpdateDescriptorSets(c->device, 2, writes, 0, NULL);
static const uint32_t want[2] = {0x13579BDFu, 0x2468ACE0u};
for (unsigned pass = 0; pass < 2; pass++) {
if (pass) *source = want[pass]; // a plain CPU write: no resolve, use, flush or shadow sync of the pool
uint64_t fence = vk_sync_probe_submit(gpa_read_record, &r);
GPA_CHECK(vk_sync_fence_wait(fence, 5000000000ull), "shader fence timed out");
GPA_CHECK(!want_mirror || r.mirrored, "source mirror setup");
vmaInvalidateAllocation(c->vma, alloc, 0, VK_WHOLE_SIZE);
uint32_t got = *(volatile uint32_t *)info.pMappedData;
if (got != want[pass]) {
printf("probe-vkmem: FAIL GPA/BDA %s read: got %#010x, expected %#010x (mirror %d)\n",
pass ? "after CPU mutation" : "initial", got, want[pass], r.mirrored);
goto done;
}
}
g_gpu_checks += 2;
printf("probe-vkmem: GPA/BDA CPU write -> shader read: initial %#010x and mutation %#010x exact (mirror %d)\n",
want[0], want[1], r.mirrored);
bad = 0;
done:
vk_sync_finish(); // after a timeout too: the objects below must outlive the shader
if (dp) vkDestroyDescriptorPool(c->device, dp, NULL);
if (r.pipe) vkDestroyPipeline(c->device, r.pipe, NULL);
if (module) vkDestroyShaderModule(c->device, module, NULL);
if (r.layout) vkDestroyPipelineLayout(c->device, r.layout, NULL);
if (dsl) vkDestroyDescriptorSetLayout(c->device, dsl, NULL);
if (out) vmaDestroyBuffer(c->vma, out, alloc);
vk_pool_fini(&pool);
#undef GPA_CHECK
return bad;
}
// test_table_store: a submission whose one part uses the device address table and records nothing else, as a part of
// the replay with such a shader ends: its prologue brings every pool of the table up to date (vk_mem_gpa_sync).
// test_table_sync, test_table_store: a guest submission of one part that records nothing. bound: its record binds the
// device address table, as the replay of a draw whose shader reads through the table does (vk_exec_draw.c ssbo_info),
// so its prologue brings every shadow of the table up to date (vk_memory.c vk_mem_shadow_sync).
static void table_part_record(VkCommandBuffer cb, void *arg) {
(void)arg;
vk_texture_new_submit();
vk_mem_mirror_begin();
vk_mem_gpa_use();
if (arg) vk_mem_shadow_table_read();
vk_mem_mirror_end(cb, vk_sync_issued() + 1);
}
static int table_part(void) {
uint64_t fence = vk_sync_probe_submit(table_part_record, NULL);
static int table_part_of(int bound) {
static int binds;
uint64_t fence = vk_sync_submit_own(table_part_record, bound ? &binds : NULL);
return vk_sync_fence_wait(fence, 5000000000ull); // its completion callbacks have run when the wait returns
}
static int table_part(void) { return table_part_of(1); }
// Shadow storage: a pool that no part ever binds or resolves, as the pools a shader reaches only through a pointer it
// read from another buffer. Its shadow, which the device address table names, starts with the bytes guest memory held
// when the pool was made; a CPU write reaches it at the next guest submission whose record binds the table, and not
// at one that does not (nothing else brings it there, which is what left the foliage's instances stale). The shadow
// is read back with a GPU copy out of the buffer a lookup names, which does not touch the pool.
static int test_table_sync(uint8_t *guest) {
VkCtx *c = vk_ctx();
if (!vk_mem_shadow_storage() || !vk_gpa_wanted()) {
printf("probe-vkmem: the sync of the device address table's shadows: skipped (%s)\n",
!vk_mem_shadow_storage() ? "no shadow storage" : "no device address table");
return 0;
}
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xA00000, .size = 0x10000,
.game_mem = guest + 0xA00000};
volatile uint32_t *w = (volatile uint32_t *)(pool.mem + 0x3000);
*w = 0x13579BDFu; // before vk_pool_init: a shadow starts with the bytes an import would show
GCHECK(vk_gpa_on() && vk_pool_init(&pool), "a CPU-visible pool in the device address table");
VkBuffer sb;
VmaAllocation sa;
volatile uint32_t *s = vk_staging(0x1000, &sb, &sa);
VkMemSpan sp;
int bad = 1;
uint32_t got;
#define TY_CHECK(cond, ...) do { if (!(cond)) { printf("probe-vkmem: FAIL table sync: "); printf(__VA_ARGS__); printf("\n"); goto done; } } while (0)
#define TY_SHADOW(into) \
do { \
TY_CHECK(vk_mem_lookup((uintptr_t)w, 4, &sp) && sp.buffer, "the pool has no buffer"); \
copy_wait(sp.buffer, sp.offset, sb, 0, 4); \
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); \
(into) = s[0]; \
} while (0)
TY_SHADOW(got);
TY_CHECK(got == 0x13579BDFu, "the new shadow holds %#010x, expected the pool's 0x13579bdf", got);
*w = 0x2468ACE0u; // a plain CPU write: nothing resolves, uses or flushes the pool
TY_CHECK(table_part_of(0), "fence timed out");
TY_SHADOW(got);
TY_CHECK(got == 0x13579BDFu, "a guest submission that does not bind the table left %#010x in the shadow, expected "
"the old 0x13579bdf (the check tests nothing if something else syncs the pool)", got);
TY_CHECK(table_part_of(1), "fence timed out");
TY_SHADOW(got);
TY_CHECK(got == 0x2468ACE0u, "after a guest submission that binds the table the shadow holds %#010x, expected the "
"CPU's 0x2468ace0", got);
*w = 0x0F1E2D3Cu; // and again, a guest submission later
TY_CHECK(table_part_of(1), "fence timed out");
TY_SHADOW(got);
TY_CHECK(got == 0x0F1E2D3Cu, "after the second such guest submission the shadow holds %#010x, expected 0x0f1e2d3c", got);
g_gpu_checks += 4;
printf("probe-vkmem: a pool that is never bound: its shadow in the device address table takes the CPU's writes at "
"each guest submission that binds the table, and at no other\n");
bad = 0;
done:
vk_sync_finish();
vmaDestroyBuffer(c->vma, sb, sa);
vk_pool_fini(&pool);
#undef TY_CHECK
#undef TY_SHADOW
return bad;
}
// Shadow storage: what a shader stores through the device address table lands in the pool's shadow copy and nothing
// notes it (vk_memory.c "Pages a shader stores into"). A GPU copy into the shadow buffer that is not noted stands in
// for the store. Word X of a page takes the stores, the CPU rewrites word Y of the same page before every part:
@@ -1143,7 +1080,7 @@ static int split_case(pool_t *pool, uint64_t off, uint64_t serial, int both, int
if (both) vk_texture_release(&d2);
if (recorded) {
SplitRec r = {6, NULL};
uint64_t fence = vk_sync_probe_submit(split_record, &r);
uint64_t fence = vk_sync_submit_own(split_record, &r);
GCHECK(vk_sync_fence_wait(fence, 5000000000ull), "the submission that looks the second descriptor's slot up timed out");
s2 = r.got;
} else {
@@ -1256,7 +1193,7 @@ static int test_gpu(void) {
vk_mem_stats(&st);
GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini");
g_gpu_checks += 2;
return test_gpa_read(h) || test_table_store(h) || test_remade(h);
return test_table_sync(h) || test_table_store(h) || test_remade(h);
}
int probe_vkmem(void) {

View File

@@ -359,13 +359,13 @@ void vk_mem_fatal_note(char *out, size_t n); // the memory mode and the lazy w
// Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU
// write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written,
// which copies the bytes back into guest memory before telling the texture cache (vk_guest_written).
void vk_mem_shadow_sync(void);
// A shader of the part being recorded reads through the device address table (vk_gpa.c), also when its descriptors
// are reused: with shadow storage the part's prologue (vk_mem_gpa_sync) then brings every pool the table names up to
// date with the CPU's writes, since the pools such a shader reaches are never resolved. fence: the submission's value,
// at which what the shaders stored through the table goes back to guest memory.
void vk_mem_gpa_use(void);
void vk_mem_gpa_sync(uint64_t fence);
// fence: the value of the submission the part belongs to (0: none, a probe's own sync), at which what shaders stored
// through the device address table goes back to guest memory (NX2NAVI_SHADOW_TABLE_STORES).
void vk_mem_shadow_sync(uint64_t fence);
// Record time: a shader of the guest submission being recorded reads the device address table (vk_gpa.c). The pools
// such a shader reaches are never resolved, so the next vk_mem_shadow_sync brings the CPU's writes into every shadow
// the table names, once per guest submission.
void vk_mem_shadow_table_read(void);
int vk_mem_shadow_storage(void); // CPU-visible pools are shadow copies of guest memory, not imports (NX2NAVI_SHADOW)
int vk_mem_shadow_table_stores(void); // shadow storage keeps what shaders store through the table (NX2NAVI_SHADOW_TABLE_STORES)
void vk_mem_gpu_pending(uint64_t addr, uint64_t size);
@@ -674,9 +674,9 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h)
// once the GPU reached it and the completion thread ran its side effects. Guest or host threads.
void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
// --probe-vkmem: record gives the submission's commands in place of the replay, under the same submission lock,
// recorder, memory prologue and fence.
uint64_t vk_sync_probe_submit(void (*record)(VkCommandBuffer, void *), void *arg);
// The probes (--probe-vkmem's table parts, the lost-device probe): one submission whose commands the caller records
// itself where the replay would, with the submission lock, the memory prologue and the fence of any other.
uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg);
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet.
@@ -782,7 +782,6 @@ typedef struct {
// from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs.
VkDescriptorUpdateTemplate tpl[2];
int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools)
int gpa; // a stage accesses guest pointers through the device address table
} VkPipeInfo;
// One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the
// bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte.

View File

@@ -145,7 +145,7 @@ VkCommandBuffer vk_exec_inline_begin(int *reopen);
void vk_exec_inline_end(int reopen);
// A test's hook (tex_probe.c): between a call with cb and one with VK_NULL_HANDLE, the calling thread counts as
// recording a replay into cb for submission `fence`, so vk_exec_inline_begin hands cb out. Called from the record
// callback of vk_sync_probe_submit, where a replay would be recorded.
// callback of vk_sync_submit_own, where a replay would be recorded.
void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence);
// vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex
// buffers): the replay continues in a new one (vk_sync_split).

View File

@@ -1388,9 +1388,13 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
// of every mirror at a table reader's draw copied stale guest bytes over what the GPU had written into the mirrors
// earlier in the frame, and Kokiri Forest's lit areas went black. A pool read only through the table is rarely a
// mirrored one (mirrors are made for pools bound as buffers, which sync then). With shadow storage every pool the
// table names is a copy of guest memory, and nothing resolves the pools such a reader reaches: bind_resources
// marks the part (vk_mem_gpa_use) and its prologue brings those copies up to date (vk_memory.c vk_mem_gpa_sync).
// table names is a copy of guest memory, and nothing resolves the pools such a reader reaches: the binding is
// noted here, and the copies are brought up to date before the part is submitted (vk_memory.c
// vk_mem_shadow_table_read). A draw that reuses the descriptors of the one before it (bind_resources) comes after
// a build in the same command buffer, and the packed descriptors of mo_desc are built in the same guest
// submission as their draws: either way this ran for the guest submission.
if (r->cbuf_slot == VK_GPA_CB_SLOT) {
vk_mem_shadow_table_read();
VkDescriptorBufferInfo t = vk_gpa_info();
return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE};
}
@@ -3231,7 +3235,6 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
static VkDescSlot d[MAX_DESC];
static VkBufferView bv[MAX_DESC];
if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS);
if (info->gpa) vk_mem_gpa_use(); // before the early returns for reused descriptors: the shader still reads the table
bind_stats(info);
int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS;
int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid;
@@ -4460,7 +4463,6 @@ static void mo_emit(VkCommandBuffer cb) {
for (int n = 0; n < MQ.n; p = &MQ.d[n++]) {
const MoDraw *q = &MQ.d[n];
const VkPipeInfo *info = q->info;
if (info->gpa) vk_mem_gpa_use(); // these draws bind packed descriptors without bind_resources, maybe in a later part
if (q->pipe != pipe) vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, q->pipe), pipe = q->pipe, MO.e_pipe++;
if (q->st != st) vk_pipeline_set_dynamic(cb, &MQ.st[q->st]), st = q->st, MO.e_dyn++;
if (!p || memcmp(&q->v, &p->v, sizeof q->v)) vkCmdSetViewportWithCount(cb, 1, &q->v);

View File

@@ -36,8 +36,8 @@
// NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0).
// NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis).
// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). With shadow storage the
// pools are copies of guest memory: a part whose shaders use the table is marked (vk_exec_draw.c bind_resources) and
// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). There
// pools are copies of guest memory, which a guest submission that binds the table (vk_exec_draw.c ssbo_info) brings up
// to date with the pages the CPU wrote, once, before the part is submitted (vk_memory.c vk_mem_shadow_sync). There
// the twin of a mirrored pool is its shadow copy, not guest memory, and what a shader stores through the table is not
// noted as a GPU write: it does not come back to guest memory, and the next whole copy-in of its page overwrites it.
// vk_memory.c "Pages a shader stores into through the device address table" has the sequence, what was measured, and

View File

@@ -86,6 +86,7 @@ typedef struct PoolRec {
uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow
uint64_t sborn; // the value of the first submission that can have used the shadow (shadow_pool)
volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded
uint32_t stab_i; // shadow_lock: 1 + its index in stab (the table's shadows), 0 when not there
// shadow_lock: base copies of the pages a noted GPU write covers (merge_page), open addressing by page address;
// csb a power of two, page 0 an empty slot
struct SBase { uint64_t page; uint8_t *base; } *sb;
@@ -112,6 +113,7 @@ static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size);
static void shadow_touch(PoolRec *r);
static void shadow_forget(PoolRec *r);
static void pool_retiring(PoolRec *r); // pool_fini: kept until pool_retire (or vk_mem_shutdown)
static void shadow_table_add(PoolRec *r);
static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record
static void grow(void **arr, int *cap, size_t elem);
@@ -336,10 +338,13 @@ static int pool_init(pool_t *p) {
phys_list[nphys++] = r;
}
plat_mutex_unlock(&pool_lock);
// The pool's pages go into the device address table: shaders that read a pointer from memory reach them there. An
// import and a shadow copy both have buf; only the import owns mem.
if (r->kind == POOL_IMPORT && r->buf && r->import_hi > r->import_lo)
// The imported pages, or the shadow that stands for them, go into the device address table: shaders that read a
// pointer from memory reach them there. A shadow in the table is kept up to date for the table's readers whether
// or not the replay resolves it (vk_mem_shadow_sync).
if (r->kind == POOL_IMPORT && (r->mem || r->smap) && r->buf && r->import_hi > r->import_lo) {
r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo);
if (r->smap && r->gpa_rec) shadow_table_add(r);
}
return 1;
}
@@ -504,7 +509,7 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touc
// touch: the GPU may read the pool's buffer in the submission being recorded, so its shadow goes on the touched list
// (shadow storage). A lookup (vk_mem_lookup) leaves it off: the caller reads guest memory itself or only asks what
// backs the address. Shaders that reach a pool through the device address table resolve nothing: the part they run in
// brings every pool of the table up to date (vk_mem_gpa_sync).
// brings every pool of the table up to date (vk_mem_shadow_sync).
static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) {
memset(out, 0, sizeof *out);
uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size;
@@ -1056,8 +1061,18 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
// invalidates the buffer, about 1.3 ms per 16 MiB at the next submit, and the import's write pinning clears the
// write watch), a CPU-visible pool gets a host-visible buffer of its own instead, at the offsets the import would have:
// - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are
// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. A part whose shaders read
// through the device address table gets every pool the table names brought up to date the same way (vk_mem_gpa_sync).
// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU.
// - The device address table (vk_gpa.c) names the shadows, and its readers reach pools the replay never resolves: the
// instanced foliage's and Navi's data, through pointers read from other buffers. Without a sync those shadows held
// what guest memory held when they were made, and Kokiri Forest's trees were bare trunks, without ferns or grass
// tufts. So a guest submission whose record binds the table (vk_mem_shadow_table_read) syncs every shadow in the
// table, once, before the first part recorded after the binding is submitted (vk_mem_shadow_sync), and those that
// have a VRAM mirror, which the table then names, queue the same pages for upload. Once is enough: the copy-ins read
// the guest submission's write-watch snapshots (snap_regions), which a second sync in the same guest submission
// would find unchanged. A sync in every part that used the table ran about three times a frame outdoors. Measured
// on Linux (RTX 4090, Kokiri Forest, by the change's author): a sync at every guest submission, about six a
// frame, took 9.8 ms and 888 write-watch queries a frame; one in the guest submissions that read the table 3.9 ms
// and 92.
// - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and
// reports it at the fence (vk_gpu_written), which copies the GPU's bytes back into guest memory.
// - Pages both write: a GPU write is noted with the whole range its binding names, and the CPU writes other fields of the
@@ -1079,24 +1094,26 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
// (vk_mem_gpu_pending): the page has no base copy and guest memory never gets the bytes. The game's culling passes
// store counts and records this way into pages whose other words the CPU rewrites. For one such page P, a word X the
// shaders store and a word Y the CPU writes:
// part A (the pass that stores): the CPU wrote Y, so A's prologue copies P in whole (shadow_copy) and queues it
// for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which no store ever
// reached. A runs and stores X = v into both.
// part B (a later part that reads X through the table), the CPU has not written P since A's write-watch snapshot:
// B's prologue leaves P alone and its draws read v.
// part B, the CPU has written Y again between the two snapshots: B's prologue copies P in whole and uploads it, X
// guest submission A (with the pass that stores): the CPU wrote Y, so A's sync copies P in whole (shadow_copy)
// and queues it for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which
// no store ever reached. A runs and stores X = v into both.
// guest submission B (a later one that reads X through the table), the CPU has not written P since A's
// write-watch snapshot:
// B's sync leaves P alone and its draws read v.
// B, the CPU has written Y again between the two snapshots: B's sync copies P in whole and uploads it, X
// is guest memory's stale word again in both copies, and the draws read that.
// An import does not have the fault: its twin is guest memory, and a page copied up from there holds the store.
// Measured (NX2NAVI_SHADOW_CHECK, a standing run outdoors, RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10): the stores
// go into one page each of ten pools of 20 and 44 KiB, and whole copy-ins overwrite one exactly four times a frame
// (240 in every 60 frames): part A's case, the copy that precedes the frame's stores, which does no harm. Part B's
// second case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the
// frame's timing differs.
// (240 in every 60 frames): A's case, the copy that precedes the frame's stores, which does no harm. B's second
// case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the frame's
// timing differs. (Counted when every part that used the table synced; a guest submission syncs once now, from the
// same write-watch snapshots, and the counts were not taken again.)
// NX2NAVI_SHADOW_TABLE_STORES=1 keeps the stores. The pages are found by their content: each whole copy-in leaves a
// hash of the page it wrote (smark), and the next one hashes the shadow page first. A different hash means something
// stored into the shadow in between, and with no base copy on the page that was a store through the table. That
// copy-in still overwrites the store (nothing tells its bytes from the CPU's), and from then on the page keeps a
// base copy: copy-ins take the CPU's bytes only, and the fence of every submission with a part that uses the table
// base copy: copy-ins take the CPU's bytes only, and the fence of every submission whose record bound the table
// writes the stored bytes of all such pages back to guest memory (table_wb_cb), as the fence of a noted write does.
// Guest memory needs them: a word the CPU resets each frame to the value it had before the stores would otherwise
// never differ from its base, and the reset would never reach the GPU.
@@ -1132,7 +1149,10 @@ static uint64_t s_marked, s_mark_ticks; // pages copied in whole with
static uint64_t s_plain_ticks; // check: the time of the whole copies made without marks
static uint64_t s_both_cpu, s_both_gpu; // check: bytes both sides changed, taken from the CPU / from the GPU
static uint64_t s_in_merge, s_in_nomirror; // check: bytes copied in by merge_page; whole, into pools without a mirror
static uint64_t s_gpa_ticks, s_gpa_syncs; // vk_mem_gpa_sync: its time (part of s_ticks) and its runs
static uint64_t s_tab_ticks, s_tab_syncs; // the sync for the table's readers: its time (part of s_ticks), its runs
static PoolRec **stab; // shadow_lock: the shadows in the device address table (PoolRec.stab_i)
static int nstab, cstab;
static uint32_t stab_want; // the last guest submission whose record bound the table
static uint64_t s_t_over_all; // shadow_lock: s_tab_over since the start
static uint64_t s_late_pages, s_late_bytes; // shadow_lock, check: write-backs into pages the CPU wrote since their copy-in
static uint64_t s_stale_pages; // shadow_lock: write-backs left out, the shadow could not hold them (shadow_write_back)
@@ -1361,6 +1381,11 @@ static void shadow_forget(PoolRec *r) {
if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; }
for (int i = 0; r->tlisted && i < ntpools; i++)
if (tpools[i] == r) { tpools[i] = tpools[--ntpools]; break; }
if (r->stab_i) { // the last one takes its place
PoolRec *last = stab[--nstab];
stab[r->stab_i - 1] = last, last->stab_i = r->stab_i;
r->stab_i = 0;
}
s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo);
s_tab_pages -= r->ntpg;
free(r->tpg), free(r->smark);
@@ -1460,23 +1485,6 @@ static void shadow_sync_pool(PoolRec *r, uint32_t epoch, int cpu) {
if (run) shadow_copy(r, run, hi);
}
void vk_mem_shadow_sync(void) {
if (!shadow_on) return;
uint64_t t0 = qpc();
uint32_t epoch = vk_texture_epoch();
int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs
plat_mutex_lock(&shadow_lock);
if (nstouch) {
plat_mutex_lock(&gen_lock);
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu), stouch[i]->stouched = 0;
plat_mutex_unlock(&gen_lock);
s_syncs += (uint64_t)nstouch;
nstouch = 0;
}
s_ticks += qpc() - t0;
plat_mutex_unlock(&shadow_lock);
}
// Record time: the GPU may write [addr, addr+size) of CPU-visible memory. With shadow storage the pages it covers get
// their base copies before the submission's copy-in (vk_mem_shadow_sync) can reach them.
void vk_mem_gpu_pending(uint64_t addr, uint64_t size) {
@@ -1576,7 +1584,7 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
return moved;
}
// Completion thread, at the fence of a submission with a part that uses the device address table (vk_mem_gpa_sync):
// Completion thread, at the fence of a submission whose record bound the device address table (vk_mem_shadow_sync):
// what the shaders stored into the pages found so far ("Pages a shader stores into" above) goes from the shadow into
// guest memory, the bytes that differ from the page's base copy. A page whose shadow equals its base is passed over
// before guest memory is touched, so a frozen region stays frozen. The pages that changed are stamped as a mirror's
@@ -1672,7 +1680,7 @@ static void shadow_table_log(uint64_t frame) {
"from the CPU at a copy-in, %llu from the GPU at a fence; write-backs into %llu pages the CPU had written "
"since their copy-in, %llu bytes",
(unsigned long long)frame, qpc_ms(s_mark_ticks), qpc_ms(s_plain_ticks), s_in_merge / 1048576.0,
s_in_nomirror / 1048576.0, (unsigned long long)s_gpa_syncs, qpc_ms(s_gpa_ticks),
s_in_nomirror / 1048576.0, (unsigned long long)s_tab_syncs, qpc_ms(s_tab_ticks),
(unsigned long long)s_both_cpu, (unsigned long long)s_both_gpu, (unsigned long long)s_late_pages,
(unsigned long long)s_late_bytes);
PoolRec *top[TABLE_LOG_TOP];
@@ -1734,7 +1742,7 @@ static void shadow_table_log(uint64_t frame) {
}
}
s_tab_over = s_tab_wb = s_tab_wb_seen = s_tab_wb_ticks = s_tab_wb_jobs = s_marked = s_mark_ticks = s_plain_ticks = 0;
s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_gpa_ticks = s_gpa_syncs = s_late_pages = s_late_bytes = 0;
s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_tab_ticks = s_tab_syncs = s_late_pages = s_late_bytes = 0;
}
static void shadow_log(uint64_t frame) {
@@ -2418,30 +2426,30 @@ static void mirror_sync(PoolRec *r) {
mrep.sync_t += qpc() - t0;
}
// ---- shadow storage: the device address table's readers
// A shader that follows a pointer through the table (vk_gpa.c) can reach any pool the table names, and the replay
// never resolves that pool, so nothing put it on the touched list and its shadow kept the bytes of its last use as a
// buffer. vk_exec_draw.c marks a part whose pipelines declare the table (vk_mem_gpa_use: at the draw or dispatch, not
// where descriptors are written, which are reused), and the part's prologue (vk_sync.c) calls vk_mem_gpa_sync: the
// shadows of every pool in the table get the pages the CPU wrote, and those of them that have a VRAM mirror, which
// the table then names, queue the same pages for upload unless this submission synced the mirror already. No mirror
// is made for a reader of the table. Without shadow storage nothing is done: an imported pool is guest memory, and
// mirrors stay unsynced for the table's readers (vk_exec_draw.c ssbo_info).
static int gpa_part_used; // replay thread: the part being recorded uses the table
void vk_mem_gpa_use(void) { gpa_part_used = 1; }
// ---- shadow storage: the copy-ins before a part is submitted, and the device address table's readers
int vk_mem_shadow_storage(void) { return shadow_on; }
static int shadow_check_on(void) { return s_check; }
int vk_mem_shadow_table_stores(void) { return s_learn; }
static int gpa_shadow_pool(const PoolRec *r) { return r->kind == POOL_IMPORT && r->gpa_rec && r->smap; }
static void shadow_table_add(PoolRec *r) {
plat_mutex_lock(&shadow_lock);
if (!r->stab_i) {
if (nstab == cstab) grow((void **)&stab, &cstab, sizeof *stab);
stab[nstab++] = r;
r->stab_i = (uint32_t)nstab;
}
plat_mutex_unlock(&shadow_lock);
}
void vk_mem_shadow_table_read(void) {
if (shadow_on) __atomic_store_n(&stab_want, vk_texture_epoch(), __ATOMIC_RELAXED);
}
// gen_lock held: nothing of the pool was stamped since its shadow's last copy-in. Its regions have this epoch's
// write-watch snapshot (vk_mem_gpa_sync takes them for all pools at once), none of them is untracked, and none holds a
// stamp newer than the shadow's sgen, which then stays as it is. With cpu the stamps are the CPU-side tables', as
// shadow_sync_pool reads them. A table sync ran shadow_sync_pool for every pool at every part that uses the table,
// 10,700 times a frame outdoors (3,772 pools, about three such parts), each with its own timer reads, snapshot walk
// and region loop; with this test 1,500 pool syncs a frame are left, and the copy-ins went from 1.32 to 1.01 ms a
// frame (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10).
// write-watch snapshot, none of them is untracked, and none holds a stamp newer than the shadow's sgen, which then
// stays as it is. With cpu the stamps are the CPU-side tables', as shadow_sync_pool reads them. The table's sync asks
// this of every shadow in the table before it runs shadow_sync_pool with its own timer reads, snapshot walk and
// region loop: outdoors about 3,800 pools are in the table and a few hundred were written.
static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) {
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
if (!epoch || r1 >= SNAP_REGIONS) return 0;
@@ -2451,54 +2459,52 @@ static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) {
return 1;
}
void vk_mem_gpa_sync(uint64_t fence) {
if (!gpa_part_used) return;
gpa_part_used = 0;
void vk_mem_shadow_sync(uint64_t fence) {
if (!shadow_on) return;
// What this part's shaders store through the table goes back to guest memory at the submission's fence, once a
// submission however many of its parts use the table (table_wb_cb). Asked before the copy-ins below: the part
// whose copy-ins find the first such page leaves it to the next part that uses the table.
static uint64_t wb_fence; // replay thread: the last fence that got the job
if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) {
wb_fence = fence;
vk_completion_add(fence, table_wb_cb, NULL);
}
uint64_t t0 = qpc(), run0 = 0, run1 = 0;
uint64_t t0 = qpc();
uint32_t epoch = vk_texture_epoch();
int cpu = vk_mirror_on && wb_no_reup(), have = 0; // before the locks: its first call logs
plat_mutex_lock_shared(&pool_lock);
plat_mutex_lock(&shadow_lock);
plat_mutex_lock(&gen_lock);
// one write-watch query per run of regions the pools cover (the index is sorted by address) instead of one a pool;
// shadow_sync_pool then finds its regions snapped
for (int i = 0; epoch && i < pool_index.n; i++) {
const PoolRec *r = pool_index.e[i].buf;
if (!gpa_shadow_pool(r)) continue;
uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT;
if (b >= SNAP_REGIONS) continue;
if (have && a <= run1 + 1) {
if (b > run1) run1 = b;
} else {
if (have) s_queries += snap_regions(run0, run1, epoch);
run0 = a, run1 = b, have = 1;
int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs
// The table's readers: the record of this guest submission bound the table, and no part of it has synced yet.
// The callers are one at a time (vk_sync.c prologue under the submission lock, and the probe).
static uint32_t tab_epoch; // the guest submission the table's shadows were last synced for
static uint64_t wb_fence; // the last fence that got the write-back job
int table = epoch && epoch != tab_epoch && __atomic_load_n(&stab_want, __ATOMIC_RELAXED) == epoch;
if (table) {
tab_epoch = epoch;
// What this submission's shaders store through the table goes back to guest memory at its fence, once
// (table_wb_cb). Asked before the copy-ins below: the sync whose copy-ins find the first such page leaves it
// to the next guest submission that binds the table.
if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) {
wb_fence = fence;
vk_completion_add(fence, table_wb_cb, NULL);
}
}
if (have) s_queries += snap_regions(run0, run1, epoch);
for (int i = 0; i < pool_index.n; i++) {
PoolRec *r = pool_index.e[i].buf;
if (gpa_shadow_pool(r) && !shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
plat_mutex_lock(&shadow_lock);
if (nstouch || (table && nstab)) {
plat_mutex_lock(&gen_lock);
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu);
uint64_t t1 = qpc();
for (int i = 0; table && i < nstab; i++) { // the shadows the replay did not resolve
PoolRec *r = stab[i];
if (r->stouched) continue;
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
if (r1 < SNAP_REGIONS) s_queries += snap_regions(r0, r1, epoch); // so that shadow_pool_clean can tell
if (!shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
}
for (int i = 0; i < nstouch; i++) stouch[i]->stouched = 0;
plat_mutex_unlock(&gen_lock);
s_syncs += (uint64_t)nstouch;
nstouch = 0;
// A shadow with a VRAM mirror: the table names the mirror, which takes the same pages unless this guest
// submission synced it already (a pool bound as a buffer). The uploads read the shadows just brought up to
// date, and the prologue records them ahead of the part, after the write-backs of the submissions before it.
// No mirror is made for a reader of the table.
for (int i = 0; table && i < nstab; i++)
if (stab[i]->mstate == MS_ON && stab[i]->acc_sub != msub) mirror_sync(stab[i]);
if (table) s_tab_ticks += qpc() - t1, s_tab_syncs++;
}
plat_mutex_unlock(&gen_lock);
// The uploads read the shadows just brought up to date, and the prologue records them ahead of the part, after
// the write-backs of the submissions before it.
for (int i = 0; i < pool_index.n; i++) {
PoolRec *r = pool_index.e[i].buf;
if (gpa_shadow_pool(r) && r->mstate == MS_ON && r->acc_sub != msub) mirror_sync(r);
}
s_gpa_ticks += qpc() - t0, s_gpa_syncs++;
s_ticks += qpc() - t0;
plat_mutex_unlock(&shadow_lock);
plat_mutex_unlock_shared(&pool_lock);
}
void vk_mem_use(VkMemSpan *sp) {
@@ -4293,7 +4299,7 @@ void vk_mem_shutdown(void) {
nsnap_pool = 0, snap_pool_bytes = 0;
plat_mutex_unlock(&snap_pool_lock);
plat_mutex_lock(&shadow_lock);
nstouch = 0, s_sb_pages = 0; // shadows too (their base copies went with their records)
nstouch = nstab = 0, s_sb_pages = 0; // shadows too (their base copies went with their records)
s_live_n = s_live_bytes = 0;
plat_mutex_unlock(&shadow_lock);
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran

View File

@@ -51,7 +51,6 @@
#include "tex.h"
#include "vk_build_gate.h"
#include "vk_cache_file.h"
#include "vk_gpa.h"
#include "vk_save_retry.h"
#include "vk_warm_hint.h"
@@ -2596,8 +2595,6 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel
p->info.push1 = l ? l->push1 : 0;
p->info.n_sets = l && l->set1 ? 2 : 1;
p->info.pool = B ? B->pool : 0;
for (uint32_t i = 0; i < p->info.n_bindings; i++)
if (p->bindings[i].kind == GFX_RES_SSBO && p->bindings[i].res->cbuf_slot == VK_GPA_CB_SLOT) p->info.gpa = 1;
if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1];
p->lay = l;
Pipe **head = &pipes[h % PIPE_BUCKETS];

View File

@@ -647,8 +647,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
// part was recorded, sub_lock held; may move slots[].
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted
vk_mem_gpa_sync(fence); // a part whose shaders read through the device address table never resolved the pools they reach
vk_mem_shadow_sync(fence); // shadow storage: the CPU's writes go in before the part is submitted
vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
int i = claim_slot(c, fence); // before reading slots: the claim may move it
@@ -710,7 +709,13 @@ uint64_t vk_sync_reserve(void) {
return v;
}
static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) {
// own: NULL for the replay of handles; else the caller's commands in the replay's place (vk_sync_submit_own).
typedef struct {
void (*record)(VkCommandBuffer, void *);
void *arg;
} OwnCommands;
static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) {
vk_sync_start();
// The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued.
// The value returned is the last one issued; a wait for it sleeps until the process ends.
@@ -733,8 +738,8 @@ static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCo
ts_begin(s->cb);
rec_begin(s->cb, fence);
vk_gpu_passes_part(s->cb, 0);
int recorded;
if (probe_record) { probe_record(s->cb, arg); recorded = 1; }
int recorded = 1;
if (own) own->record(s->cb, own->arg);
else recorded = vk_exec_record(s->cb, handles, n, fence);
s = &slots[cur_slot]; // the last part (vk_sync_split)
vk_gpu_passes_part(s->cb, 1);
@@ -781,11 +786,12 @@ static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCo
return fence;
}
uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL, NULL); }
uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL); }
// Memory probes record their shader here, not in the upload batch (which is submitted before the memory prologue).
uint64_t vk_sync_probe_submit(void (*record)(VkCommandBuffer, void *), void *arg) {
return submit(NULL, 0, record, arg);
// The upload batch is no place for a probe's commands: it is submitted before the memory prologue.
uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg) {
OwnCommands own = {record, arg};
return submit(NULL, 0, &own);
}
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
@@ -980,7 +986,7 @@ int vk_sync_probe_devlost(void) {
int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2;
volatile int32_t recorded = 0;
uint64_t before = vk_sync_issued(), t0 = plat_now_ms();
uint64_t got = vk_sync_probe_submit(probe_lost_record, (void *)&recorded);
uint64_t got = vk_sync_submit_own(probe_lost_record, (void *)&recorded);
vk_upload_flush(1);
int waited = vk_sync_exec_wait(v2, 5000000000ull);
uint64_t ms = plat_now_ms() - t0;