mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 15:49:57 -04:00
Merge the shadow sync once per guest submission that binds the device address table.
Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
@@ -286,10 +286,11 @@ well: the shadow copies are chosen for the measured cost above, not for a missin
|
||||
|
||||
The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest
|
||||
pointers into either. A shader that reads through the table resolves nothing, so the pool it reaches is not queued for
|
||||
the copy-in: a part whose pipelines declare the table is marked at its draws and dispatches, and its prologue brings
|
||||
the shadow of every pool in the table up to date with the pages the CPU wrote and queues the same pages for those
|
||||
pools' VRAM mirrors (`vk_memory.c` `vk_mem_gpa_sync`). That copies the written pages of every CPU-visible pool in
|
||||
each such part, which the cost above does not include. What a shader stores through the table lands in the shadow copy
|
||||
the copy-in: a guest submission whose record binds the table (`vk_exec_draw.c` `ssbo_info`) brings the shadow of
|
||||
every pool in the table up to date with the pages the CPU wrote, once, before the first part recorded after the
|
||||
binding is submitted, and queues the same pages for those pools' VRAM mirrors (`vk_memory.c` `vk_mem_shadow_sync`).
|
||||
That copies the written pages of every CPU-visible pool once in each such guest submission, which the cost above does
|
||||
not include. What a shader stores through the table lands in the shadow copy
|
||||
and in the pool's VRAM mirror, and no binding names it, so nothing notes it as a GPU write: it does not come back to
|
||||
guest memory, and the next whole copy-in of its page (the CPU rewrites other words of the same records) puts guest
|
||||
memory's stale bytes back over it in both copies. In the runs measured so far that copy-in always came before the
|
||||
@@ -299,8 +300,9 @@ then on like a page with a noted GPU write, and has its stores written back to g
|
||||
submission that uses the table. It is off by default (`runtime/vk_memory.c`, "Pages a shader stores into", says why).
|
||||
`NX2NAVI_SHADOW_CHECK=1` logs every 60 frames how often a copy-in overwrote such a store and for which pools, how many
|
||||
write-backs landed on pages the CPU had written since their copy-in, and what the copy-ins cost and where they went.
|
||||
`--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is never bound, and must read
|
||||
`0x13579BDF` and, after a plain CPU write, `0x2468ACE0`; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU
|
||||
`--probe-vkmem` checks the sync: the shadow of a pool that is never bound must hold `0x13579BDF`, keep it through a
|
||||
plain CPU write and a guest submission that does not bind the table, and hold the written `0x2468ACE0` after one that
|
||||
does; and with `NX2NAVI_SHADOW_TABLE_STORES=1` the store: a GPU
|
||||
write into a shadow page that nothing notes must survive the CPU's writes of that page and reach guest memory. On
|
||||
Linux, run it for each backing:
|
||||
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
; --probe-vkmem (runtime/tex_probe.c test_gpa_read): one invocation reads a 32-bit word of guest memory through the
|
||||
; device address table as the game's translated shaders do, by the table's layout (runtime/vk_gpa.c): L1 at word 16,
|
||||
; one entry per 16 MiB; L2 blocks of 4096 page entries from word 65552; word 2 holds the word index of the records,
|
||||
; four words each, the first two the buffer's device address minus the guest address of its byte 0. The pool that
|
||||
; holds the word is never a descriptor of the shader: only the table (binding 0) and a result word (binding 1) are.
|
||||
; An address no page maps gives 0. Written in SPIR-V assembly (spirv-as; no GLSL compiler in the build). As GLSL:
|
||||
; #extension GL_EXT_buffer_reference : require
|
||||
; layout(local_size_x = 1) in;
|
||||
; layout(std430, binding = 0) readonly buffer Table { uint t[]; };
|
||||
; layout(std430, binding = 1) buffer Result { uint r[]; };
|
||||
; layout(push_constant) uniform Push { uint64_t guest; } pc;
|
||||
; layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer Word { uint v; };
|
||||
; void main() {
|
||||
; uint rec = t[65552u + (t[16u + uint(pc.guest >> 24)] << 12) + (uint(pc.guest >> 12) & 4095u)];
|
||||
; if (rec == 0u) { r[0] = 0u; return; }
|
||||
; uint at = t[2] + (rec << 2);
|
||||
; r[0] = Word(pc.guest + (uint64_t(t[at]) | uint64_t(t[at + 1u]) << 32)).v;
|
||||
; }
|
||||
OpCapability Shader
|
||||
OpCapability Int64
|
||||
OpCapability PhysicalStorageBufferAddresses
|
||||
OpExtension "SPV_KHR_physical_storage_buffer"
|
||||
OpMemoryModel PhysicalStorageBuffer64 GLSL450
|
||||
OpEntryPoint GLCompute %main "main" %table %result %push
|
||||
OpExecutionMode %main LocalSize 1 1 1
|
||||
OpDecorate %_runtimearr_uint ArrayStride 4
|
||||
OpDecorate %Words Block
|
||||
OpMemberDecorate %Words 0 Offset 0
|
||||
OpDecorate %table DescriptorSet 0
|
||||
OpDecorate %table Binding 0
|
||||
OpDecorate %result DescriptorSet 0
|
||||
OpDecorate %result Binding 1
|
||||
OpDecorate %Push Block
|
||||
OpMemberDecorate %Push 0 Offset 0
|
||||
%void = OpTypeVoid
|
||||
%uint = OpTypeInt 32 0
|
||||
%ulong = OpTypeInt 64 0
|
||||
%bool = OpTypeBool
|
||||
%fn = OpTypeFunction %void
|
||||
%_runtimearr_uint = OpTypeRuntimeArray %uint
|
||||
%Words = OpTypeStruct %_runtimearr_uint
|
||||
%ptr_words = OpTypePointer StorageBuffer %Words
|
||||
%_ptr_StorageBuffer_uint = OpTypePointer StorageBuffer %uint
|
||||
%table = OpVariable %ptr_words StorageBuffer
|
||||
%result = OpVariable %ptr_words StorageBuffer
|
||||
%Push = OpTypeStruct %ulong
|
||||
%ptr_push = OpTypePointer PushConstant %Push
|
||||
%push = OpVariable %ptr_push PushConstant
|
||||
%_ptr_PushConstant_ulong = OpTypePointer PushConstant %ulong
|
||||
%_ptr_PhysicalStorageBuffer_uint = OpTypePointer PhysicalStorageBuffer %uint
|
||||
%uint_0 = OpConstant %uint 0
|
||||
%uint_1 = OpConstant %uint 1
|
||||
%uint_2 = OpConstant %uint 2
|
||||
%uint_4 = OpConstant %uint 4
|
||||
%uint_12 = OpConstant %uint 12
|
||||
%uint_16 = OpConstant %uint 16
|
||||
%uint_24 = OpConstant %uint 24
|
||||
%uint_32 = OpConstant %uint 32
|
||||
%uint_4095 = OpConstant %uint 4095
|
||||
%uint_65552 = OpConstant %uint 65552
|
||||
%main = OpFunction %void None %fn
|
||||
%entry = OpLabel
|
||||
%28 = OpAccessChain %_ptr_PushConstant_ulong %push %uint_0
|
||||
%guest = OpLoad %ulong %28
|
||||
%30 = OpShiftRightLogical %ulong %guest %uint_24
|
||||
%31 = OpUConvert %uint %30
|
||||
%59 = OpIAdd %uint %31 %uint_16
|
||||
%32 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %59
|
||||
%block = OpLoad %uint %32
|
||||
%34 = OpShiftRightLogical %ulong %guest %uint_12
|
||||
%35 = OpUConvert %uint %34
|
||||
%page = OpBitwiseAnd %uint %35 %uint_4095
|
||||
%62 = OpShiftLeftLogical %uint %block %uint_12
|
||||
%63 = OpIAdd %uint %uint_65552 %62
|
||||
%42 = OpIAdd %uint %63 %page
|
||||
%37 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %42
|
||||
%rec = OpLoad %uint %37
|
||||
%out = OpAccessChain %_ptr_StorageBuffer_uint %result %uint_0 %uint_0
|
||||
%mapped = OpINotEqual %bool %rec %uint_0
|
||||
OpSelectionMerge %done None
|
||||
OpBranchConditional %mapped %is_mapped %not_mapped
|
||||
%is_mapped = OpLabel
|
||||
%43 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %uint_2
|
||||
%recs = OpLoad %uint %43
|
||||
%45 = OpShiftLeftLogical %uint %rec %uint_2
|
||||
%at = OpIAdd %uint %recs %45
|
||||
%47 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %at
|
||||
%48 = OpLoad %uint %47
|
||||
%49 = OpIAdd %uint %at %uint_1
|
||||
%50 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %49
|
||||
%51 = OpLoad %uint %50
|
||||
%52 = OpUConvert %ulong %48
|
||||
%53 = OpUConvert %ulong %51
|
||||
%54 = OpShiftLeftLogical %ulong %53 %uint_32
|
||||
%delta = OpBitwiseOr %ulong %52 %54
|
||||
%56 = OpIAdd %ulong %guest %delta
|
||||
%ptr = OpConvertUToPtr %_ptr_PhysicalStorageBuffer_uint %56
|
||||
%value = OpLoad %uint %ptr Aligned 4
|
||||
OpStore %out %value
|
||||
OpBranch %done
|
||||
%not_mapped = OpLabel
|
||||
OpStore %out %uint_0
|
||||
OpBranch %done
|
||||
%done = OpLabel
|
||||
OpReturn
|
||||
OpFunctionEnd
|
||||
@@ -15,7 +15,6 @@
|
||||
#include "vk.h"
|
||||
#include "vk_exec.h"
|
||||
#include "vk_gpa.h"
|
||||
#include "gpa_read_spv.h"
|
||||
#include "platform/platform.h"
|
||||
|
||||
static uint64_t g_rng = 0x9E3779B97F4A7C15ull;
|
||||
@@ -372,7 +371,7 @@ static int test_import(pool_t *pool) {
|
||||
// second sync in the same epoch would not see the CPU writes made since the first), then the part's CPU inputs
|
||||
// go in before its GPU work.
|
||||
vk_texture_new_submit();
|
||||
vk_mem_shadow_sync();
|
||||
vk_mem_shadow_sync(0);
|
||||
copy_wait(sb, 0, sp.buffer, sp.offset, 0x20000);
|
||||
vk_gpu_written(a + 0x1000, 0x20000);
|
||||
GCHECK(!memcmp(m + 0x1000, s, 0x20000), "GPU copy into the imported heap range differs from the source");
|
||||
@@ -391,7 +390,7 @@ static int test_import(pool_t *pool) {
|
||||
// add to and the CPU resets (the game's lighting accumulators) depends on the reset reaching the GPU first.
|
||||
vk_mem_gpu_pending(a + 0x30000, 0x4000);
|
||||
vk_texture_new_submit();
|
||||
vk_mem_shadow_sync();
|
||||
vk_mem_shadow_sync(0);
|
||||
copy_wait(sp.buffer, sp.offset, sb, 0, 0x4000);
|
||||
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE);
|
||||
GCHECK(!memcmp(s, m + 0x30000, 0x4000), "GPU read of CPU-written heap differs");
|
||||
@@ -439,145 +438,83 @@ static int test_remade(uint8_t *guest) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// test_gpa_read's shader (runtime/shaders/gpa_read.spvasm): one invocation follows the device address table from a
|
||||
// guest address (the push constant) to the buffer that holds it and stores the 32-bit word found there in the result
|
||||
// buffer. Storage bindings: 0 the table itself, 1 the result. The pool that holds the word is never a descriptor, a
|
||||
// transfer source or a resolved span of the submission.
|
||||
|
||||
typedef struct {
|
||||
VkPipeline pipe;
|
||||
VkPipelineLayout layout;
|
||||
VkDescriptorSet set;
|
||||
uint64_t source;
|
||||
int setup_mirror, mirrored;
|
||||
} GpaRead;
|
||||
|
||||
static void gpa_read_record(VkCommandBuffer cb, void *arg) {
|
||||
GpaRead *r = arg;
|
||||
// Match vk_backend.c submit_now and vk_exec.c vk_exec_record, including epochs on descriptor reuse.
|
||||
vk_texture_new_submit();
|
||||
vk_mem_mirror_begin();
|
||||
// The only explicit use of the source pool is making its mirror, before the first read: that submission takes its
|
||||
// shadow touch and its first upload. After the CPU's write nothing resolves, uses or flushes the pool.
|
||||
if (r->setup_mirror) r->mirrored = vk_mem_mirror_probe(r->source), r->setup_mirror = 0;
|
||||
vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, r->pipe);
|
||||
vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, r->layout, 0, 1, &r->set, 0, NULL);
|
||||
vkCmdPushConstants(cb, r->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof r->source, &r->source);
|
||||
vk_mem_gpa_use(); // the shader reads the table, in the second submission through the same descriptor set
|
||||
vkCmdDispatch(cb, 1, 1, 1);
|
||||
VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
|
||||
.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
|
||||
.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT};
|
||||
VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb};
|
||||
vkCmdPipelineBarrier2(cb, &di);
|
||||
vk_mem_mirror_end(cb, vk_sync_issued() + 1); // called under the submission lock, before the value is issued
|
||||
}
|
||||
|
||||
static int test_gpa_read(uint8_t *guest) {
|
||||
VkCtx *c = vk_ctx();
|
||||
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xA00000, .size = 0x10000,
|
||||
.game_mem = guest + 0xA00000};
|
||||
if (!vk_gpa_wanted()) { // no buffer device addresses, or NX2NAVI_GPA=0: shaders read zeros through it by design
|
||||
printf("probe-vkmem: no device address table: the read through it is skipped\n");
|
||||
return 0;
|
||||
}
|
||||
volatile uint32_t *source = (volatile uint32_t *)pool.mem;
|
||||
*source = 0x13579BDFu; // before vk_pool_init: a shadow copy starts with the bytes an import would show
|
||||
GCHECK(vk_gpa_on() && vk_pool_init(&pool), "the device address table and a CPU-visible pool for the read through it");
|
||||
VkBuffer out = VK_NULL_HANDLE;
|
||||
VmaAllocation alloc = NULL;
|
||||
VkShaderModule module = VK_NULL_HANDLE;
|
||||
VkDescriptorSetLayout dsl = VK_NULL_HANDLE;
|
||||
VkDescriptorPool dp = VK_NULL_HANDLE;
|
||||
// With shadow storage the pool also gets a VRAM mirror, which the table then names: the prologue of a part that
|
||||
// uses the table keeps both up to date (vk_mem_gpa_sync). Without it the pool stays an import, guest memory itself:
|
||||
// a mirror is not synced for the table's readers there (vk_exec_draw.c ssbo_info).
|
||||
int want_mirror = vk_mirror_on && vk_mem_shadow_storage();
|
||||
GpaRead r = {.source = (uintptr_t)pool.mem, .setup_mirror = want_mirror};
|
||||
int bad = 1;
|
||||
#define GPA_CHECK(cond, what) do { if (!(cond)) { printf("probe-vkmem: FAIL GPA " what "\n"); goto done; } } while (0)
|
||||
VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 4, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT};
|
||||
VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT,
|
||||
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT};
|
||||
VmaAllocationInfo info;
|
||||
GPA_CHECK(vmaCreateBuffer(c->vma, &bi, &ai, &out, &alloc, &info) == VK_SUCCESS && info.pMappedData, "result buffer");
|
||||
VkDescriptorSetLayoutBinding bindings[2] = {
|
||||
{.binding = 0, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT},
|
||||
{.binding = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}};
|
||||
VkDescriptorSetLayoutCreateInfo dl = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .bindingCount = 2,
|
||||
.pBindings = bindings};
|
||||
GPA_CHECK(vkCreateDescriptorSetLayout(c->device, &dl, NULL, &dsl) == VK_SUCCESS, "descriptor layout");
|
||||
VkPushConstantRange push = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof r.source};
|
||||
VkPipelineLayoutCreateInfo pl = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1,
|
||||
.pSetLayouts = &dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &push};
|
||||
GPA_CHECK(vkCreatePipelineLayout(c->device, &pl, NULL, &r.layout) == VK_SUCCESS, "pipeline layout");
|
||||
VkShaderModuleCreateInfo sm = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof gpa_read_spv,
|
||||
.pCode = gpa_read_spv};
|
||||
GPA_CHECK(vkCreateShaderModule(c->device, &sm, NULL, &module) == VK_SUCCESS, "shader module");
|
||||
VkComputePipelineCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, .layout = r.layout,
|
||||
.stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT,
|
||||
.module = module, .pName = "main"}};
|
||||
GPA_CHECK(vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &cp, NULL, &r.pipe) == VK_SUCCESS, "compute pipeline");
|
||||
VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2};
|
||||
VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .maxSets = 1,
|
||||
.poolSizeCount = 1, .pPoolSizes = &ps};
|
||||
GPA_CHECK(vkCreateDescriptorPool(c->device, &pi, NULL, &dp) == VK_SUCCESS, "descriptor pool");
|
||||
VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = dp,
|
||||
.descriptorSetCount = 1, .pSetLayouts = &dsl};
|
||||
GPA_CHECK(vkAllocateDescriptorSets(c->device, &da, &r.set) == VK_SUCCESS, "descriptor set");
|
||||
VkDescriptorBufferInfo buffers[2] = {vk_gpa_info(), {out, 0, 4}};
|
||||
VkWriteDescriptorSet writes[2];
|
||||
for (unsigned i = 0; i < 2; i++)
|
||||
writes[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = r.set, .dstBinding = i,
|
||||
.descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
|
||||
.pBufferInfo = &buffers[i]};
|
||||
vkUpdateDescriptorSets(c->device, 2, writes, 0, NULL);
|
||||
static const uint32_t want[2] = {0x13579BDFu, 0x2468ACE0u};
|
||||
for (unsigned pass = 0; pass < 2; pass++) {
|
||||
if (pass) *source = want[pass]; // a plain CPU write: no resolve, use, flush or shadow sync of the pool
|
||||
uint64_t fence = vk_sync_probe_submit(gpa_read_record, &r);
|
||||
GPA_CHECK(vk_sync_fence_wait(fence, 5000000000ull), "shader fence timed out");
|
||||
GPA_CHECK(!want_mirror || r.mirrored, "source mirror setup");
|
||||
vmaInvalidateAllocation(c->vma, alloc, 0, VK_WHOLE_SIZE);
|
||||
uint32_t got = *(volatile uint32_t *)info.pMappedData;
|
||||
if (got != want[pass]) {
|
||||
printf("probe-vkmem: FAIL GPA/BDA %s read: got %#010x, expected %#010x (mirror %d)\n",
|
||||
pass ? "after CPU mutation" : "initial", got, want[pass], r.mirrored);
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
g_gpu_checks += 2;
|
||||
printf("probe-vkmem: GPA/BDA CPU write -> shader read: initial %#010x and mutation %#010x exact (mirror %d)\n",
|
||||
want[0], want[1], r.mirrored);
|
||||
bad = 0;
|
||||
done:
|
||||
vk_sync_finish(); // after a timeout too: the objects below must outlive the shader
|
||||
if (dp) vkDestroyDescriptorPool(c->device, dp, NULL);
|
||||
if (r.pipe) vkDestroyPipeline(c->device, r.pipe, NULL);
|
||||
if (module) vkDestroyShaderModule(c->device, module, NULL);
|
||||
if (r.layout) vkDestroyPipelineLayout(c->device, r.layout, NULL);
|
||||
if (dsl) vkDestroyDescriptorSetLayout(c->device, dsl, NULL);
|
||||
if (out) vmaDestroyBuffer(c->vma, out, alloc);
|
||||
vk_pool_fini(&pool);
|
||||
#undef GPA_CHECK
|
||||
return bad;
|
||||
}
|
||||
|
||||
// test_table_store: a submission whose one part uses the device address table and records nothing else, as a part of
|
||||
// the replay with such a shader ends: its prologue brings every pool of the table up to date (vk_mem_gpa_sync).
|
||||
// test_table_sync, test_table_store: a guest submission of one part that records nothing. bound: its record binds the
|
||||
// device address table, as the replay of a draw whose shader reads through the table does (vk_exec_draw.c ssbo_info),
|
||||
// so its prologue brings every shadow of the table up to date (vk_memory.c vk_mem_shadow_sync).
|
||||
static void table_part_record(VkCommandBuffer cb, void *arg) {
|
||||
(void)arg;
|
||||
vk_texture_new_submit();
|
||||
vk_mem_mirror_begin();
|
||||
vk_mem_gpa_use();
|
||||
if (arg) vk_mem_shadow_table_read();
|
||||
vk_mem_mirror_end(cb, vk_sync_issued() + 1);
|
||||
}
|
||||
|
||||
static int table_part(void) {
|
||||
uint64_t fence = vk_sync_probe_submit(table_part_record, NULL);
|
||||
static int table_part_of(int bound) {
|
||||
static int binds;
|
||||
uint64_t fence = vk_sync_submit_own(table_part_record, bound ? &binds : NULL);
|
||||
return vk_sync_fence_wait(fence, 5000000000ull); // its completion callbacks have run when the wait returns
|
||||
}
|
||||
|
||||
static int table_part(void) { return table_part_of(1); }
|
||||
|
||||
// Shadow storage: a pool that no part ever binds or resolves, as the pools a shader reaches only through a pointer it
|
||||
// read from another buffer. Its shadow, which the device address table names, starts with the bytes guest memory held
|
||||
// when the pool was made; a CPU write reaches it at the next guest submission whose record binds the table, and not
|
||||
// at one that does not (nothing else brings it there, which is what left the foliage's instances stale). The shadow
|
||||
// is read back with a GPU copy out of the buffer a lookup names, which does not touch the pool.
|
||||
static int test_table_sync(uint8_t *guest) {
|
||||
VkCtx *c = vk_ctx();
|
||||
if (!vk_mem_shadow_storage() || !vk_gpa_wanted()) {
|
||||
printf("probe-vkmem: the sync of the device address table's shadows: skipped (%s)\n",
|
||||
!vk_mem_shadow_storage() ? "no shadow storage" : "no device address table");
|
||||
return 0;
|
||||
}
|
||||
pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xA00000, .size = 0x10000,
|
||||
.game_mem = guest + 0xA00000};
|
||||
volatile uint32_t *w = (volatile uint32_t *)(pool.mem + 0x3000);
|
||||
*w = 0x13579BDFu; // before vk_pool_init: a shadow starts with the bytes an import would show
|
||||
GCHECK(vk_gpa_on() && vk_pool_init(&pool), "a CPU-visible pool in the device address table");
|
||||
VkBuffer sb;
|
||||
VmaAllocation sa;
|
||||
volatile uint32_t *s = vk_staging(0x1000, &sb, &sa);
|
||||
VkMemSpan sp;
|
||||
int bad = 1;
|
||||
uint32_t got;
|
||||
#define TY_CHECK(cond, ...) do { if (!(cond)) { printf("probe-vkmem: FAIL table sync: "); printf(__VA_ARGS__); printf("\n"); goto done; } } while (0)
|
||||
#define TY_SHADOW(into) \
|
||||
do { \
|
||||
TY_CHECK(vk_mem_lookup((uintptr_t)w, 4, &sp) && sp.buffer, "the pool has no buffer"); \
|
||||
copy_wait(sp.buffer, sp.offset, sb, 0, 4); \
|
||||
vmaInvalidateAllocation(c->vma, sa, 0, VK_WHOLE_SIZE); \
|
||||
(into) = s[0]; \
|
||||
} while (0)
|
||||
TY_SHADOW(got);
|
||||
TY_CHECK(got == 0x13579BDFu, "the new shadow holds %#010x, expected the pool's 0x13579bdf", got);
|
||||
*w = 0x2468ACE0u; // a plain CPU write: nothing resolves, uses or flushes the pool
|
||||
TY_CHECK(table_part_of(0), "fence timed out");
|
||||
TY_SHADOW(got);
|
||||
TY_CHECK(got == 0x13579BDFu, "a guest submission that does not bind the table left %#010x in the shadow, expected "
|
||||
"the old 0x13579bdf (the check tests nothing if something else syncs the pool)", got);
|
||||
TY_CHECK(table_part_of(1), "fence timed out");
|
||||
TY_SHADOW(got);
|
||||
TY_CHECK(got == 0x2468ACE0u, "after a guest submission that binds the table the shadow holds %#010x, expected the "
|
||||
"CPU's 0x2468ace0", got);
|
||||
*w = 0x0F1E2D3Cu; // and again, a guest submission later
|
||||
TY_CHECK(table_part_of(1), "fence timed out");
|
||||
TY_SHADOW(got);
|
||||
TY_CHECK(got == 0x0F1E2D3Cu, "after the second such guest submission the shadow holds %#010x, expected 0x0f1e2d3c", got);
|
||||
g_gpu_checks += 4;
|
||||
printf("probe-vkmem: a pool that is never bound: its shadow in the device address table takes the CPU's writes at "
|
||||
"each guest submission that binds the table, and at no other\n");
|
||||
bad = 0;
|
||||
done:
|
||||
vk_sync_finish();
|
||||
vmaDestroyBuffer(c->vma, sb, sa);
|
||||
vk_pool_fini(&pool);
|
||||
#undef TY_CHECK
|
||||
#undef TY_SHADOW
|
||||
return bad;
|
||||
}
|
||||
|
||||
// Shadow storage: what a shader stores through the device address table lands in the pool's shadow copy and nothing
|
||||
// notes it (vk_memory.c "Pages a shader stores into"). A GPU copy into the shadow buffer that is not noted stands in
|
||||
// for the store. Word X of a page takes the stores, the CPU rewrites word Y of the same page before every part:
|
||||
@@ -1143,7 +1080,7 @@ static int split_case(pool_t *pool, uint64_t off, uint64_t serial, int both, int
|
||||
if (both) vk_texture_release(&d2);
|
||||
if (recorded) {
|
||||
SplitRec r = {6, NULL};
|
||||
uint64_t fence = vk_sync_probe_submit(split_record, &r);
|
||||
uint64_t fence = vk_sync_submit_own(split_record, &r);
|
||||
GCHECK(vk_sync_fence_wait(fence, 5000000000ull), "the submission that looks the second descriptor's slot up timed out");
|
||||
s2 = r.got;
|
||||
} else {
|
||||
@@ -1256,7 +1193,7 @@ static int test_gpu(void) {
|
||||
vk_mem_stats(&st);
|
||||
GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini");
|
||||
g_gpu_checks += 2;
|
||||
return test_gpa_read(h) || test_table_store(h) || test_remade(h);
|
||||
return test_table_sync(h) || test_table_store(h) || test_remade(h);
|
||||
}
|
||||
|
||||
int probe_vkmem(void) {
|
||||
|
||||
21
runtime/vk.h
21
runtime/vk.h
@@ -359,13 +359,13 @@ void vk_mem_fatal_note(char *out, size_t n); // the memory mode and the lazy w
|
||||
// Shadow storage (vk_memory.c): vk_sync.c calls vk_mem_shadow_sync right before each submit; the replay notes every GPU
|
||||
// write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written,
|
||||
// which copies the bytes back into guest memory before telling the texture cache (vk_guest_written).
|
||||
void vk_mem_shadow_sync(void);
|
||||
// A shader of the part being recorded reads through the device address table (vk_gpa.c), also when its descriptors
|
||||
// are reused: with shadow storage the part's prologue (vk_mem_gpa_sync) then brings every pool the table names up to
|
||||
// date with the CPU's writes, since the pools such a shader reaches are never resolved. fence: the submission's value,
|
||||
// at which what the shaders stored through the table goes back to guest memory.
|
||||
void vk_mem_gpa_use(void);
|
||||
void vk_mem_gpa_sync(uint64_t fence);
|
||||
// fence: the value of the submission the part belongs to (0: none, a probe's own sync), at which what shaders stored
|
||||
// through the device address table goes back to guest memory (NX2NAVI_SHADOW_TABLE_STORES).
|
||||
void vk_mem_shadow_sync(uint64_t fence);
|
||||
// Record time: a shader of the guest submission being recorded reads the device address table (vk_gpa.c). The pools
|
||||
// such a shader reaches are never resolved, so the next vk_mem_shadow_sync brings the CPU's writes into every shadow
|
||||
// the table names, once per guest submission.
|
||||
void vk_mem_shadow_table_read(void);
|
||||
int vk_mem_shadow_storage(void); // CPU-visible pools are shadow copies of guest memory, not imports (NX2NAVI_SHADOW)
|
||||
int vk_mem_shadow_table_stores(void); // shadow storage keeps what shaders store through the table (NX2NAVI_SHADOW_TABLE_STORES)
|
||||
void vk_mem_gpu_pending(uint64_t addr, uint64_t size);
|
||||
@@ -674,9 +674,9 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h)
|
||||
// once the GPU reached it and the completion thread ran its side effects. Guest or host threads.
|
||||
void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value
|
||||
// --probe-vkmem: record gives the submission's commands in place of the replay, under the same submission lock,
|
||||
// recorder, memory prologue and fence.
|
||||
uint64_t vk_sync_probe_submit(void (*record)(VkCommandBuffer, void *), void *arg);
|
||||
// The probes (--probe-vkmem's table parts, the lost-device probe): one submission whose commands the caller records
|
||||
// itself where the replay would, with the submission lock, the memory prologue and the fence of any other.
|
||||
uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg);
|
||||
uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals
|
||||
int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted
|
||||
// The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet.
|
||||
@@ -782,7 +782,6 @@ typedef struct {
|
||||
// from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs.
|
||||
VkDescriptorUpdateTemplate tpl[2];
|
||||
int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools)
|
||||
int gpa; // a stage accesses guest pointers through the device address table
|
||||
} VkPipeInfo;
|
||||
// One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the
|
||||
// bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte.
|
||||
|
||||
@@ -145,7 +145,7 @@ VkCommandBuffer vk_exec_inline_begin(int *reopen);
|
||||
void vk_exec_inline_end(int reopen);
|
||||
// A test's hook (tex_probe.c): between a call with cb and one with VK_NULL_HANDLE, the calling thread counts as
|
||||
// recording a replay into cb for submission `fence`, so vk_exec_inline_begin hands cb out. Called from the record
|
||||
// callback of vk_sync_probe_submit, where a replay would be recorded.
|
||||
// callback of vk_sync_submit_own, where a replay would be recorded.
|
||||
void vk_exec_probe_inline(VkCommandBuffer cb, uint64_t fence);
|
||||
// vk_exec_draw.c: forget what the command buffer being recorded has bound (pipeline, sets, dynamic state, vertex
|
||||
// buffers): the replay continues in a new one (vk_sync_split).
|
||||
|
||||
@@ -1388,9 +1388,13 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *
|
||||
// of every mirror at a table reader's draw copied stale guest bytes over what the GPU had written into the mirrors
|
||||
// earlier in the frame, and Kokiri Forest's lit areas went black. A pool read only through the table is rarely a
|
||||
// mirrored one (mirrors are made for pools bound as buffers, which sync then). With shadow storage every pool the
|
||||
// table names is a copy of guest memory, and nothing resolves the pools such a reader reaches: bind_resources
|
||||
// marks the part (vk_mem_gpa_use) and its prologue brings those copies up to date (vk_memory.c vk_mem_gpa_sync).
|
||||
// table names is a copy of guest memory, and nothing resolves the pools such a reader reaches: the binding is
|
||||
// noted here, and the copies are brought up to date before the part is submitted (vk_memory.c
|
||||
// vk_mem_shadow_table_read). A draw that reuses the descriptors of the one before it (bind_resources) comes after
|
||||
// a build in the same command buffer, and the packed descriptors of mo_desc are built in the same guest
|
||||
// submission as their draws: either way this ran for the guest submission.
|
||||
if (r->cbuf_slot == VK_GPA_CB_SLOT) {
|
||||
vk_mem_shadow_table_read();
|
||||
VkDescriptorBufferInfo t = vk_gpa_info();
|
||||
return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE};
|
||||
}
|
||||
@@ -3231,7 +3235,6 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) {
|
||||
static VkDescSlot d[MAX_DESC];
|
||||
static VkBufferView bv[MAX_DESC];
|
||||
if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS);
|
||||
if (info->gpa) vk_mem_gpa_use(); // before the early returns for reused descriptors: the shader still reads the table
|
||||
bind_stats(info);
|
||||
int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS;
|
||||
int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid;
|
||||
@@ -4460,7 +4463,6 @@ static void mo_emit(VkCommandBuffer cb) {
|
||||
for (int n = 0; n < MQ.n; p = &MQ.d[n++]) {
|
||||
const MoDraw *q = &MQ.d[n];
|
||||
const VkPipeInfo *info = q->info;
|
||||
if (info->gpa) vk_mem_gpa_use(); // these draws bind packed descriptors without bind_resources, maybe in a later part
|
||||
if (q->pipe != pipe) vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, q->pipe), pipe = q->pipe, MO.e_pipe++;
|
||||
if (q->st != st) vk_pipeline_set_dynamic(cb, &MQ.st[q->st]), st = q->st, MO.e_dyn++;
|
||||
if (!p || memcmp(&q->v, &p->v, sizeof q->v)) vkCmdSetViewportWithCount(cb, 1, &q->v);
|
||||
|
||||
@@ -36,8 +36,8 @@
|
||||
// NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0).
|
||||
// NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis).
|
||||
// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). With shadow storage the
|
||||
// pools are copies of guest memory: a part whose shaders use the table is marked (vk_exec_draw.c bind_resources) and
|
||||
// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). There
|
||||
// pools are copies of guest memory, which a guest submission that binds the table (vk_exec_draw.c ssbo_info) brings up
|
||||
// to date with the pages the CPU wrote, once, before the part is submitted (vk_memory.c vk_mem_shadow_sync). There
|
||||
// the twin of a mirrored pool is its shadow copy, not guest memory, and what a shader stores through the table is not
|
||||
// noted as a GPU write: it does not come back to guest memory, and the next whole copy-in of its page overwrites it.
|
||||
// vk_memory.c "Pages a shader stores into through the device address table" has the sequence, what was measured, and
|
||||
|
||||
@@ -86,6 +86,7 @@ typedef struct PoolRec {
|
||||
uint32_t sgen; // gen_lock: every page write stamped up to this generation is in the shadow
|
||||
uint64_t sborn; // the value of the first submission that can have used the shadow (shadow_pool)
|
||||
volatile uint8_t stouched; // on the touched list: the GPU may use it in the submission being recorded
|
||||
uint32_t stab_i; // shadow_lock: 1 + its index in stab (the table's shadows), 0 when not there
|
||||
// shadow_lock: base copies of the pages a noted GPU write covers (merge_page), open addressing by page address;
|
||||
// csb a power of two, page 0 an empty slot
|
||||
struct SBase { uint64_t page; uint8_t *base; } *sb;
|
||||
@@ -112,6 +113,7 @@ static int shadow_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size);
|
||||
static void shadow_touch(PoolRec *r);
|
||||
static void shadow_forget(PoolRec *r);
|
||||
static void pool_retiring(PoolRec *r); // pool_fini: kept until pool_retire (or vk_mem_shutdown)
|
||||
static void shadow_table_add(PoolRec *r);
|
||||
static void pool_retire(void *arg); // completion thread: destroys a finalized pool's record
|
||||
static void grow(void **arr, int *cap, size_t elem);
|
||||
|
||||
@@ -336,10 +338,13 @@ static int pool_init(pool_t *p) {
|
||||
phys_list[nphys++] = r;
|
||||
}
|
||||
plat_mutex_unlock(&pool_lock);
|
||||
// The pool's pages go into the device address table: shaders that read a pointer from memory reach them there. An
|
||||
// import and a shadow copy both have buf; only the import owns mem.
|
||||
if (r->kind == POOL_IMPORT && r->buf && r->import_hi > r->import_lo)
|
||||
// The imported pages, or the shadow that stands for them, go into the device address table: shaders that read a
|
||||
// pointer from memory reach them there. A shadow in the table is kept up to date for the table's readers whether
|
||||
// or not the replay resolves it (vk_mem_shadow_sync).
|
||||
if (r->kind == POOL_IMPORT && (r->mem || r->smap) && r->buf && r->import_hi > r->import_lo) {
|
||||
r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo);
|
||||
if (r->smap && r->gpa_rec) shadow_table_add(r);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -504,7 +509,7 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touc
|
||||
// touch: the GPU may read the pool's buffer in the submission being recorded, so its shadow goes on the touched list
|
||||
// (shadow storage). A lookup (vk_mem_lookup) leaves it off: the caller reads guest memory itself or only asks what
|
||||
// backs the address. Shaders that reach a pool through the device address table resolve nothing: the part they run in
|
||||
// brings every pool of the table up to date (vk_mem_gpa_sync).
|
||||
// brings every pool of the table up to date (vk_mem_shadow_sync).
|
||||
static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) {
|
||||
memset(out, 0, sizeof *out);
|
||||
uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size;
|
||||
@@ -1056,8 +1061,18 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
// invalidates the buffer, about 1.3 ms per 16 MiB at the next submit, and the import's write pinning clears the
|
||||
// write watch), a CPU-visible pool gets a host-visible buffer of its own instead, at the offsets the import would have:
|
||||
// - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are
|
||||
// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. A part whose shaders read
|
||||
// through the device address table gets every pool the table names brought up to date the same way (vk_mem_gpa_sync).
|
||||
// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU.
|
||||
// - The device address table (vk_gpa.c) names the shadows, and its readers reach pools the replay never resolves: the
|
||||
// instanced foliage's and Navi's data, through pointers read from other buffers. Without a sync those shadows held
|
||||
// what guest memory held when they were made, and Kokiri Forest's trees were bare trunks, without ferns or grass
|
||||
// tufts. So a guest submission whose record binds the table (vk_mem_shadow_table_read) syncs every shadow in the
|
||||
// table, once, before the first part recorded after the binding is submitted (vk_mem_shadow_sync), and those that
|
||||
// have a VRAM mirror, which the table then names, queue the same pages for upload. Once is enough: the copy-ins read
|
||||
// the guest submission's write-watch snapshots (snap_regions), which a second sync in the same guest submission
|
||||
// would find unchanged. A sync in every part that used the table ran about three times a frame outdoors. Measured
|
||||
// on Linux (RTX 4090, Kokiri Forest, by the change's author): a sync at every guest submission, about six a
|
||||
// frame, took 9.8 ms and 888 write-watch queries a frame; one in the guest submissions that read the table 3.9 ms
|
||||
// and 92.
|
||||
// - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and
|
||||
// reports it at the fence (vk_gpu_written), which copies the GPU's bytes back into guest memory.
|
||||
// - Pages both write: a GPU write is noted with the whole range its binding names, and the CPU writes other fields of the
|
||||
@@ -1079,24 +1094,26 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) {
|
||||
// (vk_mem_gpu_pending): the page has no base copy and guest memory never gets the bytes. The game's culling passes
|
||||
// store counts and records this way into pages whose other words the CPU rewrites. For one such page P, a word X the
|
||||
// shaders store and a word Y the CPU writes:
|
||||
// part A (the pass that stores): the CPU wrote Y, so A's prologue copies P in whole (shadow_copy) and queues it
|
||||
// for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which no store ever
|
||||
// reached. A runs and stores X = v into both.
|
||||
// part B (a later part that reads X through the table), the CPU has not written P since A's write-watch snapshot:
|
||||
// B's prologue leaves P alone and its draws read v.
|
||||
// part B, the CPU has written Y again between the two snapshots: B's prologue copies P in whole and uploads it, X
|
||||
// guest submission A (with the pass that stores): the CPU wrote Y, so A's sync copies P in whole (shadow_copy)
|
||||
// and queues it for the mirror (mirror_sync). X in the shadow and in the mirror is now guest memory's X, which
|
||||
// no store ever reached. A runs and stores X = v into both.
|
||||
// guest submission B (a later one that reads X through the table), the CPU has not written P since A's
|
||||
// write-watch snapshot:
|
||||
// B's sync leaves P alone and its draws read v.
|
||||
// B, the CPU has written Y again between the two snapshots: B's sync copies P in whole and uploads it, X
|
||||
// is guest memory's stale word again in both copies, and the draws read that.
|
||||
// An import does not have the fault: its twin is guest memory, and a page copied up from there holds the store.
|
||||
// Measured (NX2NAVI_SHADOW_CHECK, a standing run outdoors, RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10): the stores
|
||||
// go into one page each of ten pools of 20 and 44 KiB, and whole copy-ins overwrite one exactly four times a frame
|
||||
// (240 in every 60 frames): part A's case, the copy that precedes the frame's stores, which does no harm. Part B's
|
||||
// second case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the
|
||||
// frame's timing differs.
|
||||
// (240 in every 60 frames): A's case, the copy that precedes the frame's stores, which does no harm. B's second
|
||||
// case would add to the four and was not seen in 1,800 frames; nothing proves it cannot happen where the frame's
|
||||
// timing differs. (Counted when every part that used the table synced; a guest submission syncs once now, from the
|
||||
// same write-watch snapshots, and the counts were not taken again.)
|
||||
// NX2NAVI_SHADOW_TABLE_STORES=1 keeps the stores. The pages are found by their content: each whole copy-in leaves a
|
||||
// hash of the page it wrote (smark), and the next one hashes the shadow page first. A different hash means something
|
||||
// stored into the shadow in between, and with no base copy on the page that was a store through the table. That
|
||||
// copy-in still overwrites the store (nothing tells its bytes from the CPU's), and from then on the page keeps a
|
||||
// base copy: copy-ins take the CPU's bytes only, and the fence of every submission with a part that uses the table
|
||||
// base copy: copy-ins take the CPU's bytes only, and the fence of every submission whose record bound the table
|
||||
// writes the stored bytes of all such pages back to guest memory (table_wb_cb), as the fence of a noted write does.
|
||||
// Guest memory needs them: a word the CPU resets each frame to the value it had before the stores would otherwise
|
||||
// never differ from its base, and the reset would never reach the GPU.
|
||||
@@ -1132,7 +1149,10 @@ static uint64_t s_marked, s_mark_ticks; // pages copied in whole with
|
||||
static uint64_t s_plain_ticks; // check: the time of the whole copies made without marks
|
||||
static uint64_t s_both_cpu, s_both_gpu; // check: bytes both sides changed, taken from the CPU / from the GPU
|
||||
static uint64_t s_in_merge, s_in_nomirror; // check: bytes copied in by merge_page; whole, into pools without a mirror
|
||||
static uint64_t s_gpa_ticks, s_gpa_syncs; // vk_mem_gpa_sync: its time (part of s_ticks) and its runs
|
||||
static uint64_t s_tab_ticks, s_tab_syncs; // the sync for the table's readers: its time (part of s_ticks), its runs
|
||||
static PoolRec **stab; // shadow_lock: the shadows in the device address table (PoolRec.stab_i)
|
||||
static int nstab, cstab;
|
||||
static uint32_t stab_want; // the last guest submission whose record bound the table
|
||||
static uint64_t s_t_over_all; // shadow_lock: s_tab_over since the start
|
||||
static uint64_t s_late_pages, s_late_bytes; // shadow_lock, check: write-backs into pages the CPU wrote since their copy-in
|
||||
static uint64_t s_stale_pages; // shadow_lock: write-backs left out, the shadow could not hold them (shadow_write_back)
|
||||
@@ -1361,6 +1381,11 @@ static void shadow_forget(PoolRec *r) {
|
||||
if (stouch[i] == r) { stouch[i] = stouch[--nstouch]; break; }
|
||||
for (int i = 0; r->tlisted && i < ntpools; i++)
|
||||
if (tpools[i] == r) { tpools[i] = tpools[--ntpools]; break; }
|
||||
if (r->stab_i) { // the last one takes its place
|
||||
PoolRec *last = stab[--nstab];
|
||||
stab[r->stab_i - 1] = last, last->stab_i = r->stab_i;
|
||||
r->stab_i = 0;
|
||||
}
|
||||
s_live_n--, s_live_bytes -= (int64_t)(r->import_hi - r->import_lo);
|
||||
s_tab_pages -= r->ntpg;
|
||||
free(r->tpg), free(r->smark);
|
||||
@@ -1460,23 +1485,6 @@ static void shadow_sync_pool(PoolRec *r, uint32_t epoch, int cpu) {
|
||||
if (run) shadow_copy(r, run, hi);
|
||||
}
|
||||
|
||||
void vk_mem_shadow_sync(void) {
|
||||
if (!shadow_on) return;
|
||||
uint64_t t0 = qpc();
|
||||
uint32_t epoch = vk_texture_epoch();
|
||||
int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
if (nstouch) {
|
||||
plat_mutex_lock(&gen_lock);
|
||||
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu), stouch[i]->stouched = 0;
|
||||
plat_mutex_unlock(&gen_lock);
|
||||
s_syncs += (uint64_t)nstouch;
|
||||
nstouch = 0;
|
||||
}
|
||||
s_ticks += qpc() - t0;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
// Record time: the GPU may write [addr, addr+size) of CPU-visible memory. With shadow storage the pages it covers get
|
||||
// their base copies before the submission's copy-in (vk_mem_shadow_sync) can reach them.
|
||||
void vk_mem_gpu_pending(uint64_t addr, uint64_t size) {
|
||||
@@ -1576,7 +1584,7 @@ static uint64_t shadow_write_back(uint64_t addr, uint64_t size) {
|
||||
return moved;
|
||||
}
|
||||
|
||||
// Completion thread, at the fence of a submission with a part that uses the device address table (vk_mem_gpa_sync):
|
||||
// Completion thread, at the fence of a submission whose record bound the device address table (vk_mem_shadow_sync):
|
||||
// what the shaders stored into the pages found so far ("Pages a shader stores into" above) goes from the shadow into
|
||||
// guest memory, the bytes that differ from the page's base copy. A page whose shadow equals its base is passed over
|
||||
// before guest memory is touched, so a frozen region stays frozen. The pages that changed are stamped as a mirror's
|
||||
@@ -1672,7 +1680,7 @@ static void shadow_table_log(uint64_t frame) {
|
||||
"from the CPU at a copy-in, %llu from the GPU at a fence; write-backs into %llu pages the CPU had written "
|
||||
"since their copy-in, %llu bytes",
|
||||
(unsigned long long)frame, qpc_ms(s_mark_ticks), qpc_ms(s_plain_ticks), s_in_merge / 1048576.0,
|
||||
s_in_nomirror / 1048576.0, (unsigned long long)s_gpa_syncs, qpc_ms(s_gpa_ticks),
|
||||
s_in_nomirror / 1048576.0, (unsigned long long)s_tab_syncs, qpc_ms(s_tab_ticks),
|
||||
(unsigned long long)s_both_cpu, (unsigned long long)s_both_gpu, (unsigned long long)s_late_pages,
|
||||
(unsigned long long)s_late_bytes);
|
||||
PoolRec *top[TABLE_LOG_TOP];
|
||||
@@ -1734,7 +1742,7 @@ static void shadow_table_log(uint64_t frame) {
|
||||
}
|
||||
}
|
||||
s_tab_over = s_tab_wb = s_tab_wb_seen = s_tab_wb_ticks = s_tab_wb_jobs = s_marked = s_mark_ticks = s_plain_ticks = 0;
|
||||
s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_gpa_ticks = s_gpa_syncs = s_late_pages = s_late_bytes = 0;
|
||||
s_both_cpu = s_both_gpu = s_in_merge = s_in_nomirror = s_tab_ticks = s_tab_syncs = s_late_pages = s_late_bytes = 0;
|
||||
}
|
||||
|
||||
static void shadow_log(uint64_t frame) {
|
||||
@@ -2418,30 +2426,30 @@ static void mirror_sync(PoolRec *r) {
|
||||
mrep.sync_t += qpc() - t0;
|
||||
}
|
||||
|
||||
// ---- shadow storage: the device address table's readers
|
||||
// A shader that follows a pointer through the table (vk_gpa.c) can reach any pool the table names, and the replay
|
||||
// never resolves that pool, so nothing put it on the touched list and its shadow kept the bytes of its last use as a
|
||||
// buffer. vk_exec_draw.c marks a part whose pipelines declare the table (vk_mem_gpa_use: at the draw or dispatch, not
|
||||
// where descriptors are written, which are reused), and the part's prologue (vk_sync.c) calls vk_mem_gpa_sync: the
|
||||
// shadows of every pool in the table get the pages the CPU wrote, and those of them that have a VRAM mirror, which
|
||||
// the table then names, queue the same pages for upload unless this submission synced the mirror already. No mirror
|
||||
// is made for a reader of the table. Without shadow storage nothing is done: an imported pool is guest memory, and
|
||||
// mirrors stay unsynced for the table's readers (vk_exec_draw.c ssbo_info).
|
||||
static int gpa_part_used; // replay thread: the part being recorded uses the table
|
||||
void vk_mem_gpa_use(void) { gpa_part_used = 1; }
|
||||
// ---- shadow storage: the copy-ins before a part is submitted, and the device address table's readers
|
||||
int vk_mem_shadow_storage(void) { return shadow_on; }
|
||||
static int shadow_check_on(void) { return s_check; }
|
||||
int vk_mem_shadow_table_stores(void) { return s_learn; }
|
||||
|
||||
static int gpa_shadow_pool(const PoolRec *r) { return r->kind == POOL_IMPORT && r->gpa_rec && r->smap; }
|
||||
static void shadow_table_add(PoolRec *r) {
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
if (!r->stab_i) {
|
||||
if (nstab == cstab) grow((void **)&stab, &cstab, sizeof *stab);
|
||||
stab[nstab++] = r;
|
||||
r->stab_i = (uint32_t)nstab;
|
||||
}
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
}
|
||||
|
||||
void vk_mem_shadow_table_read(void) {
|
||||
if (shadow_on) __atomic_store_n(&stab_want, vk_texture_epoch(), __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
// gen_lock held: nothing of the pool was stamped since its shadow's last copy-in. Its regions have this epoch's
|
||||
// write-watch snapshot (vk_mem_gpa_sync takes them for all pools at once), none of them is untracked, and none holds a
|
||||
// stamp newer than the shadow's sgen, which then stays as it is. With cpu the stamps are the CPU-side tables', as
|
||||
// shadow_sync_pool reads them. A table sync ran shadow_sync_pool for every pool at every part that uses the table,
|
||||
// 10,700 times a frame outdoors (3,772 pools, about three such parts), each with its own timer reads, snapshot walk
|
||||
// and region loop; with this test 1,500 pool syncs a frame are left, and the copy-ins went from 1.32 to 1.01 ms a
|
||||
// frame (RTX 5090 with NX2NAVI_SHADOW=1, 2026-11-10).
|
||||
// write-watch snapshot, none of them is untracked, and none holds a stamp newer than the shadow's sgen, which then
|
||||
// stays as it is. With cpu the stamps are the CPU-side tables', as shadow_sync_pool reads them. The table's sync asks
|
||||
// this of every shadow in the table before it runs shadow_sync_pool with its own timer reads, snapshot walk and
|
||||
// region loop: outdoors about 3,800 pools are in the table and a few hundred were written.
|
||||
static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) {
|
||||
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
||||
if (!epoch || r1 >= SNAP_REGIONS) return 0;
|
||||
@@ -2451,54 +2459,52 @@ static int shadow_pool_clean(const PoolRec *r, uint32_t epoch, int cpu) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
void vk_mem_gpa_sync(uint64_t fence) {
|
||||
if (!gpa_part_used) return;
|
||||
gpa_part_used = 0;
|
||||
void vk_mem_shadow_sync(uint64_t fence) {
|
||||
if (!shadow_on) return;
|
||||
// What this part's shaders store through the table goes back to guest memory at the submission's fence, once a
|
||||
// submission however many of its parts use the table (table_wb_cb). Asked before the copy-ins below: the part
|
||||
// whose copy-ins find the first such page leaves it to the next part that uses the table.
|
||||
static uint64_t wb_fence; // replay thread: the last fence that got the job
|
||||
if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) {
|
||||
wb_fence = fence;
|
||||
vk_completion_add(fence, table_wb_cb, NULL);
|
||||
}
|
||||
uint64_t t0 = qpc(), run0 = 0, run1 = 0;
|
||||
uint64_t t0 = qpc();
|
||||
uint32_t epoch = vk_texture_epoch();
|
||||
int cpu = vk_mirror_on && wb_no_reup(), have = 0; // before the locks: its first call logs
|
||||
plat_mutex_lock_shared(&pool_lock);
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
plat_mutex_lock(&gen_lock);
|
||||
// one write-watch query per run of regions the pools cover (the index is sorted by address) instead of one a pool;
|
||||
// shadow_sync_pool then finds its regions snapped
|
||||
for (int i = 0; epoch && i < pool_index.n; i++) {
|
||||
const PoolRec *r = pool_index.e[i].buf;
|
||||
if (!gpa_shadow_pool(r)) continue;
|
||||
uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT;
|
||||
if (b >= SNAP_REGIONS) continue;
|
||||
if (have && a <= run1 + 1) {
|
||||
if (b > run1) run1 = b;
|
||||
} else {
|
||||
if (have) s_queries += snap_regions(run0, run1, epoch);
|
||||
run0 = a, run1 = b, have = 1;
|
||||
int cpu = vk_mirror_on && wb_no_reup(); // before the locks: its first call logs
|
||||
// The table's readers: the record of this guest submission bound the table, and no part of it has synced yet.
|
||||
// The callers are one at a time (vk_sync.c prologue under the submission lock, and the probe).
|
||||
static uint32_t tab_epoch; // the guest submission the table's shadows were last synced for
|
||||
static uint64_t wb_fence; // the last fence that got the write-back job
|
||||
int table = epoch && epoch != tab_epoch && __atomic_load_n(&stab_want, __ATOMIC_RELAXED) == epoch;
|
||||
if (table) {
|
||||
tab_epoch = epoch;
|
||||
// What this submission's shaders store through the table goes back to guest memory at its fence, once
|
||||
// (table_wb_cb). Asked before the copy-ins below: the sync whose copy-ins find the first such page leaves it
|
||||
// to the next guest submission that binds the table.
|
||||
if (s_learn && fence && fence != wb_fence && __atomic_load_n(&ntpools, __ATOMIC_RELAXED)) {
|
||||
wb_fence = fence;
|
||||
vk_completion_add(fence, table_wb_cb, NULL);
|
||||
}
|
||||
}
|
||||
if (have) s_queries += snap_regions(run0, run1, epoch);
|
||||
for (int i = 0; i < pool_index.n; i++) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
if (gpa_shadow_pool(r) && !shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
if (nstouch || (table && nstab)) {
|
||||
plat_mutex_lock(&gen_lock);
|
||||
for (int i = 0; i < nstouch; i++) shadow_sync_pool(stouch[i], epoch, cpu);
|
||||
uint64_t t1 = qpc();
|
||||
for (int i = 0; table && i < nstab; i++) { // the shadows the replay did not resolve
|
||||
PoolRec *r = stab[i];
|
||||
if (r->stouched) continue;
|
||||
uint64_t r0 = r->import_lo >> SNAP_SHIFT, r1 = (r->import_hi - 1) >> SNAP_SHIFT;
|
||||
if (r1 < SNAP_REGIONS) s_queries += snap_regions(r0, r1, epoch); // so that shadow_pool_clean can tell
|
||||
if (!shadow_pool_clean(r, epoch, cpu)) shadow_sync_pool(r, epoch, cpu), s_syncs++;
|
||||
}
|
||||
for (int i = 0; i < nstouch; i++) stouch[i]->stouched = 0;
|
||||
plat_mutex_unlock(&gen_lock);
|
||||
s_syncs += (uint64_t)nstouch;
|
||||
nstouch = 0;
|
||||
// A shadow with a VRAM mirror: the table names the mirror, which takes the same pages unless this guest
|
||||
// submission synced it already (a pool bound as a buffer). The uploads read the shadows just brought up to
|
||||
// date, and the prologue records them ahead of the part, after the write-backs of the submissions before it.
|
||||
// No mirror is made for a reader of the table.
|
||||
for (int i = 0; table && i < nstab; i++)
|
||||
if (stab[i]->mstate == MS_ON && stab[i]->acc_sub != msub) mirror_sync(stab[i]);
|
||||
if (table) s_tab_ticks += qpc() - t1, s_tab_syncs++;
|
||||
}
|
||||
plat_mutex_unlock(&gen_lock);
|
||||
// The uploads read the shadows just brought up to date, and the prologue records them ahead of the part, after
|
||||
// the write-backs of the submissions before it.
|
||||
for (int i = 0; i < pool_index.n; i++) {
|
||||
PoolRec *r = pool_index.e[i].buf;
|
||||
if (gpa_shadow_pool(r) && r->mstate == MS_ON && r->acc_sub != msub) mirror_sync(r);
|
||||
}
|
||||
s_gpa_ticks += qpc() - t0, s_gpa_syncs++;
|
||||
s_ticks += qpc() - t0;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
plat_mutex_unlock_shared(&pool_lock);
|
||||
}
|
||||
|
||||
void vk_mem_use(VkMemSpan *sp) {
|
||||
@@ -4293,7 +4299,7 @@ void vk_mem_shutdown(void) {
|
||||
nsnap_pool = 0, snap_pool_bytes = 0;
|
||||
plat_mutex_unlock(&snap_pool_lock);
|
||||
plat_mutex_lock(&shadow_lock);
|
||||
nstouch = 0, s_sb_pages = 0; // shadows too (their base copies went with their records)
|
||||
nstouch = nstab = 0, s_sb_pages = 0; // shadows too (their base copies went with their records)
|
||||
s_live_n = s_live_bytes = 0;
|
||||
plat_mutex_unlock(&shadow_lock);
|
||||
for (int i = 0; i < nretiring; i++) destroy_rec(c, retiring[i]); // finalized ones whose callback never ran
|
||||
|
||||
@@ -51,7 +51,6 @@
|
||||
#include "tex.h"
|
||||
#include "vk_build_gate.h"
|
||||
#include "vk_cache_file.h"
|
||||
#include "vk_gpa.h"
|
||||
#include "vk_save_retry.h"
|
||||
#include "vk_warm_hint.h"
|
||||
|
||||
@@ -2596,8 +2595,6 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel
|
||||
p->info.push1 = l ? l->push1 : 0;
|
||||
p->info.n_sets = l && l->set1 ? 2 : 1;
|
||||
p->info.pool = B ? B->pool : 0;
|
||||
for (uint32_t i = 0; i < p->info.n_bindings; i++)
|
||||
if (p->bindings[i].kind == GFX_RES_SSBO && p->bindings[i].res->cbuf_slot == VK_GPA_CB_SLOT) p->info.gpa = 1;
|
||||
if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1];
|
||||
p->lay = l;
|
||||
Pipe **head = &pipes[h % PIPE_BUCKETS];
|
||||
|
||||
@@ -647,8 +647,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) {
|
||||
// part was recorded, sub_lock held; may move slots[].
|
||||
// First brings the shadow storage up to date with the CPU's writes, which needs no command buffer.
|
||||
static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) {
|
||||
vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted
|
||||
vk_mem_gpa_sync(fence); // a part whose shaders read through the device address table never resolved the pools they reach
|
||||
vk_mem_shadow_sync(fence); // shadow storage: the CPU's writes go in before the part is submitted
|
||||
vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots
|
||||
if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE;
|
||||
int i = claim_slot(c, fence); // before reading slots: the claim may move it
|
||||
@@ -710,7 +709,13 @@ uint64_t vk_sync_reserve(void) {
|
||||
return v;
|
||||
}
|
||||
|
||||
static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) {
|
||||
// own: NULL for the replay of handles; else the caller's commands in the replay's place (vk_sync_submit_own).
|
||||
typedef struct {
|
||||
void (*record)(VkCommandBuffer, void *);
|
||||
void *arg;
|
||||
} OwnCommands;
|
||||
|
||||
static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) {
|
||||
vk_sync_start();
|
||||
// The run is ending (vk_device.c vk_device_lost_check): nothing is recorded or submitted and no value is issued.
|
||||
// The value returned is the last one issued; a wait for it sleeps until the process ends.
|
||||
@@ -733,8 +738,8 @@ static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCo
|
||||
ts_begin(s->cb);
|
||||
rec_begin(s->cb, fence);
|
||||
vk_gpu_passes_part(s->cb, 0);
|
||||
int recorded;
|
||||
if (probe_record) { probe_record(s->cb, arg); recorded = 1; }
|
||||
int recorded = 1;
|
||||
if (own) own->record(s->cb, own->arg);
|
||||
else recorded = vk_exec_record(s->cb, handles, n, fence);
|
||||
s = &slots[cur_slot]; // the last part (vk_sync_split)
|
||||
vk_gpu_passes_part(s->cb, 1);
|
||||
@@ -781,11 +786,12 @@ static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCo
|
||||
return fence;
|
||||
}
|
||||
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL, NULL); }
|
||||
uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL); }
|
||||
|
||||
// Memory probes record their shader here, not in the upload batch (which is submitted before the memory prologue).
|
||||
uint64_t vk_sync_probe_submit(void (*record)(VkCommandBuffer, void *), void *arg) {
|
||||
return submit(NULL, 0, record, arg);
|
||||
// The upload batch is no place for a probe's commands: it is submitted before the memory prologue.
|
||||
uint64_t vk_sync_submit_own(void (*record)(VkCommandBuffer, void *), void *arg) {
|
||||
OwnCommands own = {record, arg};
|
||||
return submit(NULL, 0, &own);
|
||||
}
|
||||
|
||||
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
|
||||
@@ -980,7 +986,7 @@ int vk_sync_probe_devlost(void) {
|
||||
int ran = plat_cas32(&probe_ran[1], 1, 1), reached = vk_sync_fence_done() >= v2;
|
||||
volatile int32_t recorded = 0;
|
||||
uint64_t before = vk_sync_issued(), t0 = plat_now_ms();
|
||||
uint64_t got = vk_sync_probe_submit(probe_lost_record, (void *)&recorded);
|
||||
uint64_t got = vk_sync_submit_own(probe_lost_record, (void *)&recorded);
|
||||
vk_upload_flush(1);
|
||||
int waited = vk_sync_exec_wait(v2, 5000000000ull);
|
||||
uint64_t ms = plat_now_ms() - t0;
|
||||
|
||||
Reference in New Issue
Block a user