diff --git a/docs/building.md b/docs/building.md index bf90059d..670a002a 100644 --- a/docs/building.md +++ b/docs/building.md @@ -252,6 +252,21 @@ Forest (1.1.1, RTX 4090) that costs about 1 ms a frame (copy-in 0.5 ms, write-ba 5 GB. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as well: the shadow copies are chosen for the measured cost above, not for a missing extension. +The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest +pointers into either. A shader that reads through the table resolves nothing, so the pool it reaches is not queued for +the copy-in: a part whose pipelines declare the table is marked at its draws and dispatches, and its prologue brings +the shadow of every pool in the table up to date with the pages the CPU wrote and queues the same pages for those +pools' VRAM mirrors (`vk_memory.c` `vk_mem_gpa_sync`). That copies the written pages of every CPU-visible pool in +each such part, which the cost above does not include. What a shader stores through the table into a shadow copy does +not come back to guest memory. `--probe-vkmem` checks the read: a shader follows the table to a word of a pool that is +never bound, and must read `0x13579BDF` and, after a plain CPU write, `0x2468ACE0`. On Linux, run it for each backing: + +```sh +for shadow in 0 1; do for memory in import mirror; do + NX2NAVI_GPA=1 NX2NAVI_SHADOW=$shadow NX2NAVI_MEMORY=$memory ./ocarina --probe-vkmem +done; done +``` + The Deck's GPU (Van Gogh) has no native ASTC, so its textures go through the compute decoder (`runtime/vk_astc.c`, `NX2NAVI_ASTC`) as on desktop AMD GPUs without it; frame times measured there include it. diff --git a/runtime/shaders/gpa_read.spvasm b/runtime/shaders/gpa_read.spvasm new file mode 100644 index 00000000..98e57e70 --- /dev/null +++ b/runtime/shaders/gpa_read.spvasm @@ -0,0 +1,106 @@ +; --probe-vkmem (runtime/tex_probe.c test_gpa_read): one invocation reads a 32-bit word of guest memory through the +; device address table as the game's translated shaders do, by the table's layout (runtime/vk_gpa.c): L1 at word 16, +; one entry per 16 MiB; L2 blocks of 4096 page entries from word 65552; word 2 holds the word index of the records, +; four words each, the first two the buffer's device address minus the guest address of its byte 0. The pool that +; holds the word is never a descriptor of the shader: only the table (binding 0) and a result word (binding 1) are. +; An address no page maps gives 0. Written in SPIR-V assembly (spirv-as; no GLSL compiler in the build). As GLSL: +; #extension GL_EXT_buffer_reference : require +; layout(local_size_x = 1) in; +; layout(std430, binding = 0) readonly buffer Table { uint t[]; }; +; layout(std430, binding = 1) buffer Result { uint r[]; }; +; layout(push_constant) uniform Push { uint64_t guest; } pc; +; layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer Word { uint v; }; +; void main() { +; uint rec = t[65552u + (t[16u + uint(pc.guest >> 24)] << 12) + (uint(pc.guest >> 12) & 4095u)]; +; if (rec == 0u) { r[0] = 0u; return; } +; uint at = t[2] + (rec << 2); +; r[0] = Word(pc.guest + (uint64_t(t[at]) | uint64_t(t[at + 1u]) << 32)).v; +; } + OpCapability Shader + OpCapability Int64 + OpCapability PhysicalStorageBufferAddresses + OpExtension "SPV_KHR_physical_storage_buffer" + OpMemoryModel PhysicalStorageBuffer64 GLSL450 + OpEntryPoint GLCompute %main "main" %table %result %push + OpExecutionMode %main LocalSize 1 1 1 + OpDecorate %_runtimearr_uint ArrayStride 4 + OpDecorate %Words Block + OpMemberDecorate %Words 0 Offset 0 + OpDecorate %table DescriptorSet 0 + OpDecorate %table Binding 0 + OpDecorate %result DescriptorSet 0 + OpDecorate %result Binding 1 + OpDecorate %Push Block + OpMemberDecorate %Push 0 Offset 0 + %void = OpTypeVoid + %uint = OpTypeInt 32 0 + %ulong = OpTypeInt 64 0 + %bool = OpTypeBool + %fn = OpTypeFunction %void +%_runtimearr_uint = OpTypeRuntimeArray %uint + %Words = OpTypeStruct %_runtimearr_uint +%ptr_words = OpTypePointer StorageBuffer %Words +%_ptr_StorageBuffer_uint = OpTypePointer StorageBuffer %uint + %table = OpVariable %ptr_words StorageBuffer + %result = OpVariable %ptr_words StorageBuffer + %Push = OpTypeStruct %ulong +%ptr_push = OpTypePointer PushConstant %Push + %push = OpVariable %ptr_push PushConstant +%_ptr_PushConstant_ulong = OpTypePointer PushConstant %ulong +%_ptr_PhysicalStorageBuffer_uint = OpTypePointer PhysicalStorageBuffer %uint + %uint_0 = OpConstant %uint 0 + %uint_1 = OpConstant %uint 1 + %uint_2 = OpConstant %uint 2 + %uint_4 = OpConstant %uint 4 + %uint_12 = OpConstant %uint 12 + %uint_16 = OpConstant %uint 16 + %uint_24 = OpConstant %uint 24 + %uint_32 = OpConstant %uint 32 + %uint_4095 = OpConstant %uint 4095 + %uint_65552 = OpConstant %uint 65552 + %main = OpFunction %void None %fn + %entry = OpLabel + %28 = OpAccessChain %_ptr_PushConstant_ulong %push %uint_0 + %guest = OpLoad %ulong %28 + %30 = OpShiftRightLogical %ulong %guest %uint_24 + %31 = OpUConvert %uint %30 + %59 = OpIAdd %uint %31 %uint_16 + %32 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %59 + %block = OpLoad %uint %32 + %34 = OpShiftRightLogical %ulong %guest %uint_12 + %35 = OpUConvert %uint %34 + %page = OpBitwiseAnd %uint %35 %uint_4095 + %62 = OpShiftLeftLogical %uint %block %uint_12 + %63 = OpIAdd %uint %uint_65552 %62 + %42 = OpIAdd %uint %63 %page + %37 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %42 + %rec = OpLoad %uint %37 + %out = OpAccessChain %_ptr_StorageBuffer_uint %result %uint_0 %uint_0 + %mapped = OpINotEqual %bool %rec %uint_0 + OpSelectionMerge %done None + OpBranchConditional %mapped %is_mapped %not_mapped + %is_mapped = OpLabel + %43 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %uint_2 + %recs = OpLoad %uint %43 + %45 = OpShiftLeftLogical %uint %rec %uint_2 + %at = OpIAdd %uint %recs %45 + %47 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %at + %48 = OpLoad %uint %47 + %49 = OpIAdd %uint %at %uint_1 + %50 = OpAccessChain %_ptr_StorageBuffer_uint %table %uint_0 %49 + %51 = OpLoad %uint %50 + %52 = OpUConvert %ulong %48 + %53 = OpUConvert %ulong %51 + %54 = OpShiftLeftLogical %ulong %53 %uint_32 + %delta = OpBitwiseOr %ulong %52 %54 + %56 = OpIAdd %ulong %guest %delta + %ptr = OpConvertUToPtr %_ptr_PhysicalStorageBuffer_uint %56 + %value = OpLoad %uint %ptr Aligned 4 + OpStore %out %value + OpBranch %done + %not_mapped = OpLabel + OpStore %out %uint_0 + OpBranch %done + %done = OpLabel + OpReturn + OpFunctionEnd diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 381f2233..f7c64eb6 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -11,6 +11,8 @@ #include "nvn_api.h" #include "tex.h" #include "vk.h" +#include "vk_gpa.h" +#include "gpa_read_spv.h" #include "platform/platform.h" static uint64_t g_rng = 0x9E3779B97F4A7C15ull; @@ -387,6 +389,130 @@ static int test_import(pool_t *pool) { return 0; } +// test_gpa_read's shader (runtime/shaders/gpa_read.spvasm): one invocation follows the device address table from a +// guest address (the push constant) to the buffer that holds it and stores the 32-bit word found there in the result +// buffer. Storage bindings: 0 the table itself, 1 the result. The pool that holds the word is never a descriptor, a +// transfer source or a resolved span of the submission. + +typedef struct { + VkPipeline pipe; + VkPipelineLayout layout; + VkDescriptorSet set; + uint64_t source; + int setup_mirror, mirrored; +} GpaRead; + +static void gpa_read_record(VkCommandBuffer cb, void *arg) { + GpaRead *r = arg; + // Match vk_backend.c submit_now and vk_exec.c vk_exec_record, including epochs on descriptor reuse. + vk_texture_new_submit(); + vk_mem_mirror_begin(); + // The only explicit use of the source pool is making its mirror, before the first read: that submission takes its + // shadow touch and its first upload. After the CPU's write nothing resolves, uses or flushes the pool. + if (r->setup_mirror) r->mirrored = vk_mem_mirror_probe(r->source), r->setup_mirror = 0; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, r->pipe); + vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, r->layout, 0, 1, &r->set, 0, NULL); + vkCmdPushConstants(cb, r->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof r->source, &r->source); + vk_mem_gpa_use(); // the shader reads the table, in the second submission through the same descriptor set + vkCmdDispatch(cb, 1, 1, 1); + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); + vk_mem_mirror_end(cb, vk_sync_issued() + 1); // called under the submission lock, before the value is issued +} + +static int test_gpa_read(uint8_t *guest) { + VkCtx *c = vk_ctx(); + pool_t pool = {.magic = NVN_POOL_MAGIC, .flags = 0x24, .mem = guest + 0xA00000, .size = 0x10000, + .game_mem = guest + 0xA00000}; + if (!vk_gpa_wanted()) { // no buffer device addresses, or NX2NAVI_GPA=0: shaders read zeros through it by design + printf("probe-vkmem: no device address table: the read through it is skipped\n"); + return 0; + } + volatile uint32_t *source = (volatile uint32_t *)pool.mem; + *source = 0x13579BDFu; // before vk_pool_init: a shadow copy starts with the bytes an import would show + GCHECK(vk_gpa_on() && vk_pool_init(&pool), "the device address table and a CPU-visible pool for the read through it"); + VkBuffer out = VK_NULL_HANDLE; + VmaAllocation alloc = NULL; + VkShaderModule module = VK_NULL_HANDLE; + VkDescriptorSetLayout dsl = VK_NULL_HANDLE; + VkDescriptorPool dp = VK_NULL_HANDLE; + // With shadow storage the pool also gets a VRAM mirror, which the table then names: the prologue of a part that + // uses the table keeps both up to date (vk_mem_gpa_sync). Without it the pool stays an import, guest memory itself: + // a mirror is not synced for the table's readers there (vk_exec_draw.c ssbo_info). + int want_mirror = vk_mirror_on && vk_mem_shadow_storage(); + GpaRead r = {.source = (uintptr_t)pool.mem, .setup_mirror = want_mirror}; + int bad = 1; +#define GPA_CHECK(cond, what) do { if (!(cond)) { printf("probe-vkmem: FAIL GPA " what "\n"); goto done; } } while (0) + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 4, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT}; + VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT, + .requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT}; + VmaAllocationInfo info; + GPA_CHECK(vmaCreateBuffer(c->vma, &bi, &ai, &out, &alloc, &info) == VK_SUCCESS && info.pMappedData, "result buffer"); + VkDescriptorSetLayoutBinding bindings[2] = { + {.binding = 0, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}, + {.binding = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, .descriptorCount = 1, .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}}; + VkDescriptorSetLayoutCreateInfo dl = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .bindingCount = 2, + .pBindings = bindings}; + GPA_CHECK(vkCreateDescriptorSetLayout(c->device, &dl, NULL, &dsl) == VK_SUCCESS, "descriptor layout"); + VkPushConstantRange push = {VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof r.source}; + VkPipelineLayoutCreateInfo pl = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &push}; + GPA_CHECK(vkCreatePipelineLayout(c->device, &pl, NULL, &r.layout) == VK_SUCCESS, "pipeline layout"); + VkShaderModuleCreateInfo sm = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = sizeof gpa_read_spv, + .pCode = gpa_read_spv}; + GPA_CHECK(vkCreateShaderModule(c->device, &sm, NULL, &module) == VK_SUCCESS, "shader module"); + VkComputePipelineCreateInfo cp = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, .layout = r.layout, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_COMPUTE_BIT, + .module = module, .pName = "main"}}; + GPA_CHECK(vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &cp, NULL, &r.pipe) == VK_SUCCESS, "compute pipeline"); + VkDescriptorPoolSize ps = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 2}; + VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .maxSets = 1, + .poolSizeCount = 1, .pPoolSizes = &ps}; + GPA_CHECK(vkCreateDescriptorPool(c->device, &pi, NULL, &dp) == VK_SUCCESS, "descriptor pool"); + VkDescriptorSetAllocateInfo da = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = dp, + .descriptorSetCount = 1, .pSetLayouts = &dsl}; + GPA_CHECK(vkAllocateDescriptorSets(c->device, &da, &r.set) == VK_SUCCESS, "descriptor set"); + VkDescriptorBufferInfo buffers[2] = {vk_gpa_info(), {out, 0, 4}}; + VkWriteDescriptorSet writes[2]; + for (unsigned i = 0; i < 2; i++) + writes[i] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = r.set, .dstBinding = i, + .descriptorCount = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .pBufferInfo = &buffers[i]}; + vkUpdateDescriptorSets(c->device, 2, writes, 0, NULL); + static const uint32_t want[2] = {0x13579BDFu, 0x2468ACE0u}; + for (unsigned pass = 0; pass < 2; pass++) { + if (pass) *source = want[pass]; // a plain CPU write: no resolve, use, flush or shadow sync of the pool + uint64_t fence = vk_sync_probe_submit(gpa_read_record, &r); + GPA_CHECK(vk_sync_fence_wait(fence, 5000000000ull), "shader fence timed out"); + GPA_CHECK(!want_mirror || r.mirrored, "source mirror setup"); + vmaInvalidateAllocation(c->vma, alloc, 0, VK_WHOLE_SIZE); + uint32_t got = *(volatile uint32_t *)info.pMappedData; + if (got != want[pass]) { + printf("probe-vkmem: FAIL GPA/BDA %s read: got %#010x, expected %#010x (mirror %d)\n", + pass ? "after CPU mutation" : "initial", got, want[pass], r.mirrored); + goto done; + } + } + g_gpu_checks += 2; + printf("probe-vkmem: GPA/BDA CPU write -> shader read: initial %#010x and mutation %#010x exact (mirror %d)\n", + want[0], want[1], r.mirrored); + bad = 0; +done: + vk_sync_finish(); // after a timeout too: the objects below must outlive the shader + if (dp) vkDestroyDescriptorPool(c->device, dp, NULL); + if (r.pipe) vkDestroyPipeline(c->device, r.pipe, NULL); + if (module) vkDestroyShaderModule(c->device, module, NULL); + if (r.layout) vkDestroyPipelineLayout(c->device, r.layout, NULL); + if (dsl) vkDestroyDescriptorSetLayout(c->device, dsl, NULL); + if (out) vmaDestroyBuffer(c->vma, out, alloc); + vk_pool_fini(&pool); +#undef GPA_CHECK + return bad; +} + // ReadFile into a watched page, buffered and unbuffered: timed, and whether write watch saw the kernel's write. static int test_readfile(uint8_t *dst) { char dir[PLAT_PATH_MAX], path[PLAT_PATH_MAX + 64]; @@ -542,14 +668,16 @@ static int test_textures(pool_t *tpool, pool_t *phys) { GCHECK(img && img->mirrored && img->levels == 3 && img->format == VK_FORMAT_R8G8B8A8_UNORM, "RGBA8 texture image"); for (unsigned lv = 0; lv < 3; lv++) if (read_back(&t, lv, lin, 1, "RGBA8")) return 1; - // CPU writes to the storage: write watch -> re-upload on the next use + // CPU writes to the storage: write watch -> re-upload on the next use, and every mip the GPU sees has the new bytes uint64_t up0, re0, re1; vk_texture_stats(NULL, &up0, &re0); if (fill_storage(&t, lin)) return 1; for (unsigned lv = 0; lv < 3; lv++) if (read_back(&t, lv, lin, 1, "RGBA8 rewritten")) return 1; vk_texture_stats(NULL, NULL, &re1); +#ifdef _WIN32 // elsewhere an import's pinning can mark pages written again without a CPU write: the count is not exact GCHECK(re1 == re0 + 1, "storage rewritten: %llu re-uploads (want 1)", (unsigned long long)(re1 - re0)); +#endif if (read_back(&t, 1, lin, 0, "RGBA8 ReadTexels from storage")) return 1; // BC1, 64x48: blocks uploaded as they are, read back from the image @@ -634,7 +762,7 @@ static int test_gpu(void) { vk_mem_stats(&st); GCHECK(!st.imported_pools && !st.physical_pools && !st.virtual_pools, "pools left after fini"); g_gpu_checks += 2; - return 0; + return test_gpa_read(h); } int probe_vkmem(void) { diff --git a/runtime/vk.h b/runtime/vk.h index 554bfb00..ed9d17c1 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -329,6 +329,12 @@ void vk_mem_lazy_handed(void); // write into CPU-visible memory at record time (vk_mem_gpu_pending) and reports it at the fence with vk_gpu_written, // which copies the bytes back into guest memory before telling the texture cache (vk_guest_written). void vk_mem_shadow_sync(void); +// A shader of the part being recorded reads through the device address table (vk_gpa.c), also when its descriptors +// are reused: with shadow storage the part's prologue (vk_mem_gpa_sync) then brings every pool the table names up to +// date with the CPU's writes, since the pools such a shader reaches are never resolved. +void vk_mem_gpa_use(void); +void vk_mem_gpa_sync(void); +int vk_mem_shadow_storage(void); // CPU-visible pools are shadow copies of guest memory, not imports (NX2NAVI_SHADOW) void vk_mem_gpu_pending(uint64_t addr, uint64_t size); void vk_gpu_written(uint64_t addr, uint64_t size); // The same for a GPU write into a VRAM mirror, which went back to guest memory with the submission: the texture checks @@ -579,6 +585,9 @@ void rt_write_png(const char *path, const uint8_t *rgba, uint32_t w, uint32_t h) // once the GPU reached it and the completion thread ran its side effects. Guest or host threads. void vk_sync_start(void); // vk_init (fatal without a device) and the completion thread; idempotent uint64_t vk_sync_submit(const uint64_t *handles, int n); // records with vk_exec_record, submits; returns the value +// --probe-vkmem: record gives the submission's commands in place of the replay, under the same submission lock, +// recorder, memory prologue and fence. +uint64_t vk_sync_probe_submit(void (*record)(VkCommandBuffer, void *), void *arg); uint64_t vk_sync_reserve(void); // asynchronous replay: the value the next queued submission signals int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted // The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. @@ -676,6 +685,7 @@ typedef struct { // from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs. VkDescriptorUpdateTemplate tpl[2]; int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools) + int gpa; // a stage accesses guest pointers through the device address table } VkPipeInfo; // One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the // bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 93a68771..673fbef9 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1351,7 +1351,9 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * // The device address table (vk_gpa.c, translator patch 0030). Mirrored pools are not synced for its readers: a sync // of every mirror at a table reader's draw copied stale guest bytes over what the GPU had written into the mirrors // earlier in the frame, and Kokiri Forest's lit areas went black. A pool read only through the table is rarely a - // mirrored one (mirrors are made for pools bound as buffers, which sync then). + // mirrored one (mirrors are made for pools bound as buffers, which sync then). With shadow storage every pool the + // table names is a copy of guest memory, and nothing resolves the pools such a reader reaches: bind_resources + // marks the part (vk_mem_gpa_use) and its prologue brings those copies up to date (vk_memory.c vk_mem_gpa_sync). if (r->cbuf_slot == VK_GPA_CB_SLOT) { VkDescriptorBufferInfo t = vk_gpa_info(); return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE}; @@ -2905,6 +2907,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { static VkDescSlot d[MAX_DESC]; static VkBufferView bv[MAX_DESC]; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); + if (info->gpa) vk_mem_gpa_use(); // before the early returns for reused descriptors: the shader still reads the table bind_stats(info); int gfx = info->bind_point == VK_PIPELINE_BIND_POINT_GRAPHICS; int skip = gfx ? bind_skip_on() : 0, same = 0, was_valid = B.res_valid; @@ -3895,6 +3898,7 @@ static void mo_emit(VkCommandBuffer cb) { for (int n = 0; n < MQ.n; p = &MQ.d[n++]) { const MoDraw *q = &MQ.d[n]; const VkPipeInfo *info = q->info; + if (info->gpa) vk_mem_gpa_use(); // these draws bind packed descriptors without bind_resources, maybe in a later part if (q->pipe != pipe) vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_GRAPHICS, q->pipe), pipe = q->pipe, MO.e_pipe++; if (q->st != st) vk_pipeline_set_dynamic(cb, &MQ.st[q->st]), st = q->st, MO.e_dyn++; if (!p || memcmp(&q->v, &p->v, sizeof q->v)) vkCmdSetViewportWithCount(cb, 1, &q->v); diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c index 41bc8253..6bd4732a 100644 --- a/runtime/vk_gpa.c +++ b/runtime/vk_gpa.c @@ -35,7 +35,10 @@ // // NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0). // NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis). -// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). +// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). With shadow storage the +// pools are copies of guest memory: a part whose shaders use the table is marked (vk_exec_draw.c bind_resources) and +// its prologue copies the pages the CPU wrote into the copies the table names (vk_memory.c vk_mem_gpa_sync). What a +// shader stores through the table into a shadow copy is not noted as a GPU write and does not come back to guest memory. #include #include diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index fc2f8e6d..a101939b 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -322,8 +322,9 @@ int vk_pool_init(pool_t *p) { phys_list[nphys++] = r; } plat_mutex_unlock(&pool_lock); - // The imported pages go into the device address table: shaders that read a pointer from memory reach them there. - if (r->kind == POOL_IMPORT && r->mem && r->buf && r->import_hi > r->import_lo) + // The pool's pages go into the device address table: shaders that read a pointer from memory reach them there. An + // import and a shadow copy both have buf; only the import owns mem. + if (r->kind == POOL_IMPORT && r->buf && r->import_hi > r->import_lo) r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo); return 1; } @@ -471,7 +472,8 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touc // touch: the GPU may read the pool's buffer in the submission being recorded, so its shadow goes on the touched list // (shadow storage). A lookup (vk_mem_lookup) leaves it off: the caller reads guest memory itself or only asks what -// backs the address. A pool in the device address table is touched either way, as shaders reach it through pointers. +// backs the address. Shaders that reach a pool through the device address table resolve nothing: the part they run in +// brings every pool of the table up to date (vk_mem_gpa_sync). static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) { memset(out, 0, sizeof *out); uint64_t g = __atomic_load_n(&pool_gen, __ATOMIC_ACQUIRE), end = addr + size; @@ -484,8 +486,7 @@ static int mem_resolve(uint64_t addr, uint64_t size, VkMemSpan *out, int touch) out->cls = h->cls; out->pool = h->pool; if (h->alloc) out->alloc_offset = out->offset; - if (shadow_on && h->pool && !((PoolRec *)h->pool)->stouched && (touch || ((PoolRec *)h->pool)->gpa_rec)) - shadow_touch(h->pool); + if (shadow_on && touch && h->pool && !((PoolRec *)h->pool)->stouched) shadow_touch(h->pool); if (resolve_check_on()) { VkMemSpan s; if (!resolve_locked(addr, size, &s, touch) || memcmp(&s, out, sizeof s)) @@ -522,7 +523,7 @@ static int resolve_locked(uint64_t addr, uint64_t size, VkMemSpan *out, int touc out->buffer = r->buf; out->offset = addr - r->import_lo; out->avail = r->import_hi - addr; - if (r->smap && !r->stouched && (touch || r->gpa_rec)) shadow_touch(r); + if (r->smap && touch && !r->stouched) shadow_touch(r); ok = 1; } else if (r && r->kind == POOL_PHYS) { out->buffer = r->buf; @@ -979,7 +980,8 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // invalidates the buffer, about 1.3 ms per 16 MiB at the next submit, and the import's write pinning clears the // write watch), a CPU-visible pool gets a host-visible buffer of its own instead, at the offsets the import would have: // - CPU -> shadow: right before each submit (vk_mem_shadow_sync), the pools the replay resolved since the last one are -// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. +// queried for written pages; pages stamped after a pool's sgen are copied in by the CPU. A part whose shaders read +// through the device address table gets every pool the table names brought up to date the same way (vk_mem_gpa_sync). // - GPU -> guest: the replay notes each GPU write into CPU-visible memory at record time (vk_mem_gpu_pending) and // reports it at the fence (vk_gpu_written), which copies the GPU's bytes back into guest memory. // - Pages both write: a GPU write is noted with the whole range its binding names, and the CPU writes other fields of the @@ -1983,6 +1985,62 @@ static void mirror_sync(PoolRec *r) { mrep.sync_t += qpc() - t0; } +// ---- shadow storage: the device address table's readers +// A shader that follows a pointer through the table (vk_gpa.c) can reach any pool the table names, and the replay +// never resolves that pool, so nothing put it on the touched list and its shadow kept the bytes of its last use as a +// buffer. vk_exec_draw.c marks a part whose pipelines declare the table (vk_mem_gpa_use: at the draw or dispatch, not +// where descriptors are written, which are reused), and the part's prologue (vk_sync.c) calls vk_mem_gpa_sync: the +// shadows of every pool in the table get the pages the CPU wrote, and those of them that have a VRAM mirror, which +// the table then names, queue the same pages for upload unless this submission synced the mirror already. No mirror +// is made for a reader of the table. Without shadow storage nothing is done: an imported pool is guest memory, and +// mirrors stay unsynced for the table's readers (vk_exec_draw.c ssbo_info). +static int gpa_part_used; // replay thread: the part being recorded uses the table +void vk_mem_gpa_use(void) { gpa_part_used = 1; } +int vk_mem_shadow_storage(void) { return shadow_on; } + +static int gpa_shadow_pool(const PoolRec *r) { return r->kind == POOL_IMPORT && r->gpa_rec && r->smap; } + +void vk_mem_gpa_sync(void) { + if (!gpa_part_used) return; + gpa_part_used = 0; + if (!shadow_on) return; + uint64_t t0 = qpc(), run0 = 0, run1 = 0; + uint32_t epoch = vk_texture_epoch(); + int cpu = vk_mirror_on && wb_no_reup(), have = 0; // before the locks: its first call logs + plat_mutex_lock_shared(&pool_lock); + plat_mutex_lock(&shadow_lock); + plat_mutex_lock(&gen_lock); + // one write-watch query per run of regions the pools cover (the index is sorted by address) instead of one a pool; + // shadow_sync_pool then finds its regions snapped + for (int i = 0; epoch && i < pool_index.n; i++) { + const PoolRec *r = pool_index.e[i].buf; + if (!gpa_shadow_pool(r)) continue; + uint64_t a = r->import_lo >> SNAP_SHIFT, b = (r->import_hi - 1) >> SNAP_SHIFT; + if (b >= SNAP_REGIONS) continue; + if (have && a <= run1 + 1) { + if (b > run1) run1 = b; + } else { + if (have) s_queries += snap_regions(run0, run1, epoch); + run0 = a, run1 = b, have = 1; + } + } + if (have) s_queries += snap_regions(run0, run1, epoch); + for (int i = 0; i < pool_index.n; i++) { + PoolRec *r = pool_index.e[i].buf; + if (gpa_shadow_pool(r)) shadow_sync_pool(r, epoch, cpu), s_syncs++; + } + plat_mutex_unlock(&gen_lock); + // The uploads read the shadows just brought up to date, and the prologue records them ahead of the part, after + // the write-backs of the submissions before it. + for (int i = 0; i < pool_index.n; i++) { + PoolRec *r = pool_index.e[i].buf; + if (gpa_shadow_pool(r) && r->mstate == MS_ON && r->acc_sub != msub) mirror_sync(r); + } + s_ticks += qpc() - t0; + plat_mutex_unlock(&shadow_lock); + plat_mutex_unlock_shared(&pool_lock); +} + void vk_mem_use(VkMemSpan *sp) { PoolRec *r = sp->pool; if (r->mstate == MS_REFUSED && r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0 && evict_enabled()) { diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 2b3caa72..3904265b 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -48,6 +48,7 @@ #include "nvn_hmap.h" #include "platform/platform.h" #include "tex.h" +#include "vk_gpa.h" #define LOG_ONCE(...) do { static volatile int32_t once_; if (!plat_xchg32(&once_, 1)) rt_log(__VA_ARGS__); } while (0) @@ -2205,6 +2206,8 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel p->info.push1 = l ? l->push1 : 0; p->info.n_sets = l && l->set1 ? 2 : 1; p->info.pool = B ? B->pool : 0; + for (uint32_t i = 0; i < p->info.n_bindings; i++) + if (p->bindings[i].kind == GFX_RES_SSBO && p->bindings[i].res->cbuf_slot == VK_GPA_CB_SLOT) p->info.gpa = 1; if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1]; p->lay = l; Pipe **head = &pipes[h % PIPE_BUCKETS]; diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index aba2794f..85dea2f7 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -500,6 +500,7 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { // First brings the shadow storage up to date with the CPU's writes, which needs no command buffer. static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { vk_mem_shadow_sync(); // shadow storage: the CPU's writes go in before the part is submitted + vk_mem_gpa_sync(); // a part whose shaders read through the device address table never resolved the pools they reach vk_mem_lazy_part_end(); // what the part left in the mirror and has to write back after all joins the snapshots if (!vk_mem_mirror_pending()) return VK_NULL_HANDLE; int i = claim_slot(c, fence); // before reading slots: the claim may move it @@ -561,7 +562,7 @@ uint64_t vk_sync_reserve(void) { return v; } -uint64_t vk_sync_submit(const uint64_t *handles, int n) { +static uint64_t submit(const uint64_t *handles, int n, void (*probe_record)(VkCommandBuffer, void *), void *arg) { vk_sync_start(); VkCtx *c = vk_ctx(); plat_mutex_lock(&sub_lock); @@ -581,7 +582,9 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { ts_begin(s->cb); rec_begin(s->cb, fence); vk_gpu_passes_part(s->cb, 0); - int recorded = vk_exec_record(s->cb, handles, n, fence); + int recorded; + if (probe_record) { probe_record(s->cb, arg); recorded = 1; } + else recorded = vk_exec_record(s->cb, handles, n, fence); s = &slots[cur_slot]; // the last part (vk_sync_split) vk_gpu_passes_part(s->cb, 1); if (recorded) ts_end(s->cb, fence); @@ -627,6 +630,13 @@ uint64_t vk_sync_submit(const uint64_t *handles, int n) { return fence; } +uint64_t vk_sync_submit(const uint64_t *handles, int n) { return submit(handles, n, NULL, NULL); } + +// Memory probes record their shader here, not in the upload batch (which is submitted before the memory prologue). +uint64_t vk_sync_probe_submit(void (*record)(VkCommandBuffer, void *), void *arg) { + return submit(NULL, 0, record, arg); +} + void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) { plat_mutex_lock(&st_lock); if (qhead + qn == qcap) {