diff --git a/runtime/cfg.c b/runtime/cfg.c index 308a1e45..de0a8629 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -42,7 +42,7 @@ static const char *const known[] = { "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES", - "FULLSCREEN", "FXAA", "GAME_ASPECT", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", + "FULLSCREEN", "FXAA", "GAME_ASPECT", "GPA", "GPA_EMPTY", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_TEST", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", diff --git a/runtime/gfx.h b/runtime/gfx.h index f9e498c9..0a7752da 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -54,7 +54,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8 uint64_t gfx_shader_async_done(void); // translations finished by the background worker void gfx_shader_counts(uint64_t *keys, uint64_t *translated); // distinct keys in memory, translations done (NX2NAVI_TRACE=perf) // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs. -void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals); +void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, + int bda); const uint8_t *gfx_shader_key(const GfxShader *s); // the 32-byte cache key (stable while the translator is) const uint8_t *gfx_shader_input_id(const GfxShader *s); // the 32-byte input id (stable across translators) // By input id from memory or disk; xlate 1 also translates it from its stored inputs. NULL if absent. diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 85fa0571..e918809c 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -120,7 +120,7 @@ static struct { // translations in fl static char cache_dir[PATH_CAP]; static char xlate_exe[PATH_CAP]; static uint8_t xlate_id[32]; // translator identity (see the keys at the top) -static int host_f64 = 1, host_zero_locals, host_set; // gfx_shader_set_host +static int host_f64 = 1, host_zero_locals, host_bda, host_set; // gfx_shader_set_host static uint32_t host_sbuf, host_sg[3] = {32, 32, 32}, host_gather; static PlatOnce paths_once = PLAT_ONCE_INIT; @@ -316,17 +316,20 @@ static void paths(void) { plat_once(&paths_once, init_paths, NULL); } // Host answers (vk_caps.c, at device creation, before any translation). Answers other than the defaults (Float64, // exact storage offsets, 32 lanes, no gather bias, undefined locals) are mixed into the translator identity, so they // get their own cache keys. -void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals) { +void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals, + int bda) { paths(); if (host_set++) return; - host_f64 = float64, host_sbuf = sbuf_align, host_gather = gather_bits, host_zero_locals = zero_locals; + host_f64 = float64, host_sbuf = sbuf_align, host_gather = gather_bits, host_zero_locals = zero_locals, host_bda = bda; memcpy(host_sg, subgroup, sizeof host_sg); - if (float64 && !sbuf_align && subgroup[0] == 32 && subgroup[1] == 32 && subgroup[2] == 32 && !gather_bits && !zero_locals) return; + if (float64 && !sbuf_align && subgroup[0] == 32 && subgroup[1] == 32 && subgroup[2] == 32 && !gather_bits && !zero_locals && !bda) + return; uint8_t buf[32 + 96]; int n = snprintf((char *)buf + 32, 96, "TKHO f64=%d sbuf=%u sg=%u,%u,%u", float64, sbuf_align, subgroup[0], subgroup[1], subgroup[2]); if (gather_bits) n += snprintf((char *)buf + 32 + n, (size_t)(96 - n), " gather=%u", gather_bits); if (zero_locals) n += snprintf((char *)buf + 32 + n, (size_t)(96 - n), " zero_locals"); + if (bda) n += snprintf((char *)buf + 32 + n, (size_t)(96 - n), " bda"); memcpy(buf, xlate_id, 32); rt_sha256(buf, 32 + (size_t)n, xlate_id); rt_log("gfx_shader: host options %s: own shader cache keys", (char *)buf + 37); @@ -1194,6 +1197,7 @@ static size_t build_request(char *o0, const char *hex, const char *stage, const if (host_sg[si] != 32) o += sprintf(o, ",\"subgroup\":%u", host_sg[si]); if (host_gather) o += sprintf(o, ",\"gather_bias\":%u", host_gather); if (host_zero_locals) o += sprintf(o, ",\"zero_locals\":true"); + if (host_bda) o += sprintf(o, ",\"bda\":true"); // the device address table (vk_gpa.c) uint32_t magic, nvn2_size; memcpy(&magic, control, 4), memcpy(&nvn2_size, control + 0x38, 4); if (magic == 0x98761234u) { // NVN2_CONTROL_MAGIC, below diff --git a/runtime/vk_caps.c b/runtime/vk_caps.c index 499329e9..63bc0f22 100644 --- a/runtime/vk_caps.c +++ b/runtime/vk_caps.c @@ -19,6 +19,7 @@ #include "gfx.h" #include "host.h" #include "vk.h" +#include "vk_gpa.h" VkCaps vk_caps = {.float64 = 1, .sg_size = 32}; static int f64_supported; @@ -124,7 +125,8 @@ void vk_caps_init(VkCtx *c) { // undefined off Mesa, as before: zeroing moved that hash and the picture shook under DLAA (maxwell.md 3.12). // NX2NAVI_ZERO_LOCALS=1 / 0 forces either. uint32_t zero_locals = env_u32("NX2NAVI_ZERO_LOCALS", vk_caps.mesa || !cfg_bool("NX2NAVI_DLAA")); - gfx_shader_set_host(vk_caps.float64, vk_caps.sbuf_align, told, gather, zero_locals != 0); + // bda: global accesses the translator finds no storage buffer for go through the device address table (vk_gpa.c) + gfx_shader_set_host(vk_caps.float64, vk_caps.sbuf_align, told, gather, zero_locals != 0, vk_gpa_wanted_ctx(c)); if (gather) rt_log("vk caps: gather coordinates biased for %u sub-texel bits (NX2NAVI_GATHER_BIAS)", gather); char dv[32]; diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index fd6b8abe..4b8586fb 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -28,6 +28,7 @@ #define NX2NAVI_TONEMAP_STANDIN 1 // vk_tonemap.c, see untranslated_dispatch_fallback #endif #include "vk_exec.h" +#include "vk_gpa.h" #include "platform/platform.h" #include "vk_dlaa.h" @@ -1263,6 +1264,14 @@ uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r) { } static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) { + // The device address table (vk_gpa.c, translator patch 0030). Mirrored pools are not synced for its readers: a sync + // of every mirror at a table reader's draw copied stale guest bytes over what the GPU had written into the mirrors + // earlier in the frame, and Kokiri Forest's lit areas went black. A pool read only through the table is rarely a + // mirrored one (mirrors are made for pools bound as buffers, which sync then). + if (r->cbuf_slot == VK_GPA_CB_SLOT) { + VkDescriptorBufferInfo t = vk_gpa_info(); + return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE}; + } if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.sbo_max, align = LIM.sbo_align; VkDescriptorBufferInfo bi; diff --git a/runtime/vk_gpa.c b/runtime/vk_gpa.c new file mode 100644 index 00000000..691901d7 --- /dev/null +++ b/runtime/vk_gpa.c @@ -0,0 +1,259 @@ +// The device address table: GPU address pages to Vulkan buffer device addresses, for the global memory accesses the +// translator finds no storage buffer for (translator patch 0030: GlobalToStorage TryKeepDeviceAddressAccess, and the +// SPIR-V back end's GetDeviceAddressPointer, which reads this layout). Ocarina's foliage, instance and light data reach +// their buffers through pointers the shaders read from other buffers at an index (a phi of constant-buffer pointers and +// loaded ones, a pointer spilled to local memory and loaded again); those accesses read 0 before, which drew interiors +// and shadowed areas black and dropped instanced foliage. +// +// One storage buffer of 32-bit words, bound whole to every shader that declares it (res.json "table", cb_slot 0x40): +// words 0-1 the device address of a zeroed 4 KiB buffer, which an address no page maps reads and writes instead +// word 2 the word index of the records (REC_WORD) +// words 16.. L1: 65536 entries, one per 16 MiB of the 40-bit GPU address space, the index of its L2 block (0: none) +// word 65552 L2 blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds the page (0: none) +// REC_WORD records of two words: the buffer's device address minus the GPU address of its byte 0 (low word first) +// A record a page entry names is never written: a new mapping takes a new record, and a record goes back to the free +// list only once the GPU finished every submission issued while a page could still name it (vk_completion_add). Page +// and L1 entries are single 32-bit stores, so a shader that reads the table while it changes sees the old mapping or +// the new one, never half of each. The CPU keeps a copy (shadow) for its reads: the GPU copy is often device memory +// behind the PCIe BAR, slow to read back. +// +// NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0). +// NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis). +// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why). +#include +#include + +#include "cfg.h" +#include "host.h" +#include "vk.h" +#include "vk_gpa.h" +#include "platform/platform.h" + +#define L1_WORD 16u +#define L1_N 65536u +#define L2_WORD (L1_WORD + L1_N) +#define L2_N 4096u // pages per block: 16 MiB +#define BLOCKS 1024u // L2 blocks: 16 GiB of mapped GPU address space; block 0 stays zero +#define REC_WORD (L2_WORD + BLOCKS * L2_N) +#define RECS 65536u // record 0 means none +#define WORDS (REC_WORD + RECS * 2u) +#define PAGE_SHIFT 12 + +static struct { + int state; // 0 untried, 1 on, -1 off + VkBuffer buf, dummy; + VmaAllocation alloc, dummy_alloc; + volatile uint32_t *gpu; // the mapped table + uint32_t *cpu; // its shadow + uint32_t nblocks; // L2 blocks handed out (block 0 is the zero block) + uint32_t nrec; // records handed out, free ones aside + uint32_t *free_recs, nfree; // records whose fence completed + uint64_t mapped_pages, maps, unmaps, refused; +} G; +static PlatMutex gpa_lock = PLAT_MUTEX_INIT; + +static void put(uint32_t word, uint32_t v) { + G.cpu[word] = v; + __atomic_store_n(&G.gpu[word], v, __ATOMIC_RELAXED); +} + +int vk_gpa_wanted_ctx(const VkCtx *c) { + static int knob = -1; + if (knob < 0) { + const char *e = cfg_get("NX2NAVI_GPA"); + knob = !(e && !strcmp(e, "0")); + } + return knob && c && c->has_buffer_device_address; +} + +int vk_gpa_wanted(void) { return vk_gpa_wanted_ctx(vk_ctx()); } + +uint64_t vk_gpa_device_address(VkBuffer b) { + VkBufferDeviceAddressInfo ai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = b}; + return b ? vkGetBufferDeviceAddress(vk_ctx()->device, &ai) : 0; +} + +static int create(void) { + VkCtx *c = vk_ctx(); + if (!vk_gpa_wanted()) return 0; + G.cpu = calloc(WORDS, 4); + G.free_recs = malloc(RECS * sizeof *G.free_recs); + if (!G.cpu || !G.free_recs) return 0; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)WORDS * 4, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT, + .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VmaAllocationInfo info; + if (vmaCreateBuffer(c->vma, &bi, &ai, &G.buf, &G.alloc, &info) != VK_SUCCESS || !info.pMappedData) { + rt_log("vk_gpa: the %.1f MiB table could not be allocated: global accesses through pointers read 0", WORDS * 4 / 1048576.0); + return 0; + } + G.gpu = info.pMappedData; + memset((void *)G.gpu, 0, (size_t)WORDS * 4); + VkBufferCreateInfo di = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 4096, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VmaAllocationInfo dinfo; + if (vmaCreateBuffer(c->vma, &di, &ai, &G.dummy, &G.dummy_alloc, &dinfo) != VK_SUCCESS || !dinfo.pMappedData) { + rt_log("vk_gpa: the zero buffer could not be allocated"); + return 0; + } + memset(dinfo.pMappedData, 0, 4096); + uint64_t z = vk_gpa_device_address(G.dummy); + put(0, (uint32_t)z), put(1, (uint32_t)(z >> 32)), put(2, REC_WORD); + G.nblocks = 1, G.nrec = 1; + rt_log("vk_gpa: device address table on (%.1f MiB, %u blocks of 16 MiB, %u records)", WORDS * 4 / 1048576.0, BLOCKS, RECS); + return 1; +} + +int vk_gpa_on(void) { + if (__atomic_load_n(&G.state, __ATOMIC_ACQUIRE)) return G.state > 0; + plat_mutex_lock(&gpa_lock); + if (!G.state) __atomic_store_n(&G.state, create() ? 1 : -1, __ATOMIC_RELEASE); + plat_mutex_unlock(&gpa_lock); + return G.state > 0; +} + +VkDescriptorBufferInfo vk_gpa_info(void) { + if (!vk_gpa_on()) return (VkDescriptorBufferInfo){0}; + return (VkDescriptorBufferInfo){G.buf, 0, VK_WHOLE_SIZE}; +} + +// The L2 entry word of a page; 0 when its block does not exist and alloc is 0 or no block is left. +static uint32_t page_word(uint64_t page, int alloc) { + uint64_t l1 = page >> 12; // 4096 pages of 4 KiB per 16 MiB + if (l1 >= L1_N) return 0; + uint32_t blk = G.cpu[L1_WORD + l1]; + if (!blk) { + if (!alloc || G.nblocks == BLOCKS) return 0; + blk = G.nblocks++; + put(L1_WORD + (uint32_t)l1, blk); + } + return L2_WORD + blk * L2_N + (uint32_t)(page & (L2_N - 1)); +} + +static uint32_t rec_alloc(uint64_t delta) { + uint32_t r; + if (G.nfree) r = G.free_recs[--G.nfree]; + else if (G.nrec < RECS) r = G.nrec++; + else return 0; + // The record before any page names it: the page store below is ordered after these by the release fence. + put(REC_WORD + r * 2, (uint32_t)delta); + put(REC_WORD + r * 2 + 1, (uint32_t)(delta >> 32)); + __atomic_thread_fence(__ATOMIC_RELEASE); + return r; +} + +static void rec_free_cb(void *arg) { + uint32_t r = (uint32_t)(uintptr_t)arg; + plat_mutex_lock(&gpa_lock); + G.free_recs[G.nfree++] = r; + plat_mutex_unlock(&gpa_lock); +} + +uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo) { + if (!vk_gpa_on() || !buf || hi <= lo) return 0; + static int empty = -1; // NX2NAVI_GPA_EMPTY=1: nothing is mapped, every access reads the zero buffer (diagnosis) + if (empty < 0) { const char *e = cfg_get("NX2NAVI_GPA_EMPTY"); empty = e && !strcmp(e, "1"); } + if (empty) return 0; + uint64_t dev = vk_gpa_device_address(buf); + if (!dev) return 0; + // Pages the buffer holds whole: [lo, hi) rounded inwards (imports are aligned outwards, so this is all of them) + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + if (p1 <= p0) return 0; + plat_mutex_lock(&gpa_lock); + uint32_t r = rec_alloc(dev - buf_lo); + if (!r) { + G.refused++; + plat_mutex_unlock(&gpa_lock); + if (G.refused == 1) rt_log("vk_gpa: out of records: a buffer at %#llx is not in the table", (unsigned long long)lo); + return 0; + } + uint64_t n = 0; + for (uint64_t p = p0; p < p1; p++) { + uint32_t w = page_word(p, 1); + if (!w) { + if (!G.refused++) rt_log("vk_gpa: out of L2 blocks at %#llx", (unsigned long long)(p << PAGE_SHIFT)); + break; + } + put(w, r); + n++; + } + G.mapped_pages += n, G.maps++; + plat_mutex_unlock(&gpa_lock); + return r; +} + +void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint64_t page_addr)) { + if (!rec || G.state <= 0) return; + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + for (uint64_t p = p0; p < p1; p++) { + uint32_t w = page_word(p, 0); + if (w && G.cpu[w] == rec) put(w, 0), G.mapped_pages--; + } + G.unmaps++; + plat_mutex_unlock(&gpa_lock); + // Pages another live buffer also holds whole (overlapping imports of the same guest memory) get its record back. + if (cover) { + for (uint64_t p = p0; p < p1; p++) { + plat_mutex_lock(&gpa_lock); + uint32_t w = page_word(p, 0), now = w ? G.cpu[w] : 0; + plat_mutex_unlock(&gpa_lock); + if (!w || now) continue; + uint32_t other = cover(p << PAGE_SHIFT); + if (!other || other == rec) continue; + plat_mutex_lock(&gpa_lock); + if (!G.cpu[w]) put(w, other), G.mapped_pages++; + plat_mutex_unlock(&gpa_lock); + } + } + vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)rec); +} + +void vk_gpa_clear(uint64_t lo, uint64_t hi) { + if (G.state <= 0 || hi <= lo) return; + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + for (uint64_t p = p0; p < p1; p++) { + uint32_t w = page_word(p, 0); + if (w && G.cpu[w]) put(w, 0), G.mapped_pages--; + } + plat_mutex_unlock(&gpa_lock); +} + +uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) { + if (!vk_gpa_on() || !buf) return old_rec; + uint64_t dev = vk_gpa_device_address(buf); + uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT; + plat_mutex_lock(&gpa_lock); + uint32_t r = dev ? rec_alloc(dev - buf_lo) : 0; + if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; } + for (uint64_t p = p0; p < p1; p++) { + uint32_t w = page_word(p, 1); + if (w && (!G.cpu[w] || G.cpu[w] == old_rec)) { + if (!G.cpu[w]) G.mapped_pages++; + put(w, r); + } + } + plat_mutex_unlock(&gpa_lock); + if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec); + return r; +} + +void vk_gpa_report(void) { + if (G.state <= 0) return; + rt_log("vk_gpa: %llu pages (%.0f MiB) mapped, %u of %u blocks, %u records in use, %llu maps, %llu unmaps, %llu refused", + (unsigned long long)G.mapped_pages, G.mapped_pages * 4096.0 / 1048576.0, G.nblocks, BLOCKS, G.nrec - 1 - G.nfree, + (unsigned long long)G.maps, (unsigned long long)G.unmaps, (unsigned long long)G.refused); +} + +void vk_gpa_shutdown(void) { + VkCtx *c = vk_ctx(); + if (G.state <= 0 || !c) return; + vmaDestroyBuffer(c->vma, G.buf, G.alloc); + vmaDestroyBuffer(c->vma, G.dummy, G.dummy_alloc); + free(G.cpu), free(G.free_recs); + memset(&G, 0, sizeof G); + G.state = -1; +} diff --git a/runtime/vk_gpa.h b/runtime/vk_gpa.h new file mode 100644 index 00000000..ad5965ae --- /dev/null +++ b/runtime/vk_gpa.h @@ -0,0 +1,24 @@ +// The device address table (vk_gpa.c): GPU address pages to buffer device addresses for shaders' global memory accesses. +#pragma once +#include + +#include "vk.h" + +#define VK_GPA_CB_SLOT 0x40 // res.json cb_slot of the table (translator ResourceManager.DeviceAddressTableCbSlot) + +int vk_gpa_wanted(void); // the device has buffer device addresses and NX2NAVI_GPA is not 0 +int vk_gpa_wanted_ctx(const VkCtx *c); // the same during device creation, before vk_ctx() returns it +int vk_gpa_on(void); // the table exists (created on first use) +uint64_t vk_gpa_device_address(VkBuffer b); +VkDescriptorBufferInfo vk_gpa_info(void); // the whole table, for a storage descriptor +// Maps the pages [lo, hi) holds whole to buf, whose byte 0 is GPU address buf_lo. Returns the record (0: not mapped). +uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo); +// Unmaps the pages of [lo, hi) that name rec; cover (may be NULL) gives the record of another live buffer that holds a +// page whole, or 0. rec is reused once the GPU finished the submissions issued until now. +void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint64_t page_addr)); +// Unmaps every page of [lo, hi), whatever record it names (pages of a virtual pool whose physical memory goes away). +void vk_gpa_clear(uint64_t lo, uint64_t hi); +// Moves the pages of [lo, hi) that name old_rec (or none) to buf (a pool's VRAM mirror); returns the new record. +uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo); +void vk_gpa_report(void); +void vk_gpa_shutdown(void); diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 9df2a358..50661aa3 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -19,6 +19,7 @@ #include "nvn_pool_index.h" #include "vk.h" #include "vk_exec.h" +#include "vk_gpa.h" #include "wb_merge_spv.h" #include "platform/platform.h" @@ -55,6 +56,9 @@ typedef struct PoolRec { uint8_t untracked; // the last dirty scan found no write watch over it // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only. uint8_t mstate; // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED + uint32_t gpa_rec; // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none + struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps; // POOL_VIRT: its mapped ranges in the table + int nvmaps, cvmaps; VkBuffer mbuf; // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf VmaAllocation malloc_; uint32_t acc_sub; // replay: the last submission that synced the mirror (mirror_sync) @@ -102,6 +106,7 @@ static VkBufferUsageFlags pool_usage(const VkCtx *c) { if (c->has_transform_feedback) u |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT; if (c->has_conditional_rendering) u |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT; + if (vk_gpa_wanted()) u |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; // pages of the device address table return u; } @@ -153,8 +158,11 @@ static int import_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) { VkImportMemoryHostPointerInfoEXT im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, .pHostPointer = (void *)(uintptr_t)r->import_lo}; - VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &im, .allocationSize = n, - .memoryTypeIndex = (uint32_t)type}; + // A buffer with device address usage needs memory allocated for device addresses (vk_gpa.c). + VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im, + .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT}; + VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = vk_gpa_wanted() ? (void *)&fi : (void *)&im, + .allocationSize = n, .memoryTypeIndex = (uint32_t)type}; if ((res = vkAllocateMemory(c->device, &ai, NULL, &r->mem)) != VK_SUCCESS || (res = vkBindBufferMemory(c->device, r->buf, r->mem, 0)) != VK_SUCCESS) { rt_log("vk: pool %#llx+%#llx: host memory import failed (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res); @@ -295,9 +303,28 @@ int vk_pool_init(pool_t *p) { phys_list[nphys++] = r; } plat_mutex_unlock(&pool_lock); + // The imported pages go into the device address table: shaders that read a pointer from memory reach them there. + if (r->kind == POOL_IMPORT && r->mem && r->buf && r->import_hi > r->import_lo) + r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo); return 1; } +// vk_gpa_unmap: the record of another live imported pool that holds the 4 KiB page at a whole (overlapping imports +// of the same guest memory), 0 for none. +static uint32_t gpa_cover(uint64_t a) { + uint32_t rec = 0; + plat_mutex_lock_shared(&pool_lock); + for (int i = pidx_upper(&pool_index, a) - 1; i >= 0 && !rec; i--) { + const PoolRec *o = pool_index.e[i].buf; + if (o->kind == POOL_IMPORT && o->gpa_rec && o->import_lo <= a && o->import_hi >= a + 4096) rec = o->gpa_rec; + if (i && pool_index.maxhi[i - 1] <= a) break; + } + plat_mutex_unlock_shared(&pool_lock); + return rec; +} + +static void gpa_retire(void *arg) { destroy_rec(vk_ctx(), arg); } + void vk_pool_fini(pool_t *p) { VkCtx *c = vk_ctx(); if (!c) return; @@ -314,12 +341,19 @@ void vk_pool_fini(pool_t *p) { PoolRec *v = pool_index.e[i].buf; if (v->kind != POOL_VIRT) continue; for (uint64_t k = 0; k < v->npages; k++) - if (v->pt[k].phys == r) v->pt[k].phys = NULL; + if (v->pt[k].phys == r) { + v->pt[k].phys = NULL; + vk_gpa_clear(v->lo + k * NVN_VPOOL_PAGE, v->lo + (k + 1) * NVN_VPOOL_PAGE); // the buffer goes away + } } } plat_mutex_unlock(&pool_lock); if (!r) return; if (r->smap) shadow_forget(r); + // Off the device address table first; the buffer outlives the submissions that may still read the old pages. + if (r->gpa_rec) vk_gpa_unmap(r->import_lo, r->import_hi, r->gpa_rec, gpa_cover); + for (int i = 0; i < r->nvmaps; i++) vk_gpa_unmap(r->vmaps[i].lo, r->vmaps[i].hi, r->vmaps[i].rec, NULL); + free(r->vmaps), r->vmaps = NULL, r->nvmaps = 0; if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size); // the storage goes back to the guest writable // Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the // guest gets the storage back when this returns. @@ -334,9 +368,29 @@ void vk_pool_fini(pool_t *p) { // A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may // name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued). if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r); + else if (r->gpa_rec && !r->used) vk_completion_add(vk_sync_issued() + 1, gpa_retire, r); // the queue was not drained else destroy_rec(c, r); } +// pool_lock held: the device address table follows a MapVirtual request on [lo, hi) of virtual pool v. Ranges it +// covers whole leave the table (their records retire); the pages then name the physical pool's buffer at poff, or none. +static void gpa_map_virtual(PoolRec *v, uint64_t lo, uint64_t hi, PoolRec *ph, uint64_t poff) { + if (!vk_gpa_wanted()) return; + for (int i = 0; i < v->nvmaps;) + if (v->vmaps[i].lo >= lo && v->vmaps[i].hi <= hi) { + vk_gpa_unmap(v->vmaps[i].lo, v->vmaps[i].hi, v->vmaps[i].rec, NULL); + v->vmaps[i] = v->vmaps[--v->nvmaps]; + } else { + i++; + } + vk_gpa_clear(lo, hi); + if (!ph || !ph->buf) return; + uint32_t rec = vk_gpa_map(lo, hi, ph->buf, lo - poff); + if (!rec) return; + if (v->nvmaps == v->cvmaps) grow((void **)&v->vmaps, &v->cvmaps, sizeof *v->vmaps); + v->vmaps[v->nvmaps++] = (struct GpaMap){lo, hi, rec}; +} + // Requests were checked by the front end (nvn_api_core.c); apply them in order. int vk_pool_map_virtual(pool_t *v, int n, const void *requests) { const nvn_mapping_t *rq = requests; @@ -357,6 +411,8 @@ int vk_pool_map_virtual(pool_t *v, int n, const void *requests) { if (first + count > r->npages) { ok = 0; break; } for (uint64_t k = 0; k < count; k++) r->pt[first + k] = (VPage){ph, ph ? (uint64_t)rq[i].physical_offset + k * NVN_VPOOL_PAGE : 0}; + gpa_map_virtual(r, r->lo + first * NVN_VPOOL_PAGE, r->lo + (first + count) * NVN_VPOOL_PAGE, ph, + (uint64_t)rq[i].physical_offset); } plat_mutex_unlock(&pool_lock); if (!ok) rt_log("vk: MapVirtual on pool %p failed", (void *)v->mem); @@ -1344,6 +1400,8 @@ static void mirror_create(PoolRec *r) { plat_mutex_unlock(&pool_lock); r->acc_sub = msub; r->mstate = MS_ON; + // The device address table names the mirror from now on: the GPU's writes go there, and the sync copies the CPU's. + if (r->gpa_rec) r->gpa_rec = vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo); range_add(&pend, &npend, &cpend, r, 0, n); mrep.first_n++, mrep.first_bytes += n; plat_inc64(&m_live_n); @@ -2009,6 +2067,7 @@ static void mirror_log(uint64_t frame) { snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld", (long long)plat_xchg64(&v_pages, 0), (long long)plat_xchg64(&v_skipped, 0), (long long)plat_xchg64(&v_mismatch, 0)); + vk_gpa_report(); rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked " "%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, " "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches " @@ -2363,6 +2422,8 @@ void vk_mem_shutdown(void) { vk_queue_lock(); vkDeviceWaitIdle(c->device); vk_queue_unlock(); + vk_gpa_report(); + vk_gpa_shutdown(); vk_texture_shutdown(); plat_rmutex_lock(&cmd_lock); retire(c, 1); diff --git a/tools/build_xlate.py b/tools/build_xlate.py index 2e0768f1..9737014e 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 28: - sys.exit(f"expected 28 patches, found {len(patches)}") + if len(patches) != 29: + sys.exit(f"expected 29 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/xlate/AmpereLowering.cs b/tools/xlate/AmpereLowering.cs index 1f0707bd..aa9ed7db 100644 --- a/tools/xlate/AmpereLowering.cs +++ b/tools/xlate/AmpereLowering.cs @@ -428,6 +428,8 @@ static class AmpereLowering case 0x3c4: Redux(); return; case 0x81a: Sgxt(); return; case 0x806: case 0x886: Vote(op == 0x886); return; + case 0x3a1: Match(); return; + case 0xf8c: AtomsPopcInc(); return; case 0x3a8: case 0x9a8: Atomg(op == 0x9a8); return; case 0x38c: Atoms(); return; case 0x98e: Red(); return; @@ -1359,6 +1361,47 @@ static class AmpereLowering if (signedOrder) Lop32i(t, t, 0x80000000, 2); } + // MATCH.ALL Pd, Rd, Ra (op 0x3a1 with no mode bits; Ocarina's NVN2 compute programs test subgroupAllEqual with it + // before an aggregated shared atomic: @!Pd per-lane ATOMS, else REDUX and one): Pd = every active lane holds Ra's + // value, Rd = the active-lane mask then, else 0. Lowered as VOTE.ANY (the active lanes), FLO (the highest of + // them), SHFL.IDX (that lane's value), ISETP.EQ into Pd and VOTE.ALL over Pd. Only the unguarded form with a + // predicate destination is taken. + void Match() + { + uint pd = _i.Bits(81, 3); + if (_i.W1 != 0 || (_i.W2 & ~(7u << 17)) != 0 || _i.Bits(12, 4) != 7 || pd == PT) + throw Fail("MATCH mode"); + int d = Gpr((uint)Dst), src = Gpr(_i.Bits(24, 8)); + const int ballot = 240, lead = 241, value = 242; + Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY + Emit(Unguarded(0x5c30000000000000UL | ((ulong)ballot << 20) | lead)); // FLO.U32 + Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20) + | ((ulong)src << 8) | value)); // SHFL.IDX + Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)PT << 39) | ((ulong)value << 20) + | ((ulong)src << 8) | ((ulong)pd << 3) | PT)); // ISETP.EQ.U32 + Emit(Unguarded(0x50d8000000000000UL | ((ulong)pd << 45) | ((ulong)pd << 39) | RZ)); // VOTE.ALL + if (d != RZ) Sel(d, ballot, RZ, pd, false); + } + + // ATOMS.POPC.INC RZ, [Ra + URb + offset] (op 0xf8c, mode word 0x0d000000; the increment of Ocarina's aggregated + // shared counters after MATCH and REDUX): memory goes up by the number of active lanes. With no result register + // that is one ATOMS.ADD of 1 per active lane, which is what this emits; a form that wants the per-lane result (its + // rank among the active lanes) is not taken. + void AtomsPopcInc() + { + uint offset = _i.Bits(40, 24), ur = _i.Bits(64, 6); + if ((_i.W2 & ~0x3fu) != 0x0d000000u || _i.Bits(32, 8) != 0 || Dst != RZ || (offset & 3) != 0 || offset >= (1u << 22)) + throw Fail("ATOMS.POPC.INC mode"); + int addr = Gpr(_i.Bits(24, 8)); + if (ur != 0x3f) + { + Iadd3(SA, addr, Ur(ur), RZ, false); + addr = SA; + } + Mov32i(SB, 1); + EmitG(0xec00000000000000UL | RZ | ((ulong)addr << 8) | ((ulong)SB << 20) | ((ulong)(offset >> 2) << 30)); // ATOMS.ADD + } + // VOTE Rd, Pu, Pin: ballot into Rd (URd for VOTEU), mode 72-73 (ALL, ANY, EQ) over the vector predicate of bits // 87-90, result predicate 81-83 (a uniform predicate for VOTEU). Maxwell VOTE has the same fields. VOTEU's // output goes through Pk (k its UP index) with the guest's Pk saved around it; no uniform predicate is read. diff --git a/tools/xlate/Program.cs b/tools/xlate/Program.cs index 08ae3b0c..8bca05a2 100644 --- a/tools/xlate/Program.cs +++ b/tools/xlate/Program.cs @@ -129,6 +129,9 @@ static class SwshXlate public bool QueryHostSupportsNonConstantTextureOffset() => false; // "softfloat64": true lowers FP64 operations to FP32 (the translator's DoubleToFloat pass), for a host without shaderFloat64. public bool QueryHostSupportsShaderFloat64() => _job["softfloat64"] is not JsonValue v || !(v.TryGetValue(out bool b) && b); + // The runtime binds the device address table (runtime/vk_gpa.c) when the device has buffer device addresses: + // global accesses the translator finds no storage buffer for go through it (patch 0030). + public bool QueryHostSupportsBufferDeviceAddress() => _job["bda"] is JsonValue v && v.TryGetValue(out bool b) && b; public bool QueryHostSupportsTextureShadowLod() => true; // Sub-texel precision bits of a host whose gathers round differently from Maxwell (AMD, Intel): the translator then // biases gather coordinates so the same texels are picked (TexturePass.InsertCoordGatherBias). 0: no bias. @@ -234,6 +237,7 @@ static class SwshXlate var o = new JsonObject { ["binding"] = b.Binding, ["slot"] = b.Slot, ["cb_slot"] = pointer ? b.SbCbSlot & 0x1f : b.SbCbSlot, ["cb_offset"] = pointer ? b.SbCbOffset & 0x3ff : b.SbCbOffset, ["flags"] = (int)b.Flags }; if (pointer) o["ptr"] = (b.SbCbOffset >> 10) * 8; + if (b.SbCbSlot == 0x40) o["table"] = true; // the device address table (patch 0030, ResourceManager.DeviceAddressTableCbSlot) res["sbufs"].AsArray().Add((JsonNode)o); } foreach (var (list, name) in new[] { (p.Info.Textures, "textures"), (p.Info.Images, "images") }) diff --git a/tools/xlate/patches/0030-global-memory-through-device-addresses.patch b/tools/xlate/patches/0030-global-memory-through-device-addresses.patch new file mode 100644 index 00000000..5793ac5b --- /dev/null +++ b/tools/xlate/patches/0030-global-memory-through-device-addresses.patch @@ -0,0 +1,327 @@ +--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs ++++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs +@@ -1820,6 +1820,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + + private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore) + { ++ if (IsGlobalStorageKind(operation.StorageKind)) ++ { ++ return GenerateDeviceAddressLoadOrStore(context, operation, isStore); ++ } ++ + SpvInstruction pointer = GetStoragePointer(context, operation, out AggregateType varType); + + if (isStore) +@@ -1834,10 +1839,137 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + } + } + ++ private static bool IsGlobalStorageKind(StorageKind storageKind) ++ { ++ return storageKind is StorageKind.GlobalMemory or ++ StorageKind.GlobalMemoryS8 or ++ StorageKind.GlobalMemoryS16 or ++ StorageKind.GlobalMemoryU8 or ++ StorageKind.GlobalMemoryU16; ++ } ++ ++ // NX2NAVI: the device address table (Translation/Optimizations/GlobalToStorage.cs TryKeepDeviceAddressAccess), a ++ // storage buffer of 32-bit words the host keeps (runtime/vk_gpa.c): ++ // words 0-1 the device address of a zeroed 4 KiB buffer, for addresses no page maps ++ // word 2 the word index of the records ++ // words 16.. 65536 first-level entries, one per 16 MiB of the 40-bit GPU address space: the index of the ++ // second-level block, 0 for none (block 0 stays zero) ++ // word 65552 second-level blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds ++ // the page, 0 for none ++ // records two words each, the buffer's device address minus its GPU address (low word first); the host ++ // never changes a record a page names, so a page entry read at any time gives a whole one ++ // The pointer addresses the 32-bit word at the address's 4-byte-aligned word. ++ private static SpvInstruction GetDeviceAddressPointer(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high) ++ { ++ SpvInstruction u32 = context.TypeU32(); ++ SpvInstruction u64 = context.TypeInt(64, 0); ++ SpvInstruction tableVar = context.StorageBuffers[binding]; ++ SpvInstruction elemPtr = context.TypePointer(StorageClass.Uniform, u32); ++ SpvInstruction zero = context.Constant(context.TypeS32(), 0); ++ ++ SpvInstruction Word(SpvInstruction index) => context.Load(u32, context.AccessChain(elemPtr, tableVar, zero, index)); ++ SpvInstruction U(uint value) => context.Constant(u32, value); ++ SpvInstruction Wide(SpvInstruction lo, SpvInstruction hi) => ++ context.BitwiseOr(u64, context.UConvert(u64, lo), context.ShiftLeftLogical(u64, context.UConvert(u64, hi), U(32))); ++ ++ SpvInstruction inRange = context.ULessThan(context.TypeBool(), high, U(256)); ++ SpvInstruction l1 = context.BitwiseOr(u32, context.ShiftLeftLogical(u32, high, U(8)), context.ShiftRightLogical(u32, low, U(24))); ++ l1 = context.Select(u32, inRange, l1, U(0)); ++ SpvInstruction block = Word(context.IAdd(u32, l1, U(16))); ++ SpvInstruction page = context.BitwiseAnd(u32, context.ShiftRightLogical(u32, low, U(12)), U(0xfff)); ++ SpvInstruction record = Word(context.IAdd(u32, context.IAdd(u32, context.IMul(u32, block, U(4096)), U(65552)), page)); ++ SpvInstruction mapped = context.INotEqual(context.TypeBool(), record, U(0)); ++ SpvInstruction recordWord = context.IAdd(u32, Word(U(2)), context.IMul(u32, record, U(2))); ++ SpvInstruction deltaLow = Word(recordWord); ++ SpvInstruction deltaHigh = Word(context.IAdd(u32, recordWord, U(1))); ++ ++ SpvInstruction wordLow = context.BitwiseAnd(u32, low, U(0xfffffffc)); ++ SpvInstruction device = context.IAdd(u64, Wide(wordLow, high), Wide(deltaLow, deltaHigh)); ++ SpvInstruction dummy = context.IAdd(u64, Wide(Word(U(0)), Word(U(1))), context.UConvert(u64, context.BitwiseAnd(u32, low, U(0xffc)))); ++ SpvInstruction address = context.Select(u64, mapped, device, dummy); ++ ++ return context.ConvertUToPtr(context.TypePointer(StorageClass.PhysicalStorageBuffer, u32), address); ++ } ++ ++ private static SpvInstruction LoadAligned(CodeGenContext context, SpvInstruction type, SpvInstruction pointer) ++ { ++ SpvInstruction load = context.Load(type, pointer, MemoryAccessMask.Aligned); ++ load.AddOperand((Spv.Generator.LiteralInteger)4); ++ return load; ++ } ++ ++ private static void StoreAligned(CodeGenContext context, SpvInstruction pointer, SpvInstruction value) ++ { ++ SpvInstruction store = context.Store(pointer, value, MemoryAccessMask.Aligned); ++ store.AddOperand((Spv.Generator.LiteralInteger)4); ++ } ++ ++ // Sources: the table binding, the address's low and high words, the value for a store. 8- and 16-bit accesses ++ // read the word and extract the bytes; their stores clear and set them with two atomics. ++ private static OperationResult GenerateDeviceAddressLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore) ++ { ++ int binding = ((AstOperand)operation.GetSource(0)).Value; ++ SpvInstruction u32 = context.TypeU32(); ++ SpvInstruction low = context.Get(AggregateType.U32, operation.GetSource(1)); ++ SpvInstruction high = context.Get(AggregateType.U32, operation.GetSource(2)); ++ SpvInstruction pointer = GetDeviceAddressPointer(context, binding, low, high); ++ StorageKind kind = operation.StorageKind; ++ int bits = kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryU8 ? 8 ++ : kind is StorageKind.GlobalMemoryS16 or StorageKind.GlobalMemoryU16 ? 16 : 32; ++ ++ if (bits == 32) ++ { ++ if (isStore) ++ { ++ StoreAligned(context, pointer, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1))); ++ return OperationResult.Invalid; ++ } ++ ++ return new OperationResult(AggregateType.U32, LoadAligned(context, u32, pointer)); ++ } ++ ++ SpvInstruction shift = context.ShiftLeftLogical(u32, context.BitwiseAnd(u32, low, context.Constant(u32, bits == 8 ? 3u : 2u)), context.Constant(u32, 3)); ++ SpvInstruction fieldMask = context.Constant(u32, bits == 8 ? 0xffu : 0xffffu); ++ ++ if (isStore) ++ { ++ SpvInstruction value = context.BitwiseAnd(u32, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1)), fieldMask); ++ SpvInstruction mask = context.ShiftLeftLogical(u32, fieldMask, shift); ++ SpvInstruction scope = context.Constant(u32, 1); // device ++ SpvInstruction semantics = context.Constant(u32, 0); ++ context.AtomicAnd(u32, pointer, scope, semantics, context.Not(u32, mask)); ++ context.AtomicOr(u32, pointer, scope, semantics, context.ShiftLeftLogical(u32, value, shift)); ++ return OperationResult.Invalid; ++ } ++ ++ SpvInstruction word = LoadAligned(context, u32, pointer); ++ ++ if (kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryS16) ++ { ++ SpvInstruction s32 = context.TypeS32(); ++ SpvInstruction field = context.BitFieldSExtract(s32, context.Bitcast(s32, word), shift, context.Constant(u32, bits)); ++ return new OperationResult(AggregateType.S32, field); ++ } ++ ++ return new OperationResult(AggregateType.U32, context.BitwiseAnd(u32, context.ShiftRightLogical(u32, word, shift), fieldMask)); ++ } ++ + private static SpvInstruction GetStoragePointer(CodeGenContext context, AstOperation operation, out AggregateType varType) + { + StorageKind storageKind = operation.StorageKind; + ++ // NX2NAVI: atomics on the device address table's pointers (GenerateDeviceAddressLoadOrStore covers loads and ++ // stores): sources are the table binding and the address's two words. ++ if (storageKind == StorageKind.GlobalMemory) ++ { ++ varType = AggregateType.U32; ++ return GetDeviceAddressPointer( ++ context, ++ ((AstOperand)operation.GetSource(0)).Value, ++ context.Get(AggregateType.U32, operation.GetSource(1)), ++ context.Get(AggregateType.U32, operation.GetSource(2))); ++ } ++ + StorageClass storageClass; + SpvInstruction baseObj; + int srcIndex = 0; +--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs ++++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs +@@ -45,7 +45,18 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv + + context.AddCapability(Capability.Shader); + +- context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450); ++ if (context.Properties.UsesDeviceAddresses) ++ { ++ // NX2NAVI: global memory through the device address table (Instructions.cs GetDeviceAddressPointer) ++ context.AddCapability(Capability.PhysicalStorageBufferAddresses); ++ context.AddCapability(Capability.Int64); ++ context.AddExtension("SPV_KHR_physical_storage_buffer"); ++ context.SetMemoryModel(AddressingModel.PhysicalStorageBuffer64, MemoryModel.GLSL450); ++ } ++ else ++ { ++ context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450); ++ } + + context.AddCapability(Capability.GroupNonUniformBallot); + context.AddCapability(Capability.GroupNonUniformShuffle); +--- a/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs ++++ b/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs +@@ -347,6 +347,15 @@ namespace Ryujinx.Graphics.Shader + { + return true; + } ++ ++ /// ++ /// NX2NAVI: queries whether the host binds a device address table (GPU address pages to buffer device addresses) ++ /// for global memory accesses whose storage buffer the translator cannot find. ++ /// ++ bool QueryHostSupportsBufferDeviceAddress() ++ { ++ return false; ++ } + bool QueryHostSupportsShaderNonUniformIndexing() + { + return false; +--- a/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs ++++ b/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs +@@ -19,6 +19,10 @@ namespace Ryujinx.Graphics.Shader.StructuredIr + public IReadOnlyDictionary LocalMemories => _localMemories; + public IReadOnlyDictionary SharedMemories => _sharedMemories; + ++ // NX2NAVI: global memory operations go through the device address table (ResourceManager ++ // TryGetDeviceAddressTableBinding); the SPIR-V module uses the PhysicalStorageBuffer64 addressing model. ++ public bool UsesDeviceAddresses { get; set; } ++ + public ShaderProperties() + { + _constantBuffers = new Dictionary(); +--- a/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs +@@ -273,6 +273,11 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations + // The returned value being null means that the global memory replacement failed, + // so we just make loads read 0 and stores do nothing. + ++ if (TryKeepDeviceAddressAccess(resourceManager, gpuAccessor, node, operation)) ++ { ++ continue; ++ } ++ + gpuAccessor.Log($"Failed to reserve storage buffer for global memory operation \"{operation.Inst}\"."); + + if (operation.Dest != null) +@@ -323,6 +328,51 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations + } + } + ++ // NX2NAVI: a global access no storage buffer serves (its address is a pointer the program read from memory at an ++ // index, a phi of such pointers and constant buffer ones, or past the slots) stays a global access, with the device ++ // address table's binding as a new first source: the SPIR-V back end looks the address's page up in the table ++ // and accesses the buffer device address it gives (CodeGen/Spirv/Instructions.cs GetDeviceAddressPointer). An ++ // access the table cannot serve (a float atomic, which needs the loop of TryGenerateStorageOp) keeps the old ++ // fallback. The operation is replaced in place, so the pass goes on from the same node. ++ private static bool TryKeepDeviceAddressAccess( ++ ResourceManager resourceManager, ++ IGpuAccessor gpuAccessor, ++ LinkedListNode node, ++ Operation operation) ++ { ++ if (!gpuAccessor.QueryHostSupportsBufferDeviceAddress() || ++ (operation.Inst & Instruction.FP32) != 0 || ++ (operation.Inst & Instruction.Mask) == Instruction.AtomicCompareAndSwap && operation.StorageKind != StorageKind.GlobalMemory || ++ operation.SourcesCount < 2 || ++ (operation.StorageKind != StorageKind.GlobalMemory && operation.Inst.IsAtomic()) || ++ !resourceManager.TryGetDeviceAddressTableBinding(out int binding)) ++ { ++ return false; ++ } ++ ++ Operand[] sources = new Operand[operation.SourcesCount + 1]; ++ sources[0] = Const(binding); ++ ++ for (int index = 0; index < operation.SourcesCount; index++) ++ { ++ sources[index + 1] = operation.GetSource(index); ++ } ++ ++ Operand dest = operation.Dest; ++ ++ for (int index = 0; index < operation.SourcesCount; index++) ++ { ++ operation.SetSource(index, null); ++ } ++ ++ operation.Dest = null; ++ node.Value = new Operation(operation.Inst, operation.StorageKind, dest, sources); ++ ++ gpuAccessor.Log($"Global memory operation \"{operation.Inst}\" goes through the device address table."); ++ ++ return true; ++ } ++ + private static bool IsGlobalMemory(StorageKind storageKind) + { + return storageKind is StorageKind.GlobalMemory or +@@ -660,6 +710,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations + Queue phis = new(); + HashSet visited = []; + List targetCbs = []; ++ bool unresolved = false; // NX2NAVI: a phi source with no constant buffer base (a pointer from memory) + + Operand globalAddress = operation.GetSource(0); + +@@ -714,9 +765,22 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations + { + phis.Enqueue(phi2); + } ++ else if (phiSource.Type != OperandType.Undefined && phiSource.AsgOp is not PhiNode) ++ { ++ unresolved = true; ++ } + } + } + ++ // NX2NAVI: with the device address table, an address that may come from a pointer the program read from memory ++ // goes through the table (the caller keeps it global): a range check against the constant buffer candidates ++ // would send it to one of them, or read 0. ++ if (gpuAccessor.QueryHostSupportsBufferDeviceAddress() && (targetCbs.Count == 0 || unresolved)) ++ { ++ functionId = 0; ++ return false; ++ } ++ + if (targetCbs.Count == 0) + { + AddTableCandidates(resourceManager, operation.GetSource(0), targetCbs); +--- a/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs +@@ -176,6 +176,22 @@ namespace Ryujinx.Graphics.Shader.Translation + return setAndBinding.Binding; + } + ++ // NX2NAVI: the storage buffer slot of the device address table, with a constant buffer slot no program has ++ // (NVN banks are 0-17, GlobalToStorage.PointerSlotFlag is 0x20); tools/xlate/Program.cs writes it as "table". ++ public const int DeviceAddressTableCbSlot = 0x40; ++ ++ public bool TryGetDeviceAddressTableBinding(out int binding) ++ { ++ if (!TryGetStorageBufferBinding(DeviceAddressTableCbSlot, 0, false, out binding)) ++ { ++ return false; ++ } ++ ++ Properties.UsesDeviceAddresses = true; ++ ++ return true; ++ } ++ + public bool TryGetStorageBufferBinding(int sbCbSlot, int sbCbOffset, bool write, out int binding) + { + if (!TryGetSbSlot((byte)sbCbSlot, (ushort)sbCbOffset, out int slot))