Reach buffers through a device address table where the translator finds no storage buffer, and lower MATCH.ALL and ATOMS.POPC.INC.

Kokiri Forest drew black, in tiles and in shadowed and indoor areas, where shaders read light, probe and instance data through pointers they load from other buffers at an index (a phi of a pushed pointer and a loaded one, a pointer spilled to local memory): those reads returned 0. Translator patch 0030 keeps such accesses global, with a new first source naming the table, and the SPIR-V back end looks the address's 4 KiB page up in it and loads through the buffer device address it gives (PhysicalStorageBuffer64). runtime/vk_gpa.c keeps the table: two levels over the 40-bit GPU address space, page entries naming immutable records, records reused only after the GPU finished with them. Imported pools, their VRAM mirrors and the pages of virtual pools go in it; pools in it are destroyed after the submissions that may still read them. The last failing compute program needed MATCH.ALL (subgroupAllEqual) and the aggregated shared increment ATOMS.POPC.INC, now lowered in AmpereLowering.cs; Kokiri Forest translates without a failure.

NX2NAVI_GPA=0 turns the table off and NX2NAVI_GPA_EMPTY=1 keeps it empty. A first try synced every VRAM mirror at each table reader's draw, which overwrote what the GPU had written into them and made the black worse; mirrors sync only when bound.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

runtime/cfg.c

@@ -42,7 +42,7 @@ static const char *const known[] = {
     "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR",
     "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "LOG_RT_SIZES", "DUMP_WHITE", "DUMP_AFTER", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES",
     "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES",
-    "FULLSCREEN", "FXAA", "GAME_ASPECT", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE",
+    "FULLSCREEN", "FXAA", "GAME_ASPECT", "GPA", "GPA_EMPTY", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE",
     "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
     "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MENU_LANG", "MENU_TEST",
     "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",

runtime/gfx.h

@@ -54,7 +54,8 @@ const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8
 uint64_t gfx_shader_async_done(void);                      // translations finished by the background worker
 void gfx_shader_counts(uint64_t *keys, uint64_t *translated);   // distinct keys in memory, translations done (NX2NAVI_TRACE=perf)
 // Translator answers for this device (vk_caps.c): shaderFloat64, storage offset rounding, subgroup size per vs/fs/cs.
-void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals);
+void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals,
+                         int bda);
 const uint8_t *gfx_shader_key(const GfxShader *s);          // the 32-byte cache key (stable while the translator is)
 const uint8_t *gfx_shader_input_id(const GfxShader *s);     // the 32-byte input id (stable across translators)
 // By input id from memory or disk; xlate 1 also translates it from its stored inputs. NULL if absent.

runtime/gfx_shader.c

@@ -120,7 +120,7 @@ static struct {                                            // translations in fl
 static char cache_dir[PATH_CAP];
 static char xlate_exe[PATH_CAP];
 static uint8_t xlate_id[32];                               // translator identity (see the keys at the top)
-static int host_f64 = 1, host_zero_locals, host_set;                         // gfx_shader_set_host
+static int host_f64 = 1, host_zero_locals, host_bda, host_set;                         // gfx_shader_set_host
 static uint32_t host_sbuf, host_sg[3] = {32, 32, 32}, host_gather;
 static PlatOnce paths_once = PLAT_ONCE_INIT;
 
@@ -316,17 +316,20 @@ static void paths(void) { plat_once(&paths_once, init_paths, NULL); }
 // Host answers (vk_caps.c, at device creation, before any translation). Answers other than the defaults (Float64,
 // exact storage offsets, 32 lanes, no gather bias, undefined locals) are mixed into the translator identity, so they
 // get their own cache keys.
-void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals) {
+void gfx_shader_set_host(int float64, uint32_t sbuf_align, const uint32_t subgroup[3], uint32_t gather_bits, int zero_locals,
+                         int bda) {
     paths();
     if (host_set++) return;
-    host_f64 = float64, host_sbuf = sbuf_align, host_gather = gather_bits, host_zero_locals = zero_locals;
+    host_f64 = float64, host_sbuf = sbuf_align, host_gather = gather_bits, host_zero_locals = zero_locals, host_bda = bda;
     memcpy(host_sg, subgroup, sizeof host_sg);
-    if (float64 && !sbuf_align && subgroup[0] == 32 && subgroup[1] == 32 && subgroup[2] == 32 && !gather_bits && !zero_locals) return;
+    if (float64 && !sbuf_align && subgroup[0] == 32 && subgroup[1] == 32 && subgroup[2] == 32 && !gather_bits && !zero_locals && !bda)
+        return;
     uint8_t buf[32 + 96];
     int n = snprintf((char *)buf + 32, 96, "TKHO f64=%d sbuf=%u sg=%u,%u,%u", float64, sbuf_align, subgroup[0], subgroup[1],
                      subgroup[2]);
     if (gather_bits) n += snprintf((char *)buf + 32 + n, (size_t)(96 - n), " gather=%u", gather_bits);
     if (zero_locals) n += snprintf((char *)buf + 32 + n, (size_t)(96 - n), " zero_locals");
+    if (bda) n += snprintf((char *)buf + 32 + n, (size_t)(96 - n), " bda");
     memcpy(buf, xlate_id, 32);
     rt_sha256(buf, 32 + (size_t)n, xlate_id);
     rt_log("gfx_shader: host options %s: own shader cache keys", (char *)buf + 37);
@@ -1194,6 +1197,7 @@ static size_t build_request(char *o0, const char *hex, const char *stage, const
     if (host_sg[si] != 32) o += sprintf(o, ",\"subgroup\":%u", host_sg[si]);
     if (host_gather) o += sprintf(o, ",\"gather_bias\":%u", host_gather);
     if (host_zero_locals) o += sprintf(o, ",\"zero_locals\":true");
+    if (host_bda) o += sprintf(o, ",\"bda\":true");   // the device address table (vk_gpa.c)
     uint32_t magic, nvn2_size;
     memcpy(&magic, control, 4), memcpy(&nvn2_size, control + 0x38, 4);
     if (magic == 0x98761234u) {   // NVN2_CONTROL_MAGIC, below

runtime/vk_caps.c

@@ -19,6 +19,7 @@
 #include "gfx.h"
 #include "host.h"
 #include "vk.h"
+#include "vk_gpa.h"
 
 VkCaps vk_caps = {.float64 = 1, .sg_size = 32};
 static int f64_supported;
@@ -124,7 +125,8 @@ void vk_caps_init(VkCtx *c) {
     // undefined off Mesa, as before: zeroing moved that hash and the picture shook under DLAA (maxwell.md 3.12).
     // NX2NAVI_ZERO_LOCALS=1 / 0 forces either.
     uint32_t zero_locals = env_u32("NX2NAVI_ZERO_LOCALS", vk_caps.mesa || !cfg_bool("NX2NAVI_DLAA"));
-    gfx_shader_set_host(vk_caps.float64, vk_caps.sbuf_align, told, gather, zero_locals != 0);
+    // bda: global accesses the translator finds no storage buffer for go through the device address table (vk_gpa.c)
+    gfx_shader_set_host(vk_caps.float64, vk_caps.sbuf_align, told, gather, zero_locals != 0, vk_gpa_wanted_ctx(c));
     if (gather) rt_log("vk caps: gather coordinates biased for %u sub-texel bits (NX2NAVI_GATHER_BIAS)", gather);
 
     char dv[32];

runtime/vk_exec_draw.c

@@ -28,6 +28,7 @@
 #define NX2NAVI_TONEMAP_STANDIN 1   // vk_tonemap.c, see untranslated_dispatch_fallback
 #endif
 #include "vk_exec.h"
+#include "vk_gpa.h"
 #include "platform/platform.h"
 #include "vk_dlaa.h"
 
@@ -1263,6 +1264,14 @@ uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r) {
 }
 
 static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) {
+    // The device address table (vk_gpa.c, translator patch 0030). Mirrored pools are not synced for its readers: a sync
+    // of every mirror at a table reader's draw copied stale guest bytes over what the GPU had written into the mirrors
+    // earlier in the frame, and Kokiri Forest's lit areas went black. A pool read only through the table is rarely a
+    // mirrored one (mirrors are made for pools bound as buffers, which sync then).
+    if (r->cbuf_slot == VK_GPA_CB_SLOT) {
+        VkDescriptorBufferInfo t = vk_gpa_info();
+        return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE};
+    }
     if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init();
     uint64_t max = LIM.sbo_max, align = LIM.sbo_align;
     VkDescriptorBufferInfo bi;

runtime/vk_gpa.c

@@ -0,0 +1,259 @@
+// The device address table: GPU address pages to Vulkan buffer device addresses, for the global memory accesses the
+// translator finds no storage buffer for (translator patch 0030: GlobalToStorage TryKeepDeviceAddressAccess, and the
+// SPIR-V back end's GetDeviceAddressPointer, which reads this layout). Ocarina's foliage, instance and light data reach
+// their buffers through pointers the shaders read from other buffers at an index (a phi of constant-buffer pointers and
+// loaded ones, a pointer spilled to local memory and loaded again); those accesses read 0 before, which drew interiors
+// and shadowed areas black and dropped instanced foliage.
+//
+// One storage buffer of 32-bit words, bound whole to every shader that declares it (res.json "table", cb_slot 0x40):
+//   words 0-1    the device address of a zeroed 4 KiB buffer, which an address no page maps reads and writes instead
+//   word 2       the word index of the records (REC_WORD)
+//   words 16..   L1: 65536 entries, one per 16 MiB of the 40-bit GPU address space, the index of its L2 block (0: none)
+//   word 65552   L2 blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds the page (0: none)
+//   REC_WORD     records of two words: the buffer's device address minus the GPU address of its byte 0 (low word first)
+// A record a page entry names is never written: a new mapping takes a new record, and a record goes back to the free
+// list only once the GPU finished every submission issued while a page could still name it (vk_completion_add). Page
+// and L1 entries are single 32-bit stores, so a shader that reads the table while it changes sees the old mapping or
+// the new one, never half of each. The CPU keeps a copy (shadow) for its reads: the GPU copy is often device memory
+// behind the PCIe BAR, slow to read back.
+//
+// NX2NAVI_GPA=0 turns the table off: the translator is told the host has no device addresses (those accesses read 0).
+// NX2NAVI_GPA_EMPTY=1 keeps it empty, so every access through it reads the zero buffer (diagnosis).
+// Mirrored pools are not synced for the table's readers (vk_exec_draw.c ssbo_info says why).
+#include <stdlib.h>
+#include <string.h>
+
+#include "cfg.h"
+#include "host.h"
+#include "vk.h"
+#include "vk_gpa.h"
+#include "platform/platform.h"
+
+#define L1_WORD 16u
+#define L1_N 65536u
+#define L2_WORD (L1_WORD + L1_N)
+#define L2_N 4096u                     // pages per block: 16 MiB
+#define BLOCKS 1024u                   // L2 blocks: 16 GiB of mapped GPU address space; block 0 stays zero
+#define REC_WORD (L2_WORD + BLOCKS * L2_N)
+#define RECS 65536u                    // record 0 means none
+#define WORDS (REC_WORD + RECS * 2u)
+#define PAGE_SHIFT 12
+
+static struct {
+    int state;                         // 0 untried, 1 on, -1 off
+    VkBuffer buf, dummy;
+    VmaAllocation alloc, dummy_alloc;
+    volatile uint32_t *gpu;            // the mapped table
+    uint32_t *cpu;                     // its shadow
+    uint32_t nblocks;                  // L2 blocks handed out (block 0 is the zero block)
+    uint32_t nrec;                     // records handed out, free ones aside
+    uint32_t *free_recs, nfree;        // records whose fence completed
+    uint64_t mapped_pages, maps, unmaps, refused;
+} G;
+static PlatMutex gpa_lock = PLAT_MUTEX_INIT;
+
+static void put(uint32_t word, uint32_t v) {
+    G.cpu[word] = v;
+    __atomic_store_n(&G.gpu[word], v, __ATOMIC_RELAXED);
+}
+
+int vk_gpa_wanted_ctx(const VkCtx *c) {
+    static int knob = -1;
+    if (knob < 0) {
+        const char *e = cfg_get("NX2NAVI_GPA");
+        knob = !(e && !strcmp(e, "0"));
+    }
+    return knob && c && c->has_buffer_device_address;
+}
+
+int vk_gpa_wanted(void) { return vk_gpa_wanted_ctx(vk_ctx()); }
+
+uint64_t vk_gpa_device_address(VkBuffer b) {
+    VkBufferDeviceAddressInfo ai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = b};
+    return b ? vkGetBufferDeviceAddress(vk_ctx()->device, &ai) : 0;
+}
+
+static int create(void) {
+    VkCtx *c = vk_ctx();
+    if (!vk_gpa_wanted()) return 0;
+    G.cpu = calloc(WORDS, 4);
+    G.free_recs = malloc(RECS * sizeof *G.free_recs);
+    if (!G.cpu || !G.free_recs) return 0;
+    VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)WORDS * 4,
+                             .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
+    VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT,
+                                  .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE};
+    VmaAllocationInfo info;
+    if (vmaCreateBuffer(c->vma, &bi, &ai, &G.buf, &G.alloc, &info) != VK_SUCCESS || !info.pMappedData) {
+        rt_log("vk_gpa: the %.1f MiB table could not be allocated: global accesses through pointers read 0", WORDS * 4 / 1048576.0);
+        return 0;
+    }
+    G.gpu = info.pMappedData;
+    memset((void *)G.gpu, 0, (size_t)WORDS * 4);
+    VkBufferCreateInfo di = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 4096,
+                             .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT,
+                             .sharingMode = VK_SHARING_MODE_EXCLUSIVE};
+    VmaAllocationInfo dinfo;
+    if (vmaCreateBuffer(c->vma, &di, &ai, &G.dummy, &G.dummy_alloc, &dinfo) != VK_SUCCESS || !dinfo.pMappedData) {
+        rt_log("vk_gpa: the zero buffer could not be allocated");
+        return 0;
+    }
+    memset(dinfo.pMappedData, 0, 4096);
+    uint64_t z = vk_gpa_device_address(G.dummy);
+    put(0, (uint32_t)z), put(1, (uint32_t)(z >> 32)), put(2, REC_WORD);
+    G.nblocks = 1, G.nrec = 1;
+    rt_log("vk_gpa: device address table on (%.1f MiB, %u blocks of 16 MiB, %u records)", WORDS * 4 / 1048576.0, BLOCKS, RECS);
+    return 1;
+}
+
+int vk_gpa_on(void) {
+    if (__atomic_load_n(&G.state, __ATOMIC_ACQUIRE)) return G.state > 0;
+    plat_mutex_lock(&gpa_lock);
+    if (!G.state) __atomic_store_n(&G.state, create() ? 1 : -1, __ATOMIC_RELEASE);
+    plat_mutex_unlock(&gpa_lock);
+    return G.state > 0;
+}
+
+VkDescriptorBufferInfo vk_gpa_info(void) {
+    if (!vk_gpa_on()) return (VkDescriptorBufferInfo){0};
+    return (VkDescriptorBufferInfo){G.buf, 0, VK_WHOLE_SIZE};
+}
+
+// The L2 entry word of a page; 0 when its block does not exist and alloc is 0 or no block is left.
+static uint32_t page_word(uint64_t page, int alloc) {
+    uint64_t l1 = page >> 12;   // 4096 pages of 4 KiB per 16 MiB
+    if (l1 >= L1_N) return 0;
+    uint32_t blk = G.cpu[L1_WORD + l1];
+    if (!blk) {
+        if (!alloc || G.nblocks == BLOCKS) return 0;
+        blk = G.nblocks++;
+        put(L1_WORD + (uint32_t)l1, blk);
+    }
+    return L2_WORD + blk * L2_N + (uint32_t)(page & (L2_N - 1));
+}
+
+static uint32_t rec_alloc(uint64_t delta) {
+    uint32_t r;
+    if (G.nfree) r = G.free_recs[--G.nfree];
+    else if (G.nrec < RECS) r = G.nrec++;
+    else return 0;
+    // The record before any page names it: the page store below is ordered after these by the release fence.
+    put(REC_WORD + r * 2, (uint32_t)delta);
+    put(REC_WORD + r * 2 + 1, (uint32_t)(delta >> 32));
+    __atomic_thread_fence(__ATOMIC_RELEASE);
+    return r;
+}
+
+static void rec_free_cb(void *arg) {
+    uint32_t r = (uint32_t)(uintptr_t)arg;
+    plat_mutex_lock(&gpa_lock);
+    G.free_recs[G.nfree++] = r;
+    plat_mutex_unlock(&gpa_lock);
+}
+
+uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo) {
+    if (!vk_gpa_on() || !buf || hi <= lo) return 0;
+    static int empty = -1;   // NX2NAVI_GPA_EMPTY=1: nothing is mapped, every access reads the zero buffer (diagnosis)
+    if (empty < 0) { const char *e = cfg_get("NX2NAVI_GPA_EMPTY"); empty = e && !strcmp(e, "1"); }
+    if (empty) return 0;
+    uint64_t dev = vk_gpa_device_address(buf);
+    if (!dev) return 0;
+    // Pages the buffer holds whole: [lo, hi) rounded inwards (imports are aligned outwards, so this is all of them)
+    uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT;
+    if (p1 <= p0) return 0;
+    plat_mutex_lock(&gpa_lock);
+    uint32_t r = rec_alloc(dev - buf_lo);
+    if (!r) {
+        G.refused++;
+        plat_mutex_unlock(&gpa_lock);
+        if (G.refused == 1) rt_log("vk_gpa: out of records: a buffer at %#llx is not in the table", (unsigned long long)lo);
+        return 0;
+    }
+    uint64_t n = 0;
+    for (uint64_t p = p0; p < p1; p++) {
+        uint32_t w = page_word(p, 1);
+        if (!w) {
+            if (!G.refused++) rt_log("vk_gpa: out of L2 blocks at %#llx", (unsigned long long)(p << PAGE_SHIFT));
+            break;
+        }
+        put(w, r);
+        n++;
+    }
+    G.mapped_pages += n, G.maps++;
+    plat_mutex_unlock(&gpa_lock);
+    return r;
+}
+
+void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint64_t page_addr)) {
+    if (!rec || G.state <= 0) return;
+    uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT;
+    plat_mutex_lock(&gpa_lock);
+    for (uint64_t p = p0; p < p1; p++) {
+        uint32_t w = page_word(p, 0);
+        if (w && G.cpu[w] == rec) put(w, 0), G.mapped_pages--;
+    }
+    G.unmaps++;
+    plat_mutex_unlock(&gpa_lock);
+    // Pages another live buffer also holds whole (overlapping imports of the same guest memory) get its record back.
+    if (cover) {
+        for (uint64_t p = p0; p < p1; p++) {
+            plat_mutex_lock(&gpa_lock);
+            uint32_t w = page_word(p, 0), now = w ? G.cpu[w] : 0;
+            plat_mutex_unlock(&gpa_lock);
+            if (!w || now) continue;
+            uint32_t other = cover(p << PAGE_SHIFT);
+            if (!other || other == rec) continue;
+            plat_mutex_lock(&gpa_lock);
+            if (!G.cpu[w]) put(w, other), G.mapped_pages++;
+            plat_mutex_unlock(&gpa_lock);
+        }
+    }
+    vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)rec);
+}
+
+void vk_gpa_clear(uint64_t lo, uint64_t hi) {
+    if (G.state <= 0 || hi <= lo) return;
+    uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT;
+    plat_mutex_lock(&gpa_lock);
+    for (uint64_t p = p0; p < p1; p++) {
+        uint32_t w = page_word(p, 0);
+        if (w && G.cpu[w]) put(w, 0), G.mapped_pages--;
+    }
+    plat_mutex_unlock(&gpa_lock);
+}
+
+uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo) {
+    if (!vk_gpa_on() || !buf) return old_rec;
+    uint64_t dev = vk_gpa_device_address(buf);
+    uint64_t p0 = (lo + 4095) >> PAGE_SHIFT, p1 = hi >> PAGE_SHIFT;
+    plat_mutex_lock(&gpa_lock);
+    uint32_t r = dev ? rec_alloc(dev - buf_lo) : 0;
+    if (!r) { plat_mutex_unlock(&gpa_lock); return old_rec; }
+    for (uint64_t p = p0; p < p1; p++) {
+        uint32_t w = page_word(p, 1);
+        if (w && (!G.cpu[w] || G.cpu[w] == old_rec)) {
+            if (!G.cpu[w]) G.mapped_pages++;
+            put(w, r);
+        }
+    }
+    plat_mutex_unlock(&gpa_lock);
+    if (old_rec) vk_completion_add(vk_sync_issued() + 1, rec_free_cb, (void *)(uintptr_t)old_rec);
+    return r;
+}
+
+void vk_gpa_report(void) {
+    if (G.state <= 0) return;
+    rt_log("vk_gpa: %llu pages (%.0f MiB) mapped, %u of %u blocks, %u records in use, %llu maps, %llu unmaps, %llu refused",
+           (unsigned long long)G.mapped_pages, G.mapped_pages * 4096.0 / 1048576.0, G.nblocks, BLOCKS, G.nrec - 1 - G.nfree,
+           (unsigned long long)G.maps, (unsigned long long)G.unmaps, (unsigned long long)G.refused);
+}
+
+void vk_gpa_shutdown(void) {
+    VkCtx *c = vk_ctx();
+    if (G.state <= 0 || !c) return;
+    vmaDestroyBuffer(c->vma, G.buf, G.alloc);
+    vmaDestroyBuffer(c->vma, G.dummy, G.dummy_alloc);
+    free(G.cpu), free(G.free_recs);
+    memset(&G, 0, sizeof G);
+    G.state = -1;
+}

runtime/vk_gpa.h

@@ -0,0 +1,24 @@
+// The device address table (vk_gpa.c): GPU address pages to buffer device addresses for shaders' global memory accesses.
+#pragma once
+#include <stdint.h>
+
+#include "vk.h"
+
+#define VK_GPA_CB_SLOT 0x40   // res.json cb_slot of the table (translator ResourceManager.DeviceAddressTableCbSlot)
+
+int vk_gpa_wanted(void);                   // the device has buffer device addresses and NX2NAVI_GPA is not 0
+int vk_gpa_wanted_ctx(const VkCtx *c);     // the same during device creation, before vk_ctx() returns it
+int vk_gpa_on(void);                       // the table exists (created on first use)
+uint64_t vk_gpa_device_address(VkBuffer b);
+VkDescriptorBufferInfo vk_gpa_info(void);  // the whole table, for a storage descriptor
+// Maps the pages [lo, hi) holds whole to buf, whose byte 0 is GPU address buf_lo. Returns the record (0: not mapped).
+uint32_t vk_gpa_map(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo);
+// Unmaps the pages of [lo, hi) that name rec; cover (may be NULL) gives the record of another live buffer that holds a
+// page whole, or 0. rec is reused once the GPU finished the submissions issued until now.
+void vk_gpa_unmap(uint64_t lo, uint64_t hi, uint32_t rec, uint32_t (*cover)(uint64_t page_addr));
+// Unmaps every page of [lo, hi), whatever record it names (pages of a virtual pool whose physical memory goes away).
+void vk_gpa_clear(uint64_t lo, uint64_t hi);
+// Moves the pages of [lo, hi) that name old_rec (or none) to buf (a pool's VRAM mirror); returns the new record.
+uint32_t vk_gpa_remap(uint64_t lo, uint64_t hi, uint32_t old_rec, VkBuffer buf, uint64_t buf_lo);
+void vk_gpa_report(void);
+void vk_gpa_shutdown(void);

runtime/vk_memory.c

@@ -19,6 +19,7 @@
 #include "nvn_pool_index.h"
 #include "vk.h"
 #include "vk_exec.h"
+#include "vk_gpa.h"
 #include "wb_merge_spv.h"
 #include "platform/platform.h"
 
@@ -55,6 +56,9 @@ typedef struct PoolRec {
     uint8_t untracked;                 // the last dirty scan found no write watch over it
     // VRAM mirror (POOL_IMPORT, "VRAM mirror" below). mstate and mbuf change on the replay thread only.
     uint8_t mstate;                    // MS_NONE until the first buffer use, then MS_ON or MS_REFUSED
+    uint32_t gpa_rec;                  // POOL_IMPORT: its record in the device address table (vk_gpa.c), 0 none
+    struct GpaMap { uint64_t lo, hi; uint32_t rec; } *vmaps;   // POOL_VIRT: its mapped ranges in the table
+    int nvmaps, cvmaps;
     VkBuffer mbuf;                     // MS_ON: device-local copy of [import_lo, import_hi), same offsets as buf
     VmaAllocation malloc_;
     uint32_t acc_sub;                  // replay: the last submission that synced the mirror (mirror_sync)
@@ -102,6 +106,7 @@ static VkBufferUsageFlags pool_usage(const VkCtx *c) {
     if (c->has_transform_feedback)
         u |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT | VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT;
     if (c->has_conditional_rendering) u |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT;
+    if (vk_gpa_wanted()) u |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;   // pages of the device address table
     return u;
 }
 
@@ -153,8 +158,11 @@ static int import_pool(VkCtx *c, PoolRec *r, uint64_t addr, uint64_t size) {
     VkImportMemoryHostPointerInfoEXT im = {.sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
                                            .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
                                            .pHostPointer = (void *)(uintptr_t)r->import_lo};
-    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &im, .allocationSize = n,
-                               .memoryTypeIndex = (uint32_t)type};
+    // A buffer with device address usage needs memory allocated for device addresses (vk_gpa.c).
+    VkMemoryAllocateFlagsInfo fi = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, .pNext = &im,
+                                    .flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT};
+    VkMemoryAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = vk_gpa_wanted() ? (void *)&fi : (void *)&im,
+                               .allocationSize = n, .memoryTypeIndex = (uint32_t)type};
     if ((res = vkAllocateMemory(c->device, &ai, NULL, &r->mem)) != VK_SUCCESS ||
         (res = vkBindBufferMemory(c->device, r->buf, r->mem, 0)) != VK_SUCCESS) {
         rt_log("vk: pool %#llx+%#llx: host memory import failed (%d)", (unsigned long long)addr, (unsigned long long)size, (int)res);
@@ -295,9 +303,28 @@ int vk_pool_init(pool_t *p) {
         phys_list[nphys++] = r;
     }
     plat_mutex_unlock(&pool_lock);
+    // The imported pages go into the device address table: shaders that read a pointer from memory reach them there.
+    if (r->kind == POOL_IMPORT && r->mem && r->buf && r->import_hi > r->import_lo)
+        r->gpa_rec = vk_gpa_map(r->import_lo, r->import_hi, r->buf, r->import_lo);
     return 1;
 }
 
+// vk_gpa_unmap: the record of another live imported pool that holds the 4 KiB page at a whole (overlapping imports
+// of the same guest memory), 0 for none.
+static uint32_t gpa_cover(uint64_t a) {
+    uint32_t rec = 0;
+    plat_mutex_lock_shared(&pool_lock);
+    for (int i = pidx_upper(&pool_index, a) - 1; i >= 0 && !rec; i--) {
+        const PoolRec *o = pool_index.e[i].buf;
+        if (o->kind == POOL_IMPORT && o->gpa_rec && o->import_lo <= a && o->import_hi >= a + 4096) rec = o->gpa_rec;
+        if (i && pool_index.maxhi[i - 1] <= a) break;
+    }
+    plat_mutex_unlock_shared(&pool_lock);
+    return rec;
+}
+
+static void gpa_retire(void *arg) { destroy_rec(vk_ctx(), arg); }
+
 void vk_pool_fini(pool_t *p) {
     VkCtx *c = vk_ctx();
     if (!c) return;
@@ -314,12 +341,19 @@ void vk_pool_fini(pool_t *p) {
             PoolRec *v = pool_index.e[i].buf;
             if (v->kind != POOL_VIRT) continue;
             for (uint64_t k = 0; k < v->npages; k++)
-                if (v->pt[k].phys == r) v->pt[k].phys = NULL;
+                if (v->pt[k].phys == r) {
+                    v->pt[k].phys = NULL;
+                    vk_gpa_clear(v->lo + k * NVN_VPOOL_PAGE, v->lo + (k + 1) * NVN_VPOOL_PAGE);   // the buffer goes away
+                }
         }
     }
     plat_mutex_unlock(&pool_lock);
     if (!r) return;
     if (r->smap) shadow_forget(r);
+    // Off the device address table first; the buffer outlives the submissions that may still read the old pages.
+    if (r->gpa_rec) vk_gpa_unmap(r->import_lo, r->import_hi, r->gpa_rec, gpa_cover);
+    for (int i = 0; i < r->nvmaps; i++) vk_gpa_unmap(r->vmaps[i].lo, r->vmaps[i].hi, r->vmaps[i].rec, NULL);
+    free(r->vmaps), r->vmaps = NULL, r->nvmaps = 0;
     if (r->kind == POOL_IMPORT) mem_thaw(r->lo, r->size);   // the storage goes back to the guest writable
     // Images cut from this range go first; a used buffer or memory waits until the GPU is done with it, since the
     // guest gets the storage back when this returns.
@@ -334,9 +368,29 @@ void vk_pool_fini(pool_t *p) {
     // A mirrored pool's record and buffers outlive the submission being recorded, whose sync copies and write-backs may
     // name them (the game frees a pool only once the GPU is done with it, but its CPU writes may still be queued).
     if (r->mbuf) mirror_retiring(r), vk_completion_add(vk_sync_issued() + 1, mirror_retire, r);
+    else if (r->gpa_rec && !r->used) vk_completion_add(vk_sync_issued() + 1, gpa_retire, r);   // the queue was not drained
     else destroy_rec(c, r);
 }
 
+// pool_lock held: the device address table follows a MapVirtual request on [lo, hi) of virtual pool v. Ranges it
+// covers whole leave the table (their records retire); the pages then name the physical pool's buffer at poff, or none.
+static void gpa_map_virtual(PoolRec *v, uint64_t lo, uint64_t hi, PoolRec *ph, uint64_t poff) {
+    if (!vk_gpa_wanted()) return;
+    for (int i = 0; i < v->nvmaps;)
+        if (v->vmaps[i].lo >= lo && v->vmaps[i].hi <= hi) {
+            vk_gpa_unmap(v->vmaps[i].lo, v->vmaps[i].hi, v->vmaps[i].rec, NULL);
+            v->vmaps[i] = v->vmaps[--v->nvmaps];
+        } else {
+            i++;
+        }
+    vk_gpa_clear(lo, hi);
+    if (!ph || !ph->buf) return;
+    uint32_t rec = vk_gpa_map(lo, hi, ph->buf, lo - poff);
+    if (!rec) return;
+    if (v->nvmaps == v->cvmaps) grow((void **)&v->vmaps, &v->cvmaps, sizeof *v->vmaps);
+    v->vmaps[v->nvmaps++] = (struct GpaMap){lo, hi, rec};
+}
+
 // Requests were checked by the front end (nvn_api_core.c); apply them in order.
 int vk_pool_map_virtual(pool_t *v, int n, const void *requests) {
     const nvn_mapping_t *rq = requests;
@@ -357,6 +411,8 @@ int vk_pool_map_virtual(pool_t *v, int n, const void *requests) {
         if (first + count > r->npages) { ok = 0; break; }
         for (uint64_t k = 0; k < count; k++)
             r->pt[first + k] = (VPage){ph, ph ? (uint64_t)rq[i].physical_offset + k * NVN_VPOOL_PAGE : 0};
+        gpa_map_virtual(r, r->lo + first * NVN_VPOOL_PAGE, r->lo + (first + count) * NVN_VPOOL_PAGE, ph,
+                        (uint64_t)rq[i].physical_offset);
     }
     plat_mutex_unlock(&pool_lock);
     if (!ok) rt_log("vk: MapVirtual on pool %p failed", (void *)v->mem);
@@ -1344,6 +1400,8 @@ static void mirror_create(PoolRec *r) {
     plat_mutex_unlock(&pool_lock);
     r->acc_sub = msub;
     r->mstate = MS_ON;
+    // The device address table names the mirror from now on: the GPU's writes go there, and the sync copies the CPU's.
+    if (r->gpa_rec) r->gpa_rec = vk_gpa_remap(r->import_lo, r->import_hi, r->gpa_rec, r->mbuf, r->import_lo);
     range_add(&pend, &npend, &cpend, r, 0, n);
     mrep.first_n++, mrep.first_bytes += n;
     plat_inc64(&m_live_n);
@@ -2009,6 +2067,7 @@ static void mirror_log(uint64_t frame) {
         snprintf(ver, sizeof ver, "; verify: pages compared %lld, skipped %lld (written since), mismatches %lld",
                  (long long)plat_xchg64(&v_pages, 0), (long long)plat_xchg64(&v_skipped, 0),
                  (long long)plat_xchg64(&v_mismatch, 0));
+    vk_gpa_report();
     rt_log("mem: mirror frame %llu: pools %lld, %.1f MiB device-local; stay imported: budget %lld (%.1f MiB), untracked "
            "%lld, overlapping %lld, failed %lld; per frame: submits %.2f, syncs %.1f, upload %.3f MiB in %.1f ranges, "
            "first use %.2f pools %.3f MiB, write-back %.3f MiB in %.1f ranges, merge %.3f MiB in %.1f dispatches "
@@ -2363,6 +2422,8 @@ void vk_mem_shutdown(void) {
     vk_queue_lock();
     vkDeviceWaitIdle(c->device);
     vk_queue_unlock();
+    vk_gpa_report();
+    vk_gpa_shutdown();
     vk_texture_shutdown();
     plat_rmutex_lock(&cmd_lock);
     retire(c, 1);

tools/build_xlate.py

@@ -99,8 +99,8 @@ def prepare_sources() -> Path:
     for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")):
         shutil.rmtree(junk)
     patches = sorted((XLATE / "patches").glob("*.patch"))
-    if len(patches) != 28:
-        sys.exit(f"expected 28 patches, found {len(patches)}")
+    if len(patches) != 29:
+        sys.exit(f"expected 29 patches, found {len(patches)}")
     for patch in patches:
         # git apply works outside a repository when given a directory; --check first so a failure names the patch.
         for extra in (["--check"], []):

tools/xlate/AmpereLowering.cs

@@ -428,6 +428,8 @@ static class AmpereLowering
                 case 0x3c4: Redux(); return;
                 case 0x81a: Sgxt(); return;
                 case 0x806: case 0x886: Vote(op == 0x886); return;
+                case 0x3a1: Match(); return;
+                case 0xf8c: AtomsPopcInc(); return;
                 case 0x3a8: case 0x9a8: Atomg(op == 0x9a8); return;
                 case 0x38c: Atoms(); return;
                 case 0x98e: Red(); return;
@@ -1359,6 +1361,47 @@ static class AmpereLowering
             if (signedOrder) Lop32i(t, t, 0x80000000, 2);
         }
 
+        // MATCH.ALL Pd, Rd, Ra (op 0x3a1 with no mode bits; Ocarina's NVN2 compute programs test subgroupAllEqual with it
+        // before an aggregated shared atomic: @!Pd per-lane ATOMS, else REDUX and one): Pd = every active lane holds Ra's
+        // value, Rd = the active-lane mask then, else 0. Lowered as VOTE.ANY (the active lanes), FLO (the highest of
+        // them), SHFL.IDX (that lane's value), ISETP.EQ into Pd and VOTE.ALL over Pd. Only the unguarded form with a
+        // predicate destination is taken.
+        void Match()
+        {
+            uint pd = _i.Bits(81, 3);
+            if (_i.W1 != 0 || (_i.W2 & ~(7u << 17)) != 0 || _i.Bits(12, 4) != 7 || pd == PT)
+                throw Fail("MATCH mode");
+            int d = Gpr((uint)Dst), src = Gpr(_i.Bits(24, 8));
+            const int ballot = 240, lead = 241, value = 242;
+            Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot));        // VOTE.ANY
+            Emit(Unguarded(0x5c30000000000000UL | ((ulong)ballot << 20) | lead));                                    // FLO.U32
+            Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)lead << 20)
+                           | ((ulong)src << 8) | value));                                                            // SHFL.IDX
+            Emit(Unguarded(0x5b60000000000000UL | (2UL << 49) | ((ulong)PT << 39) | ((ulong)value << 20)
+                           | ((ulong)src << 8) | ((ulong)pd << 3) | PT));                                            // ISETP.EQ.U32
+            Emit(Unguarded(0x50d8000000000000UL | ((ulong)pd << 45) | ((ulong)pd << 39) | RZ));                     // VOTE.ALL
+            if (d != RZ) Sel(d, ballot, RZ, pd, false);
+        }
+
+        // ATOMS.POPC.INC RZ, [Ra + URb + offset] (op 0xf8c, mode word 0x0d000000; the increment of Ocarina's aggregated
+        // shared counters after MATCH and REDUX): memory goes up by the number of active lanes. With no result register
+        // that is one ATOMS.ADD of 1 per active lane, which is what this emits; a form that wants the per-lane result (its
+        // rank among the active lanes) is not taken.
+        void AtomsPopcInc()
+        {
+            uint offset = _i.Bits(40, 24), ur = _i.Bits(64, 6);
+            if ((_i.W2 & ~0x3fu) != 0x0d000000u || _i.Bits(32, 8) != 0 || Dst != RZ || (offset & 3) != 0 || offset >= (1u << 22))
+                throw Fail("ATOMS.POPC.INC mode");
+            int addr = Gpr(_i.Bits(24, 8));
+            if (ur != 0x3f)
+            {
+                Iadd3(SA, addr, Ur(ur), RZ, false);
+                addr = SA;
+            }
+            Mov32i(SB, 1);
+            EmitG(0xec00000000000000UL | RZ | ((ulong)addr << 8) | ((ulong)SB << 20) | ((ulong)(offset >> 2) << 30));   // ATOMS.ADD
+        }
+
         // VOTE Rd, Pu, Pin: ballot into Rd (URd for VOTEU), mode 72-73 (ALL, ANY, EQ) over the vector predicate of bits
         // 87-90, result predicate 81-83 (a uniform predicate for VOTEU). Maxwell VOTE has the same fields. VOTEU's
         // output goes through Pk (k its UP index) with the guest's Pk saved around it; no uniform predicate is read.

tools/xlate/Program.cs

@@ -129,6 +129,9 @@ static class SwshXlate
         public bool QueryHostSupportsNonConstantTextureOffset() => false;
         // "softfloat64": true lowers FP64 operations to FP32 (the translator's DoubleToFloat pass), for a host without shaderFloat64.
         public bool QueryHostSupportsShaderFloat64() => _job["softfloat64"] is not JsonValue v || !(v.TryGetValue<bool>(out bool b) && b);
+        // The runtime binds the device address table (runtime/vk_gpa.c) when the device has buffer device addresses:
+        // global accesses the translator finds no storage buffer for go through it (patch 0030).
+        public bool QueryHostSupportsBufferDeviceAddress() => _job["bda"] is JsonValue v && v.TryGetValue<bool>(out bool b) && b;
         public bool QueryHostSupportsTextureShadowLod() => true;
         // Sub-texel precision bits of a host whose gathers round differently from Maxwell (AMD, Intel): the translator then
         // biases gather coordinates so the same texels are picked (TexturePass.InsertCoordGatherBias). 0: no bias.
@@ -234,6 +237,7 @@ static class SwshXlate
                     var o = new JsonObject { ["binding"] = b.Binding, ["slot"] = b.Slot, ["cb_slot"] = pointer ? b.SbCbSlot & 0x1f : b.SbCbSlot,
                         ["cb_offset"] = pointer ? b.SbCbOffset & 0x3ff : b.SbCbOffset, ["flags"] = (int)b.Flags };
                     if (pointer) o["ptr"] = (b.SbCbOffset >> 10) * 8;
+                    if (b.SbCbSlot == 0x40) o["table"] = true;   // the device address table (patch 0030, ResourceManager.DeviceAddressTableCbSlot)
                     res["sbufs"].AsArray().Add((JsonNode)o);
                 }
                 foreach (var (list, name) in new[] { (p.Info.Textures, "textures"), (p.Info.Images, "images") })

tools/xlate/patches/0030-global-memory-through-device-addresses.patch

@@ -0,0 +1,327 @@
+--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs
++++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs
+@@ -1820,6 +1820,11 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
+ 
+         private static OperationResult GenerateLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore)
+         {
++            if (IsGlobalStorageKind(operation.StorageKind))
++            {
++                return GenerateDeviceAddressLoadOrStore(context, operation, isStore);
++            }
++
+             SpvInstruction pointer = GetStoragePointer(context, operation, out AggregateType varType);
+ 
+             if (isStore)
+@@ -1834,10 +1839,137 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
+             }
+         }
+ 
++        private static bool IsGlobalStorageKind(StorageKind storageKind)
++        {
++            return storageKind is StorageKind.GlobalMemory or
++                StorageKind.GlobalMemoryS8 or
++                StorageKind.GlobalMemoryS16 or
++                StorageKind.GlobalMemoryU8 or
++                StorageKind.GlobalMemoryU16;
++        }
++
++        // NX2NAVI: the device address table (Translation/Optimizations/GlobalToStorage.cs TryKeepDeviceAddressAccess), a
++        // storage buffer of 32-bit words the host keeps (runtime/vk_gpa.c):
++        //   words 0-1   the device address of a zeroed 4 KiB buffer, for addresses no page maps
++        //   word 2      the word index of the records
++        //   words 16..  65536 first-level entries, one per 16 MiB of the 40-bit GPU address space: the index of the
++        //               second-level block, 0 for none (block 0 stays zero)
++        //   word 65552  second-level blocks of 4096 entries, one per 4 KiB page: the record of the buffer that holds
++        //               the page, 0 for none
++        //   records     two words each, the buffer's device address minus its GPU address (low word first); the host
++        //               never changes a record a page names, so a page entry read at any time gives a whole one
++        // The pointer addresses the 32-bit word at the address's 4-byte-aligned word.
++        private static SpvInstruction GetDeviceAddressPointer(CodeGenContext context, int binding, SpvInstruction low, SpvInstruction high)
++        {
++            SpvInstruction u32 = context.TypeU32();
++            SpvInstruction u64 = context.TypeInt(64, 0);
++            SpvInstruction tableVar = context.StorageBuffers[binding];
++            SpvInstruction elemPtr = context.TypePointer(StorageClass.Uniform, u32);
++            SpvInstruction zero = context.Constant(context.TypeS32(), 0);
++
++            SpvInstruction Word(SpvInstruction index) => context.Load(u32, context.AccessChain(elemPtr, tableVar, zero, index));
++            SpvInstruction U(uint value) => context.Constant(u32, value);
++            SpvInstruction Wide(SpvInstruction lo, SpvInstruction hi) =>
++                context.BitwiseOr(u64, context.UConvert(u64, lo), context.ShiftLeftLogical(u64, context.UConvert(u64, hi), U(32)));
++
++            SpvInstruction inRange = context.ULessThan(context.TypeBool(), high, U(256));
++            SpvInstruction l1 = context.BitwiseOr(u32, context.ShiftLeftLogical(u32, high, U(8)), context.ShiftRightLogical(u32, low, U(24)));
++            l1 = context.Select(u32, inRange, l1, U(0));
++            SpvInstruction block = Word(context.IAdd(u32, l1, U(16)));
++            SpvInstruction page = context.BitwiseAnd(u32, context.ShiftRightLogical(u32, low, U(12)), U(0xfff));
++            SpvInstruction record = Word(context.IAdd(u32, context.IAdd(u32, context.IMul(u32, block, U(4096)), U(65552)), page));
++            SpvInstruction mapped = context.INotEqual(context.TypeBool(), record, U(0));
++            SpvInstruction recordWord = context.IAdd(u32, Word(U(2)), context.IMul(u32, record, U(2)));
++            SpvInstruction deltaLow = Word(recordWord);
++            SpvInstruction deltaHigh = Word(context.IAdd(u32, recordWord, U(1)));
++
++            SpvInstruction wordLow = context.BitwiseAnd(u32, low, U(0xfffffffc));
++            SpvInstruction device = context.IAdd(u64, Wide(wordLow, high), Wide(deltaLow, deltaHigh));
++            SpvInstruction dummy = context.IAdd(u64, Wide(Word(U(0)), Word(U(1))), context.UConvert(u64, context.BitwiseAnd(u32, low, U(0xffc))));
++            SpvInstruction address = context.Select(u64, mapped, device, dummy);
++
++            return context.ConvertUToPtr(context.TypePointer(StorageClass.PhysicalStorageBuffer, u32), address);
++        }
++
++        private static SpvInstruction LoadAligned(CodeGenContext context, SpvInstruction type, SpvInstruction pointer)
++        {
++            SpvInstruction load = context.Load(type, pointer, MemoryAccessMask.Aligned);
++            load.AddOperand((Spv.Generator.LiteralInteger)4);
++            return load;
++        }
++
++        private static void StoreAligned(CodeGenContext context, SpvInstruction pointer, SpvInstruction value)
++        {
++            SpvInstruction store = context.Store(pointer, value, MemoryAccessMask.Aligned);
++            store.AddOperand((Spv.Generator.LiteralInteger)4);
++        }
++
++        // Sources: the table binding, the address's low and high words, the value for a store. 8- and 16-bit accesses
++        // read the word and extract the bytes; their stores clear and set them with two atomics.
++        private static OperationResult GenerateDeviceAddressLoadOrStore(CodeGenContext context, AstOperation operation, bool isStore)
++        {
++            int binding = ((AstOperand)operation.GetSource(0)).Value;
++            SpvInstruction u32 = context.TypeU32();
++            SpvInstruction low = context.Get(AggregateType.U32, operation.GetSource(1));
++            SpvInstruction high = context.Get(AggregateType.U32, operation.GetSource(2));
++            SpvInstruction pointer = GetDeviceAddressPointer(context, binding, low, high);
++            StorageKind kind = operation.StorageKind;
++            int bits = kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryU8 ? 8
++                : kind is StorageKind.GlobalMemoryS16 or StorageKind.GlobalMemoryU16 ? 16 : 32;
++
++            if (bits == 32)
++            {
++                if (isStore)
++                {
++                    StoreAligned(context, pointer, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1)));
++                    return OperationResult.Invalid;
++                }
++
++                return new OperationResult(AggregateType.U32, LoadAligned(context, u32, pointer));
++            }
++
++            SpvInstruction shift = context.ShiftLeftLogical(u32, context.BitwiseAnd(u32, low, context.Constant(u32, bits == 8 ? 3u : 2u)), context.Constant(u32, 3));
++            SpvInstruction fieldMask = context.Constant(u32, bits == 8 ? 0xffu : 0xffffu);
++
++            if (isStore)
++            {
++                SpvInstruction value = context.BitwiseAnd(u32, context.Get(AggregateType.U32, operation.GetSource(operation.SourcesCount - 1)), fieldMask);
++                SpvInstruction mask = context.ShiftLeftLogical(u32, fieldMask, shift);
++                SpvInstruction scope = context.Constant(u32, 1);   // device
++                SpvInstruction semantics = context.Constant(u32, 0);
++                context.AtomicAnd(u32, pointer, scope, semantics, context.Not(u32, mask));
++                context.AtomicOr(u32, pointer, scope, semantics, context.ShiftLeftLogical(u32, value, shift));
++                return OperationResult.Invalid;
++            }
++
++            SpvInstruction word = LoadAligned(context, u32, pointer);
++
++            if (kind is StorageKind.GlobalMemoryS8 or StorageKind.GlobalMemoryS16)
++            {
++                SpvInstruction s32 = context.TypeS32();
++                SpvInstruction field = context.BitFieldSExtract(s32, context.Bitcast(s32, word), shift, context.Constant(u32, bits));
++                return new OperationResult(AggregateType.S32, field);
++            }
++
++            return new OperationResult(AggregateType.U32, context.BitwiseAnd(u32, context.ShiftRightLogical(u32, word, shift), fieldMask));
++        }
++
+         private static SpvInstruction GetStoragePointer(CodeGenContext context, AstOperation operation, out AggregateType varType)
+         {
+             StorageKind storageKind = operation.StorageKind;
+ 
++            // NX2NAVI: atomics on the device address table's pointers (GenerateDeviceAddressLoadOrStore covers loads and
++            // stores): sources are the table binding and the address's two words.
++            if (storageKind == StorageKind.GlobalMemory)
++            {
++                varType = AggregateType.U32;
++                return GetDeviceAddressPointer(
++                    context,
++                    ((AstOperand)operation.GetSource(0)).Value,
++                    context.Get(AggregateType.U32, operation.GetSource(1)),
++                    context.Get(AggregateType.U32, operation.GetSource(2)));
++            }
++
+             StorageClass storageClass;
+             SpvInstruction baseObj;
+             int srcIndex = 0;
+--- a/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
++++ b/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/SpirvGenerator.cs
+@@ -45,7 +45,18 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv
+ 
+             context.AddCapability(Capability.Shader);
+ 
+-            context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450);
++            if (context.Properties.UsesDeviceAddresses)
++            {
++                // NX2NAVI: global memory through the device address table (Instructions.cs GetDeviceAddressPointer)
++                context.AddCapability(Capability.PhysicalStorageBufferAddresses);
++                context.AddCapability(Capability.Int64);
++                context.AddExtension("SPV_KHR_physical_storage_buffer");
++                context.SetMemoryModel(AddressingModel.PhysicalStorageBuffer64, MemoryModel.GLSL450);
++            }
++            else
++            {
++                context.SetMemoryModel(AddressingModel.Logical, MemoryModel.GLSL450);
++            }
+ 
+             context.AddCapability(Capability.GroupNonUniformBallot);
+             context.AddCapability(Capability.GroupNonUniformShuffle);
+--- a/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs
++++ b/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs
+@@ -347,6 +347,15 @@ namespace Ryujinx.Graphics.Shader
+         {
+             return true;
+         }
++
++        /// <summary>
++        /// NX2NAVI: queries whether the host binds a device address table (GPU address pages to buffer device addresses)
++        /// for global memory accesses whose storage buffer the translator cannot find.
++        /// </summary>
++        bool QueryHostSupportsBufferDeviceAddress()
++        {
++            return false;
++        }
+         bool QueryHostSupportsShaderNonUniformIndexing()
+         {
+             return false;
+--- a/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs
++++ b/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs
+@@ -19,6 +19,10 @@ namespace Ryujinx.Graphics.Shader.StructuredIr
+         public IReadOnlyDictionary<int, MemoryDefinition> LocalMemories => _localMemories;
+         public IReadOnlyDictionary<int, MemoryDefinition> SharedMemories => _sharedMemories;
+ 
++        // NX2NAVI: global memory operations go through the device address table (ResourceManager
++        // TryGetDeviceAddressTableBinding); the SPIR-V module uses the PhysicalStorageBuffer64 addressing model.
++        public bool UsesDeviceAddresses { get; set; }
++
+         public ShaderProperties()
+         {
+             _constantBuffers = new Dictionary<int, BufferDefinition>();
+--- a/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs
++++ b/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs
+@@ -273,6 +273,11 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
+                             // The returned value being null means that the global memory replacement failed,
+                             // so we just make loads read 0 and stores do nothing.
+ 
++                            if (TryKeepDeviceAddressAccess(resourceManager, gpuAccessor, node, operation))
++                            {
++                                continue;
++                            }
++
+                             gpuAccessor.Log($"Failed to reserve storage buffer for global memory operation \"{operation.Inst}\".");
+ 
+                             if (operation.Dest != null)
+@@ -323,6 +328,51 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
+             }
+         }
+ 
++        // NX2NAVI: a global access no storage buffer serves (its address is a pointer the program read from memory at an
++        // index, a phi of such pointers and constant buffer ones, or past the slots) stays a global access, with the device
++        // address table's binding as a new first source: the SPIR-V back end looks the address's page up in the table
++        // and accesses the buffer device address it gives (CodeGen/Spirv/Instructions.cs GetDeviceAddressPointer). An
++        // access the table cannot serve (a float atomic, which needs the loop of TryGenerateStorageOp) keeps the old
++        // fallback. The operation is replaced in place, so the pass goes on from the same node.
++        private static bool TryKeepDeviceAddressAccess(
++            ResourceManager resourceManager,
++            IGpuAccessor gpuAccessor,
++            LinkedListNode<INode> node,
++            Operation operation)
++        {
++            if (!gpuAccessor.QueryHostSupportsBufferDeviceAddress() ||
++                (operation.Inst & Instruction.FP32) != 0 ||
++                (operation.Inst & Instruction.Mask) == Instruction.AtomicCompareAndSwap && operation.StorageKind != StorageKind.GlobalMemory ||
++                operation.SourcesCount < 2 ||
++                (operation.StorageKind != StorageKind.GlobalMemory && operation.Inst.IsAtomic()) ||
++                !resourceManager.TryGetDeviceAddressTableBinding(out int binding))
++            {
++                return false;
++            }
++
++            Operand[] sources = new Operand[operation.SourcesCount + 1];
++            sources[0] = Const(binding);
++
++            for (int index = 0; index < operation.SourcesCount; index++)
++            {
++                sources[index + 1] = operation.GetSource(index);
++            }
++
++            Operand dest = operation.Dest;
++
++            for (int index = 0; index < operation.SourcesCount; index++)
++            {
++                operation.SetSource(index, null);
++            }
++
++            operation.Dest = null;
++            node.Value = new Operation(operation.Inst, operation.StorageKind, dest, sources);
++
++            gpuAccessor.Log($"Global memory operation \"{operation.Inst}\" goes through the device address table.");
++
++            return true;
++        }
++
+         private static bool IsGlobalMemory(StorageKind storageKind)
+         {
+             return storageKind is StorageKind.GlobalMemory or
+@@ -660,6 +710,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
+             Queue<PhiNode> phis = new();
+             HashSet<PhiNode> visited = [];
+             List<uint> targetCbs = [];
++            bool unresolved = false;   // NX2NAVI: a phi source with no constant buffer base (a pointer from memory)
+ 
+             Operand globalAddress = operation.GetSource(0);
+ 
+@@ -714,9 +765,22 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations
+                     {
+                         phis.Enqueue(phi2);
+                     }
++                    else if (phiSource.Type != OperandType.Undefined && phiSource.AsgOp is not PhiNode)
++                    {
++                        unresolved = true;
++                    }
+                 }
+             }
+ 
++            // NX2NAVI: with the device address table, an address that may come from a pointer the program read from memory
++            // goes through the table (the caller keeps it global): a range check against the constant buffer candidates
++            // would send it to one of them, or read 0.
++            if (gpuAccessor.QueryHostSupportsBufferDeviceAddress() && (targetCbs.Count == 0 || unresolved))
++            {
++                functionId = 0;
++                return false;
++            }
++
+             if (targetCbs.Count == 0)
+             {
+                 AddTableCandidates(resourceManager, operation.GetSource(0), targetCbs);
+--- a/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs
++++ b/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs
+@@ -176,6 +176,22 @@ namespace Ryujinx.Graphics.Shader.Translation
+             return setAndBinding.Binding;
+         }
+ 
++        // NX2NAVI: the storage buffer slot of the device address table, with a constant buffer slot no program has
++        // (NVN banks are 0-17, GlobalToStorage.PointerSlotFlag is 0x20); tools/xlate/Program.cs writes it as "table".
++        public const int DeviceAddressTableCbSlot = 0x40;
++
++        public bool TryGetDeviceAddressTableBinding(out int binding)
++        {
++            if (!TryGetStorageBufferBinding(DeviceAddressTableCbSlot, 0, false, out binding))
++            {
++                return false;
++            }
++
++            Properties.UsesDeviceAddresses = true;
++
++            return true;
++        }
++
+         public bool TryGetStorageBufferBinding(int sbCbSlot, int sbCbOffset, bool write, out int binding)
+         {
+             if (!TryGetSbSlot((byte)sbCbSlot, (ushort)sbCbOffset, out int slot))