From 618b9ba4e5b1826e59567d47b10a6c2f679e0dee Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 6 Nov 2026 11:02:48 -0500 Subject: [PATCH] Fix Ocarina's white dialogue frames, the fountain floor speckle and the button prompt text The texture pool descriptors are now a ring of sets, one per submission, brought up to date from a change log, and the first submission of each frame waits for its GPU work before the game goes on: the GPU ran late and part of a frame read data of the next one, and distant characters drew NaN that bloom spread over the frame. The window acquire also waits for the image's previous frame. Function locals start at zero unless DLAA is on, which removes the per-warp light tiles on the Great Fairy Fountain floor and Link's dark, speckled look. Texture slots follow only live entries of their own target, so the volumetric fog gets its 3D volume. The pool refresh walks only the slots that changed. The translator implements PRMT (patch 0021), which it only logged before; the button prompts render again. HFMA2.RELU no longer depends on it, HMMA.F16 rounds once and HMMA.F32 is lowered, checked against GPU vectors with a new Maxwell warp emulator. The frame dump gains a re-arming white-frame trigger, a trigger on a depth target's first pass, NaN pixels in magenta, image handles and view targets, and depth copies of sampled depth textures. Co-Authored-By: Claude Opus 5.5 --- docs/building.md | 5 + docs/maxwell.md | 2 +- runtime/cfg.c | 2 +- runtime/flight.h | 3 +- runtime/nvn_api.h | 1 + runtime/nvn_api_core.c | 18 + runtime/nvn_api_obj.c | 35 + runtime/vk.h | 5 + runtime/vk_backend.c | 2 +- runtime/vk_caps.c | 11 +- runtime/vk_dump.c | 127 +- runtime/vk_exec.c | 10 + runtime/vk_exec.h | 4 + runtime/vk_exec_draw.c | 176 ++- runtime/vk_exec_xfer.c | 49 +- runtime/vk_texture.c | 63 +- tests/maxwell_warp_emu.py | 1306 +++++++++++++++++ tests/maxwell_warp_emu_selftest.py | 351 +++++ tests/tensor_lowering_check.py | 441 ++++++ tools/build_xlate.py | 4 +- tools/xlate/AmpereLowering.cs | 115 +- tools/xlate/Program.cs | 49 + .../patches/0021-prmt-byte-permute.patch | 161 ++ 23 files changed, 2845 insertions(+), 95 deletions(-) create mode 100644 tests/maxwell_warp_emu.py create mode 100644 tests/maxwell_warp_emu_selftest.py create mode 100644 tests/tensor_lowering_check.py create mode 100644 tools/xlate/patches/0021-prmt-byte-permute.patch diff --git a/docs/building.md b/docs/building.md index 7eead6c4..704eb064 100644 --- a/docs/building.md +++ b/docs/building.md @@ -257,6 +257,11 @@ Shaders: `.venv-py314\Scripts\python tools\shader_census.py --version --tr to `build\\shaders\` and runs it through the translator and `spirv-val`; failure groups go to `failures.txt`. `tools\xlate_smoke.py` checks the translator alone, without game files. +A change to the SM86 tensor lowering (`tools/xlate/AmpereLowering.cs` Hmma, Ldsm, Movm, Half) passes +`python tests\tensor_lowering_check.py`: it lowers GPU-captured vectors with `nx2navi_xlate --lower` and runs them on +`tests\maxwell_warp_emu.py`, a model of what the Ryujinx translator makes of Maxwell code (the docstring names the +vector file and what is checked). `tests\maxwell_warp_emu_selftest.py` checks the emulator itself. No game files. + ## Reverse engineering (optional) - Ghidra 12.1.2 with Adubbz's Ghidra-Switch-Loader in `toolchain\ghidra\` (JDK 21) opens the game modules (NSO). diff --git a/docs/maxwell.md b/docs/maxwell.md index bcc2dfd1..7f3648be 100644 --- a/docs/maxwell.md +++ b/docs/maxwell.md @@ -88,7 +88,7 @@ Source shorthands: | 3.9 | **Integer vertex inputs read as raw bits** (bone indices `R8UI`). | Float-declared inputs convert integers. | NXN Mario: stretched characters; re-translated with `uint` inputs. Ours: attribute types passed to `QueryGraphicsState` (`Program.cs`). | NXN | | 3.10 | **BFE / BFI fields past bit 31** (as PTX `bfe` / `bfi`): a field that runs past bit 31 is cut there; one that starts past it extracts 0 (signed: bit 31 repeated) and inserts nothing. NX2navi 1.4.3's Depths terrain vertex shaders (42 variants) take the position from a register, `((2y + x) * 8) & 0xff` with 5-bit x and y, so up to 248. | `OpBitField*Extract` / `OpBitFieldInsert` undefined when offset or offset + count exceeds 32; AMD's bitfield instructions take offset and width modulo 32. | Ryujinx passes the fields through. UltraTears saw the Depths walls and ground missing on the Steam Deck and a Linux RX 570 (RADV). Ours: clamped where the field is not a constant inside the word (translator patch 0012). RX 9070 on Windows drew the Depths the same before and after. | PTX ISA `bfe`, `bfi` | | 3.11 | **SHF by 0** gives the unshifted word (C for `SHF.L`, A for `SHF.R`). | Ryujinx's funnel shifts the other word by 32: undefined in SPIR-V; AMD shifts take the low 5 bits and give A \| C. | the hardware fixed Depths terrain gaps in its funnel shift (PR 10459). Ours: a select on a zero amount (translator patch 0013). In 1.4.3 every register SHF pairs its word with 0, so the result was right already. | — | -| 3.12 | **Registers read before any write** on a path (paths the structurizer adds included) have no defined value in the game's code; UltraTears reports 0 from the NVIDIA driver. | An `OpVariable` without an initializer loads an undefined value; Mesa folds undefined values freely. | Ours: on Mesa drivers (RADV, ANV, NVK; `vk_caps_driver`) every function-local variable starts at 0 (`OpConstantNull`, job flag `zero_locals`, translator patch 0016; `NX2NAVI_ZERO_LOCALS` overrides). Elsewhere the load stays undefined: a zero start on every driver (the old patch 0014) made the whole picture shake by about 0.9 output pixels a frame with DLSS or DLAA on an RTX 4080 SUPER (#612). It also moved the SPIR-V hash of the game's FXAA shader out of `FXAA_FS` (`vk_dlaa.c` `vk_dlaa_skip_draw`), the likely cause of the shake; that path is not traced. | — | +| 3.12 | **Registers read before any write** on a path (paths the structurizer adds included) have no defined value in the game's code; UltraTears reports 0 from the NVIDIA driver. | An `OpVariable` without an initializer loads an undefined value; Mesa folds undefined values freely. | Ours: every function-local variable starts at 0 (`OpConstantNull`, job flag `zero_locals`, translator patch 0016; `NX2NAVI_ZERO_LOCALS` overrides) on Mesa drivers (RADV, ANV, NVK; `vk_caps_driver`) and, with DLAA off, on every other driver too: on NVIDIA an undefined read takes another warp lane's value, and Ocarina's clustered-light loop lit its floors in 4x8-pixel tiles (one warp each) and darkened Link (runtime/vk_caps.c). With DLAA on, off Mesa, the load stays undefined: a zero start on every driver (the old patch 0014) made the whole picture shake by about 0.9 output pixels a frame with DLSS or DLAA on an RTX 4080 SUPER (#612). It also moved the SPIR-V hash of the game's FXAA shader out of `FXAA_FS` (`vk_dlaa.c` `vk_dlaa_skip_draw`), the likely cause of the shake; that path is not traced. | — | ## 4. Memory model diff --git a/runtime/cfg.c b/runtime/cfg.c index 5603cae7..b22e027b 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -40,7 +40,7 @@ static const char *const known[] = { "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", - "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", + "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "DUMP_WHITE", "DUMP_AFTER", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", diff --git a/runtime/flight.h b/runtime/flight.h index f2f2d8ae..9ed9d476 100644 --- a/runtime/flight.h +++ b/runtime/flight.h @@ -6,7 +6,8 @@ #define FLIGHT_THUMB_W 320 #define FLIGHT_THUMB_H 180 -extern int flight_on; // NX2NAVI_FLIGHT=1, set once by flight_init before the first present +extern int flight_on; +extern int vk_dump_white; // vk_dump.c, NX2NAVI_DUMP_WHITE // NX2NAVI_FLIGHT=1, set once by flight_init before the first present void flight_init(void); // prof_start calls it // perf_present, once per present on the render thread: the interval since the previous present, the render thread's diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 903cd3e0..06182937 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -45,6 +45,7 @@ const pool_t *nvn_pool_find(uint64_t addr, uint64_t len); // GPU-visible pool h uint64_t nvn_new_serial(void); // next tex_t / prog_t serial, from 1 uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first) int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0 +extern int nvn_submits_in_frame; // nvnQueueSubmitCommands calls since the last present (NX2NAVI_SUBMIT_FINISH) void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the game wrote pool storage: back end re-reads // NX2NAVI_TRACE=perf: MemoryPoolFlushMappedRange [0] and BufferFlushMappedRange [1] calls [0] and bytes [1] by the pool's // CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call. diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index f83dcec5..d7fdb2ad 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -622,6 +622,7 @@ NVN_API(nvnQueueFinish) { // lod_entered / lod_issued count submits that entered QueueSubmitCommands / reached the back end; lod_fence holds // the fence of the last LOD_RING. NX2NAVI_LOD_GUARD=0 turns the wait off. #define LOD_RING 64 +int nvn_submits_in_frame; // nvnQueueSubmitCommands calls since the last nvnQueuePresentTexture (NX2NAVI_SUBMIT_FINISH) static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING]; // void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its @@ -655,6 +656,23 @@ NVN_API(nvnQueueSubmitCommands) { while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} __atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED); __atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST); + // The first submission of a frame (since the last present) returns once the GPU has run it. NVN starts the GPU when + // the submit returns and it is through a frame's first submission (Ocarina's frame-start compute: light grid, probes) + // within a few milliseconds; here the replay records first and the GPU runs later. Ocarina rewrites data those + // passes and the frame's lit materials read once it has submitted its later work, with no fence: with the GPU late, + // part of a frame drew with the next frame's data, distant characters wrote NaN and bloom spread it (white dialogue + // frames, 3% of them in 100 s of dialogue; none with this wait, at the same 58-59 fps: the GPU is idle at that + // point). Copies of the data at submit made it worse, a mix of two frames' versions. + // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits + // after every submission, 0 never. + static long long finish_mask = -2; + if (finish_mask == -2) { + const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); + finish_mask = e && *e ? strtoll(e, NULL, 0) : 1; + if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all + } + int idx = nvn_submits_in_frame++; + if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX); } // The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index 5410b440..4152288d 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -756,6 +756,19 @@ static void sleep_until(uint64_t deadline) { } } +// The submit fence of each window image's newest present (win_t has no room: a side table for the first windows). +static struct { const win_t *w; uint64_t fence[4]; } img_fence[4]; +static uint64_t last_present_fence; // the submit fence at the newest present of any window + +static uint64_t *img_fence_of(const win_t *o, int idx) { + if (idx < 0 || idx >= 4) return NULL; + for (int i = 0; i < 4; i++) { + if (img_fence[i].w == o) return &img_fence[i].fence[idx]; + if (!img_fence[i].w) return img_fence[i].w = o, &img_fence[i].fence[idx]; + } + return NULL; +} + // NVNwindowAcquireTextureResult nvnWindowAcquireTexture(NVNwindow *w, NVNsync *s, int *index): blocks here, not in the // present, until the image's previous present plus one period has passed (the game often presents with a lock held). // That is the time the next frame goes to the display, so the image is free as soon as that time is reached. @@ -772,6 +785,24 @@ NVN_API(nvnWindowAcquireTexture) { sleep_until(at); perf_end(PERF_PACE, p0); } + // On the Switch the image comes back only after the display has shown it, which needs the GPU to have finished the + // frame drawn into it, so the CPU never runs more than the swapchain's depth ahead of the GPU. Here the pacer frees + // images by the clock alone, so the acquire also waits for the GPU to finish the image's newest present. With the + // GPU slowed (frame dump logging), this alone cut Ocarina's NaN frames from 56 to 23 of 61; the wait after a frame's + // first submission (nvn_api_core.c nvnQueueSubmitCommands) removes the rest. NX2NAVI_ACQUIRE_WAIT=0 leaves it to the + // clock; =2 waits for the newest present of any image (one frame in flight at most). + static int acq_wait = -1; + if (acq_wait < 0) acq_wait = rt_knob("ACQUIRE_WAIT", "1"); + uint64_t *fe = img_fence_of(o, idx), want = fe ? *fe : 0; + if (acq_wait == 2) want = last_present_fence; + if (acq_wait && want) { + const GfxBackend *b = gfx_backend(); + if (b->fence_done() < want) { + uint64_t p0 = perf_begin(); + b->fence_wait(want, UINT64_MAX); + perf_end(PERF_PACE, p0); + } + } // The sync means "the display no longer reads this image". Presents are copies queued in submit order, so it is // already satisfied: fenced = 3 with fence 0. if (s) { @@ -794,12 +825,16 @@ NVN_API(nvnQueuePresentTexture) { if (o->magic != NVN_WIN_MAGIC || idx < 0 || idx >= o->ntex || idx >= 4 || !o->tex[idx]) rt_fatal("nvn: QueuePresentTexture of image %d on window %p", idx, (void *)o); o->presents++; + nvn_submits_in_frame = 0; const tex_t *t = (const tex_t *)o->tex[idx]; uint64_t period = present_period(o), now = kern_now_ns(); uint64_t slot = now + pace(&o->pace, now, period); gfx_backend()->present(t, o->crop, slot); o->ticket[idx] = o->presents; o->avail[idx] = slot + period; + uint64_t *fe = img_fence_of(o, idx); + if (fe) *fe = nvn_fence_last(); // the frame drawn into it (nvnWindowAcquireTexture waits for it) + last_present_fence = nvn_fence_last(); frame_step_present(slot, o->interval); // frame_rate.c: the real-time step follows the display times rt_frame_presented(t, o->presents); } diff --git a/runtime/vk.h b/runtime/vk.h index a870a732..6914bab9 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -425,6 +425,11 @@ VkImage vk_tonemap_record(VkCommandBuffer cb, VkImage src, VkFormat sf, uint32_t // its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it // guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0. uint64_t vk_texture_slot_token(int id); +// Slots whose token may have changed since *pos (0 at first; then the value these calls leave): fn is called for each +// (some more than once) and *pos advanced. 0 when too many changes came since: the caller looks at every slot. *pos +// from vk_texture_dirty_pos_locked after such a walk. Texture lock held. +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg); +uint64_t vk_texture_dirty_pos_locked(void); void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 2402c47b..247894fb 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -58,7 +58,7 @@ static void present_main(void *arg) { busy = 1; plat_cond_broadcast(&pcv); plat_mutex_unlock(&pl); - if (flight_on) vk_thumbs_frame(j.frame); + if (flight_on || vk_dump_white) vk_thumbs_frame(j.frame); // NX2NAVI_DUMP_WHITE reads them too vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.format, j.w, j.h, j.has_crop ? j.crop : NULL); uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too if (j.value && f != j.value) diff --git a/runtime/vk_caps.c b/runtime/vk_caps.c index 0b85e18b..89b148a7 100644 --- a/runtime/vk_caps.c +++ b/runtime/vk_caps.c @@ -116,9 +116,14 @@ void vk_caps_init(VkCtx *c) { // BotW and NX2navi on AMD). The translator biases gather coordinates by the host's sub-texel precision; NVIDIA rounds // as Maxwell does. NX2NAVI_GATHER_BIAS= forces it (0: off). uint32_t gather = env_u32("NX2NAVI_GATHER_BIAS", c->props.vendorID == 0x10de ? 0 : l->subTexelPrecisionBits); - // A register read before any write is an undefined load in SPIR-V, and Mesa folds undefined values freely, so on - // Mesa function-local variables start at 0 (docs/maxwell.md 3.12). NX2NAVI_ZERO_LOCALS=1 forces it, 0 turns it off. - uint32_t zero_locals = env_u32("NX2NAVI_ZERO_LOCALS", (uint32_t)vk_caps.mesa); + // A register read before any write is an undefined load in SPIR-V, so function-local variables start at 0 + // (docs/maxwell.md 3.12). Mesa folds undefined values freely; NVIDIA's compiler gives such a read a value of another + // lane of the warp: Ocarina's clustered-light loop reads registers that only some paths write, and on an RTX 5090 + // the Great Fairy Fountain floor took an extra light term in 4x8-pixel tiles (one 32-lane warp each) and Link went + // dark with green speckles. With DLAA on (vk_dlaa.c, which finds the game's FXAA by its SPIR-V hash) they stay + // undefined off Mesa, as before: zeroing moved that hash and the picture shook under DLAA (maxwell.md 3.12). + // NX2NAVI_ZERO_LOCALS=1 / 0 forces either. + uint32_t zero_locals = env_u32("NX2NAVI_ZERO_LOCALS", vk_caps.mesa || !cfg_bool("NX2NAVI_DLAA")); gfx_shader_set_host(vk_caps.float64, vk_caps.sbuf_align, told, gather, zero_locals != 0); if (gather) rt_log("vk caps: gather coordinates biased for %u sub-texel bits (NX2NAVI_GATHER_BIAS)", gather); diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index 19c85e13..316fbc8c 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -5,7 +5,8 @@ // written in the frame are copied once each, slices side by side. Files in NX2NAVI_DUMP_DIR (default "dump"): // pass__t_x__.png colour target i (td: depth; img: storage image; lut: 3D; // a BC target drawn through a block view: its blocks as that view) -// ..._tm.png float formats: x/(1+x) of the same image (the plain one clamps) +// ..._tm.png float formats: x/(1+x) of the same image (the plain one clamps); +// in both, a pixel with a NaN channel is magenta // passes.txt per pass or dispatch: targets, draws, each distinct pipeline + texture set with its shaders' SPIR-V // hashes (sh_.spv written once) and the bound textures with their samplers // images.txt per image: channel min/max, distinct texel values over the image and over its top quarter @@ -35,6 +36,7 @@ #include "vk.h" #include "vk_exec.h" +#include "flight.h" uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r); // vk_exec_draw.c int vk_exec_raw_9e5(VkImage img, VkFormat f); // vk_exec_draw.c #include "platform/platform.h" @@ -49,6 +51,10 @@ int vk_exec_raw_9e5(VkImage img, VkFormat f); // v int vk_dump_armed; static uint64_t dump_frame, log_from; // log_from: NX2NAVI_DUMP_LOG, first frame logged to passes.txt without images static char dir[400]; +static char base_dir[400]; // NX2NAVI_DUMP_DIR; with NX2NAVI_DUMP_WHITE each attempt goes to base_dir/f +static uint64_t white_last; // NX2NAVI_DUMP_WHITE: the frame dumped last (0: none), checked 3 presents on +static int white_tries, white_caught; +static uint64_t dump_after; // NX2NAVI_DUMP_AFTER static FILE *passes_f, *images_f; static int seq, finished, copied; // copied: passes and dispatches of the dump frame whose images were copied static int skip_runs, dump_ubo; // NX2NAVI_DUMP_SKIP_RUNS, NX2NAVI_DUMP_UBO (vec4 per block, 0 for none) @@ -221,8 +227,8 @@ static void write_images(const Job *j) { uint32_t pw = j->w * j->d, ph = j->h; uint64_t n = (uint64_t)pw * ph; float *px = malloc(n * 16); - uint8_t *out = malloc(n * 4); - if (!px || !out) rt_fatal("vk_dump: out of memory for %s", j->name); + uint8_t *out = malloc(n * 4), *isnan_ = calloc(n, 1); // isnan_: a pixel with a NaN channel, drawn magenta + if (!px || !out || !isnan_) rt_fatal("vk_dump: out of memory for %s", j->name); float mn[4] = {INFINITY, INFINITY, INFINITY, INFINITY}, mx[4] = {-INFINITY, -INFINITY, -INFINITY, -INFINITY}; uint64_t nans = 0; for (uint32_t z = 0; z < j->d; z++) @@ -231,7 +237,7 @@ static void write_images(const Job *j) { float *o = px + 4 * ((uint64_t)y * pw + (uint64_t)z * j->w + x); decode(j->f, j->map + (((uint64_t)z * j->h + y) * j->w + x) * j->f->bytes, o); for (int c = 0; c < 4; c++) { - if (isnan(o[c])) { nans++, o[c] = 0; continue; } + if (isnan(o[c])) { nans++, o[c] = 0, isnan_[(uint64_t)y * pw + (uint64_t)z * j->w + x] = 1; continue; } if (o[c] < mn[c]) mn[c] = o[c]; if (o[c] > mx[c]) mx[c] = o[c]; } @@ -255,6 +261,7 @@ static void write_images(const Job *j) { out[4 * i + c] = u8(x); } out[4 * i + 3] = 255; + if (isnan_[i]) out[4 * i] = 255, out[4 * i + 1] = 0, out[4 * i + 2] = 255; } char path[600]; snprintf(path, sizeof path, "%s/%s%s.png", dir, j->name, pass ? "_tm" : ""); @@ -270,6 +277,7 @@ static void write_images(const Job *j) { } plat_mutex_unlock(&img_lock); free(out); + free(isnan_); free(px); } @@ -527,9 +535,12 @@ static void flush_luts(VkExec *ex, int n) { VkFormat bf = block_format(l->format); snprintf(name, sizeof name, "pass_%04d_%s%d_%ux%ux%u_%d_%s%s", n, l->is2d ? "tex" : "lut", l->id, l->w, l->h, l->d, (int)l->format, fmt_name(l->format, fn), bf ? "_blocks" : ""); + // a sampled depth texture copies its depth aspect (the colour aspect of a depth image read as zeros) + VkImageAspectFlags asp = l->format >= VK_FORMAT_D16_UNORM && l->format <= VK_FORMAT_D32_SFLOAT_S8_UINT + ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT; const char *why = bf ? copy_region(ex, l->image, bf, VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 0, (VkExtent3D){l->w, l->h, l->d}, (l->w + 3) / 4, (l->h + 3) / 4, l->d, name) - : copy_out(ex, l->image, l->format, VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, l->w, l->h, l->d, name); + : copy_out(ex, l->image, l->format, asp, 0, 0, l->w, l->h, l->d, name); fprintf(passes_f, " %s tex %d -> %s.png%s%s\n", l->is2d ? "2D" : "3D", l->id, name, why ? " skipped: " : "", why ? why : ""); } @@ -572,8 +583,12 @@ static void describe_bindings(Str *b, const VkExec *ex, const VkPipeInfo *info) } if (st == 5) sp(b, " -> %#llx", (unsigned long long)vk_exec_cs_storage_addr(ex, r)); for (int k = 0; k < 16; k++) - if (ex->ssbo[st][k].addr) - sp(b, " %d=%#llx+%#llx", k, (unsigned long long)ex->ssbo[st][k].addr, (unsigned long long)ex->ssbo[st][k].size); + if (ex->ssbo[st][k].addr) { + VkDescriptorBufferInfo cb; // "(copy)": a binding that only reads it uses the submit-time copy (vk_usnap.c) + int copy = vk_usnap_find(ex->ssbo[st][k].addr, ex->ssbo[st][k].size, 1, &cb); + sp(b, " %d=%#llx+%#llx%s", k, (unsigned long long)ex->ssbo[st][k].addr, (unsigned long long)ex->ssbo[st][k].size, + copy ? "(copy)" : ""); + } sp(b, "\n"); // NX2NAVI_DUMP_UBO: also the small storage buffers' guest bytes (GPU results as of the last write-back); per-view // constants (exposure and such) live in such buffers. @@ -624,10 +639,11 @@ static void describe_bindings(Str *b, const VkExec *ex, const VkPipeInfo *info) const VkTexImage *t = vk_texture_by_id(NVN_HANDLE_TEX(h), &v, &hv); if (!t) { sp(b, "tex %d unresolved (dummy)\n", NVN_HANDLE_TEX(h)); continue; } static const char *vt[] = {"1D", "2D", "3D", "cube", "1DArray", "2DArray", "cubeArray"}; - sp(b, "tex %d %s(%d) %ux%ux%u levels %u layers %u image %s", NVN_HANDLE_TEX(h), fmt_name(t->format, fn), + sp(b, "tex %d %s(%d) %ux%ux%u levels %u layers %u image %s vkimage %p", NVN_HANDLE_TEX(h), fmt_name(t->format, fn), (int)t->format, t->width, t->height, t->depth, t->levels, t->layers, - t->view_type <= 6 ? vt[t->view_type] : "?"); + t->view_type <= 6 ? vt[t->view_type] : "?", (void *)t->image); if (hv && v.set_format) sp(b, " view-nvn-fmt %#x", v.format); + if (hv && v.set_target) sp(b, " view-target %u", v.target); if (hv && v.set_levels) sp(b, " view-levels %u+%u", v.base_level, v.nlevels); if (hv && v.set_layers) sp(b, " view-layers %u+%u", v.min_layer, v.nlayers); samp_par s; @@ -730,9 +746,9 @@ static void target_line(VkExec *ex, const VkExecTarget *t, const char *tag, int t->height, 1, name) : t->is3d ? "3D slice (the volume is dumped as lut where it is sampled)" : copy_out(ex, t->image, t->format, t->aspect, t->level, t->base_layer, t->width, t->height, 1, name); - fprintf(passes_f, " %s: nvn %#x %s(%d) %ux%u level %u layers %u+%u samples %u addr %#llx -> %s.png%s%s\n", tag, - t->nvn_format, fmt_name(t->format, fn), (int)t->format, t->width, t->height, t->level, t->base_layer, - t->layers, t->samples, (unsigned long long)(uintptr_t)t->tex->mem, name, why ? " skipped: " : "", + fprintf(passes_f, " %s: nvn %#x %s(%d) %ux%u level %u layers %u+%u samples %u addr %#llx vkimage %p -> %s.png%s%s\n", + tag, t->nvn_format, fmt_name(t->format, fn), (int)t->format, t->width, t->height, t->level, t->base_layer, + t->layers, t->samples, (unsigned long long)(uintptr_t)t->tex->mem, (void *)t->image, name, why ? " skipped: " : "", why ? why : ""); } @@ -755,6 +771,11 @@ static void pass_decide(int same_targets) { void vk_dump_pass_end(VkExec *ex) { plat_mutex_lock(&rec_lock); + if (dump_after && dump_frame == UINT64_MAX && ex->depth.tex && (uint64_t)(uintptr_t)ex->depth.tex->mem == dump_after) { + dump_frame = frame_now(); // this pass and the rest of its frame + rt_log("vk_dump: a pass drew into %#llx in frame %llu: it and the rest of the frame go to %s", + (unsigned long long)dump_after, (unsigned long long)dump_frame, dir); + } if (!active()) goto out; int n = seq++; uint64_t sig = targets_sig(ex); @@ -893,9 +914,34 @@ out: static void make_dirs(char *path) { plat_file_mkdir_p(path); } +// NX2NAVI_DUMP_WHITE=1 (diagnosis): no frame is named; from frame 1800 each present looks at the thumbnail of the frame +// three presents back (gfx_present.c vk_thumbs_*), and the first one whose picture left of the overlay and above the +// dialogue box is blank white makes the dump take the next frame (white flashes come in runs of several frames). +int vk_dump_white; // vk_backend.c asks for a thumbnail of every present while it is set +#define dump_white vk_dump_white + +// passes.txt and images.txt in dir (a new folder per NX2NAVI_DUMP_WHITE attempt) +static void open_files(void) { + make_dirs(dir); + char path[600]; + snprintf(path, sizeof path, "%s/passes.txt", dir); + passes_f = fopen(path, "w"); + snprintf(path, sizeof path, "%s/images.txt", dir); + images_f = fopen(path, "w"); + if (!passes_f || !images_f) rt_fatal("vk_dump: cannot write to %s", dir); +} + void vk_dump_init(void) { const char *e = cfg_get("NX2NAVI_DUMP"); - if (!e || !*e || !(dump_frame = strtoull(e, NULL, 10))) return; + dump_white = rt_knob("DUMP_WHITE", "0"); + // NX2NAVI_DUMP_AFTER=
: the first render pass whose depth target lives at that address and the rest of its frame + // (a target drawn once, at an area load) + const char *da = cfg_get("NX2NAVI_DUMP_AFTER"); + dump_after = da && *da ? strtoull(da, NULL, 0) : 0; + if (dump_white || dump_after) { + dump_frame = UINT64_MAX; + vk_thumbs_enable(16); + } else if (!e || !*e || !(dump_frame = strtoull(e, NULL, 10))) return; const char *lf = cfg_get("NX2NAVI_DUMP_LOG"); if (lf && *lf) log_from = strtoull(lf, NULL, 10); const char *sr = cfg_get("NX2NAVI_DUMP_SKIP_RUNS"); @@ -912,14 +958,10 @@ void vk_dump_init(void) { w[1] = *q == 'x' ? (uint32_t)strtoul(q + 1, &q, 10) : 0; } const char *d = cfg_get("NX2NAVI_DUMP_DIR"); - snprintf(dir, sizeof dir, "%s", d && *d ? d : "dump"); - make_dirs(dir); + snprintf(base_dir, sizeof base_dir, "%s", d && *d ? d : "dump"); + snprintf(dir, sizeof dir, "%s", base_dir); char path[600]; - snprintf(path, sizeof path, "%s/passes.txt", dir); - passes_f = fopen(path, "w"); - snprintf(path, sizeof path, "%s/images.txt", dir); - images_f = fopen(path, "w"); - if (!passes_f || !images_f) rt_fatal("vk_dump: cannot write to %s", dir); + if (!dump_white) open_files(); const char *bf = cfg_get("NX2NAVI_DUMP_BUF"); if (bf && *bf) { char *q; @@ -937,7 +979,43 @@ void vk_dump_init(void) { rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); } +// The thumbnail of frame f is blank white left of the overlay and above the dialogue box: -1 not available, 0 no, 1 yes. +static int thumb_white(uint64_t f) { + static uint8_t px[FLIGHT_THUMB_W * FLIGHT_THUMB_H * 4]; + if (!vk_thumbs_read(f, px)) return -1; + int n = 0, bright = 0; + for (int y = 0; y < 105; y++) + for (int x = 0; x < 250; x++, n++) { + const uint8_t *q = px + 4 * (y * FLIGHT_THUMB_W + x); + bright += (q[0] * 30 + q[1] * 59 + q[2] * 11) / 100 > 245; + } + return bright * 100 > n * 97; +} + +static void white_check(uint64_t frame) { + static int run; // consecutive white thumbnails: white runs are mostly 1-2 frames, too short for a dump 4 frames on + if (frame < 1800) return; + int w = thumb_white(frame - 3); + if (w < 0) return; + run = w ? run + 1 : 0; + if (run >= 3) { // frames frame-5..frame-3 white: the run is likely still on 4 frames later + run = 0; + dump_frame = frame + 1; + snprintf(dir, sizeof dir, "%s/f%llu", base_dir, (unsigned long long)dump_frame); + open_files(); + rt_log("vk_dump: frames %llu-%llu were white; frame %llu goes to %s", (unsigned long long)(frame - 5), + (unsigned long long)(frame - 3), (unsigned long long)dump_frame, dir); + } +} + void vk_dump_present(uint64_t frame) { + if (dump_white && white_last && frame == white_last + 3) { // was the dumped frame itself white? + int w = thumb_white(white_last); + rt_log("vk_dump: dumped frame %llu was %s", (unsigned long long)white_last, w > 0 ? "white: done" : "not white: trying again"); + if (w > 0 || ++white_tries >= 8) white_caught = 1; + white_last = 0; + } + if (dump_white && !white_caught && !white_last && dump_frame == UINT64_MAX) white_check(frame); if (frame != dump_frame) return; plat_mutex_lock(&rec_lock); finished = 1; @@ -952,4 +1030,13 @@ void vk_dump_present(uint64_t frame) { passes_f = NULL; rt_log("vk_dump: frame %llu: %d passes and dispatches, about %llu MB of PNGs in %s", (unsigned long long)frame, seq, (unsigned long long)(png_bytes >> 20), dir); + if (dump_white) { // ready for another attempt (vk_dump_present checks this one's thumbnail 3 presents on) + white_last = frame; + plat_mutex_lock(&rec_lock); + seq = copied = 0, last_targets = 0, pass_skip = NULL, png_bytes = 0; + nluts = nluts_done = nents = ents_over = pass_draws = 0; + dump_frame = UINT64_MAX; + finished = 0; + plat_mutex_unlock(&rec_lock); + } } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index e2af956b..ecaaf947 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -63,6 +63,7 @@ void vk_exec_frame_written(const tex_t *t) { } void vk_exec_frame_log(const tex_t *p, uint64_t frame) { + vk_exec_cond_frame_end(frame); // a frame boundary in replay order (present_image runs on the replay thread) if (!(rt_trace & RT_TRACE_GFX)) return; VkExecFrame *f = &vk_exec_frame; int hit = 0; @@ -743,6 +744,13 @@ static void clear_attachment(VkExec *ex, const VkClearAttachment *a) { static void clear_cond_log(const VkExec *ex, const char *what, int index) { static int on = -1, n; if (on < 0) on = rt_knob("LOG_RCOND", "0"); + if (on == 2 && index < 0 && ex->has_render_cond && ex->depth.width >= 1600) { // full-size depth clears, all of them + char why[256]; + vk_exec_cond_why(&ex->render_cond, why, sizeof why); + rt_log("SEQ depth clear %#llx %s; %s", ex->depth.img ? (unsigned long long)ex->depth.img->addr : 0ull, + ex->render_disabled ? "skipped" : "RUN", why); + return; + } if (!on || !ex->has_render_cond || n >= 400) return; n++; const VkExecTarget *t = index < 0 ? &ex->depth : index < (int)ex->pass_ncolor ? &ex->color[index] : NULL; @@ -754,6 +762,7 @@ static void clear_cond_log(const VkExec *ex, const char *what, int index) { static void clear_color(VkExec *ex, const p_clear_color *c) { vk_exec_frame.clears++; clear_cond_log(ex, "colour", c->index); + if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : vk_cond_clears_run++; if (ex->render_disabled) return; resolve_rt(ex); int i = c->index; @@ -785,6 +794,7 @@ static void clear_color(VkExec *ex, const p_clear_color *c) { static void clear_ds(VkExec *ex, const p_clear_ds *c) { clear_cond_log(ex, "depth", -1); + if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : (vk_cond_clears_run++, vk_cond_depth_run++); if (ex->render_disabled) return; resolve_rt(ex); const VkExecTarget *t = &ex->depth; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index ad437edd..e694760c 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -38,6 +38,10 @@ typedef struct { uint8_t scale; // VkTexImage.scale } VkExecTarget; +void vk_exec_cond_frame_end(uint64_t frame); // vk_exec_xfer.c: a present, in replay order (NX2NAVI_LOG_RCOND) +extern unsigned vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run; +void vk_exec_cond_why(const p_render_cond *c, char *buf, size_t cap); // NX2NAVI_LOG_RCOND=ab + // The replay state. One instance lives for the run: NVN state is queue state and carries over from one submission to // the next, as in MacNativeGames' and LAS Recomp's replays. The "per command buffer" fields are reset by vk_exec_record. typedef struct VkExec { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a8f3f2fa..a8d0893c 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1526,13 +1526,33 @@ static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int sto // rewrites the entries whose slot changed (vk_texture_slot_token) and fills the kinds a new pipeline asked for; with // update-after-bind (descriptorIndexing) a write never invalidates a command buffer being recorded. Each submit's first // sync also looks every registered slot up once, the freshness check a bound texture gets (vk_texture_by_id_locked). -// The entries are not double-buffered: a slot registered again while a frame using it is in flight is read new by that -// frame, as on the Switch, where the pool is memory the GPU reads as it goes. +// The set is one of a ring of POOL_SETS, one per submission: on the Switch the GPU reads the pool as it goes, but it +// runs a submission right after nvnQueueSubmitCommands, so what it reads is the pool as of about the submit. Here the +// GPU runs a submission several milliseconds later, and Ocarina registers slots for its next submission right after +// submitting: with one set, an in-flight submission sampled a texture registered for the next one, the lit materials +// of distant characters read the wrong texture as their light or probe data, wrote NaN and bloom spread it (white +// dialogue frames, a fifth of them; none once each submission waited for its GPU work, NX2NAVI_SUBMIT_FINISH). A +// submission's first pool_sync takes the next set (pool_take_set), waits for the GPU only if that set's last +// submission is still running, and brings it up to date from the change log (every entry written since the set was +// last current; the whole set when the log has wrapped). Writes within a submission go to its own set at once, as +// before. volatile uint32_t vk_pool_kinds_used; +#define POOL_SETS 4 +#define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write static struct { VkDescriptorSetLayout layout, empty; VkDescriptorPool dpool; - VkDescriptorSet set; + VkDescriptorSet set; // the current submission's (sets[cur]) + VkDescriptorSet sets[POOL_SETS]; + uint64_t set_fence[POOL_SETS]; // the timeline value of the last submission that bound it + uint64_t set_synced[POOL_SETS]; // the change-log position up to which it holds every write + int nsets, cur, all; // all: the first fill, written to every set + uint64_t cur_fence; // the submission sets[cur] serves + uint32_t *log; // POOL_LOG entries + uint64_t logn; // writes so far + uint64_t *last; // per entry: the log position of its newest write (catch-up writes it once, there) + uint64_t switches, waits, catchup, written, rewrites, catch_ns, sync_ns, n_slots, n_refill, n_fresh; + uint64_t dirty_pos; // vk_texture_dirty_slots_locked position (0: the next sync walks every slot) int ready, uab; uint32_t kinds, gen, epoch, sgen; int retry; // the last sync left slots to the creation budget @@ -1567,19 +1587,33 @@ static VkDescriptorSetLayout pool_layout_make(void) { if (vkCreateDescriptorSetLayout(c->device, &li, NULL, &l) != VK_SUCCESS) rt_fatal("vk: the texture pool layout failed"); VkDescriptorPoolSize ps[2] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX}, {VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}}; - VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0, - .maxSets = 1, .poolSizeCount = 2, .pPoolSizes = ps}; - if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) rt_fatal("vk: the texture pool descriptor pool failed"); - VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = PL.dpool, - .descriptorSetCount = 1, .pSetLayouts = &l}; - if (vkAllocateDescriptorSets(c->device, &ai, &PL.set) != VK_SUCCESS) rt_fatal("vk: the texture pool set failed"); + // POOL_SETS sets, fewer where the device has no room for them (one at the least, as before) + for (int n = POOL_SETS; n >= 1 && !PL.nsets; n--) { + VkDescriptorPoolSize psn[2] = {{ps[0].type, ps[0].descriptorCount * (uint32_t)n}, + {ps[1].type, ps[1].descriptorCount * (uint32_t)n}}; + VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, + .flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0, + .maxSets = (uint32_t)n, .poolSizeCount = 2, .pPoolSizes = psn}; + if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) continue; + VkDescriptorSetLayout ls[POOL_SETS]; + for (int i = 0; i < n; i++) ls[i] = l; + VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = PL.dpool, + .descriptorSetCount = (uint32_t)n, .pSetLayouts = ls}; + if (vkAllocateDescriptorSets(c->device, &ai, PL.sets) == VK_SUCCESS) PL.nsets = n; + else vkDestroyDescriptorPool(c->device, PL.dpool, NULL), PL.dpool = VK_NULL_HANDLE; + } + if (!PL.nsets) rt_fatal("vk: the texture pool set failed"); + PL.set = PL.sets[0]; + PL.log = malloc(sizeof *PL.log * POOL_LOG); + PL.last = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP, sizeof *PL.last); + if (!PL.log || !PL.last) rt_fatal("vk: out of memory"); PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok); PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view); PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp); if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory"); - rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s", VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, - PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)"); + rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)", + VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, + PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets); return l; } @@ -1620,16 +1654,83 @@ static void pool_flush(void) { pool_nw = 0; } -static void pool_write(uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { +static void pool_write_to(VkDescriptorSet set, uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { if (pool_nw == POOL_WBATCH) pool_flush(); pool_ii[pool_nw] = (VkDescriptorImageInfo){samp, view, VK_IMAGE_LAYOUT_GENERAL}; - pool_w[pool_nw] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = PL.set, + pool_w[pool_nw] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = set, .dstBinding = binding, .dstArrayElement = id, .descriptorCount = 1, .descriptorType = view ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, .pImageInfo = &pool_ii[pool_nw]}; pool_nw++; } +// An entry whose wanted value PL.view / PL.samp already hold: into the current set, and into the log for the others +// (the first fill goes into every set). +static void pool_write(uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { + if (PL.all) { + for (int i = 0; i < PL.nsets; i++) pool_write_to(PL.sets[i], binding, id, view, samp); + return; + } + pool_write_to(PL.set, binding, id, view, samp); + uint32_t e = binding * VK_POOL_TEX + id; + PL.last[e] = PL.logn; + PL.log[PL.logn++ % POOL_LOG] = e; +} + +// Writes entry e (binding * VK_POOL_TEX + id) of set as PL.view / PL.samp hold it now. +static void pool_entry_to(VkDescriptorSet set, uint32_t e) { + uint32_t binding = e / VK_POOL_TEX, id = e % VK_POOL_TEX; + if (binding < VK_POOL_KINDS) { + VkImageView v = PL.view[(size_t)binding * VK_POOL_TEX + id]; + if (v) pool_write_to(set, binding, id, v, VK_NULL_HANDLE); + } else if (id < VK_POOL_SAMP && PL.samp[id]) { + pool_write_to(set, binding, id, VK_NULL_HANDLE, PL.samp[id]); + } +} + +// A submission's first pool use: the next set of the ring, once the GPU is done with it. 1 when it switched; *from is +// the log position the set holds writes up to (pool_catch_up brings it level once pool_sync has refreshed the entries: +// a view written into a set must be alive, and the refresh is what replaces the views of destroyed textures). +static int pool_take_set(VkExec *ex, uint64_t *from) { + if (PL.cur_fence == ex->fence) return 0; + int i = (PL.cur + 1) % PL.nsets; + if (PL.nsets > 1) { + const GfxBackend *b = gfx_backend(); + if (b->fence_done() < PL.set_fence[i]) PL.waits++, b->fence_wait(PL.set_fence[i], UINT64_MAX); + } + *from = PL.set_synced[i]; + PL.cur = i, PL.set = PL.sets[i], PL.cur_fence = ex->fence, PL.set_fence[i] = ex->fence; + return PL.nsets > 1; +} + +// The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now. +static void pool_catch_up(uint64_t from, uint64_t to) { + VkDescriptorSet set = PL.set; + if (to - from > POOL_LOG) { // the log has wrapped since: every entry + PL.rewrites++; + for (uint32_t e = 0; e < VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP; e++) pool_entry_to(set, e); + } else { + // each entry once, at its newest write: an entry written again later (also by this submission's own sync, + // which wrote it into this set already) is skipped at the older positions + PL.catchup += to - from; + for (uint64_t q = from; q < to; q++) { + uint32_t e = PL.log[q % POOL_LOG]; + if (PL.last[e] == q) pool_entry_to(set, e), PL.written++; + } + } + pool_flush(); + PL.switches++; + if (PL.switches % 3600 == 0) + rt_log("vk: texture pool sets: %llu switches, %llu waited for the GPU, %.1f log entries and %.1f writes per " + "catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.0f slots, %.0f " + "refilled, %.0f checked)", + (unsigned long long)PL.switches, (unsigned long long)PL.waits, (double)PL.catchup / (double)PL.switches, + (double)PL.written / (double)PL.switches, (unsigned long long)PL.rewrites, + PL.catch_ns / 1e6 / (double)PL.switches, PL.sync_ns / 1e6 / (double)PL.switches, + (double)PL.n_slots / (double)PL.switches, (double)PL.n_refill / (double)PL.switches, + (double)PL.n_fresh / (double)PL.switches); +} + // Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the // slot's texture was left to a later sync by the creation budget (pool_sync). // NX2NAVI_SKIP_SHADERS=,...: draws and dispatches whose shader's SPIR-V has one of these FNV-1a hashes (the "vs"/"fs"/ @@ -1684,8 +1785,38 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { return !vk_texture_create_deferred_locked(); } +static void pool_sync_entries(VkExec *ex); + static void pool_sync(VkExec *ex) { if (!PL.layout) vk_pool_set_layout(); + uint64_t from = 0, pre = PL.logn; + int switched = pool_take_set(ex, &from); // every submission its own set, before the shortcut + uint64_t t0 = kern_now_ns(); + pool_sync_entries(ex); + uint64_t t1 = kern_now_ns(); + PL.sync_ns += t1 - t0; + if (switched) { + vk_texture_lock(); + pool_catch_up(from, pre); // what the other sets got since; this sync's own writes went into it already + vk_texture_unlock(); + PL.catch_ns += kern_now_ns() - t1; + } + PL.set_synced[PL.cur] = PL.logn; +} + +typedef struct { VkExec *ex; uint32_t used; int n, retry; } PoolDirty; + +static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_locked: a slot whose token may have changed + PoolDirty *d = arg; + if (id >= d->n) return; + uint64_t t = vk_texture_slot_token(id); + if (t == PL.tok[id]) return; + PL.n_refill++; + if (pool_fill(d->ex, (uint32_t)id, d->used, t)) PL.tok[id] = t; + else d->retry = 1; +} + +static void pool_sync_entries(VkExec *ex) { uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED); uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); if (PL.ready && !PL.retry && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return; @@ -1694,24 +1825,37 @@ static void pool_sync(VkExec *ex) { int n = vk_texture_slot_count_locked(); if (n > VK_POOL_TEX) n = VK_POOL_TEX; if (!PL.ready) { // every entry a stand-in first: partially bound arrays are still read out of range by guests + PL.all = 1; for (uint32_t id = 0; id < VK_POOL_TEX; id++) pool_fill(ex, id, 0xffffu, 0); for (uint32_t id = 0; id < VK_POOL_SAMP; id++) PL.samp[id] = Z.samp, pool_write(VK_POOL_KINDS, id, VK_NULL_HANDLE, Z.samp); + pool_flush(); + PL.all = 0; PL.ready = 1, PL.sgen = 0; } int retry = 0; // slots whose texture the creation budget left: filled again by the next sync if (fresh || gen != PL.gen || ep != PL.epoch || PL.retry) { vk_texture_create_budget_locked(POOL_CREATE_BUDGET); - for (int id = 0; id < n; id++) { // slots past n were never registered: stand-ins since the first sync + // Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that + // changed (vk_texture_dirty_slots_locked), not the whole table + PoolDirty pd = {ex, used, n, 0}; + int walked = !fresh && ep == PL.epoch && !PL.retry && PL.dirty_pos && + vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, &pd); + retry = pd.retry; + if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(); + for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync uint64_t t = vk_texture_slot_token(id); if (t != PL.tok[id]) { + PL.n_refill++; if (pool_fill(ex, (uint32_t)id, used, t)) PL.tok[id] = t; else retry = 1; } else if (t && fresh) { pool_fill(ex, (uint32_t)id, fresh, t); } else if (t && ep != PL.epoch) { + PL.n_fresh++; vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check } } + if (!walked) PL.n_slots += (uint64_t)n; vk_texture_create_budget_locked(-1); } PL.retry = retry; @@ -2620,7 +2764,7 @@ static int draw_skipped(const VkExec *ex, int mesh, int why) { // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { - if (ex->render_disabled) return draw_skipped(ex, mesh, SK_DISABLED); + if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED); cb_res(ex); dummies_init(); if (ex->pass_noatt) { // no attachment: the pass covers this draw's viewport or scissor, reopened when it needs more diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index c218288f..e6dce958 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -124,9 +124,29 @@ static void counter_cb(void *arg) { // report counts as pending until then, not until its fence passes: the callback runs after the fence, and in between // the record still held what the CPU wrote (Ocarina re-arms its condition records with 1 for the next frame), so a // condition read 1 where the GPU had written 0, a depth clear ran after the depth copy and frames showed only sky. -typedef struct { uint64_t addr, seq; volatile uint64_t written; int unknown, type; } Pending; +// frame: the present count (vk_exec_cond_frame_end, in replay order) when the report was replayed. The game arms its +// condition records from the CPU between frames, so within a frame a timestamp report decides the record's value even +// after it reached memory: Ocarina's CPU, running a frame ahead, had re-armed a record with 1 by the time a later depth +// clear of the frame read it, the clear ran after the depth copy and the frame showed only sky (white flashes). +typedef struct { uint64_t addr, seq, frame; volatile uint64_t written; int unknown, type; } Pending; static Pending pending[PENDING]; static uint64_t report_seq; +static uint64_t cond_frame; + +// Per-frame counts of conditional work (NX2NAVI_LOG_RCOND=1, vk_exec_cond_frame_end logs them): clears and +// draws issued under a render condition, run or skipped. +unsigned vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run; + +void vk_exec_cond_frame_end(uint64_t frame) { + static int on = -1; + if (on < 0) on = rt_knob("LOG_RCOND", "0"); + if (on && (vk_cond_clears_run || vk_cond_clears_skip || vk_cond_draws_skip)) + rt_log("vk: frame %llu conditions: clears run %u skipped %u, draws skipped %u, depth clears run %u", + (unsigned long long)frame, vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run); + vk_cond_clears_run = vk_cond_clears_skip = vk_cond_draws_skip = vk_cond_depth_run = 0; + if (on == 2) rt_log("SEQ present %llu", (unsigned long long)frame); + cond_frame++; +} static Pending *pending_slot(uint64_t addr, int insert); @@ -149,11 +169,12 @@ static Pending *pending_slot(uint64_t addr, int insert) { if (a == addr) return e; if (!insert) continue; if (!a) { if (!free_) free_ = e; continue; } - if (__atomic_load_n(&e->written, __ATOMIC_ACQUIRE) >= e->seq && (!victim || e->seq < victim->seq)) victim = e; + if (__atomic_load_n(&e->written, __ATOMIC_ACQUIRE) >= e->seq && e->frame != cond_frame && (!victim || e->seq < victim->seq)) + victim = e; // done, and not from this frame (whose reports still decide their records) } Pending *e = free_ ? free_ : victim; if (!e) return NULL; // every probed entry still has a report in flight - e->seq = 0, e->unknown = 0, e->type = -1; + e->seq = 0, e->frame = 0, e->unknown = 0, e->type = -1; __atomic_store_n(&e->written, 0, __ATOMIC_RELEASE); __atomic_store_n(&e->addr, addr, __ATOMIC_RELEASE); return e; @@ -342,7 +363,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { vk_completion_add(ex->fence, counter_cb, a); } Pending *e = pending_slot(p->addr, 1); - if (e) e->seq = report_seq, e->unknown = samples, e->type = (int)p->type; + if (e) e->seq = report_seq, e->frame = cond_frame, e->unknown = samples, e->type = (int)p->type; else LOG_ONCE("vk: a counter report's table neighbourhood is all in flight: conditions on %#llx read memory", (unsigned long long)p->addr); } @@ -352,6 +373,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { // 1: *v read, 0: not readable, 2: a samples-passed report not in memory yet (unknown) static int counter_word(uint64_t addr, uint64_t *v) { const Pending *e = pending_slot(addr, 0); // the last report to addr decides + if (e && !e->unknown && e->frame == cond_frame) return *v = 0, 1; // a timestamp/statistics report of this frame if (e && __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq) return *v = 0, e->unknown ? 2 : 1; VkMemSpan m; if (nvn_gpu_only(addr) || !vk_mem_resolve(addr, sizeof *v, &m)) return 0; @@ -366,6 +388,21 @@ static int last_type(uint64_t addr) { return e ? e->type : -1; } +// NX2NAVI_LOG_RCOND=ab (diagnosis): the words a condition compares and where each came from: "f" a report replayed +// this frame (0), "p" a pending report (0 or unknown), "m" memory, with the newest report's type and frame. +void vk_exec_cond_why(const p_render_cond *c, char *buf, size_t cap) { + int o = snprintf(buf, cap, "%#llx mode %d:", (unsigned long long)c->addr, c->mode); + for (int k = 0; k < 2 && o < (int)cap; k++) { + uint64_t a = c->addr + 16 * (uint64_t)k; + const Pending *e = pending_slot(a, 0); + const char *src = !e ? "m" : !e->unknown && e->frame == cond_frame ? "f" + : __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq ? "p" : "m"; + o += snprintf(buf + o, cap - (size_t)o, " %llu %s(type %d, frame %lld of %llu)", + (unsigned long long)*(const uint64_t *)(uintptr_t)a, src, e ? e->type : -1, + e ? (long long)e->frame : -1LL, (unsigned long long)cond_frame); + } +} + static int cond_enabled(const p_render_cond *c) { uint64_t x, y; int rx = counter_word(c->addr, &x), ry = counter_word(c->addr + 16, &y); @@ -546,6 +583,10 @@ static void copy_t2t(VkExec *ex, const p_copy_t2t *p) { } ex->recorded++; vk_texture_gpu_written(p->dst); + static int lg = -1; + if (lg < 0) lg = rt_knob("LOG_RCOND", "0"); + if (lg == 2 && (d.img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) && d.w >= 1600) + rt_log("SEQ depth copy %#llx -> %#llx", (unsigned long long)s.img->addr, (unsigned long long)d.img->addr); } // ---- buffer to buffer diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 4dc2d9cd..119960c6 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -126,11 +126,59 @@ static uint32_t reg_serial; static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); } +// Slots that may resolve differently since a position, for the texture pool descriptors (vk_exec_draw.c +// pool_sync_entries), which otherwise looked at every registered slot after each change: Ocarina registers slots all +// through a submission, and the refresh walked the whole table about twenty times per submission (3.5 ms). A ring of +// changes, each a slot id (registered) or an address bucket (an entry at that agen index changed: every slot registered +// over it, kept in per-bucket lists). Lock held for all of it. +#define DIRTY_N (1u << 16) +#define DIRTY_BUCKET 0x80000000u +static uint32_t dirty[DIRTY_N]; +static uint64_t ndirty; +static int32_t bucket_head[AGEN_N]; +static int32_t *slot_next, *slot_prev; // per slot id: its bucket list links (slot_prev -2: in no list) +static uint16_t *slot_bucket; + +static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; } + static void tex_gen_bump(uint64_t addr) { if (!++tex_gen) tex_gen = 1; slots_gen++; - uint32_t *g = &agen[agen_idx(addr)]; + uint32_t b = agen_idx(addr); + uint32_t *g = &agen[b]; __atomic_store_n(g, *g + 1, __ATOMIC_RELEASE); + dirty_push(DIRTY_BUCKET | b); +} + +// Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register). +static void slot_rebucket(int id, uint32_t b) { + if (slot_prev[id] != -2) { // out of its old list + if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id]; + else bucket_head[slot_bucket[id]] = slot_next[id]; + if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id]; + } + slot_bucket[id] = (uint16_t)b; + slot_prev[id] = -1; + slot_next[id] = bucket_head[b]; + if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id; + bucket_head[b] = id; +} + +uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; } + +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) { + if (!slot_next || ndirty - *pos > DIRTY_N) { + *pos = ndirty; + return 0; + } + for (uint64_t q = *pos; q < ndirty; q++) { + uint32_t x = dirty[q % DIRTY_N]; + if (x & DIRTY_BUCKET) + for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg); + else fn((int)x, arg); + } + *pos = ndirty; + return 1; } static volatile int32_t tex_ready; @@ -1109,7 +1157,11 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { } if (!sinfo) { uint64_t *p = calloc(1 << 20, sizeof *p); - if (!p) rt_fatal("vk: out of memory for texture slot tokens"); + slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20); + slot_bucket = calloc(1 << 20, sizeof *slot_bucket); + if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); + for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1; + for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1; __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); } TexSlot *s = &tslots[id]; @@ -1119,6 +1171,8 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); + slot_rebucket(id, agen_idx((uintptr_t)t->mem)); + dirty_push((uint32_t)id); s->has_view = v != NULL; if (v) s->v = *v; else memset(&s->v, 0, sizeof s->v); @@ -1149,6 +1203,11 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { e = NULL; for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) { if (p->released) continue; + // Only an entry of the slot's target: Ocarina's volumetric fog registers a 3D two-channel volume (32x24x64) + // over the storage of the bloom pass's 480x270 2D target; followed, the fog computes and the fog pass got + // that 2D image, no 3D view can be made of it (vk_exec_draw.c get_view) and they read and wrote stand-ins. + // With the target kept they get the volume itself. + if (p->key.d.target != k.d.target) continue; if (!any) any = p; if (same_but_swizzle(&p->key, &k)) { e = p; break; } } diff --git a/tests/maxwell_warp_emu.py b/tests/maxwell_warp_emu.py new file mode 100644 index 00000000..52fb98e2 --- /dev/null +++ b/tests/maxwell_warp_emu.py @@ -0,0 +1,1306 @@ +"""A 32-lane Maxwell warp emulator for checking the SM86 tensor lowering on the CPU. + +tools/xlate/AmpereLowering.cs rewrites SM86 instructions as Maxwell (64-bit) instructions, and the patched Ryujinx shader +translator (build/xlate/ryujinx/src/Ryujinx.Graphics.Shader) turns those into SPIR-V. This file executes the Maxwell +words the way that translator does: fields are decoded exactly as Ryujinx's Decoders/InstDecoders.cs, opcodes are +matched with the bit patterns of Decoders/InstTable.cs, and each instruction does what Ryujinx's Instructions/*.cs +emit for it (not what a real Maxwell GPU does where the two differ; see "Ryujinx semantics" below). With it, the result +of one lowered SM86 instruction (HMMA, LDSM, MOVM, HFMA2.RELU, ...) can be compared with vectors captured on real +hardware without running the game. + +State per lane: R0-R254 (RZ = R255 reads 0, writes are dropped), P0-P6 (PT = P7 is true, writes dropped), the lane id, +local memory; per warp: shared memory. Every lane is active at the start; a guard (bits 16-19: predicate 16-18, inverted +19) masks the writes of one instruction, EXIT retires lanes. Branches must be uniform across the active lanes. + +Ryujinx semantics this follows (and real hardware may not): + * Half-precision arithmetic (HADD2, HMUL2, HFMA2) works in F32 on the unpacked halves and rounds to F16 once at the + end (Ryujinx InstEmitFloatArithmetic EmitHadd2Hmul2 / EmitHfma2, InstEmitHelper GetHalfPacked: PackHalf2x16). + HFMA2 is therefore an F32 fused multiply-add rounded to F32, then to F16. + * That rounding is skipped when the packed value goes straight into another unpack: Ryujinx Optimizer.PropagatePack + turns UnpackHalf2x16(PackHalf2x16(a, b)) into a (or b). So F2F.F16.F32 then F2F.F32.F16 of the same register, or + HFMA2 feeding HFMA2, passes the F32 value on unrounded; an integer instruction (SHFL, LOP, ISCADD, ...) in between + keeps the rounding, a MOV does not. Modelled per register (Warp.halves); a guarded write and a branch end it. + * FTZ, FMZ and rounding-mode fields of FFMA / FMUL / FADD / HFMA2 / HMUL2 are ignored, as Ryujinx ignores them. + Denormals are kept (IEEE), saturation is clamp(x, 0, 1) with NaN giving 0 (GLSL FClamp as min(max(x, 0), 1) on + a GPU whose min/max return the non-NaN operand). + * PRMT is not implemented in Ryujinx 1.3.2: InstEmit.PrmtR/PrmtI/PrmtC/PrmtRc only log "not implemented", so the + destination keeps its old value. That is the default here (prmt="ryujinx"); prmt="maxwell" gives the hardware + byte permute instead, to see what the lowering intended. + * SHFL follows Ryujinx's HelperFunctionManager GenerateShuffle* helpers (clamp and segment mask from operand C, the + lane's own value where the source lane is out of range). + * LDS / LDL read the 32-bit word at (address >> 2) and extract a sub-word at (address & 3) * 8 (InstEmitMemory + EmitLoad); STS of 8 and 16 bits stores those bytes at the address. + * Condition codes (.CC, .X) are not modelled: an instruction that writes or reads them raises. + * NaN results carry numpy's payload and sign. Neither is defined in SPIR-V, so result comparisons (same_value) let + a NaN match any NaN for float results; see "comparing results". + +Library use: + regs = run(words, regs_in, smem=None) # words: Maxwell 64-bit words; returns a (32, 256) uint32 array + warp = run_warp(words, regs_in, smem=None) # the whole Warp (regs, preds, smem, warnings) + +CLI (vectors written by the tensor ground-truth capture, see main()): + python tests/maxwell_warp_emu.py [--xlate EXE] [--prmt ryujinx|maxwell] [--case NAME] [--trace] + [--fill HEX] [--exact] +tests/tensor_lowering_check.py runs the full check of the tensor lowering on top of this (guards, untouched registers, +several initial fills). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_XLATE = ROOT.parent.parent / "build" / "xlate" / ("nx2navi_xlate.exe" if os.name == "nt" else "nx2navi_xlate") + +LANES = 32 +RZ = 255 +PT = 7 +U32 = np.uint32 +F32 = np.float32 +F16 = np.float16 + + +class UnknownOpcode(Exception): + """A Maxwell word this emulator has no decoder entry for.""" + + +class Unsupported(Exception): + """A known instruction in a mode or form the emulator does not implement.""" + + +# ---- bit-level helpers -------------------------------------------------------------------------------------------- + +def u32(x) -> np.ndarray: + """A lane vector (or broadcast scalar) as uint32, wrapping Python ints to 32 bits.""" + if isinstance(x, (int, np.integer)): + return np.full(LANES, int(x) & 0xFFFFFFFF, dtype=U32) + return np.asarray(x).astype(U32, copy=True) + + +def sext(value: int, bits: int) -> int: + """Sign-extends a bits-wide field to a Python int.""" + value &= (1 << bits) - 1 + return value - (1 << bits) if value >> (bits - 1) else value + + +def as_f32(u: np.ndarray) -> np.ndarray: + return u32(u).view(F32) + + +def from_f32(f: np.ndarray) -> np.ndarray: + return np.asarray(f, dtype=F32).view(U32).copy() + + +def half_lo(u: np.ndarray) -> np.ndarray: + """UnpackHalf2x16Low: the low 16 bits as F16, widened to F32 (exact).""" + return (u32(u) & U32(0xFFFF)).astype(np.uint16).view(F16).astype(F32) + + +def half_hi(u: np.ndarray) -> np.ndarray: + """UnpackHalf2x16High.""" + return (u32(u) >> U32(16)).astype(np.uint16).view(F16).astype(F32) + + +def pack_half(lo: np.ndarray, hi: np.ndarray) -> np.ndarray: + """PackHalf2x16: each F32 rounded to F16 (round to nearest even, numpy's conversion), lo in bits 0-15.""" + l = np.asarray(lo, dtype=F32).astype(F16).view(np.uint16).astype(U32) + h = np.asarray(hi, dtype=F32).astype(F16).view(np.uint16).astype(U32) + return l | (h << U32(16)) + + +def fma_f32(a: np.ndarray, b: np.ndarray, c: np.ndarray) -> np.ndarray: + """F32 fused multiply-add, rounded once (FPFusedMultiplyAdd -> GLSL.std.450 Fma, fused on the target GPUs). + + a * b is exact in F64 (2 x 24 significant bits). The F64 sum s may be inexact; TwoSum gives its error e exactly + (s + e == a * b + c). Rounding s to F32 is then correct except when s lies exactly halfway between two F32 + values, where the sign of e decides the direction. F32 halfway points are F64 values, so no other case can + double-round.""" + a64 = np.asarray(a, dtype=F32).astype(np.float64) + b64 = np.asarray(b, dtype=F32).astype(np.float64) + c64 = np.asarray(c, dtype=F32).astype(np.float64) + p = a64 * b64 + s = p + c64 + bb = s - p + e = (p - (s - bb)) + (c64 - bb) + r = s.astype(F32) + # The F32 neighbour of r on the side of s. + toward = np.nextafter(r, np.where(s > r.astype(np.float64), F32(np.inf), F32(-np.inf)).astype(F32)) + mid = (r.astype(np.float64) + toward.astype(np.float64)) * 0.5 + fix = np.isfinite(s) & np.isfinite(r) & np.isfinite(toward) & (e != 0) & (s == mid) + if fix.any(): + # At a tie, the exact value lies on the side of e: take the neighbour when e points towards it. + up = (toward.astype(np.float64) > r.astype(np.float64)) + take = fix & (up == (e > 0)) + r = np.where(take, toward, r).astype(F32) + return r + + +def saturate(x: np.ndarray) -> np.ndarray: + """FPSaturate: clamp to [0, 1], NaN to 0 (fmax/fmin return the non-NaN operand).""" + return np.fmin(np.fmax(np.asarray(x, dtype=F32), F32(0)), F32(1)).astype(F32) + + +def abs_neg(x: np.ndarray, absolute: bool, negate: bool) -> np.ndarray: + """FPAbsNeg: abs first, then negate (sign-bit operations in SPIR-V; numpy's abs/negative act on the sign too).""" + x = np.asarray(x, dtype=F32) + if absolute: + x = np.abs(x) + if negate: + x = np.negative(x) + return x + + +class Ins: + """One Maxwell word with the field accessors InstDecoders.cs uses.""" + + def __init__(self, word: int, index: int): + self.w = word + self.index = index + + def bits(self, offset: int, count: int) -> int: + return (self.w >> offset) & ((1 << count) - 1) + + def bit(self, offset: int) -> bool: + return (self.w >> offset) & 1 == 1 + + # Fields shared by most decoders. + @property + def rd(self): return self.bits(0, 8) + @property + def ra(self): return self.bits(8, 8) + @property + def rb(self): return self.bits(20, 8) + @property + def rc(self): return self.bits(39, 8) + @property + def imm20(self): # InstDecoders Imm20: bits 20-38, bit 19 from bit 56 + return ((self.w >> 37) & 0x80000) | ((self.w >> 20) & 0x7FFFF) + @property + def imm20_sint(self): return sext(self.imm20, 20) # InstEmitHelper Imm20ToSInt + @property + def imm20_float(self): return (self.imm20 << 12) & 0xFFFFFFFF # Imm20ToFloat + @property + def imm32(self): return (self.w >> 20) & 0xFFFFFFFF + @property + def imm24_sint(self): return sext(self.bits(20, 24), 24) + + def __repr__(self): + return f"[{self.index}] {self.w:016x}" + + +# ---- warp state --------------------------------------------------------------------------------------------------- + +class Warp: + def __init__(self, smem: bytes | None = None, smem_size: int = 0x10000, local_size: int = 0x2000, + cbufs: dict | None = None, prmt: str = "ryujinx", fill: int = 0, trace: bool = False): + # Registers not given by the caller start as `fill` (SPIR-V locals are undefined unless the job zeroes them; + # a poison value such as 0x7fc0dead shows reads of registers the lowering never wrote). + self.r = np.full((LANES, 256), fill & 0xFFFFFFFF, dtype=U32) + self.r[:, RZ] = 0 + self.p = np.zeros((LANES, 8), dtype=bool) + self.p[:, PT] = True + self.smem = bytearray(smem) if smem is not None else bytearray(smem_size) + self.local = [bytearray(local_size) for _ in range(LANES)] + self.cbufs = {int(k): bytes(v) for k, v in (cbufs or {}).items()} + self.lane = np.arange(LANES, dtype=U32) + self.active = np.ones(LANES, dtype=bool) + if prmt not in ("ryujinx", "maxwell"): + raise ValueError(f"prmt mode {prmt!r}") + self.prmt = prmt + self.trace = trace + self.warnings: list[str] = [] + # Ryujinx Optimizer.PropagatePack turns an UnpackHalf2x16 of a PackHalf2x16 result into the pack's F32 + # operand, so that half is never rounded to F16. Per lane and register: the F32 halves of the pack that wrote + # the register (sh_ok), kept until the register is written otherwise. A write under a guard ends it in every + # lane: Ryujinx puts the instruction in an if-block and merges old and new value in a phi, which is no pack. + self.sh_lo = np.zeros((LANES, 256), dtype=F32) + self.sh_hi = np.zeros((LANES, 256), dtype=F32) + self.sh_ok = np.zeros((LANES, 256), dtype=bool) + self.unconditional = True # the current instruction's guard is PT (run_warp sets it) + + # Register file access: R(n) reads a lane vector, W(n, v, mask) writes the lanes in mask. halves: the F32 values + # a PackHalf2x16 rounded into v (see sh_ok); copy_from: v is a copy of that register (MOV), pack and all. + def R(self, n: int) -> np.ndarray: + if n == RZ: + return np.zeros(LANES, dtype=U32) + return self.r[:, n].copy() + + def W(self, n: int, v, mask: np.ndarray, halves=None, copy_from: int | None = None): + if n == RZ: + return + v = u32(v) + self.r[mask, n] = v[mask] + if not self.unconditional: + self.sh_ok[:, n] = False + elif halves is not None: + self.sh_lo[mask, n] = np.broadcast_to(np.asarray(halves[0], dtype=F32), (LANES,))[mask] + self.sh_hi[mask, n] = np.broadcast_to(np.asarray(halves[1], dtype=F32), (LANES,))[mask] + self.sh_ok[mask, n] = True + elif copy_from is not None and copy_from != RZ: + for arr in (self.sh_lo, self.sh_hi, self.sh_ok): + arr[mask, n] = arr[mask, copy_from] + else: + self.sh_ok[mask, n] = False + + def halves(self, n: int): + """UnpackHalf2x16Low / High of register n as F32, after PropagatePack (the pack's operand where one wrote n).""" + u = self.R(n) + lo, hi = half_lo(u), half_hi(u) + if n != RZ: + ok = self.sh_ok[:, n] + lo = np.where(ok, self.sh_lo[:, n], lo).astype(F32) + hi = np.where(ok, self.sh_hi[:, n], hi).astype(F32) + return lo, hi + + def P(self, n: int, inv: bool = False) -> np.ndarray: + v = np.ones(LANES, dtype=bool) if n == PT else self.p[:, n].copy() + return ~v if inv else v + + def WP(self, n: int, v, mask: np.ndarray): + if n == PT: + return + v = np.broadcast_to(np.asarray(v, dtype=bool), (LANES,)) + self.p[mask, n] = v[mask] + + def cbuf_word(self, slot: int, word_offset: int) -> np.ndarray: + """c[slot][word_offset * 4] (InstDecoders CbufOffset counts words).""" + data = self.cbufs.get(slot) + at = word_offset * 4 + if data is None or at + 4 > len(data): + raise Unsupported(f"constant buffer c[{slot}][0x{at:x}] not provided") + return u32(int.from_bytes(data[at:at + 4], "little")) + + +# ---- instruction semantics ---------------------------------------------------------------------------------------- +# Each handler takes (warp, ins, mask): mask = the active lanes whose guard holds. Lanes outside mask must not change. + +def _no_cc(i: Ins, cc_bit: int | None, x_bit: int | None, name: str): + if cc_bit is not None and i.bit(cc_bit): + raise Unsupported(f"{name}.CC (condition codes not modelled) {i}") + if x_bit is not None and i.bit(x_bit): + raise Unsupported(f"{name}.X (carry not modelled) {i}") + + +def op_nop(w, i, m): + pass + + +def op_mov_r(w, i, m): # InstEmitMove MovR: SrcA in bits 20-27; a Copy, which PropagateCopy sees through + w.W(i.rd, w.R(i.rb), m, copy_from=i.rb) + + +def op_mov_i(w, i, m): # MovI: GetSrcImm(op.Imm20), the raw 20-bit field (not shifted or sign-extended) + w.W(i.rd, u32(i.imm20), m) + + +def op_mov_c(w, i, m): + w.W(i.rd, w.cbuf_word(i.bits(34, 5), i.bits(20, 14)), m) + + +def op_mov32i(w, i, m): + w.W(i.rd, u32(i.imm32), m) + + +def _iadd(w, i, m, a, b, avg): # InstEmitIntegerArithmetic EmitIadd: AvgMode 1 negates B, 2 negates A + if avg == 2: + a = U32(0) - a + if avg == 1: + b = U32(0) - b + w.W(i.rd, a + b, m) + + +def op_iadd_r(w, i, m): + _no_cc(i, 47, 43, "IADD") + _iadd(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(48, 2)) + + +def op_iadd_i(w, i, m): + _no_cc(i, 47, 43, "IADD") + _iadd(w, i, m, w.R(i.ra), u32(i.imm20_sint), i.bits(48, 2)) + + +def op_iadd32i(w, i, m): + _no_cc(i, 52, 53, "IADD32I") + _iadd(w, i, m, w.R(i.ra), u32(i.imm32), i.bits(55, 2)) + + +def _iadd3(w, i, m, a, b, c, lrs, parts): + def extend(v, part): # EmitIadd3 Extend: HalfSelect 1 = low 16 bits, 2 = high 16 bits + if part == 1: + return v & U32(0xFFFF) + if part == 2: + return v >> U32(16) + return v + a, b, c = (extend(v, p) for v, p in zip((a, b, c), parts)) + if i.bit(51): + a = U32(0) - a + if i.bit(50): + b = U32(0) - b + if i.bit(49): + c = U32(0) - c + res = a + b + if lrs == 2: + res = res << U32(16) + elif lrs == 1: + res = res >> U32(16) + w.W(i.rd, res + c, m) + + +def op_iadd3_r(w, i, m): + _no_cc(i, 47, 48, "IADD3") + _iadd3(w, i, m, w.R(i.ra), w.R(i.rb), w.R(i.rc), i.bits(37, 2), (i.bits(35, 2), i.bits(33, 2), i.bits(31, 2))) + + +def op_iadd3_i(w, i, m): + _no_cc(i, 47, 48, "IADD3") + _iadd3(w, i, m, w.R(i.ra), u32(i.imm20_sint), w.R(i.rc), 0, (0, 0, 0)) + + +def _iscadd(w, i, m, a, b, shift, avg): # EmitIscadd: (A << shift), then the AvgMode negations, then A + B + a = a << U32(shift) + if avg == 2: + a = U32(0) - a + if avg == 1: + b = U32(0) - b + w.W(i.rd, a + b, m) + + +def op_iscadd_r(w, i, m): + _no_cc(i, 47, None, "ISCADD") + _iscadd(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(39, 5), i.bits(48, 2)) + + +def op_iscadd_i(w, i, m): + _no_cc(i, 47, None, "ISCADD") + _iscadd(w, i, m, w.R(i.ra), u32(i.imm20_sint), i.bits(39, 5), i.bits(48, 2)) + + +def op_iscadd32i(w, i, m): + _no_cc(i, 52, None, "ISCADD32I") + _iscadd(w, i, m, w.R(i.ra), u32(i.imm32), i.bits(53, 5), 0) + + +def _shl(w, i, m, a, b): # InstEmitShift EmitShl: .M masks the count to 5 bits, else counts >= 32 give 0 + _no_cc(i, 47, 43, "SHL") + if i.bit(39): + b = b & U32(31) + w.W(i.rd, a << b, m) + else: + w.W(i.rd, np.where(b < 32, a << (b & U32(31)), U32(0)), m) + + +def _shr(w, i, m, a, b): # EmitShr: .BREV (40) reverses A first, .S32 (48), .M (39) + _no_cc(i, 47, None, "SHR") + if i.bits(43, 2): + raise Unsupported(f"SHR.X {i}") + if i.bit(40): + a = u32([int(f"{int(v):032b}"[::-1], 2) for v in a]) + signed = i.bit(48) + if i.bit(39): + b = b & U32(31) + sh = b & U32(31) + res = (a.view(np.int32) >> sh.astype(np.int32)).view(U32) if signed else a >> sh + if not i.bit(39): + big = (a.view(np.int32) >> 31).view(U32) if signed else np.zeros(LANES, dtype=U32) + res = np.where(b < 32, res, big) + w.W(i.rd, res, m) + + +def op_shl_r(w, i, m): _shl(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_shl_i(w, i, m): _shl(w, i, m, w.R(i.ra), u32(i.imm20_sint)) +def op_shr_r(w, i, m): _shr(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_shr_i(w, i, m): _shr(w, i, m, w.R(i.ra), u32(i.imm20_sint)) + + +def _lop_pred(w, i, m, res, pred_op, dest_pred): # EmitLopPredWrite + if dest_pred == PT: + return + v = {0: np.zeros(LANES, bool), 1: np.ones(LANES, bool), 2: res == 0, 3: res != 0}[pred_op] + w.WP(dest_pred, v, m) + + +def _lop(w, i, m, a, b, logic, inv_a, inv_b, pred_op, dest_pred): # InstEmitIntegerLogical EmitLop + if inv_a: + a = ~a + if inv_b: + b = ~b + res = {0: a & b, 1: a | b, 2: a ^ b, 3: b}[logic] + _lop_pred(w, i, m, res, pred_op, dest_pred) + w.W(i.rd, res, m) + + +def op_lop_r(w, i, m): + _no_cc(i, 47, 43, "LOP") + _lop(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(41, 2), i.bit(39), i.bit(40), i.bits(44, 2), i.bits(48, 3)) + + +def op_lop_i(w, i, m): + _no_cc(i, 47, 43, "LOP") + _lop(w, i, m, w.R(i.ra), u32(i.imm20_sint), i.bits(41, 2), i.bit(39), i.bit(40), i.bits(44, 2), i.bits(48, 3)) + + +def op_lop32i(w, i, m): # Lop32i: no predicate output (DestPred PT) + _no_cc(i, 52, 57, "LOP32I") + _lop(w, i, m, w.R(i.ra), u32(i.imm32), i.bits(53, 2), i.bit(55), i.bit(56), 0, PT) + + +def _lop3(a, b, c, lut): + """The LOP3 truth table (Lop3Expression.GetFromTruthTable): bit (a << 2 | b << 1 | c) of lut, per bit.""" + res = np.zeros(LANES, dtype=U32) + for k in range(8): + if lut >> k & 1: + ta = a if k & 4 else ~a + tb = b if k & 2 else ~b + tc = c if k & 1 else ~c + res |= ta & tb & tc + return res + + +def op_lop3_r(w, i, m): + _no_cc(i, 47, 38, "LOP3") + res = _lop3(w.R(i.ra), w.R(i.rb), w.R(i.rc), i.bits(28, 8)) + _lop_pred(w, i, m, res, i.bits(36, 2), i.bits(48, 3)) + w.W(i.rd, res, m) + + +def op_lop3_i(w, i, m): + _no_cc(i, 47, 57, "LOP3") + w.W(i.rd, _lop3(w.R(i.ra), u32(i.imm20_sint), w.R(i.rc), i.bits(48, 8)), m) + + +def _prmt(w, i, m, a, sel, c): + if w.prmt == "ryujinx": + # Ryujinx InstEmit.PrmtR / PrmtI: "not implemented", nothing is emitted, the destination keeps its value. + w.warnings.append(f"PRMT at {i} is a no-op in Ryujinx 1.3.2 (destination R{i.rd} unchanged)") + return + if i.bits(48, 4): + raise Unsupported(f"PRMT mode {i.bits(48, 4)} {i}") + res = np.zeros(LANES, dtype=U32) + src = [(a >> U32(8 * k)) & U32(0xFF) for k in range(4)] + [(c >> U32(8 * k)) & U32(0xFF) for k in range(4)] + for byte in range(4): + nib = (sel >> U32(4 * byte)) & U32(0xF) + val = np.zeros(LANES, dtype=U32) + for k in range(8): + val = np.where((nib & U32(7)) == k, src[k], val) + val = np.where((nib & U32(8)) != 0, np.where(val & U32(0x80), U32(0xFF), U32(0)), val) + res |= val << U32(8 * byte) + w.W(i.rd, res, m) + + +def op_prmt_r(w, i, m): _prmt(w, i, m, w.R(i.ra), w.R(i.rb), w.R(i.rc)) +def op_prmt_i(w, i, m): _prmt(w, i, m, w.R(i.ra), u32(i.imm20_sint), w.R(i.rc)) + + +def op_sel_r(w, i, m): # InstEmitMove EmitSel: P ? A : B + p = w.P(i.bits(39, 3), i.bit(42)) + w.W(i.rd, np.where(p, w.R(i.ra), w.R(i.rb)), m) + + +def op_sel_i(w, i, m): + p = w.P(i.bits(39, 3), i.bit(42)) + w.W(i.rd, np.where(p, w.R(i.ra), u32(i.imm20_sint)), m) + + +def _isetp(w, i, m, a, b): # InstEmitIntegerComparison EmitIsetp (not extended) + if i.bit(43): + raise Unsupported(f"ISETP.X {i}") + cmp, signed = i.bits(49, 3), i.bit(48) + x, y = (a.view(np.int32), b.view(np.int32)) if signed else (a, b) + r = {0: np.zeros(LANES, bool), 1: x < y, 2: x == y, 3: x <= y, 4: x > y, 5: x != y, 6: x >= y, + 7: np.ones(LANES, bool)}[cmp] + pred = w.P(i.bits(39, 3), i.bit(42)) + op = i.bits(45, 2) + def comb(v): + if op == 0: + return v & pred + if op == 1: + return v | pred + if op == 2: + return v ^ pred + raise Unsupported(f"ISETP bool op {op} {i}") + w.WP(i.bits(3, 3), comb(r), m) + w.WP(i.bits(0, 3), comb(~r), m) + + +def op_isetp_r(w, i, m): _isetp(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_isetp_i(w, i, m): _isetp(w, i, m, w.R(i.ra), u32(i.imm20_sint)) + + +def _p2r(w, i, m, a, mask): # InstEmitPredicate EmitP2r + if i.bit(40): + raise Unsupported(f"P2R.CC {i}") + shift = i.bits(41, 2) * 8 + mask = (mask & U32(0xFF)) << U32(shift) + insert = np.zeros(LANES, dtype=U32) + for k in range(7): + insert |= np.where(w.p[:, k], U32(1 << (k + shift)), U32(0)) + w.W(i.rd, (a & ~mask) | (insert & mask), m) + + +def op_p2r_r(w, i, m): _p2r(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_p2r_i(w, i, m): _p2r(w, i, m, w.R(i.ra), u32(i.imm20)) # P2rI: the raw Imm20 as mask + + +def _r2p(w, i, m, value, mask): # InstEmitMove EmitR2p + if i.bit(40): + raise Unsupported(f"R2P.CC {i}") + shift = i.bits(41, 2) * 8 + for k in range(7): + sel = (mask >> U32(k)) & U32(1) != 0 + bit = (value >> U32(k + shift)) & U32(1) != 0 + w.WP(k, np.where(sel, bit, w.p[:, k]), m) + + +def op_r2p_r(w, i, m): _r2p(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_r2p_i(w, i, m): _r2p(w, i, m, w.R(i.ra), u32(i.imm20_sint)) # R2pI: Imm20ToSInt + + +def op_s2r(w, i, m): # InstEmitMove S2r: SR_LANEID; one warp, so TID.X is the lane and TID.Y/Z are 0 + sreg = i.bits(20, 8) + if sreg in (0, 33): + v = w.lane + elif sreg in (34, 35): + v = np.zeros(LANES, dtype=U32) + elif sreg == 32: + v = w.lane + else: + raise Unsupported(f"S2R SR {sreg} {i}") + w.W(i.rd, v, m) + + +def op_shfl(w, i, m): # InstEmitWarp Shfl + HelperFunctionManager GenerateShuffle* (subgroup size 32) + a = w.R(i.ra) + b = u32(i.bits(20, 5)) if i.bit(28) else w.R(i.rb) + c = u32(i.bits(34, 13)) if i.bit(29) else w.R(i.rc) + lane = w.lane + clamp = c & U32(0x1F) + seg = (c >> U32(8)) & U32(0x1F) + min_t = lane & seg + max_t = (clamp & ~seg) | min_t + mode = i.bits(30, 2) + if mode == 0: # IDX + src = (b & ~seg) | min_t + valid = src <= max_t + elif mode == 1: # UP: signed compare (ICompareGreaterOrEqual) + src = lane - b + valid = src.view(np.int32) >= min_t.view(np.int32) + elif mode == 2: # DOWN + src = lane + b + valid = src <= max_t + else: # BFLY + src = lane ^ b + valid = src <= max_t + idx = np.where(valid, src, lane).astype(np.int64) + # A predicated SHFL becomes an if-block in SPIR-V: reading a lane outside it is undefined. + bad = m & valid & ~m[np.clip(idx, 0, LANES - 1)] + if bad.any(): + w.warnings.append(f"SHFL at {i} reads inactive lanes {sorted(set(idx[bad].tolist()))}") + res = np.where(valid, a[np.clip(idx, 0, LANES - 1)], a) + w.W(i.rd, res, m) + w.WP(i.bits(48, 3), valid, m) + + +def _mem_load(w, i, m, mem_of, size, base, offset, what): + """InstEmitMemory EmitLoad: per word, the word at (address >> 2), sub-words extracted at (address & 3) * 8.""" + if size > 6: + raise Unsupported(f"{what} size {size} {i}") + count = {5: 2, 6: 4}.get(size, 1) + for k in range(count): + if i.rd + k >= RZ: + break + vals = np.zeros(LANES, dtype=U32) + for lane in np.nonzero(m)[0]: + mem = mem_of(lane) + addr = (int(base[lane]) + offset + 4 * k) & 0xFFFFFFFF + at = (addr >> 2) * 4 + if at + 4 > len(mem): + raise IndexError(f"{what} lane {lane} address 0x{addr:x} outside {len(mem)} bytes {i}") + v = int.from_bytes(mem[at:at + 4], "little") >> ((addr & 3) * 8) + if size == 0: + v &= 0xFF + elif size == 1: + v = sext(v, 8) & 0xFFFFFFFF + elif size == 2: + v &= 0xFFFF + elif size == 3: + v = sext(v, 16) & 0xFFFFFFFF + vals[lane] = v + w.W(i.rd + k, vals, m) + + +def _mem_store(w, i, m, mem_of, size, base, offset, what, local): + """EmitStore: 32-bit words at (address >> 2); shared 8/16-bit stores write those bytes at the address, local + ones insert into the word at (address & 3) * 8. Lanes store in lane order (the last lane wins a conflict).""" + if size > 6: + raise Unsupported(f"{what} size {size} {i}") + count = {5: 2, 6: 4}.get(size, 1) + for k in range(count): + reg = i.rd if i.rd + k >= RZ else i.rd + k + vals = w.R(reg) + for lane in np.nonzero(m)[0]: + mem = mem_of(lane) + addr = (int(base[lane]) + offset + 4 * k) & 0xFFFFFFFF + v = int(vals[lane]) + if size < 4 and not local: + n = 1 if size < 2 else 2 + if addr + n > len(mem): + raise IndexError(f"{what} lane {lane} address 0x{addr:x} outside {len(mem)} bytes {i}") + mem[addr:addr + n] = (v & ((1 << (8 * n)) - 1)).to_bytes(n, "little") + continue + at = (addr >> 2) * 4 + if at + 4 > len(mem): + raise IndexError(f"{what} lane {lane} address 0x{addr:x} outside {len(mem)} bytes {i}") + if size < 4: + n = 8 if size < 2 else 16 + sh = (addr & 3) * 8 + old = int.from_bytes(mem[at:at + 4], "little") + fmask = ((1 << n) - 1) << sh + v = (old & ~fmask & 0xFFFFFFFF) | ((v << sh) & fmask) + mem[at:at + 4] = (v & 0xFFFFFFFF).to_bytes(4, "little") + + +def op_lds(w, i, m): + _mem_load(w, i, m, lambda lane: w.smem, i.bits(48, 3), w.R(i.ra), i.imm24_sint, "LDS") + + +def op_sts(w, i, m): + _mem_store(w, i, m, lambda lane: w.smem, i.bits(48, 3), w.R(i.ra), i.imm24_sint, "STS", False) + + +def op_ldl(w, i, m): + _mem_load(w, i, m, lambda lane: w.local[lane], i.bits(48, 3), w.R(i.ra), i.imm24_sint, "LDL") + + +def op_stl(w, i, m): + _mem_store(w, i, m, lambda lane: w.local[lane], i.bits(48, 3), w.R(i.ra), i.imm24_sint, "STL", True) + + +def op_f2f_r(w, i, m): # InstEmitConversion F2fR / EmitF2F + _no_cc(i, 47, None, "F2F") + dst, src = i.bits(8, 2), i.bits(10, 2) + if dst not in (1, 2) or src not in (1, 2): + raise Unsupported(f"F2F with F64 {i}") + x = w.halves(i.rb)[1 if i.bit(41) else 0] if src == 1 else as_f32(w.R(i.rb)) + x = abs_neg(x, i.bit(49), i.bit(45)) + rnd = (i.bits(42, 1) << 2) | i.bits(39, 2) # IntegerRound: 4 round, 5 floor, 6 ceil, 7 trunc + if src == dst: + x = {4: np.rint, 5: np.floor, 6: np.ceil, 7: np.trunc}.get(rnd, lambda v: v)(x).astype(F32) + if i.bit(50): + x = saturate(x) + if dst == 2: + w.W(i.rd, from_f32(x), m) + else: # WriteFP: PackHalf2x16(x, 0) + zero = np.zeros(LANES, F32) + w.W(i.rd, pack_half(x, zero), m, halves=(x, zero)) + + +def _ffma(w, i, m, a, b, c, neg_b, neg_c, sat): # EmitFfma (rounding mode and FMZ ignored) + r = fma_f32(as_f32(a), abs_neg(as_f32(b), False, neg_b), abs_neg(as_f32(c), False, neg_c)) + w.W(i.rd, from_f32(saturate(r) if sat else r), m) + + +def op_ffma_r(w, i, m): + _no_cc(i, 47, None, "FFMA") + _ffma(w, i, m, w.R(i.ra), w.R(i.rb), w.R(i.rc), i.bit(48), i.bit(49), i.bit(50)) + + +def op_ffma_i(w, i, m): + _no_cc(i, 47, None, "FFMA") + _ffma(w, i, m, w.R(i.ra), u32(i.imm20_float), w.R(i.rc), i.bit(48), i.bit(49), i.bit(50)) + + +def op_ffma32i(w, i, m): # Ffma32i: C is the destination register + _no_cc(i, 52, None, "FFMA32I") + _ffma(w, i, m, w.R(i.ra), u32(i.imm32), w.R(i.rd), i.bit(56), i.bit(57), i.bit(55)) + + +def _fmul(w, i, m, a, b, scale, neg_b, sat): # EmitFmul: A times the scale first, then A * B + a = as_f32(a) + if scale: + a = (a * F32({1: 0.5, 2: 0.25, 3: 0.125, 4: 8.0, 5: 4.0, 6: 2.0}.get(scale, 1.0))).astype(F32) + r = (a * abs_neg(as_f32(b), False, neg_b)).astype(F32) + w.W(i.rd, from_f32(saturate(r) if sat else r), m) + + +def op_fmul_r(w, i, m): + _no_cc(i, 47, None, "FMUL") + _fmul(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(41, 3), i.bit(48), i.bit(50)) + + +def op_fmul_i(w, i, m): + _no_cc(i, 47, None, "FMUL") + _fmul(w, i, m, w.R(i.ra), u32(i.imm20_float), i.bits(41, 3), i.bit(48), i.bit(50)) + + +def op_fmul32i(w, i, m): + _no_cc(i, 52, None, "FMUL32I") + _fmul(w, i, m, w.R(i.ra), u32(i.imm32), 0, False, i.bit(55)) + + +def _fadd(w, i, m, a, b, neg_a, neg_b, abs_a, abs_b, sat): # EmitFadd + r = (abs_neg(as_f32(a), abs_a, neg_a) + abs_neg(as_f32(b), abs_b, neg_b)).astype(F32) + w.W(i.rd, from_f32(saturate(r) if sat else r), m) + + +def op_fadd_r(w, i, m): + _no_cc(i, 47, None, "FADD") + _fadd(w, i, m, w.R(i.ra), w.R(i.rb), i.bit(48), i.bit(45), i.bit(46), i.bit(49), i.bit(50)) + + +def op_fadd_i(w, i, m): + _no_cc(i, 47, None, "FADD") + _fadd(w, i, m, w.R(i.ra), u32(i.imm20_float), i.bit(48), i.bit(45), i.bit(46), i.bit(49), i.bit(50)) + + +def op_fadd32i(w, i, m): + _no_cc(i, 52, None, "FADD32I") + _fadd(w, i, m, w.R(i.ra), u32(i.imm32), i.bit(56), i.bit(53), i.bit(54), i.bit(57), False) + + +# Half precision: InstEmitHelper GetHalfSrc / GetHalfUnpacked / GetHalfPacked. +def half_src(w: Warp, reg: int, swizzle: int, neg: bool = False, absolute: bool = False): + """The two F32 values of a packed-half register operand: swizzle 0 (H1_H0), 1 (F32: the register as one F32 + twice), 2 (H0_H0), 3 (H1_H1); then abs, neg. The halves come through Warp.halves (PropagatePack).""" + if swizzle == 1: + x = as_f32(w.R(reg)) + pair = (x, x) + else: + lo, hi = w.halves(reg) + pair = {0: (lo, hi), 2: (lo, lo), 3: (hi, hi)}[swizzle] + return [abs_neg(x, absolute, neg) for x in pair] + + +def half_imm_pair(h0: int, h1: int): + """GetHalfSrc(immH0, immH1): each 10-bit field shifted left 6 as an F16.""" + return [np.full(LANES, np.array([(v << 6) & 0xFFFF], dtype=np.uint16).view(F16)[0], dtype=F32) for v in (h0, h1)] + + +def half_imm32(imm: int): + return [np.full(LANES, np.array([(imm >> s) & 0xFFFF], dtype=np.uint16).view(F16)[0], dtype=F32) for s in (0, 16)] + + +def half_pack(w: Warp, ofmt: int, res, rd: int): + """GetHalfPacked: 0 F16 pair, 1 F32 (lane 0's result), 2 MRG_H0 (high half kept), 3 MRG_H1 (low half kept). + Returns the register value and the F32 halves PackHalf2x16 rounded (None for the F32 format).""" + if ofmt == 1: + return from_f32(res[0]), None + if ofmt == 0: + lo, hi = res[0], res[1] + else: + old_lo, old_hi = w.halves(rd) # GetHalfDest: an UnpackHalf2x16 of Rd + lo, hi = (res[0], old_hi) if ofmt == 2 else (old_lo, res[1]) + return pack_half(lo, hi), (lo, hi) + + +def _hadd_hmul(w, i, m, a, b, add, ofmt, sat): # EmitHadd2Hmul2: F32 add/multiply per half, saturate, pack + res = [] + for k in range(2): + r = (a[k] + b[k]).astype(F32) if add else (a[k] * b[k]).astype(F32) + res.append(saturate(r) if sat else r) + v, halves = half_pack(w, ofmt, res, i.rd) + w.W(i.rd, v, m, halves=halves) + + +def _hfma(w, i, m, a, b, c, ofmt, sat): # EmitHfma2: F32 fused multiply-add per half, saturate, pack + res = [] + for k in range(2): + r = fma_f32(a[k], b[k], c[k]) + res.append(saturate(r) if sat else r) + v, halves = half_pack(w, ofmt, res, i.rd) + w.W(i.rd, v, m, halves=halves) + + +def op_hadd2_r(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2), i.bit(43), i.bit(44)) + b = half_src(w, i.rb, i.bits(28, 2), i.bit(31), i.bit(30)) + _hadd_hmul(w, i, m, a, b, True, i.bits(49, 2), i.bit(32)) + + +def op_hadd2_i(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2), i.bit(43), i.bit(44)) + b = half_imm_pair(i.bits(20, 10), ((i.w >> 47) & 0x200) | i.bits(30, 9)) + _hadd_hmul(w, i, m, a, b, True, i.bits(49, 2), i.bit(52)) + + +def op_hadd232i(w, i, m): + a = half_src(w, i.ra, i.bits(53, 2), i.bit(56), False) + _hadd_hmul(w, i, m, a, half_imm32(i.imm32), True, 0, i.bit(52)) + + +def op_hmul2_r(w, i, m): # Hmul2R: the NegA bit (31) negates B + a = half_src(w, i.ra, i.bits(47, 2), False, i.bit(44)) + b = half_src(w, i.rb, i.bits(28, 2), i.bit(31), i.bit(30)) + _hadd_hmul(w, i, m, a, b, False, i.bits(49, 2), i.bit(32)) + + +def op_hmul2_i(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2), i.bit(43), i.bit(44)) + b = half_imm_pair(i.bits(20, 10), ((i.w >> 47) & 0x200) | i.bits(30, 9)) + _hadd_hmul(w, i, m, a, b, False, i.bits(49, 2), i.bit(52)) + + +def op_hmul232i(w, i, m): + a = half_src(w, i.ra, i.bits(53, 2)) + _hadd_hmul(w, i, m, a, half_imm32(i.imm32), False, 0, i.bit(52)) + + +def op_hfma2_r(w, i, m): # Hfma2R: NegA (31) negates B, NegC (30) + a = half_src(w, i.ra, i.bits(47, 2)) + b = half_src(w, i.rb, i.bits(28, 2), i.bit(31)) + c = half_src(w, i.rc, i.bits(35, 2), i.bit(30)) + _hfma(w, i, m, a, b, c, i.bits(49, 2), i.bit(32)) + + +def op_hfma2_i(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2)) + b = half_imm_pair(i.bits(20, 10), ((i.w >> 47) & 0x200) | i.bits(30, 9)) + c = half_src(w, i.rc, i.bits(53, 2), i.bit(51)) + _hfma(w, i, m, a, b, c, i.bits(49, 2), i.bit(52)) + + +def op_hfma232i(w, i, m): # Hfma232i: C is the destination register, H1_H0 + a = half_src(w, i.ra, i.bits(47, 2)) + c = half_src(w, i.rd, 0, i.bit(51)) + _hfma(w, i, m, a, half_imm32(i.imm32), c, 0, False) + + +# Control flow: handled by the executor (EXIT retires lanes, BRA moves the warp). +def op_exit(w, i, m): + if i.bits(0, 5) != 15: + raise Unsupported(f"EXIT with condition code test {i.bits(0, 5)} {i}") + w.active &= ~m + + +OP_BRA = "bra" + +# The decoder: patterns as Ryujinx Decoders/InstTable.cs writes them (bit 63 first, x = any). The most specific +# matching pattern wins, as InstTable's XBits ordering does. +TABLE = [ + ("0101000010110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "NOP", op_nop), + ("111000110000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "EXIT", op_exit), + ("111000100100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "BRA", OP_BRA), + ("0101110010011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV", op_mov_r), + ("0011100x10011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV.I", op_mov_i), + ("0100110010011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV.C", op_mov_c), + ("000000010000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV32I", op_mov32i), + ("0101110000010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD", op_iadd_r), + ("0011100x00010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD.I", op_iadd_i), + ("0001110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD32I", op_iadd32i), + ("010111001100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD3", op_iadd3_r), + ("0011100x1100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD3.I", op_iadd3_i), + ("0101110000011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISCADD", op_iscadd_r), + ("0011100x00011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISCADD.I", op_iscadd_i), + ("000101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISCADD32I", op_iscadd32i), + ("0101110001001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHL", op_shl_r), + ("0011100x01001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHL.I", op_shl_i), + ("0101110000101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHR", op_shr_r), + ("0011100x00101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHR.I", op_shr_i), + ("0101110001000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP", op_lop_r), + ("0011100x01000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP.I", op_lop_i), + ("000001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP32I", op_lop32i), + ("0101101111100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP3", op_lop3_r), + ("001111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP3.I", op_lop3_i), + ("010110111100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "PRMT", op_prmt_r), + ("0011011x1100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "PRMT.I", op_prmt_i), + ("0101110010100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SEL", op_sel_r), + ("0011100x10100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SEL.I", op_sel_i), + ("010110110110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISETP", op_isetp_r), + ("0011011x0110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISETP.I", op_isetp_i), + ("0101110011101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "P2R", op_p2r_r), + ("0011100x11101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "P2R.I", op_p2r_i), + ("0101110011110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "R2P", op_r2p_r), + ("0011100x11110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "R2P.I", op_r2p_i), + ("1111000011001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "S2R", op_s2r), + ("1110111100010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHFL", op_shfl), + ("1110111101001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LDS", op_lds), + ("1110111101011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "STS", op_sts), + ("1110111101000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LDL", op_ldl), + ("1110111101010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "STL", op_stl), + ("0101110010101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "F2F", op_f2f_r), + ("010110011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA", op_ffma_r), + ("0011001x1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA.I", op_ffma_i), + ("000011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA32I", op_ffma32i), + ("0101110001101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FMUL", op_fmul_r), + ("0011100x01101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FMUL.I", op_fmul_i), + ("00011110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FMUL32I", op_fmul32i), + ("0101110001011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FADD", op_fadd_r), + ("0011100x01011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FADD.I", op_fadd_i), + ("000010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FADD32I", op_fadd32i), + ("0101110100010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HADD2", op_hadd2_r), + ("0111101x0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HADD2.I", op_hadd2_i), + ("0010110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HADD2_32I", op_hadd232i), + ("0101110100001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HMUL2", op_hmul2_r), + ("0111100x0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HMUL2.I", op_hmul2_i), + ("0010101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HMUL2_32I", op_hmul232i), + ("0101110100000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HFMA2", op_hfma2_r), + ("01110xxx0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HFMA2.I", op_hfma2_i), + ("0010100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HFMA2_32I", op_hfma232i), +] + + +def _compile_table(): + out = [] + for pattern, name, fn in TABLE: + assert len(pattern) == 64, name + mask = value = 0 + for k, ch in enumerate(pattern): + bit = 63 - k + if ch != "x": + mask |= 1 << bit + value |= (ch == "1") << bit + out.append((mask, value, bin(mask).count("1"), name, fn)) + return out + + +_DECODE = _compile_table() + + +def decode(word: int): + """(name, handler) of a Maxwell word, by the most specific matching InstTable pattern.""" + best = None + for mask, value, fixed, name, fn in _DECODE: + if word & mask == value and (best is None or fixed > best[0]): + best = (fixed, name, fn) + if best is None: + raise UnknownOpcode(f"unknown Maxwell opcode 0x{word:016x}") + return best[1], best[2] + + +def slot_address(k: int) -> int: + """Byte address of instruction k in a compute program as AmpereLowering.Lower lays it out (a scheduling word + first in every 32 bytes; --lower prints the instruction words only).""" + return 8 * (1 + k + k // 3) + + +def run_warp(words, regs_in=None, smem=None, max_steps: int = 200000, **opts) -> Warp: + """Executes the Maxwell words on one warp and returns its final state. regs_in in any form load_regs takes.""" + words = [w if isinstance(w, int) else int(str(w), 16) for w in words] + w = Warp(smem=parse_bytes(smem) if smem is not None else None, **opts) + for lane, regs in _normalised(regs_in or {}).items(): + for reg, val in regs.items(): + kind, n = reg + if kind == "R": + w.r[lane, n] = val & 0xFFFFFFFF if n != RZ else 0 + elif kind == "P" and n != PT: + w.p[lane, n] = bool(val) + addr_to_index = {slot_address(k): k for k in range(len(words) + 1)} + # Where control flow joins (branch targets) Ryujinx merges registers in phis, and an UnpackHalf2x16 of a phi is + # not folded: the PropagatePack state (Warp.sh_ok) is dropped there and at every BRA (conservative: a register no + # path writes needs no phi). + joins = set() + for k, word in enumerate(words): + if decode(word)[1] is OP_BRA: + joins.add(addr_to_index.get(slot_address(k) + 8 + Ins(word, k).imm24_sint)) + pc = steps = 0 + with np.errstate(all="ignore"): + while pc < len(words) and w.active.any(): + steps += 1 + if steps > max_steps: + raise RuntimeError(f"more than {max_steps} steps (loop?)") + ins = Ins(words[pc], pc) + name, fn = decode(ins.w) + guard = w.P(ins.bits(16, 3), ins.bit(19)) + mask = w.active & guard + w.unconditional = ins.bits(16, 3) == PT and not ins.bit(19) + if pc in joins or fn is OP_BRA: + w.sh_ok[:] = False + if w.trace: + print(f" {pc:4d} {ins.w:016x} {name:<10} lanes {int(mask.sum())}", file=sys.stderr) + if fn is OP_BRA: + if ins.bits(0, 5) != 15: + raise Unsupported(f"BRA with condition code test {ins.bits(0, 5)} {ins}") + if mask.any(): + if (mask != w.active).any(): + raise Unsupported(f"divergent BRA {ins}") + target = slot_address(pc) + 8 + ins.imm24_sint + if target not in addr_to_index: + raise Unsupported(f"BRA to 0x{target:x}, not an instruction slot {ins}") + pc = addr_to_index[target] + continue + else: + fn(w, ins, mask) + pc += 1 + return w + + +def run(words, regs_in=None, smem=None, **opts) -> np.ndarray: + """Executes the Maxwell words; returns the per-lane register file, a (32, 256) uint32 array (row = lane).""" + return run_warp(words, regs_in, smem, **opts).r + + +# ---- vector files and the --lower bridge -------------------------------------------------------------------------- + +def parse_int(v) -> int: + """An int, or a hex string (with or without 0x).""" + if isinstance(v, bool): + return int(v) + if isinstance(v, (int, np.integer)): + return int(v) + s = str(v).strip().lower() + return int(s[2:] if s.startswith("0x") else s, 16) + + +def parse_bytes(v) -> bytes: + if isinstance(v, (bytes, bytearray)): + return bytes(v) + if isinstance(v, list): + return bytes(parse_int(x) & 0xFF for x in v) + s = re.sub(r"\s+", "", str(v)) + return bytes.fromhex(s[2:] if s.lower().startswith("0x") else s) + + +def parse_reg(name, urbase: int | None = None): + """("R", n) or ("P", n) for a register name: an int, "R12", "12", "RZ", "UR3" (needs urbase), "P0", "PT".""" + if isinstance(name, (int, np.integer)): + return ("R", int(name)) + s = str(name).strip().upper() + if s in ("RZ", "R255"): + return ("R", RZ) + if s == "PT": + return ("P", PT) + if s.startswith("UR"): + if s == "URZ": + return ("R", RZ) + if urbase is None: + raise ValueError(f"{name}: uniform register without a urbase") + return ("R", urbase + int(s[2:])) + if s.startswith("P"): + return ("P", int(s[1:])) + if s.startswith("R"): + s = s[1:] + return ("R", int(s)) + + +def load_regs(spec, urbase: int | None = None) -> dict[int, dict]: + """Normalises register specs to {lane: {(kind, n): int}}. Accepts {lane: {reg: value}} with lane keys as ints or + strings (and "*" / "all" for every lane), or a list with one entry per lane; each lane entry a dict {reg: value} + or a list of values indexed by register number; values ints or hex strings.""" + out: dict[int, dict] = {lane: {} for lane in range(LANES)} + if isinstance(spec, list): + items = list(enumerate(spec)) + else: + items = list(spec.items()) + # "*" first so per-lane entries override it. + items.sort(key=lambda kv: 0 if str(kv[0]).strip().lower() in ("*", "all") else 1) + for lane_key, regs in items: + key = str(lane_key).strip().lower() + lanes = range(LANES) if key in ("*", "all") else [int(key)] + if isinstance(regs, list): + regs = {n: v for n, v in enumerate(regs) if v is not None} + for lane in lanes: + for reg, val in (regs or {}).items(): + out[lane][parse_reg(reg, urbase)] = parse_int(val) + return out + + +def _normalised(spec) -> dict[int, dict]: + """load_regs output passes through unchanged (keys (kind, n)); anything else is loaded.""" + if isinstance(spec, dict) and all(isinstance(k, int) for k in spec) and \ + all(isinstance(r, tuple) for regs in spec.values() if isinstance(regs, dict) for r in regs): + return spec + return load_regs(spec) + + +def lower(sm86, exe: Path = DEFAULT_XLATE): + """Runs nx2navi_xlate --lower on SM86 instructions (a list of [w0, w1, w2, w3] or one such list); returns + (Maxwell words, urbase, local bytes).""" + if sm86 and not isinstance(sm86[0], (list, tuple)): + sm86 = [sm86] + text = "".join(" ".join(f"{parse_int(x) & 0xFFFFFFFF:08x}" for x in ins) + "\n" for ins in sm86) + fd, path = tempfile.mkstemp(suffix=".sm86.txt") + try: + with os.fdopen(fd, "w") as f: + f.write(text) + p = subprocess.run([str(exe), "--lower", path], capture_output=True, text=True) + finally: + os.unlink(path) + if p.returncode != 0: + raise RuntimeError(f"--lower failed: {p.stderr.strip()}") + words, urbase, local = [], None, 0 + for line in p.stdout.splitlines(): + f = line.split() + if not f: + continue + if f[0] == "urbase": + urbase = int(f[1]) + elif f[0] == "local": + local = int(f[1]) + else: + words.append(int(f[0], 16)) + return words, urbase, local + + +def _fmt(v: int) -> str: + """A register value as hex plus its F32 and half-pair readings.""" + f = np.array([v], dtype=U32).view(F32)[0] + h = np.array([v & 0xFFFF, v >> 16], dtype=np.uint16).view(F16) + return f"{v:08x} (f32 {f:.6g}, h {float(h[0]):.5g}/{float(h[1]):.5g})" + + +# ---- comparing results ---------------------------------------------------------------------------------------------- +# A float result is right when its bits are, except that any NaN matches any NaN: SPIR-V leaves the payload and sign +# of a NaN to the host (Vulkan spec, "Precision and Operation of SPIR-V Instructions"), and this emulator's numpy +# arithmetic picks its own, while SM86 hardware writes 0x7fff for every F16 NaN result. Whether a value is NaN is the +# lowering's doing; which NaN is not. Moves (LDSM, MOVM) and integer results compare bit for bit. + +def result_kind(sm86) -> str: + """How the destination registers of an SM86 instruction compare: "f16x2" (two F16 values), "f32" or "bits". + sm86 is [w0, w1, w2, w3] or a list of them (the last one decides).""" + ins = sm86[-1] if isinstance(sm86[0], (list, tuple)) else sm86 + w0, w2 = parse_int(ins[0]), parse_int(ins[2]) + op = w0 & 0xFFF + if op == 0x23C: # HMMA: .F32 (bit 76) or .F16 results + return "f32" if (w2 >> 12) & 1 else "f16x2" + if (op & 0x1FF) in (0x030, 0x031, 0x032): # HADD2 / HFMA2 / HMUL2, F32 results with bit 78 + return "f32" if (w2 >> 14) & 1 else "f16x2" + return "bits" + + +def _nan16(h: int) -> bool: + return (h & 0x7FFF) > 0x7C00 + + +def _nan32(v: int) -> bool: + return (v & 0x7FFFFFFF) > 0x7F800000 + + +def same_value(got: int, want: int, kind: str) -> bool: + """got matches want: equal bits, or for float kinds NaN where want is NaN (see above).""" + got, want = got & 0xFFFFFFFF, want & 0xFFFFFFFF + if got == want: + return True + if kind == "f32": + return _nan32(got) and _nan32(want) + if kind == "f16x2": + return all(g == w or (_nan16(g) and _nan16(w)) + for g, w in ((got & 0xFFFF, want & 0xFFFF), (got >> 16, want >> 16))) + return False + + +def ulps(got: int, want: int, kind: str) -> int: + """Distance in units in the last place (the larger of the two halves for "f16x2"); 0 for matching NaNs, a huge + number for NaN against a number. +0 and -0 are 0 apart (same_value still tells them apart).""" + def order(v, sign_bit, nan): + if nan(v): + return None + mag = v & (sign_bit - 1) + return -mag if v & sign_bit else mag + def one(g, w, sign_bit, nan): + og, ow = order(g, sign_bit, nan), order(w, sign_bit, nan) + if og is None or ow is None: + return 0 if og is None and ow is None else 1 << 32 + return abs(og - ow) + got, want = got & 0xFFFFFFFF, want & 0xFFFFFFFF + if kind == "f32": + return one(got, want, 1 << 31, _nan32) + if kind == "f16x2": + return max(one(got & 0xFFFF, want & 0xFFFF, 1 << 15, _nan16), one(got >> 16, want >> 16, 1 << 15, _nan16)) + return 0 if got == want else 1 << 32 + + +def run_case(case: dict, exe: Path, prmt: str, trace: bool = False, fill: int = 0, exact: bool = False): + """Lowers, runs and compares one vector case; returns (ok, lines of report). Float results compare with + same_value (a NaN matches a NaN) unless exact.""" + kind = "bits" if exact else result_kind(case["sm86"]) + words, urbase, _ = lower(case["sm86"], exe) + regs_in = load_regs(case.get("regs_in", {}), urbase) + expected = load_regs(case.get("regs_out", {}), urbase) + # LDSM cases (work/tensor-gt/README.md): "lane_addresses" hold Ra per lane when regs_in does not, relative to + # "smem", which sits at shared address "smem_base". Shared memory is padded to 64 KiB so stray reads stay in range. + smem = None + if case.get("smem") is not None: + base = case.get("smem_base", 0) + base = base if isinstance(base, int) else parse_int(base) + data = parse_bytes(case["smem"]) + buf = bytearray(max(0x10000, base + len(data))) + buf[base:base + len(data)] = data + smem = bytes(buf) + sm86 = case["sm86"] if isinstance(case["sm86"][0], (list, tuple)) else [case["sm86"]] + if case.get("lane_addresses") is not None and len(sm86) == 1: + ra = (parse_int(sm86[0][0]) >> 24) & 0xFF + base = case.get("smem_base", 0) + base = base if isinstance(base, int) else parse_int(base) + for lane, a in enumerate(case["lane_addresses"]): + regs_in[lane].setdefault(("R", ra), (parse_int(a) + base) & 0xFFFFFFFF) + warp = run_warp(words, regs_in, smem, prmt=prmt, trace=trace, fill=fill) + diffs = [] + for lane in range(LANES): + for (reg_kind, n), want in sorted(expected[lane].items()): + if reg_kind == "R": + got = int(warp.r[lane, n]) + if not same_value(got, want, kind): + diffs.append((lane, reg_kind, n, got, want)) + elif int(warp.p[lane, n]) != int(bool(want)): + diffs.append((lane, reg_kind, n, int(warp.p[lane, n]), want)) + if "smem_out" in case: + want_mem = parse_bytes(case["smem_out"]) + if bytes(warp.smem[:len(want_mem)]) != want_mem: + diffs.append((-1, "smem", 0, 0, 0)) + report = [f" {len(words)} Maxwell words, urbase {urbase}"] + report += [f" warning: {m}" for m in dict.fromkeys(warp.warnings)] + if not diffs: + return True, report + lanes = sorted({d[0] for d in diffs}) + report.append(f" {len(diffs)} differing values in lanes {lanes[:16]}{' ...' if len(lanes) > 16 else ''}") + if kind != "bits": + worst = max(ulps(got, want, kind) for _, reg_kind, _, got, want in diffs if reg_kind == "R") + report.append(f" largest difference: {'NaN against a number' if worst >= 1 << 32 else f'{worst} ulp'} ({kind})") + for lane, reg_kind, n, got, want in diffs[:12]: + if reg_kind == "smem": + report.append(" shared memory differs from smem_out") + elif reg_kind == "P": + report.append(f" lane {lane:2d} P{n}: got {got} want {int(bool(want))}") + else: + report.append(f" lane {lane:2d} R{n}: got {_fmt(got)} want {_fmt(want & 0xFFFFFFFF)}") + return False, report + + +def load_cases(path: Path) -> list[dict]: + data = json.loads(path.read_text(encoding="utf-8")) + if isinstance(data, dict): + data = data.get("cases", data.get("vectors", list(data.values()) if all(isinstance(v, dict) for v in data.values()) else [])) + return data + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Lower SM86 vector cases with nx2navi_xlate --lower, run them on the " + "Maxwell warp emulator and compare with the captured results.") + ap.add_argument("vectors", type=Path, help="JSON list of cases: name, sm86 [w0..w3], regs_in, smem?, regs_out") + ap.add_argument("--xlate", type=Path, default=DEFAULT_XLATE, help="nx2navi_xlate executable") + ap.add_argument("--prmt", choices=("ryujinx", "maxwell"), default="ryujinx", + help="PRMT as the translator runs it (no-op) or as Maxwell hardware does") + ap.add_argument("--fill", default="0", help="initial value (hex) of registers regs_in does not set") + ap.add_argument("--case", action="append", help="run only cases whose name contains this (repeatable)") + ap.add_argument("--trace", action="store_true", help="print every executed Maxwell instruction") + ap.add_argument("--exact", action="store_true", help="compare float results bit for bit (NaN payloads too)") + args = ap.parse_args(argv) + cases = load_cases(args.vectors) + passed = failed = 0 + for idx, case in enumerate(cases): + name = case.get("name", f"case{idx}") + if args.case and not any(c in name for c in args.case): + continue + try: + ok, report = run_case(case, args.xlate, args.prmt, args.trace, parse_int(args.fill), args.exact) + except Exception as e: # one broken case does not stop the run + ok, report = False, [f" error: {type(e).__name__}: {e}"] + print(f"{'PASS' if ok else 'FAIL'} {name}") + if not ok or any("warning" in r for r in report): + print("\n".join(report)) + passed += ok + failed += not ok + print(f"{passed} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/maxwell_warp_emu_selftest.py b/tests/maxwell_warp_emu_selftest.py new file mode 100644 index 00000000..042f5463 --- /dev/null +++ b/tests/maxwell_warp_emu_selftest.py @@ -0,0 +1,351 @@ +"""Self-test of tests/maxwell_warp_emu.py: hand-encoded Maxwell sequences against hand-computed results. + +Part 1 runs Maxwell words encoded here from the field layouts of Ryujinx Decoders/InstDecoders.cs (integer and logic +ops, SHFL.IDX with clamp and segment masks, LDS/STS sizes, F2F, HFMA2, FFMA rounding, PRMT, predicates, EXIT, BRA). +Part 2, when build/xlate/nx2navi_xlate.exe exists, lowers SM86 HMMA.1688 / HMMA.16816, LDSM and MOVM with +nx2navi_xlate --lower and checks the emulated result against exactly representable reference values (small +integers, so every F16 rounding is exact and only the fragment layout can be wrong), and HFMA2.RELU as the translator +runs it (Ryujinx 1.3.2 drops PRMT, so AmpereLowering.cs Half must not use it). tests/tensor_lowering_check.py checks +the same lowerings against hardware results. + + python tests/maxwell_warp_emu_selftest.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import maxwell_warp_emu as emu # noqa: E402 + +PT = 7 << 16 # guard field: predicate PT +LANES = 32 +lane = np.arange(LANES, dtype=np.int64) +failures: list[str] = [] +checks = 0 + + +def check(name: str, got, want): + global checks + checks += 1 + got = np.asarray(got).astype(np.int64) + want = np.broadcast_to(np.asarray(want).astype(np.int64), got.shape) + if not np.array_equal(got, want): + bad = np.nonzero(got != want)[0][:6] + failures.append(f"{name}: " + ", ".join(f"lane {b} got {int(got[b]):#x} want {int(want[b]):#x}" for b in bad)) + + +# ---- Maxwell encoders (InstDecoders.cs field positions) ----------------------------------------------------------- + +def guard(p: int, inv: bool = False) -> int: + return (p | (8 if inv else 0)) << 16 + + +def iadd32i(d, a, imm, g=PT): return 0x1C00000000000000 | (imm & 0xFFFFFFFF) << 20 | a << 8 | d | g +def mov32i(d, imm, g=PT): return 0x0100000000000000 | 0xF << 12 | (imm & 0xFFFFFFFF) << 20 | d | g +def mov(d, s, g=PT): return 0x5C98000000000000 | 0xF << 39 | s << 20 | d | g +def s2r(d, sr=0): return 0xF0C8000000000000 | sr << 20 | d | PT +def lop32i(d, a, imm, op): return 0x0400000000000000 | op << 53 | (imm & 0xFFFFFFFF) << 20 | a << 8 | d | PT +def lop3(d, a, b, c, lut): return 0x5BE0000000000000 | 7 << 48 | c << 39 | lut << 28 | b << 20 | a << 8 | d | PT +def iscadd(d, a, b, sh): return 0x5C18000000000000 | sh << 39 | b << 20 | a << 8 | d | PT +def iadd3(d, a, b, c, neg_b=False): return 0x5CC0000000000000 | c << 39 | b << 20 | a << 8 | d | (1 << 50 if neg_b else 0) | PT +def shl_i(d, a, n): return 0x3848000000000000 | n << 20 | a << 8 | d | PT +def shr_i(d, a, n, signed=False): return 0x3828000000000000 | (1 << 48 if signed else 0) | n << 20 | a << 8 | d | PT +def shr_r(d, a, b): return 0x5C28000000000000 | b << 20 | a << 8 | d | PT +def prmt(d, a, sel, c): return 0x5BC0000000000000 | c << 39 | sel << 20 | a << 8 | d | PT +def isetp_i(pd, a, imm, cmp, signed=False): # ISETP.cmp Pd, PT, Ra, imm, PT (AND) + return 0x3660000000000000 | cmp << 49 | (1 << 48 if signed else 0) | 7 << 39 | (imm & 0x7FFFF) << 20 | a << 8 | pd << 3 | 7 | PT +def shfl(d, a, b, c, mode=0, b_imm=False, c_imm=False, pd=7): + return (0xEF10000000000000 | pd << 48 | (c << 34 if c_imm else c << 39) | (1 << 29 if c_imm else 0) + | (1 << 28 if b_imm else 0) | mode << 30 | b << 20 | a << 8 | d | PT) +def lds(d, a, off, size): return 0xEF48000000000000 | size << 48 | (off & 0xFFFFFF) << 20 | a << 8 | d | PT +def sts(v, a, off, size): return 0xEF58000000000000 | size << 48 | (off & 0xFFFFFF) << 20 | a << 8 | v | PT +def f2f(d, b, dst, src, high=False): return 0x5CA8000000000000 | dst << 8 | src << 10 | b << 20 | (1 << 41 if high else 0) | d | PT +def hfma2(d, a, b, c, neg_c=False, sat=False): + return 0x5D00000000000000 | c << 39 | (1 << 30 if neg_c else 0) | (1 << 32 if sat else 0) | b << 20 | a << 8 | d | PT +def ffma(d, a, b, c): return 0x5980000000000000 | c << 39 | b << 20 | a << 8 | d | PT +def bra(offset): return 0xE24000000000000F | (offset & 0xFFFFFF) << 20 | PT +EXIT = 0xE30000000007000F + + +def f32bits(x): return int(np.array([x], dtype=np.float32).view(np.uint32)[0]) +def h16bits(x): return int(np.array([x], dtype=np.float16).view(np.uint16)[0]) +def hpair(lo, hi): return h16bits(lo) | h16bits(hi) << 16 + + +def regs_all(**regs): + return {"*": {k: v for k, v in regs.items()}} + + +def per_lane(reg, values): + return {l: {reg: int(v) & 0xFFFFFFFF} for l, v in enumerate(values)} + + +def merge(*specs): + out: dict = {} + for s in specs: + for k, v in s.items(): + out.setdefault(k, {}).update(v) + return out + + +# ---- part 1: hand-encoded sequences ------------------------------------------------------------------------------- + +def part1(): + # IADD32I, MOV32I, IADD3 with a negated operand, ISCADD. + r = emu.run([mov32i(1, 0x7FFFFFFF), iadd32i(2, 0, 5), iadd32i(3, 1, 1), iadd32i(4, 0, -3), + iadd3(5, 0, 2, 1, neg_b=True), iscadd(6, 0, 2, 4), EXIT], per_lane("R0", lane * 3)) + check("IADD32I", r[:, 2], lane * 3 + 5) + check("IADD32I wraps", r[:, 3], 0x80000000) + check("IADD32I negative", r[:, 4], (lane.astype(np.int64) * 3 - 3) & 0xFFFFFFFF) + check("IADD3 -B", r[:, 5], (lane.astype(np.int64) * 3 - (lane * 3 + 5) + 0x7FFFFFFF) & 0xFFFFFFFF) + check("ISCADD", r[:, 6], ((lane.astype(np.int64) * 3) << 4) + lane * 3 + 5) + check("RZ stays 0", r[:, 255], 0) + + # S2R lane id, LOP32I, SHL/SHR, LOP3, SHR by register. + r = emu.run([s2r(1), lop32i(2, 1, 3, 0), shl_i(3, 1, 3), shr_i(4, 1, 2), lop32i(5, 1, 0xF0, 1), + lop32i(6, 1, 0x5, 2), lop3(7, 1, 2, 3, 0x96), mov32i(8, 0x80000000), shr_i(9, 8, 4, signed=True), + shr_r(10, 3, 2), EXIT], {}) + check("S2R SR_LANEID", r[:, 1], lane) + check("LOP32I.AND", r[:, 2], lane & 3) + check("SHL imm", r[:, 3], lane << 3) + check("SHR imm", r[:, 4], lane >> 2) + check("LOP32I.OR", r[:, 5], lane | 0xF0) + check("LOP32I.XOR", r[:, 6], lane ^ 5) + check("LOP3 0x96 (xor3)", r[:, 7], lane ^ (lane & 3) ^ (lane << 3)) + check("SHR.S32", r[:, 9], 0xF8000000) + check("SHR by register", r[:, 10], (lane << 3) >> (lane & 3)) + + # SHFL.IDX: the whole warp by a lane register; a segment of 4 lanes; out of range lanes keep their own value. + vals = lane * 0x101 + 7 + seq = [s2r(1), iadd32i(2, 1, 1), lop32i(2, 2, 31, 0), + shfl(3, 0, 2, 31, c_imm=True), # (lane + 1) % 32 + shfl(4, 0, 1, 0x1C03, b_imm=True, c_imm=True), # segments of 4: lane (t & ~3) | 1 + shfl(5, 0, 5, 0x1803, b_imm=True, c_imm=True, pd=0), # segments of 8, clamp 3: index 5 invalid + shfl(6, 0, 2, 0x1803, b_imm=True, c_imm=True, pd=1), # index 2 valid + shfl(7, 0, 1, 0x1F, mode=3, b_imm=True, c_imm=True), # BFLY 1 + shfl(8, 0, 2, 0x00, mode=2, b_imm=True, c_imm=True), # DOWN 2, clamp 0: lanes > 0... invalid + shfl(9, 0, 2, 0x1F, mode=2, b_imm=True, c_imm=True), # DOWN 2 + shfl(10, 0, 1, 0, mode=1, b_imm=True, c_imm=True), # UP 1 + EXIT] + w = emu.run_warp(seq, per_lane("R0", vals)) + r = w.r + check("SHFL.IDX lane register", r[:, 3], vals[(lane + 1) % 32]) + check("SHFL.IDX segment 4", r[:, 4], vals[(lane & ~3) | 1]) + check("SHFL.IDX clamp invalid keeps own value", r[:, 5], vals) + check("SHFL.IDX clamp invalid predicate", w.p[:, 0], False) + check("SHFL.IDX in clamp", r[:, 6], vals[(lane & ~7) | 2]) + check("SHFL.IDX valid predicate", w.p[:, 1], True) + check("SHFL.BFLY", r[:, 7], vals[lane ^ 1]) + check("SHFL.DOWN clamp 0", r[:, 8], vals) + check("SHFL.DOWN", r[:, 9], np.where(lane + 2 <= 31, vals[np.minimum(lane + 2, 31)], vals)) + check("SHFL.UP", r[:, 10], np.where(lane >= 1, vals[np.maximum(lane.astype(np.int64) - 1, 0)], vals)) + + # STS / LDS: 32, 16 and 8 bits, sign extension, 64 and 128 bits, the immediate offset. + seq = [s2r(1), shl_i(2, 1, 2), # R2 = 4 * lane + sts(1, 2, 0x100, 4), # word [0x100 + 4t] = t + lds(3, 2, 0x100, 4), + lds(4, 2, 0x100, 2), lds(5, 2, 0x102, 2), lds(6, 2, 0x101, 0), + mov32i(7, 0x8001F0F0), sts(7, 255, 0x40, 4), lds(8, 255, 0x40, 3), lds(9, 255, 0x42, 3), + lds(10, 255, 0x40, 1), lds(11, 255, 0x41, 0), + lds(12, 255, 0x100, 6), # R12-R15 = words 0..3 of the table + mov32i(16, 0xABCD), sts(16, 255, 0x82, 2), lds(17, 255, 0x80, 4), + EXIT] + r = emu.run(seq, {}) + check("LDS.32", r[:, 3], lane) + check("LDS.U16", r[:, 4], lane) + check("LDS.U16 high half", r[:, 5], 0) + check("LDS.U8 byte 1", r[:, 6], 0) + check("LDS.S16", r[:, 8], 0xFFFFF0F0) + check("LDS.S16 high", r[:, 9], 0xFFFF8001) + check("LDS.S8", r[:, 10], 0xFFFFFFF0) + check("LDS.U8", r[:, 11], 0xF0) + for k in range(4): + check(f"LDS.128 word {k}", r[:, 12 + k], k) + check("STS.U16 at +2", r[:, 17], 0xABCD0000) + + # F2F: F32 from either F16 half (exact), F16 from F32 (round to nearest even, high half zero). + src = hpair(1.5, -0.333251953125) + seq = [mov32i(1, src), f2f(2, 1, 2, 1), f2f(3, 1, 2, 1, high=True), + mov32i(4, f32bits(1 / 3)), f2f(5, 4, 1, 2), + mov32i(6, f32bits(1 + 2 ** -11)), f2f(7, 6, 1, 2), # tie: to even (1.0) + mov32i(8, f32bits(1 + 3 * 2 ** -11)), f2f(9, 8, 1, 2), # tie: to even (1 + 2^-9) + mov32i(10, f32bits(70000.0)), f2f(11, 10, 1, 2), # overflow: inf + EXIT] + r = emu.run(seq, {}) + check("F2F.F32.F16 H0", r[:, 2], f32bits(1.5)) + check("F2F.F32.F16 H1", r[:, 3], f32bits(-0.333251953125)) + check("F2F.F16.F32 1/3", r[:, 5], 0x3555) + check("F2F.F16.F32 tie down", r[:, 7], 0x3C00) + check("F2F.F16.F32 tie up", r[:, 9], 0x3C02) + check("F2F.F16.F32 overflow", r[:, 11], 0x7C00) + + # HFMA2: per half F32 FMA then F16; neg C; saturate. + seq = [mov32i(1, hpair(1.5, 2.0)), mov32i(2, hpair(2.0, -0.5)), mov32i(3, hpair(0.25, 1.0)), + hfma2(4, 1, 2, 3), hfma2(5, 1, 2, 3, neg_c=True), hfma2(6, 1, 2, 3, sat=True), + mov32i(7, hpair(1 + 2 ** -10, 1.0)), mov32i(8, hpair(1 + 2 ** -10, 2 ** -11)), mov32i(9, hpair(0.0, 1.0)), + hfma2(10, 7, 8, 9), EXIT] + r = emu.run(seq, {}) + check("HFMA2", r[:, 4], hpair(3.25, 0.0)) + check("HFMA2 -C", r[:, 5], hpair(2.75, -2.0)) + check("HFMA2.SAT", r[:, 6], hpair(1.0, 0.0)) + # (1 + 2^-10)^2 = 1 + 2^-9 + 2^-20 -> F16 1 + 2^-9; 1 + 2^-11 is an F16 tie -> 1.0. + check("HFMA2 rounding", r[:, 10], hpair(1 + 2 ** -9, 1.0)) + + # Ryujinx Optimizer.PropagatePack: an F16 result unpacked again without an integer operation in between is the + # unrounded F32 value (directly or through a MOV); LOP in between, or a guarded pack, keeps the F16 rounding. + third, third16 = f32bits(1 / 3), f32bits(float(np.float16(1 / 3))) + seq = [mov32i(1, third), f2f(2, 1, 1, 2), f2f(3, 2, 2, 1), # pack, unpack: folded + mov(4, 2), f2f(5, 4, 2, 1), # through a MOV: folded + lop32i(6, 2, 0xFFFFFFFF, 0), f2f(7, 6, 2, 1), # through LOP: rounded + s2r(8), isetp_i(0, 8, 32, 1), # P0 = lane < 32, true everywhere + f2f(9, 1, 1, 2) & ~(0xF << 16) | guard(0), f2f(10, 9, 2, 1), # guarded pack: rounded + mov32i(11, hpair(40000.0, 1.0)), mov32i(12, hpair(2.0, 2.0)), mov32i(13, 0), mov32i(16, hpair(0.5, 0.5)), + hfma2(14, 11, 12, 13), hfma2(15, 14, 16, 13), # 40000 * 2 = inf as F16, * 0.5 again + EXIT] + r = emu.run(seq, {}) + check("F2F.F16 then F2F.F32: unrounded (PropagatePack)", r[:, 3], third) + check("... through MOV: unrounded", r[:, 5], third) + check("... through LOP: rounded", r[:, 7], third16) + check("... guarded pack: rounded", r[:, 10], third16) + check("HFMA2 overflow stored as F16 inf", r[:, 14], hpair(np.inf, 2.0)) + check("HFMA2 into HFMA2: the unrounded 80000 halved", r[:, 15], hpair(40000.0, 1.0)) + + # FFMA: rounded once. Exact a*b + c lies just below an F32 halfway point; F64 alone would round to the tie and + # then to even (0x3f800002), the fused result is 0x3f800001. + seq = [mov32i(1, 0x39800001), mov32i(2, 0x397FFFFE), mov32i(3, 0x3F800001), ffma(4, 1, 2, 3), + mov32i(5, f32bits(3.0)), mov32i(6, f32bits(0.5)), ffma(7, 5, 6, 5), EXIT] + r = emu.run(seq, {}) + check("FFMA single rounding at a tie", r[:, 4], 0x3F800001) + check("FFMA", r[:, 7], f32bits(4.5)) + + # PRMT: a no-op as Ryujinx translates it, the Maxwell byte permute otherwise (0xbb99: sign of bytes 1 and 3). + seq = [mov32i(1, 0x7F80C000), mov32i(2, 0xBB99), mov32i(3, 0x11111111), prmt(3, 1, 2, 255), + mov32i(4, 0x3210), mov32i(5, 0x22222222), prmt(5, 1, 4, 255), EXIT] + r = emu.run(seq, {}, prmt="ryujinx") + check("PRMT (Ryujinx: no-op)", r[:, 3], 0x11111111) + r = emu.run(seq, {}, prmt="maxwell") + check("PRMT sign replicate (Maxwell)", r[:, 3], 0x0000FFFF) + check("PRMT identity (Maxwell)", r[:, 5], 0x7F80C000) + + # Predicates: ISETP, a guarded write, an inverted guard, EXIT of some lanes, a uniform BRA over one instruction. + seq = [s2r(1), isetp_i(0, 1, 16, 1), # P0 = lane < 16 + mov32i(2, 0xAAAA, g=guard(0)), mov32i(3, 0xBBBB, g=guard(0, inv=True)), + bra(emu.slot_address(6) - emu.slot_address(4) - 8), mov32i(4, 0xDEAD), # skipped + mov32i(5, 0x5555), + (EXIT & ~(0xF << 16)) | guard(0, inv=True), # lanes >= 16 exit + mov32i(6, 0x6666), EXIT] + w = emu.run_warp(seq, regs_all(R2=1, R3=1, R6=1)) + check("ISETP.LT", w.p[:, 0], lane < 16) + check("guarded write", w.r[:, 2], np.where(lane < 16, 0xAAAA, 1)) + check("inverted guard", w.r[:, 3], np.where(lane < 16, 1, 0xBBBB)) + check("BRA skips", w.r[:, 4], 0) + check("after BRA", w.r[:, 5], 0x5555) + check("guarded EXIT", w.r[:, 6], np.where(lane < 16, 0x6666, 1)) + + # Unknown opcodes raise with the word. + global checks + checks += 1 + try: + emu.run([0xFFFFFFFFFFFFFFFF], {}) + failures.append("unknown opcode did not raise") + except emu.UnknownOpcode as e: + if "0xffffffffffffffff" not in str(e): + failures.append(f"unknown opcode message: {e}") + + +# ---- part 2: lowered SM86 tensor instructions --------------------------------------------------------------------- + +def hmma_case(k16: bool, rng): + """HMMA.{1688,16816}.F16 R4, R8, R12, R16 with small integers; returns (sm86, regs_in, expected D regs).""" + K = 16 if k16 else 8 + A = rng.integers(-3, 4, size=(16, K)).astype(np.float64) + B = rng.integers(-3, 4, size=(K, 8)).astype(np.float64) + C = rng.integers(-8, 9, size=(16, 8)).astype(np.float64) + D = A @ B + C + regs_in, want = {}, {} + for t in range(LANES): + g, q = t // 4, t % 4 + regs = {} + for j in range(K // 8): + for r in range(2): + regs[f"R{8 + 2 * j + r}"] = hpair(A[g + 8 * r, 8 * j + 2 * q], A[g + 8 * r, 8 * j + 2 * q + 1]) + regs[f"R{12 + j}"] = hpair(B[8 * j + 2 * q, g], B[8 * j + 2 * q + 1, g]) + for r in range(2): + regs[f"R{16 + r}"] = hpair(C[g + 8 * r, 2 * q], C[g + 8 * r, 2 * q + 1]) + regs_in[t] = regs + want[t] = {4 + r: hpair(D[g + 8 * r, 2 * q], D[g + 8 * r, 2 * q + 1]) for r in range(2)} + w0 = 0x23C | 7 << 12 | 4 << 16 | 8 << 24 + sm86 = [w0, 12, 16 | (1 << 11 if k16 else 0), 0] + return sm86, regs_in, want + + +def part2(exe: Path): + rng = np.random.default_rng(86) + for k16 in (False, True): + sm86, regs_in, want = hmma_case(k16, rng) + words, urbase, _ = emu.lower(sm86, exe) + r = emu.run(words, regs_in) + for reg in (4, 5): + check(f"HMMA.{'16816' if k16 else '1688'} D R{reg} (lowered, {len(words)} words)", + r[:, reg], [want[t][reg] for t in range(LANES)]) + + # LDSM.16.M88.4 R4, [R1] and LDSM.16.MT88.1 R8, [R1]: matrix m row r at 0x200 * m + 0x20 * r (16 bytes used), + # element (m, row, col) = 0x1000 * m + 0x100 * row + col. + smem = bytearray(0x1000) + for m in range(4): + for row in range(8): + for col in range(8): + at = 0x200 * m + 0x20 * row + 2 * col + smem[at:at + 2] = (0x1000 * m + 0x100 * row + col).to_bytes(2, "little") + addr = {t: {"R1": 0x200 * (t // 8) + 0x20 * (t % 8)} for t in range(LANES)} + el = lambda m, row, col: 0x1000 * m + 0x100 * row + col + for transpose, size, dst in ((False, 2, 4), (True, 0, 8), (True, 2, 12), (False, 0, 16)): + sm86 = [0x83B | 7 << 12 | dst << 16 | 1 << 24, 0, size << 8 | (0x4000 if transpose else 0), 0] + words, _, _ = emu.lower(sm86, exe) + r = emu.run(words, addr, bytes(smem)) + for m in range(1 << size): + if transpose: # column t/4 of rows 2(t%4) (low) and 2(t%4)+1 (high) + want = [el(m, 2 * (t % 4), t // 4) | el(m, 2 * (t % 4) + 1, t // 4) << 16 for t in range(LANES)] + else: # row t/4, columns 2(t%4), 2(t%4)+1 + want = [el(m, t // 4, 2 * (t % 4)) | el(m, t // 4, 2 * (t % 4) + 1) << 16 for t in range(LANES)] + check(f"LDSM.16.{'MT88' if transpose else 'M88'}.{1 << size} matrix {m}", r[:, dst + m], want) + + # MOVM.16.MT88 R2, R1: element (row, col) = 8 * row + col at lane 4 row + col / 2, half col % 2. + regs_in = {t: {"R1": (8 * (t // 4) + 2 * (t % 4)) | (8 * (t // 4) + 2 * (t % 4) + 1) << 16} for t in range(LANES)} + words, _, _ = emu.lower([0x23A | 7 << 12 | 2 << 16 | 1 << 24, 0, 0, 0], exe) + r = emu.run(words, regs_in) + want = [(8 * (2 * (t % 4)) + t // 4) | (8 * (2 * (t % 4) + 1) + t // 4) << 16 for t in range(LANES)] + check("MOVM.16.MT88", r[:, 2], want) + + # HFMA2.RELU R0, R1, R2, R3 (PT) as the translator runs it (no PRMT: AmpereLowering.cs Half builds the mask with + # integer operations), whatever the scratch registers held before: negative sums become +0, positive ones stay. + sm86 = [0x231 | 7 << 12 | 0 << 16 | 1 << 24, 2, 3 | 0x8000 | 0x03800000, 0] + words, _, _ = emu.lower(sm86, exe) + for a, b, c, want in (((1.0, -2.0), (-3.0, 0.5), (1.0, 0.25), (0.0, 0.0)), # sums -2.0, -0.75 + ((1.0, 2.0), (3.0, -0.5), (1.0, 0.25), (4.0, 0.0))): # sums 4.0, -0.75 + regs_in = regs_all(R1=hpair(*a), R2=hpair(*b), R3=hpair(*c)) + for fill in (0, 0x7FC0DEAD): + w = emu.run_warp(words, regs_in, prmt="ryujinx", fill=fill) + check(f"HFMA2.RELU -> {want} as translated (fill {fill:#x})", w.r[:, 0], hpair(*want)) + check(f"HFMA2.RELU uses no PRMT (fill {fill:#x})", [len(w.warnings)], 0) + + +def main() -> int: + part1() + exe = emu.DEFAULT_XLATE + if exe.exists(): + part2(exe) + else: + print(f" skip: part 2 needs {exe}") + if failures: + print("\n".join("FAIL " + f for f in failures)) + print(f"{checks - len(failures)}/{checks} checks passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tensor_lowering_check.py b/tests/tensor_lowering_check.py new file mode 100644 index 00000000..7f21a20c --- /dev/null +++ b/tests/tensor_lowering_check.py @@ -0,0 +1,441 @@ +"""Checks the lowered SM86 tensor instructions against results captured on an NVIDIA GPU, in one command. + +tools/xlate/AmpereLowering.cs rewrites the SM86 instructions of the game's neural-network passes as Maxwell code: +Lowerer.Hmma (HMMA.1688 / HMMA.16816, .F16 and .F32), Lowerer.Ldsm (LDSM.16.M88 / MT88 .1/.2/.4), Lowerer.Movm +(MOVM.16.MT88) and Lowerer.Half (HFMA2, HFMA2.RELU). This script lowers every case of the vector file with +`nx2navi_xlate --lower`, runs the Maxwell words on tests/maxwell_warp_emu.py (which executes them as the Ryujinx +translator does) and compares with the registers the GPU wrote. Exit status 1 on any failure. The vector file names +the GPU that produced it ("device"). So far that is an RTX 5090 running the compute_86 PTX through the driver's JIT, +as no SM86 part was at hand: the results follow the PTX semantics but need not show SM86's internal precision. + + C:/Users/PC/AppData/Local/Programs/Python/Python312/python tests/tensor_lowering_check.py [vectors.json] + [--xlate EXE] [--case NAME] [-v] + +Defaults: vectors.json is /work/tensor-gt/vectors.json (/work/tensor-gt/gen_vectors.py writes +it: each case is one SM86 instruction in inline PTX, run on one warp), the translator +/build/xlate/nx2navi_xlate.exe (python tools/build_xlate.py builds it); is the directory two +levels above this repository. numpy is needed. + +Every case runs several times and must pass each run: + * with the registers the vectors do not set first 0, then 0x7fc0dead, so a read of a register the lowering never + wrote shows; + * then under @P0 (the guard field of the SM86 word set to P0) with P0 false in every third lane: lanes with P0 get + the GPU's result, the others keep their old destination values. The other lanes still supply their fragments, + rows and addresses, which works only if the shuffles run in every lane and just the destination writes take the + guard, the rule AmpereLowering.cs follows. +Each run checks: + * destination registers bit for bit, except that a NaN matches any NaN in a float result (maxwell_warp_emu + same_value: SPIR-V leaves NaN payloads to the host, the GPU writes 0x7fff). HMMA gets no tolerance: the GPU rounds + every element once (f16 or f32 of the exact A*B+C), and so does the lowering for these values, whose sums are + exact in F32; the largest difference in ulp is reported; + * every other register R0-R236 (guest and uniform registers; R237 up is the lowering's own) and P0-P6 keep their + values, shared memory is untouched, and the emulator has nothing to report (a shuffle reading a lane its guard + left out, a PRMT the translator drops); + * urbase, the register the lowering mapped UR0 to, is the highest register the instruction names plus two, rounded + down to even: AmpereLowering HighestGpr counted every register of the instruction's fragments and quads. +Read off the lowered words (both lowerings): every write to a guest register carries the instruction's guard and no +SHFL is guarded. +Derived cases, beyond the hardware vectors, for forms the game uses: reference models of HFMA2(.RELU) and LDSM, each +first checked against every hardware case of its instruction, give the expected values of HFMA2.RELU with an +immediate B (opcode 0x831, as in the game's HFMA2.RELU Rd, Ra, 1, 1, Rc) and LDSM .M88.4 / .MT88.4 with an offset. +""" + +from __future__ import annotations + +import argparse +import sys +from fractions import Fraction +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import maxwell_warp_emu as emu # noqa: E402 + +LANES = 32 +RZ = 255 +GUEST_REGS = 237 # R0-R236: guest and uniform registers; AmpereLowering.cs uses R237 up itself +FILLS = (0x00000000, 0x7FC0DEAD) +EXIT = 0xE30000000007000F # Maxwell EXIT: runs no code, gives the emulator's initial state +DEFAULT_VECTORS = emu.ROOT.parent.parent / "work" / "tensor-gt" / "vectors.json" + + +def guard_lanes() -> np.ndarray: + """P0 for the guarded run: false in lanes 1, 4, 7, ... so every quad and row group mixes both.""" + return np.arange(LANES) % 3 != 1 + + +# ---- the instructions of the vector file ---------------------------------------------------------------------- + +def words_of(sm86) -> list[int]: + return [emu.parse_int(x) & 0xFFFFFFFF for x in sm86] + + +def operands(sm86) -> tuple[list[int], list[int]]: + """(destination registers, source registers) one SM86 instruction names, from its fields (work/tensor-gt/README.md + "Encoding bits"; the same fields AmpereLowering.cs decodes).""" + w0, w1, w2, _ = words_of(sm86) + op = w0 & 0xFFF + rd, ra, rb, rc = (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF, w1 & 0xFF, w2 & 0xFF + + def regs(base, n): + return [] if base == RZ else list(range(base, base + n)) + if op == 0x23C: # HMMA: shape bit 75, .F32 bit 76 + k16, f32 = (w2 >> 11) & 1, (w2 >> 12) & 1 + cd = 4 if f32 else 2 + return regs(rd, cd), regs(ra, 4 if k16 else 2) + regs(rb, 2 if k16 else 1) + regs(rc, cd) + if op == 0x83B: # LDSM: count 1 << bits 72-73 + return regs(rd, 1 << ((w2 >> 8) & 3)), regs(ra, 1) + if op == 0x23A: # MOVM + return regs(rd, 1), regs(ra, 1) + if op in (0x231, 0x831): # HFMA2: B a register (0x231) or an immediate (0x831) + return regs(rd, 1), regs(ra, 1) + (regs(rb, 1) if op == 0x231 else []) + regs(rc, 1) + raise ValueError(f"no operand table for SM86 opcode 0x{op:03x}") + + +def expected_urbase(sm86) -> int: + dests, srcs = operands(sm86) + return (max(dests + srcs + [0]) + 2) & ~1 + + +def shared_memory(case: dict) -> bytes | None: + """The case's shared memory as maxwell_warp_emu.run_case places it: smem at smem_base, 64 KiB at least.""" + if case.get("smem") is None: + return None + base = emu.parse_int(case.get("smem_base", 0)) + data = emu.parse_bytes(case["smem"]) + buf = bytearray(max(0x10000, base + len(data))) + buf[base:base + len(data)] = data + return bytes(buf) + + +def consistency(case: dict) -> list[str]: + """Problems of the case itself: regs_out not exactly the destinations, LDSM addresses that disagree.""" + problems = [] + dests, _ = operands(case["sm86"]) + for lane, regs in emu.load_regs(case["regs_out"]).items(): + if sorted(n for kind, n in regs if kind == "R") != sorted(dests): + problems.append(f"lane {lane}: regs_out has {sorted(n for _, n in regs)}, the instruction writes {dests}") + break + if case.get("lane_addresses") is not None: + ra = (words_of(case["sm86"])[0] >> 24) & 0xFF + base = emu.parse_int(case.get("smem_base", 0)) + regs_in = emu.load_regs(case["regs_in"]) + for lane, a in enumerate(case["lane_addresses"]): + given = regs_in[lane].get(("R", ra)) + if given is not None and given != (emu.parse_int(a) + base) & 0xFFFFFFFF: + problems.append(f"lane {lane}: R{ra} = {given:#x} but lane_addresses says {emu.parse_int(a) + base:#x}") + break + return problems + + +# ---- running and comparing -------------------------------------------------------------------------------------- + +class Outcome: + def __init__(self): + self.problems: list[str] = [] + self.worst_ulp = 0 + self.nan_matches = 0 + self.runs = 0 + + +def check_run(out: Outcome, label: str, words, case: dict, fill: int, guard: np.ndarray | None, verbose: bool): + """One run of the lowered words; appends what is wrong to out.problems.""" + sm86 = case["sm86"] + dests, _ = operands(sm86) + kind = emu.result_kind(sm86) + regs_in = emu.load_regs(case["regs_in"]) + if case.get("lane_addresses") is not None: + ra = (words_of(sm86)[0] >> 24) & 0xFF + base = emu.parse_int(case.get("smem_base", 0)) + for lane, a in enumerate(case["lane_addresses"]): + regs_in[lane].setdefault(("R", ra), (emu.parse_int(a) + base) & 0xFFFFFFFF) + if guard is not None: + for lane in range(LANES): + regs_in[lane][("P", 0)] = bool(guard[lane]) + expected = emu.load_regs(case["regs_out"]) + smem = shared_memory(case) + start = emu.run_warp([EXIT], regs_in, smem, fill=fill) + warp = emu.run_warp(words, regs_in, smem, fill=fill) + out.runs += 1 + where = f"{label}, fill {fill:#010x}" + bad = [] + for lane in range(LANES): + on = guard is None or guard[lane] + for n in dests: + got = int(warp.r[lane, n]) + want = expected[lane][("R", n)] if on else int(start.r[lane, n]) + k = kind if on else "bits" + if k != "bits": + out.worst_ulp = max(out.worst_ulp, emu.ulps(got, want, k)) + if not emu.same_value(got, want, k): + bad.append(f"lane {lane:2d} R{n}: got {emu._fmt(got)} want {emu._fmt(want & 0xFFFFFFFF)}" + + ("" if on else " (guard false: old value)")) + elif got != want: + out.nan_matches += 1 + if bad: + out.problems.append(f"{where}: {len(bad)} wrong destination values") + out.problems += [" " + b for b in bad[:12 if verbose else 4]] + keep = [n for n in range(GUEST_REGS) if n not in dests] + lanes, cols = np.nonzero(warp.r[:, keep] != start.r[:, keep]) + if len(lanes): + regs = sorted({keep[c] for c in cols}) + out.problems.append(f"{where}: registers outside the destination changed: R{regs[:8]} in lanes {sorted(set(lanes.tolist()))[:8]}") + if (warp.p[:, :7] != start.p[:, :7]).any(): + out.problems.append(f"{where}: guest predicates P0-P6 changed") + if bytes(warp.smem) != bytes(start.smem): + out.problems.append(f"{where}: shared memory changed") + for m in dict.fromkeys(warp.warnings): + out.problems.append(f"{where}: emulator: {m}") + + +# Maxwell instructions of maxwell_warp_emu.TABLE without a destination register in bits 0-7. +NO_REGISTER_RESULT = {"EXIT", "BRA", "NOP", "STS", "STL", "ISETP", "ISETP.I", "R2P", "R2P.I"} + + +def guard_rule_problems(words, guard: int, label: str) -> list[str]: + """AmpereLowering.cs's guard rule read off the lowered words: a write to a guest register (below R237) carries the + SM86 instruction's guard (Maxwell guard field `guard`), a shuffle runs unguarded, in every lane.""" + problems = [] + for k, w in enumerate(words): + name, _ = emu.decode(w) + g = (w >> 16) & 0xF + if name == "SHFL" and g != 7: + problems.append(f"{label}: word {k} {w:016x}: SHFL under a guard") + if name not in NO_REGISTER_RESULT and (w & 0xFF) < GUEST_REGS and g != guard: + problems.append(f"{label}: word {k} {w:016x}: {name} writes guest register R{w & 0xFF} without the instruction's guard") + return problems + + +def check_case(case: dict, exe: Path, verbose: bool = False) -> Outcome: + out = Outcome() + out.problems += [f"vector file: {p}" for p in consistency(case)] + try: + words, urbase, _ = emu.lower(case["sm86"], exe) + except RuntimeError as e: + out.problems.append(str(e)) + return out + if urbase != expected_urbase(case["sm86"]): + out.problems.append(f"urbase {urbase}, expected {expected_urbase(case['sm86'])}: HighestGpr misses a register range") + out.problems += guard_rule_problems(words, (words_of(case["sm86"])[0] >> 12) & 0xF, "lowered words") + for fill in FILLS: + check_run(out, "all lanes", words, case, fill, None, verbose) + # The same instruction under @P0: SM86 guard bits 12-15 = 0. + w = words_of(case["sm86"]) + guarded = dict(case, sm86=[w[0] & ~0xF000] + w[1:]) + gwords, _, _ = emu.lower(guarded["sm86"], exe) + out.problems += guard_rule_problems(gwords, 0, "lowered words under @P0") + check_run(out, "@P0", gwords, guarded, FILLS[1], guard_lanes(), verbose) + return out + + +# ---- reference models for the derived cases ------------------------------------------------------------------------ + +def half_value(h: int): + """An F16 bit pattern as ("nan"), ("inf", sign) or ("num", Fraction, sign).""" + sign = h >> 15 + e, m = (h >> 10) & 0x1F, h & 0x3FF + if e == 0x1F: + return ("nan",) if m else ("inf", sign) + v = Fraction(m, 1 << 24) if e == 0 else Fraction(0x400 | m, 1 << 25) * (Fraction(2) ** e) + return ("num", -v if sign else v, sign) + + +def round_half(x: Fraction) -> int: + """x (not zero) rounded to F16, to nearest even; beyond the range infinity.""" + sign = 0x8000 if x < 0 else 0 + m = abs(x) + e = m.numerator.bit_length() - m.denominator.bit_length() + if Fraction(2) ** e > m: + e -= 1 + elif Fraction(2) ** (e + 1) <= m: + e += 1 + quantum = Fraction(2) ** (max(e, -14) - 10) + q = m / quantum + n = q.numerator // q.denominator + rest = q - n + if rest > Fraction(1, 2) or (rest == Fraction(1, 2) and n & 1): + n += 1 + value = n * quantum + if value >= 65536: + return sign | 0x7C00 + return sign | int(np.array([float(value)], dtype=np.float64).astype(np.float16).view(np.uint16)[0]) + + +def fma_half(a: int, b: int, c: int) -> int: + """F16 a * b + c rounded once (IEEE 754 fusedMultiplyAdd, round to nearest even); a NaN result is 0x7fff.""" + va, vb, vc = half_value(a), half_value(b), half_value(c) + if "nan" in (va[0], vb[0], vc[0]): + return 0x7FFF + zero = lambda v: v[0] == "num" and v[1] == 0 + if (va[0] == "inf" and zero(vb)) or (vb[0] == "inf" and zero(va)): + return 0x7FFF + psign = (a >> 15) ^ (b >> 15) + if va[0] == "inf" or vb[0] == "inf": + if vc[0] == "inf" and vc[1] != psign: + return 0x7FFF + return 0x7C00 | psign << 15 + if vc[0] == "inf": + return 0x7C00 | vc[1] << 15 + p = va[1] * vb[1] + s = p + vc[1] + if s == 0: # exact zero: -0 only when both terms are -0 + return 0x8000 if (p == 0 and psign and vc[2]) else 0 + return round_half(s) + + +def hfma2_model(a: int, b: int, c: int, relu: bool) -> int: + """HFMA2 (.RELU) on both halves (H1_H0 operands): RELU makes negative numbers and -0 +0, keeps NaN as 0x7fff.""" + res = 0 + for shift in (0, 16): + h = fma_half((a >> shift) & 0xFFFF, (b >> shift) & 0xFFFF, (c >> shift) & 0xFFFF) + if relu and h != 0x7FFF and h & 0x8000: + h = 0 + res |= h << shift + return res + + +def hfma2_operands(case: dict, lane: int, b_imm: int | None = None): + w0, w1, w2, _ = words_of(case["sm86"]) + regs = emu.load_regs(case["regs_in"])[lane] + get = lambda n: 0 if n == RZ else regs[("R", n)] + b = b_imm if b_imm is not None else get(w1 & 0xFF) + return get((w0 >> 24) & 0xFF), b, get(w2 & 0xFF), bool((w2 >> 15) & 1) + + +def ldsm_model(smem: bytes, addresses: list[int], count: int, transpose: bool, offset: int) -> list[list[int]]: + """LDSM per work/tensor-gt/README.md "Fragment layouts": out[lane][m] is lane's value of Rd + m.""" + out = [[0] * count for _ in range(LANES)] + rd16 = lambda at: int.from_bytes(smem[at:at + 2], "little") + for m in range(count): + rows = [addresses[8 * m + r] + offset for r in range(8)] + for t in range(LANES): + if transpose: + out[t][m] = rd16(rows[2 * (t % 4)] + 2 * (t // 4)) | rd16(rows[2 * (t % 4) + 1] + 2 * (t // 4)) << 16 + else: + at = rows[t // 4] + 4 * (t % 4) + out[t][m] = int.from_bytes(smem[at:at + 4], "little") + return out + + +def ldsm_fields(sm86): + w0, w1, w2, _ = words_of(sm86) + return (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF, 1 << ((w2 >> 8) & 3), bool((w2 >> 14) & 1), (w1 >> 8) & 0xFFFFFF + + +def model_problems(cases: list[dict]) -> dict[str, list[str]]: + """Each model against every hardware case of its instruction (exact bits: the models write 0x7fff for NaN).""" + found = {} + for case in cases: + w0 = words_of(case["sm86"])[0] + op = w0 & 0xFFF + expected = emu.load_regs(case["regs_out"]) + bad = [] + if op == 0x231: + rd = (w0 >> 16) & 0xFF + for lane in range(LANES): + want = expected[lane][("R", rd)] + got = hfma2_model(*hfma2_operands(case, lane)) + if got != want: + bad.append(f"lane {lane}: model {got:08x}, GPU {want:08x}") + elif op == 0x83B: + rd, ra, count, transpose, offset = ldsm_fields(case["sm86"]) + regs = emu.load_regs(case["regs_in"]) + model = ldsm_model(shared_memory(case), [regs[t][("R", ra)] for t in range(LANES)], count, transpose, offset) + for t in range(LANES): + for m in range(count): + if model[t][m] != expected[t][("R", rd + m)]: + bad.append(f"lane {t} R{rd + m}: model {model[t][m]:08x}, GPU {expected[t][('R', rd + m)]:08x}") + else: + continue + found[case["name"]] = bad + return found + + +def derived_cases(cases: list[dict]) -> list[dict]: + by_name = {c["name"]: c for c in cases} + out = [] + relu = by_name.get("hfma2_relu") + if relu is not None: + w0, _, w2, w3 = words_of(relu["sm86"]) + rb = words_of(relu["sm86"])[1] & 0xFF + for imm in (0x3C003C00, 0x4000BC00): # (1, 1) as the game's networks use it; (-1, 2) + regs_in = [{k: v for k, v in lane.items() if k != f"R{rb}"} for lane in relu["regs_in"]] + sm86 = [(w0 & ~0xFFF) | 0x831, imm, w2, w3] + rd = (w0 >> 16) & 0xFF + regs_out = [] + for lane in range(LANES): + a, b, c, r = hfma2_operands(relu, lane, imm) + regs_out.append({f"R{rd}": f"{hfma2_model(a, b, c, r):08x}"}) + out.append({"name": f"derived_hfma2_relu_imm_{imm:08x}", "sm86": [f"{x:08x}" for x in sm86], + "regs_in": regs_in, "regs_out": regs_out}) + src = by_name.get("ldsm_m88_x4") + if src is not None: + w0, _, w2, w3 = words_of(src["sm86"]) + regs = emu.load_regs(src["regs_in"]) + rd, ra = (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF + addresses = [regs[t][("R", ra)] for t in range(LANES)] + for transpose in (False, True): + sm86 = [w0, 0x40 << 8, (w2 & ~0x4000) | (0x4000 if transpose else 0), w3] + model = ldsm_model(shared_memory(src), addresses, 4, transpose, 0x40) + regs_out = [{f"R{rd + m}": f"{model[t][m]:08x}" for m in range(4)} for t in range(LANES)] + out.append(dict(src, name=f"derived_ldsm_{'mt88' if transpose else 'm88'}_x4_off40", + sm86=[f"{x:08x}" for x in sm86], regs_out=regs_out)) + return out + + +# ---- driver ------------------------------------------------------------------------------------------------------- + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("vectors", nargs="?", type=Path, default=DEFAULT_VECTORS, help=f"vector file (default {DEFAULT_VECTORS})") + ap.add_argument("--xlate", type=Path, default=emu.DEFAULT_XLATE, help="nx2navi_xlate executable") + ap.add_argument("--case", action="append", help="only cases whose name contains this (repeatable)") + ap.add_argument("-v", "--verbose", action="store_true", help="more differing values per failure") + args = ap.parse_args(argv) + if not args.vectors.exists(): + print(f"no vector file {args.vectors}") + return 1 + if not args.xlate.exists(): + print(f"no translator {args.xlate} (python tools/build_xlate.py)") + return 1 + cases = emu.load_cases(args.vectors) + failed = 0 + + models = model_problems(cases) + for name, bad in models.items(): + if bad: + failed += 1 + print(f"FAIL model vs GPU on {name}: " + "; ".join(bad[:4])) + model_ok = not any(models.values()) + print(f"reference models (HFMA2, LDSM) agree with the GPU on {sum(1 for b in models.values() if not b)}/{len(models)} cases") + + todo = cases + (derived_cases(cases) if model_ok else []) + passed = 0 + for case in todo: + name = case["name"] + if args.case and not any(c in name for c in args.case): + continue + try: + out = check_case(case, args.xlate, args.verbose) + except Exception as e: # one broken case does not stop the run + out = Outcome() + out.problems.append(f"error: {type(e).__name__}: {e}") + kind = emu.result_kind(case["sm86"]) + notes = [] + if kind != "bits": + notes.append("largest difference " + ("NaN against a number" if out.worst_ulp >= 1 << 32 else f"{out.worst_ulp} ulp")) + if out.nan_matches: + notes.append(f"{out.nan_matches} NaN values with another NaN encoding than the GPU's") + ok = not out.problems + passed += ok + failed += not ok + print(f"{'PASS' if ok else 'FAIL'} {name} ({out.runs} runs{', ' + ', '.join(notes) if notes else ''})") + for p in out.problems: + print(" " + p) + print(f"{passed} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/build_xlate.py b/tools/build_xlate.py index a6bcacf4..b1bbec6b 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 19: - sys.exit(f"expected 19 patches, found {len(patches)}") + if len(patches) != 20: + sys.exit(f"expected 20 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/xlate/AmpereLowering.cs b/tools/xlate/AmpereLowering.cs index 0a4be334..4e347b22 100644 --- a/tools/xlate/AmpereLowering.cs +++ b/tools/xlate/AmpereLowering.cs @@ -130,6 +130,7 @@ static class AmpereLowering int highest = HighestGpr(code); if (spillAt < 0 && highest >= UrLimit) throw new RegisterPressureException($"guest registers up to R{highest}"); UrBase = spillAt >= 0 ? SpillFrom : (highest + 2) & ~1; + LastUrBase = UrBase; } // Spill mode: the temporary that holds guest register r for this instruction, loading its group on first use. @@ -168,13 +169,14 @@ static class AmpereLowering if (b is >= 0x140 and <= 0x15f) continue; // branches, calls, exits, barriers: no register fields see(i.Bits(16, 8)); see(i.Bits(24, 8)); // Register ranges the base fields do not show: LDSM writes Rd..Rd+count-1; TXD reads Ra..Ra+2 and Rb..Rb+3 - // and writes the second destination pair; HMMA reads A, B and C fragments and writes the D pair. + // and writes the second destination pair; HMMA reads A (K/4 registers), B (K/8) and C and writes D, where + // C and D are a pair (.F16) or four registers (.F32, bit 76). if (op == 0x83b && i.Bits(16, 8) != RZ) see(i.Bits(16, 8) + (1u << (int)i.Bits(72, 2)) - 1); if (op == 0xb6c) { see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 2); see(i.Bits(32, 8) + 3); see(i.Bits(64, 8) + 1); } if (op == 0x23c) { - uint k16 = i.Bits(75, 1); - see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + 1); + uint k16 = i.Bits(75, 1), cd = i.Bit(76) ? 3u : 1u; + see(i.Bits(16, 8) + cd); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + cd); } if (b < 0x080 || b is >= 0x160 and <= 0x16f) see(i.Bits(64, 8)); // C of ALU forms, TEX second destinations if (i.Bits(9, 3) == 1) see(i.Bits(32, 8)); @@ -1507,10 +1509,13 @@ static class AmpereLowering } // HADD2 / HMUL2 / HFMA2 with packed-half results: saturate 77, FTZ 80, bit 76 the FMZ precision of HMUL2/HFMA2. - // HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered: the sum - // goes to TexOut, PRMT replicates each lane's sign bit over that lane (selector 0xbb99) and LOP.AND with the - // inverted mask clears the negative lanes (negative NaN becomes 0 too). F32 results (78), the H0_NH1 source (86) - // and absolute values where Maxwell has none are not lowered. HADD2's second operand is B, or C for forms 2/3. + // HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered. Measured + // on an NVIDIA GPU (tests/tensor_lowering_check.py): a negative result or -0 becomes +0, +inf stays, -inf + // becomes 0, a NaN result stays NaN (0x7fff). The sum goes to TexOut and integer operations build the mask of + // the half lanes to clear. Maxwell PRMT would be the obvious tool, but the translator emits nothing for it + // (Ryujinx 1.3.2, Instructions/InstEmit.cs, PrmtR only logs "not implemented"). + // F32 results (78), the H0_NH1 source (86) and absolute values where Maxwell has none are not lowered. HADD2's + // second operand is B, or C for forms 2/3. void Half(ulong maxwell) { uint op = _i.Bits(0, 12) & 0x1ff; @@ -1548,11 +1553,20 @@ static class AmpereLowering | ((ulong)c.reg << 39); } if (!relu) { EmitG(w); return; } - Emit(Unguarded(w)); - Mov32i(TexOut + 1, 0xbb99); // PRMT selector: bytes 0-1 the sign of byte 1, bytes 2-3 the sign of byte 3 - Emit(Unguarded(0x5bc0000000000000UL | ((ulong)RZ << 39) | ((ulong)(TexOut + 1) << 20) | ((ulong)TexOut << 8) | (uint)(TexOut + 2))); - // LOP.AND Rd, sum, ~mask (InvB bit 40), no predicate output (PT at 48) - EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)(TexOut + 2) << 20) | ((ulong)TexOut << 8) | (uint)Gpr((uint)Dst)); + // RELU on both half lanes of the sum x at once: each half h becomes |h| (its sign bit cleared), or 0 when h is + // a negative number (sign set, not NaN). |h| > 0x7c00 means NaN, so |h| + 0x3ff carries into bit 15 of its + // half exactly for a NaN, and never into the other half (|h| + 0x3ff < 0x10000). A NaN keeps the payload the + // host computed, with the sign cleared (the hardware's is 0x7fff). Scratch writes unguarded, Rd guarded. + const int x = TexOut, y = TexOut + 1, t = TexOut + 2; + Emit(Unguarded(w)); // HFMA2 x + Lop32i(y, x, 0x7fff7fff, 0); // y = |h| per half + Emit(Unguarded(0x1c00000000000000UL | (0x03ff03ffUL << 20) | ((ulong)y << 8) | t)); // IADD32I t = y + 0x03ff03ff + Emit(Unguarded(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)x << 8) | t)); // LOP.AND t = x & ~t + Emit(Unguarded(0x3828000000000000UL | (15UL << 20) | ((ulong)t << 8) | t)); // SHR t >>= 15 + Lop32i(t, t, 0x00010001, 0); // bit 0 / 16: clear that half + Emit(Unguarded(0x5c18000000000000UL | (16UL << 39) | (1UL << 48) | ((ulong)t << 20) | ((ulong)t << 8) | t)); // ISCADD t = (t << 16) - t + // LOP.AND Rd = y & ~t (InvB bit 40), no predicate output (PT at 48) + EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)y << 8) | (uint)Gpr((uint)Dst)); } // MOVM.16.MT88 Rd, Ra (0x23a): an 8x8 matrix of 16-bit elements spread over the warp (lane t holds row t/4, @@ -1973,33 +1987,39 @@ static class AmpereLowering } } - // HMMA.1688.F16 / HMMA.16816.F16 Rd, Ra, Rb, Rc: a warp-wide matrix multiply-accumulate D(16x8) = A(16xK) * - // B(Kx8) + C(16x8) on F16 elements, K = 8, or 16 when bit 75 is set. Fields: D 16-23 (a register pair), A 24-31 - // (K/4 registers), B 32-39 (K/8 registers), C 64-71 (a pair; RZ means a zero accumulator). With g = lane / 4 and - // q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every register): + // HMMA.1688 / HMMA.16816 Rd, Ra, Rb, Rc with F16 inputs: a warp-wide matrix multiply-accumulate D(16x8) = + // A(16xK) * B(Kx8) + C(16x8), K = 8, or 16 when bit 75 is set. Bit 76 selects F32 accumulators and results + // (.F32: C and D are four registers each) instead of F16 ones (.F16: a register pair). Fields: D 16-23, A 24-31 + // (K/4 registers), B 32-39 (K/8 registers), C 64-71 (RZ means a zero accumulator). With g = lane / 4 and + // q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every packed register): // A register 2j + r: row g + 8r, k 8j + 2q and 8j + 2q + 1 (j is 0 only when K = 8) // B register j: k 8j + 2q and 8j + 2q + 1, column g - // C / D register r: row g + 8r, columns 2q and 2q + 1 + // C / D, .F16: register r holds row g + 8r, columns 2q and 2q + 1 + // C / D, .F32: register 2r + c holds row g + 8r, column 2q + c // Maxwell has no tensor instructions, so the product is expanded per lane. For each k pair (2p, 2p + 1) the lane // fetches the A word of its own rows from lane 4g + p % 4 and, for each of its two columns n = 2q + c, the B word - // from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA (an - // F16 x F16 product is exact in F32). The accumulators are rounded to F16 after every eight k (one 1688 step of - // the hardware) and widened again when more follow; the final F16 values are packed as F2fp in this file does - // (F2F.F16 writes the low half and zeroes the high half, ISCADD merges the column pair). - // The work runs unguarded because every lane must supply its fragments to the shuffles; only the two result - // writes take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest - // operands are never there (guest and uniform registers end below R237, spill temporaries are R204-R235). + // from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA in + // F32 (an F16 x F16 product is exact in F32, so the step rounds once whether or not the host fuses it). + // The .F16 form rounds to F16 once, after all K products, as the hardware does: it does not round between the + // two eight-k halves of HMMA.16816. Measured on an NVIDIA GPU (tests/tensor_lowering_check.py compares this + // lowering with those results): a first-half sum of 65536 that the second half cancels gives 0, which an F16 + // rounding after the first half turns into +inf, and 1 + 2^-11 + 2^-11 split over the halves gives 1 + 2^-10, + // which that rounding turns into 1. The F16 results are then packed as F2fp in this file does (F2F.F16 writes + // the low half and zeroes the high half, ISCADD merges the column pair). + // The work runs unguarded because every lane must supply its fragments to the shuffles; only the result writes + // take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest operands + // are never there (guest and uniform registers end below R237, spill temporaries are R188-R235). void Hmma() { - bool k16 = _i.Bit(75); + bool k16 = _i.Bit(75), f32 = _i.Bit(76); uint d = (uint)Dst, a = _i.Bits(24, 8), b = _i.Bits(32, 8), c = _i.Bits(64, 8); - int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1; + int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1, cdRegs = f32 ? 4 : 2; // A fragment base aligned to its size and the whole fragment below RZ (RZ itself reads as zeros). bool fits(uint reg, int n) => reg == RZ || (reg % (uint)n == 0 && reg + (uint)n <= RZ); - // Word 1 above B, word 2 other than C and the shape bit, and word 3 bits 96-104 / 126-127 select modes - // (F32 accumulators and others) that are not lowered. - if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x8ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0 - || !fits(d, 2) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, 2)) + // Word 1 above B, word 2 other than C, the shape (75) and the accumulator type (76), and word 3 bits 96-104 / + // 126-127 select modes (bit 74 another shape, HMMA.1684; other input types) that are not lowered. + if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x18ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0 + || !fits(d, cdRegs) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, cdRegs)) throw Fail("HMMA mode"); if (d == RZ) return; // no result and no other effect @@ -2027,13 +2047,14 @@ static class AmpereLowering return lane; } - // Accumulators from C (both halves of each row register), or zero. - for (int r = 0; r < 2; r++) - for (int col = 0; col < 2; col++) - { - if (c == RZ) Mov32i(acc + 2 * r + col, 0); - else widen(acc + 2 * r + col, part(c, r), col == 1); - } + // Accumulators from C, or zero: accumulator 2r + col is register 2r + col of an .F32 C, half col of register + // r of an .F16 C. + for (int k = 0; k < 4; k++) + { + if (c == RZ) Mov32i(acc + k, 0); + else if (f32) Mov(acc + k, part(c, k), false); + else widen(acc + k, part(c, k >> 1), (k & 1) == 1); + } // Lane bases: 8q first (it needs the raw lane id), then 4g. Put(0xf0c8000000000000UL | laneA, false); // S2R SR_LANEID @@ -2065,15 +2086,17 @@ static class AmpereLowering Emit(Unguarded(0x5980000000000000UL | ((ulong)t << 39) | ((ulong)(bw + h) << 20) | ((ulong)(aw + 2 * r + h) << 8) | (uint)t)); } } - if ((p & 3) == 3) // end of an eight-k step: round to F16, widen again if another step follows - for (int t = acc; t < acc + 4; t++) - { - narrow(t); - if (p + 1 < pairs) widen(t, t, false); - } } - // D register r = column 2q + 1 in the high half, column 2q in the low half (ISCADD hi << 16 + lo). + // .F32: D register 2r + col is accumulator 2r + col. + if (f32) + { + for (int k = 0; k < 4; k++) Mov(part(d, k), acc + k); + return; + } + // .F16: one rounding of each sum to F16, then D register r = column 2q + 1 in the high half, column 2q in the + // low half (ISCADD hi << 16 + lo). + for (int t = acc; t < acc + 4; t++) narrow(t); for (int r = 0; r < 2; r++) EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)(acc + 2 * r) << 20) | ((ulong)(acc + 2 * r + 1) << 8) | (uint)part(d, r)); } @@ -2328,6 +2351,10 @@ static class AmpereLowering // c1: the program's c1 constants (BRX jump tables); set before Lower by the caller. [ThreadStatic] public static byte[] C1Bytes; + // Diagnostics only (Program.cs LowerFile, the --lower mode): the register UR0 was mapped to by the last Lowerer + // built on this thread (Lowerer.UrBase; SpillFrom in spill mode). Lowering never reads it. + [ThreadStatic] public static int LastUrBase; + // localBytes: the local memory size the lowered program needs when that changed (spill mode), else 0. Graphics // programs carry it in their header already; compute and task/mesh programs need it from the translator's host // (job "local_mem"). diff --git a/tools/xlate/Program.cs b/tools/xlate/Program.cs index 3777a6a0..c375589c 100644 --- a/tools/xlate/Program.cs +++ b/tools/xlate/Program.cs @@ -6,6 +6,7 @@ // nx2navi_xlate --serve one request per stdin line: {"job": {...}, "out": ""}; one reply line per request: // "ok " or "fail "; a malformed line gets "fail ? " and the loop continues. // nx2navi_xlate batch mode: a JSON array of jobs, prints a summary, exit 1 if any failed. +// nx2navi_xlate --lower lowers SM86 instructions given as hex words and prints the Maxwell words (LowerFile). // job: { "key", "stage": "vs|ps|cs", "code": path (graphics = 0x50-byte SPH + instructions, compute = instructions), "c1": hex, // "sm86": bool + "size": bytes (an NVN2 program, lowered by AmpereLowering.cs), "nvn2_table": [start, textures, // images] (NVN2 compute binding table), "lowered_out": path, "mesh": {"stage": 6|7, "threads", "maxv", "maxp", @@ -231,8 +232,56 @@ static class SwshXlate catch (Exception e) { if (Environment.GetEnvironmentVariable("DMSH_STACK") != null) Console.Error.WriteLine(e.StackTrace); return $"{e.GetType().Name}: {e.Message.Split('\n')[0]}"; } } + // --lower : runs AmpereLowering on a few SM86 instructions, for executing single instructions on the CPU + // (tests/maxwell_warp_emu.py). Each line of the file is one instruction as four hex 32-bit words "w0 w1 w2 w3" + // (w0 = bits 0-31, optional 0x prefix; blank lines and '#' comments skipped). They are lowered as a compute program + // (AmpereLowering.Lower with graphics false), with an SM86 EXIT appended unless the last instruction is an + // unpredicated EXIT. Prints the lowered Maxwell instruction words in program order, one 16-digit hex word per line: + // the scheduling words Lower puts first in every 32 bytes are left out, the closing EXIT Lower appends is kept. Then + // "urbase N" (the register UR0 maps to, AmpereLowering.LastUrBase) and, in spill mode only, "local N" (the local + // memory bytes the program needs). Errors go to stderr with exit code 1. + static int LowerFile(string path) + { + try + { + var bytes = new List(); + uint[] last = null; + foreach (string raw in File.ReadAllLines(path)) + { + string line = raw.Split('#')[0].Trim(); + if (line.Length == 0) continue; + string[] f = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + if (f.Length != 4) throw new FormatException($"expected four hex words: \"{line}\""); + last = f.Select(s => Convert.ToUInt32(s.StartsWith("0x", StringComparison.OrdinalIgnoreCase) ? s[2..] : s, 16)).ToArray(); + foreach (uint w in last) bytes.AddRange(BitConverter.GetBytes(w)); + } + // SM86 EXIT: opcode 0x94d under PT (bits 12-15 = 7), second condition PT (bits 87-90 = 7); AmpereLowering Exit. + bool exits = last != null && (last[0] & 0xffff) == 0x794d && ((last[2] >> 23) & 15) == 7 && ((last[2] >> 20) & 3) == 0; + if (!exits) foreach (uint w in new uint[] { 0x0000794d, 0, 0x03800000, 0x000fea00 }) bytes.AddRange(BitConverter.GetBytes(w)); + byte[] code = bytes.ToArray(); + byte[] lowered = AmpereLowering.Lower(code, code.Length, false, out int local); + // Compute layout (AmpereLowering.Lower): a scheduling word at every 32-byte boundary, instructions in the other + // slots, zero padding after the closing EXIT. Instruction words are never 0, so the last nonzero one ends it. + var words = new List(); + int end = 0; + for (int a = 0; a + 8 <= lowered.Length; a += 8) + { + if ((a & 0x1f) == 0) continue; + ulong w = BitConverter.ToUInt64(lowered, a); + words.Add(w); + if (w != 0) end = words.Count; + } + foreach (ulong w in words.Take(end)) Console.WriteLine(w.ToString("x16")); + Console.WriteLine($"urbase {AmpereLowering.LastUrBase}"); + if (local != 0) Console.WriteLine($"local {local}"); + return 0; + } + catch (Exception e) { Console.Error.WriteLine($"{e.GetType().Name}: {e.Message}"); return 1; } + } + static int Main(string[] args) { + if (args.Length == 2 && args[0] == "--lower") return LowerFile(args[1]); if (args.Length == 1 && args[0] == "--serve") { // One request per line: {"job": {...}, "out": ""}; one reply line "ok " / "fail ". diff --git a/tools/xlate/patches/0021-prmt-byte-permute.patch b/tools/xlate/patches/0021-prmt-byte-permute.patch new file mode 100644 index 00000000..97b35aab --- /dev/null +++ b/tools/xlate/patches/0021-prmt-byte-permute.patch @@ -0,0 +1,161 @@ +diff --git a/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs b/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs +index d77621d..462c0ca 100644 +--- a/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs ++++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs +@@ -215,34 +215,6 @@ namespace Ryujinx.Graphics.Shader.Instructions + context.TranslatorContext.GpuAccessor.Log("Shader instruction Pret is not implemented."); + } + +- public static void PrmtR(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtR is not implemented."); +- } +- +- public static void PrmtI(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtI is not implemented."); +- } +- +- public static void PrmtC(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtC is not implemented."); +- } +- +- public static void PrmtRc(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtRc is not implemented."); +- } +- + public static void R2b(EmitterContext context) + { + context.GetOp(); +diff --git a/src/Ryujinx.Graphics.Shader/Instructions/InstEmitPermute.cs b/src/Ryujinx.Graphics.Shader/Instructions/InstEmitPermute.cs +new file mode 100644 +index 0000000..b79ad3e +--- /dev/null ++++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmitPermute.cs +@@ -0,0 +1,116 @@ ++using Ryujinx.Graphics.Shader.Decoders; ++using Ryujinx.Graphics.Shader.IntermediateRepresentation; ++using Ryujinx.Graphics.Shader.Translation; ++ ++using static Ryujinx.Graphics.Shader.Instructions.InstEmitHelper; ++using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper; ++ ++namespace Ryujinx.Graphics.Shader.Instructions ++{ ++ // NX2NAVI: PRMT (byte permute). Upstream Ryujinx only logged it as not implemented, so the destination kept its old ++ // value; the SM86 lowering (tools/xlate/AmpereLowering.cs Prmt, the HFMA2.RELU sign mask in Half) emits it for every ++ // byte permute of the Switch 2 shaders (82 pixel, 303 vertex and 277 compute programs of Ocarina), which lost them. ++ // Rd byte i is byte k of the 8 bytes {C, A} (A bytes 0-3, C bytes 4-7). In the default mode k is nibble i of B and ++ // its bit 3 replaces the byte with its sign (0x00 or 0xff); the other modes take k from B bits 0-1 alone, as the PTX ++ // prmt modes f4e, b4e, rc8, ecl, ecr and rc16 define. ++ static partial class InstEmit ++ { ++ public static void PrmtR(EmitterContext context) ++ { ++ InstPrmtR op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = GetSrcReg(context, op.SrcB); ++ Operand srcC = GetSrcReg(context, op.SrcC); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ public static void PrmtI(EmitterContext context) ++ { ++ InstPrmtI op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = Const(op.Imm20); ++ Operand srcC = GetSrcReg(context, op.SrcC); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ public static void PrmtC(EmitterContext context) ++ { ++ InstPrmtC op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = GetSrcCbuf(context, op.CbufSlot, op.CbufOffset); ++ Operand srcC = GetSrcReg(context, op.SrcC); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ public static void PrmtRc(EmitterContext context) ++ { ++ InstPrmtRc op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = GetSrcReg(context, op.SrcC); ++ Operand srcC = GetSrcCbuf(context, op.CbufSlot, op.CbufOffset); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ private static Operand EmitPrmt(EmitterContext context, Operand srcA, Operand srcB, Operand srcC, PMode mode) ++ { ++ Operand result = Const(0); ++ Operand low = context.BitwiseAnd(srcB, Const(3)); // the start byte of the special modes ++ ++ for (int i = 0; i < 4; i++) ++ { ++ Operand index; // 0-7: the source byte of result byte i ++ Operand sign = null; // non-zero: replicate that byte's sign bit (default mode only) ++ ++ switch (mode) ++ { ++ case PMode.F4e: ++ index = context.BitwiseAnd(context.IAdd(low, Const(i)), Const(7)); ++ break; ++ case PMode.B4e: ++ index = context.BitwiseAnd(context.IAdd(low, Const(8 - i)), Const(7)); ++ break; ++ case PMode.Rc8: ++ index = low; ++ break; ++ case PMode.Ecl: ++ index = context.IMaximumU32(low, Const(i)); ++ break; ++ case PMode.Ecr: ++ index = context.IMinimumU32(low, Const(i)); ++ break; ++ case PMode.Rc16: ++ index = context.BitwiseOr(context.ShiftLeft(context.BitwiseAnd(low, Const(1)), Const(1)), Const(i & 1)); ++ break; ++ default: ++ Operand nibble = context.BitfieldExtractU32(srcB, Const(i * 4), Const(4)); ++ index = context.BitwiseAnd(nibble, Const(7)); ++ sign = context.BitwiseAnd(nibble, Const(8)); ++ break; ++ } ++ ++ Operand word = context.ConditionalSelect(context.ICompareLessUnsigned(index, Const(4)), srcA, srcC); ++ Operand shift = context.ShiftLeft(context.BitwiseAnd(index, Const(3)), Const(3)); ++ Operand value = context.BitfieldExtractU32(word, shift, Const(8)); ++ ++ if (sign != null) ++ { ++ Operand negative = context.ICompareNotEqual(context.BitwiseAnd(value, Const(0x80)), Const(0)); ++ Operand replicated = context.ConditionalSelect(negative, Const(0xff), Const(0)); ++ value = context.ConditionalSelect(context.ICompareNotEqual(sign, Const(0)), replicated, value); ++ } ++ ++ result = context.BitwiseOr(result, context.ShiftLeft(value, Const(i * 8))); ++ } ++ ++ return result; ++ } ++ } ++}