diff --git a/docs/building.md b/docs/building.md index 7eead6c4..704eb064 100644 --- a/docs/building.md +++ b/docs/building.md @@ -257,6 +257,11 @@ Shaders: `.venv-py314\Scripts\python tools\shader_census.py --version --tr to `build\\shaders\` and runs it through the translator and `spirv-val`; failure groups go to `failures.txt`. `tools\xlate_smoke.py` checks the translator alone, without game files. +A change to the SM86 tensor lowering (`tools/xlate/AmpereLowering.cs` Hmma, Ldsm, Movm, Half) passes +`python tests\tensor_lowering_check.py`: it lowers GPU-captured vectors with `nx2navi_xlate --lower` and runs them on +`tests\maxwell_warp_emu.py`, a model of what the Ryujinx translator makes of Maxwell code (the docstring names the +vector file and what is checked). `tests\maxwell_warp_emu_selftest.py` checks the emulator itself. No game files. + ## Reverse engineering (optional) - Ghidra 12.1.2 with Adubbz's Ghidra-Switch-Loader in `toolchain\ghidra\` (JDK 21) opens the game modules (NSO). diff --git a/docs/maxwell.md b/docs/maxwell.md index bcc2dfd1..7f3648be 100644 --- a/docs/maxwell.md +++ b/docs/maxwell.md @@ -88,7 +88,7 @@ Source shorthands: | 3.9 | **Integer vertex inputs read as raw bits** (bone indices `R8UI`). | Float-declared inputs convert integers. | NXN Mario: stretched characters; re-translated with `uint` inputs. Ours: attribute types passed to `QueryGraphicsState` (`Program.cs`). | NXN | | 3.10 | **BFE / BFI fields past bit 31** (as PTX `bfe` / `bfi`): a field that runs past bit 31 is cut there; one that starts past it extracts 0 (signed: bit 31 repeated) and inserts nothing. NX2navi 1.4.3's Depths terrain vertex shaders (42 variants) take the position from a register, `((2y + x) * 8) & 0xff` with 5-bit x and y, so up to 248. | `OpBitField*Extract` / `OpBitFieldInsert` undefined when offset or offset + count exceeds 32; AMD's bitfield instructions take offset and width modulo 32. | Ryujinx passes the fields through. UltraTears saw the Depths walls and ground missing on the Steam Deck and a Linux RX 570 (RADV). Ours: clamped where the field is not a constant inside the word (translator patch 0012). RX 9070 on Windows drew the Depths the same before and after. | PTX ISA `bfe`, `bfi` | | 3.11 | **SHF by 0** gives the unshifted word (C for `SHF.L`, A for `SHF.R`). | Ryujinx's funnel shifts the other word by 32: undefined in SPIR-V; AMD shifts take the low 5 bits and give A \| C. | the hardware fixed Depths terrain gaps in its funnel shift (PR 10459). Ours: a select on a zero amount (translator patch 0013). In 1.4.3 every register SHF pairs its word with 0, so the result was right already. | — | -| 3.12 | **Registers read before any write** on a path (paths the structurizer adds included) have no defined value in the game's code; UltraTears reports 0 from the NVIDIA driver. | An `OpVariable` without an initializer loads an undefined value; Mesa folds undefined values freely. | Ours: on Mesa drivers (RADV, ANV, NVK; `vk_caps_driver`) every function-local variable starts at 0 (`OpConstantNull`, job flag `zero_locals`, translator patch 0016; `NX2NAVI_ZERO_LOCALS` overrides). Elsewhere the load stays undefined: a zero start on every driver (the old patch 0014) made the whole picture shake by about 0.9 output pixels a frame with DLSS or DLAA on an RTX 4080 SUPER (#612). It also moved the SPIR-V hash of the game's FXAA shader out of `FXAA_FS` (`vk_dlaa.c` `vk_dlaa_skip_draw`), the likely cause of the shake; that path is not traced. | — | +| 3.12 | **Registers read before any write** on a path (paths the structurizer adds included) have no defined value in the game's code; UltraTears reports 0 from the NVIDIA driver. | An `OpVariable` without an initializer loads an undefined value; Mesa folds undefined values freely. | Ours: every function-local variable starts at 0 (`OpConstantNull`, job flag `zero_locals`, translator patch 0016; `NX2NAVI_ZERO_LOCALS` overrides) on Mesa drivers (RADV, ANV, NVK; `vk_caps_driver`) and, with DLAA off, on every other driver too: on NVIDIA an undefined read takes another warp lane's value, and Ocarina's clustered-light loop lit its floors in 4x8-pixel tiles (one warp each) and darkened Link (runtime/vk_caps.c). With DLAA on, off Mesa, the load stays undefined: a zero start on every driver (the old patch 0014) made the whole picture shake by about 0.9 output pixels a frame with DLSS or DLAA on an RTX 4080 SUPER (#612). It also moved the SPIR-V hash of the game's FXAA shader out of `FXAA_FS` (`vk_dlaa.c` `vk_dlaa_skip_draw`), the likely cause of the shake; that path is not traced. | — | ## 4. Memory model diff --git a/runtime/cfg.c b/runtime/cfg.c index 5603cae7..b22e027b 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -40,7 +40,7 @@ static const char *const known[] = { "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", - "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", + "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "DUMP_WHITE", "DUMP_AFTER", "SUBMIT_FINISH", "ACQUIRE_WAIT", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", diff --git a/runtime/flight.h b/runtime/flight.h index f2f2d8ae..9ed9d476 100644 --- a/runtime/flight.h +++ b/runtime/flight.h @@ -6,7 +6,8 @@ #define FLIGHT_THUMB_W 320 #define FLIGHT_THUMB_H 180 -extern int flight_on; // NX2NAVI_FLIGHT=1, set once by flight_init before the first present +extern int flight_on; +extern int vk_dump_white; // vk_dump.c, NX2NAVI_DUMP_WHITE // NX2NAVI_FLIGHT=1, set once by flight_init before the first present void flight_init(void); // prof_start calls it // perf_present, once per present on the render thread: the interval since the previous present, the render thread's diff --git a/runtime/nvn_api.h b/runtime/nvn_api.h index 903cd3e0..06182937 100644 --- a/runtime/nvn_api.h +++ b/runtime/nvn_api.h @@ -45,6 +45,7 @@ const pool_t *nvn_pool_find(uint64_t addr, uint64_t len); // GPU-visible pool h uint64_t nvn_new_serial(void); // next tex_t / prog_t serial, from 1 uint64_t nvn_fence_last(void); // fence value of the newest submit (0 before the first) int nvn_depth_minus_w(void); // 1 after DeviceSetDepthMode(NEAR_IS_MINUS_W), else 0 +extern int nvn_submits_in_frame; // nvnQueueSubmitCommands calls since the last present (NX2NAVI_SUBMIT_FINISH) void nvn_guest_written(uint64_t addr, uint64_t size); // the host or the game wrote pool storage: back end re-reads // NX2NAVI_TRACE=perf: MemoryPoolFlushMappedRange [0] and BufferFlushMappedRange [1] calls [0] and bytes [1] by the pool's // CPU class (0 CPU_CACHED, 1 CPU_UNCACHED, 2 other) since the previous call. diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index f83dcec5..d7fdb2ad 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -622,6 +622,7 @@ NVN_API(nvnQueueFinish) { // lod_entered / lod_issued count submits that entered QueueSubmitCommands / reached the back end; lod_fence holds // the fence of the last LOD_RING. NX2NAVI_LOD_GUARD=0 turns the wait off. #define LOD_RING 64 +int nvn_submits_in_frame; // nvnQueueSubmitCommands calls since the last nvnQueuePresentTexture (NX2NAVI_SUBMIT_FINISH) static uint64_t lod_entered, lod_issued, lod_fence[LOD_RING]; // void nvnQueueSubmitCommands(NVNqueue *q, int n, const NVNcommandHandle *handles): replayed on this thread with its @@ -655,6 +656,23 @@ NVN_API(nvnQueueSubmitCommands) { while (f > prev && !__atomic_compare_exchange_n(&last_fence, &prev, f, 1, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} __atomic_store_n(&lod_fence[no % LOD_RING], f, __ATOMIC_RELAXED); __atomic_store_n(&lod_issued, no, __ATOMIC_SEQ_CST); + // The first submission of a frame (since the last present) returns once the GPU has run it. NVN starts the GPU when + // the submit returns and it is through a frame's first submission (Ocarina's frame-start compute: light grid, probes) + // within a few milliseconds; here the replay records first and the GPU runs later. Ocarina rewrites data those + // passes and the frame's lit materials read once it has submitted its later work, with no fence: with the GPU late, + // part of a frame drew with the next frame's data, distant characters wrote NaN and bloom spread it (white dialogue + // frames, 3% of them in 100 s of dialogue; none with this wait, at the same 58-59 fps: the GPU is idle at that + // point). Copies of the data at submit made it worse, a mix of two frames' versions. + // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits + // after every submission, 0 never. + static long long finish_mask = -2; + if (finish_mask == -2) { + const char *e = cfg_get("NX2NAVI_SUBMIT_FINISH"); + finish_mask = e && *e ? strtoll(e, NULL, 0) : 1; + if (e && *e && finish_mask == 1 && !(e[0] == '0' && (e[1] == 'x' || e[1] == 'X'))) finish_mask = -1; // "1": all + } + int idx = nvn_submits_in_frame++; + if (finish_mask && f && (finish_mask == -1 || (idx < 63 && (finish_mask >> idx & 1)))) gfx_backend()->fence_wait(f, UINT64_MAX); } // The producer (1.2.1 main f_70008c6460) releases the read lock on obj+0x538 at lr 0x70008c70c8; when diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index 5410b440..4152288d 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -756,6 +756,19 @@ static void sleep_until(uint64_t deadline) { } } +// The submit fence of each window image's newest present (win_t has no room: a side table for the first windows). +static struct { const win_t *w; uint64_t fence[4]; } img_fence[4]; +static uint64_t last_present_fence; // the submit fence at the newest present of any window + +static uint64_t *img_fence_of(const win_t *o, int idx) { + if (idx < 0 || idx >= 4) return NULL; + for (int i = 0; i < 4; i++) { + if (img_fence[i].w == o) return &img_fence[i].fence[idx]; + if (!img_fence[i].w) return img_fence[i].w = o, &img_fence[i].fence[idx]; + } + return NULL; +} + // NVNwindowAcquireTextureResult nvnWindowAcquireTexture(NVNwindow *w, NVNsync *s, int *index): blocks here, not in the // present, until the image's previous present plus one period has passed (the game often presents with a lock held). // That is the time the next frame goes to the display, so the image is free as soon as that time is reached. @@ -772,6 +785,24 @@ NVN_API(nvnWindowAcquireTexture) { sleep_until(at); perf_end(PERF_PACE, p0); } + // On the Switch the image comes back only after the display has shown it, which needs the GPU to have finished the + // frame drawn into it, so the CPU never runs more than the swapchain's depth ahead of the GPU. Here the pacer frees + // images by the clock alone, so the acquire also waits for the GPU to finish the image's newest present. With the + // GPU slowed (frame dump logging), this alone cut Ocarina's NaN frames from 56 to 23 of 61; the wait after a frame's + // first submission (nvn_api_core.c nvnQueueSubmitCommands) removes the rest. NX2NAVI_ACQUIRE_WAIT=0 leaves it to the + // clock; =2 waits for the newest present of any image (one frame in flight at most). + static int acq_wait = -1; + if (acq_wait < 0) acq_wait = rt_knob("ACQUIRE_WAIT", "1"); + uint64_t *fe = img_fence_of(o, idx), want = fe ? *fe : 0; + if (acq_wait == 2) want = last_present_fence; + if (acq_wait && want) { + const GfxBackend *b = gfx_backend(); + if (b->fence_done() < want) { + uint64_t p0 = perf_begin(); + b->fence_wait(want, UINT64_MAX); + perf_end(PERF_PACE, p0); + } + } // The sync means "the display no longer reads this image". Presents are copies queued in submit order, so it is // already satisfied: fenced = 3 with fence 0. if (s) { @@ -794,12 +825,16 @@ NVN_API(nvnQueuePresentTexture) { if (o->magic != NVN_WIN_MAGIC || idx < 0 || idx >= o->ntex || idx >= 4 || !o->tex[idx]) rt_fatal("nvn: QueuePresentTexture of image %d on window %p", idx, (void *)o); o->presents++; + nvn_submits_in_frame = 0; const tex_t *t = (const tex_t *)o->tex[idx]; uint64_t period = present_period(o), now = kern_now_ns(); uint64_t slot = now + pace(&o->pace, now, period); gfx_backend()->present(t, o->crop, slot); o->ticket[idx] = o->presents; o->avail[idx] = slot + period; + uint64_t *fe = img_fence_of(o, idx); + if (fe) *fe = nvn_fence_last(); // the frame drawn into it (nvnWindowAcquireTexture waits for it) + last_present_fence = nvn_fence_last(); frame_step_present(slot, o->interval); // frame_rate.c: the real-time step follows the display times rt_frame_presented(t, o->presents); } diff --git a/runtime/vk.h b/runtime/vk.h index a870a732..6914bab9 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -425,6 +425,11 @@ VkImage vk_tonemap_record(VkCommandBuffer cb, VkImage src, VkFormat sf, uint32_t // its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it // guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0. uint64_t vk_texture_slot_token(int id); +// Slots whose token may have changed since *pos (0 at first; then the value these calls leave): fn is called for each +// (some more than once) and *pos advanced. 0 when too many changes came since: the caller looks at every slot. *pos +// from vk_texture_dirty_pos_locked after such a walk. Texture lock held. +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg); +uint64_t vk_texture_dirty_pos_locked(void); void vk_sampler_dlss_bias(float bias); // texture LOD bias added for DLSS upscaling (material samplers) uint32_t vk_sampler_gen(void); // changes whenever a sampler slot is registered void vk_texture_gpu_written(const tex_t *t); // rendered or copied into by the GPU: no re-upload from storage diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 2402c47b..247894fb 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -58,7 +58,7 @@ static void present_main(void *arg) { busy = 1; plat_cond_broadcast(&pcv); plat_mutex_unlock(&pl); - if (flight_on) vk_thumbs_frame(j.frame); + if (flight_on || vk_dump_white) vk_thumbs_frame(j.frame); // NX2NAVI_DUMP_WHITE reads them too vk_present_blit(j.image, VK_IMAGE_LAYOUT_GENERAL, j.format, j.w, j.h, j.has_crop ? j.crop : NULL); uint64_t f = vk_sync_submit(NULL, 0); // a timeline value after the blit: deferred destruction waits for the blit too if (j.value && f != j.value) diff --git a/runtime/vk_caps.c b/runtime/vk_caps.c index 0b85e18b..89b148a7 100644 --- a/runtime/vk_caps.c +++ b/runtime/vk_caps.c @@ -116,9 +116,14 @@ void vk_caps_init(VkCtx *c) { // BotW and NX2navi on AMD). The translator biases gather coordinates by the host's sub-texel precision; NVIDIA rounds // as Maxwell does. NX2NAVI_GATHER_BIAS= forces it (0: off). uint32_t gather = env_u32("NX2NAVI_GATHER_BIAS", c->props.vendorID == 0x10de ? 0 : l->subTexelPrecisionBits); - // A register read before any write is an undefined load in SPIR-V, and Mesa folds undefined values freely, so on - // Mesa function-local variables start at 0 (docs/maxwell.md 3.12). NX2NAVI_ZERO_LOCALS=1 forces it, 0 turns it off. - uint32_t zero_locals = env_u32("NX2NAVI_ZERO_LOCALS", (uint32_t)vk_caps.mesa); + // A register read before any write is an undefined load in SPIR-V, so function-local variables start at 0 + // (docs/maxwell.md 3.12). Mesa folds undefined values freely; NVIDIA's compiler gives such a read a value of another + // lane of the warp: Ocarina's clustered-light loop reads registers that only some paths write, and on an RTX 5090 + // the Great Fairy Fountain floor took an extra light term in 4x8-pixel tiles (one 32-lane warp each) and Link went + // dark with green speckles. With DLAA on (vk_dlaa.c, which finds the game's FXAA by its SPIR-V hash) they stay + // undefined off Mesa, as before: zeroing moved that hash and the picture shook under DLAA (maxwell.md 3.12). + // NX2NAVI_ZERO_LOCALS=1 / 0 forces either. + uint32_t zero_locals = env_u32("NX2NAVI_ZERO_LOCALS", vk_caps.mesa || !cfg_bool("NX2NAVI_DLAA")); gfx_shader_set_host(vk_caps.float64, vk_caps.sbuf_align, told, gather, zero_locals != 0); if (gather) rt_log("vk caps: gather coordinates biased for %u sub-texel bits (NX2NAVI_GATHER_BIAS)", gather); diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index 19c85e13..316fbc8c 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -5,7 +5,8 @@ // written in the frame are copied once each, slices side by side. Files in NX2NAVI_DUMP_DIR (default "dump"): // pass__t_x__.png colour target i (td: depth; img: storage image; lut: 3D; // a BC target drawn through a block view: its blocks as that view) -// ..._tm.png float formats: x/(1+x) of the same image (the plain one clamps) +// ..._tm.png float formats: x/(1+x) of the same image (the plain one clamps); +// in both, a pixel with a NaN channel is magenta // passes.txt per pass or dispatch: targets, draws, each distinct pipeline + texture set with its shaders' SPIR-V // hashes (sh_.spv written once) and the bound textures with their samplers // images.txt per image: channel min/max, distinct texel values over the image and over its top quarter @@ -35,6 +36,7 @@ #include "vk.h" #include "vk_exec.h" +#include "flight.h" uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r); // vk_exec_draw.c int vk_exec_raw_9e5(VkImage img, VkFormat f); // vk_exec_draw.c #include "platform/platform.h" @@ -49,6 +51,10 @@ int vk_exec_raw_9e5(VkImage img, VkFormat f); // v int vk_dump_armed; static uint64_t dump_frame, log_from; // log_from: NX2NAVI_DUMP_LOG, first frame logged to passes.txt without images static char dir[400]; +static char base_dir[400]; // NX2NAVI_DUMP_DIR; with NX2NAVI_DUMP_WHITE each attempt goes to base_dir/f +static uint64_t white_last; // NX2NAVI_DUMP_WHITE: the frame dumped last (0: none), checked 3 presents on +static int white_tries, white_caught; +static uint64_t dump_after; // NX2NAVI_DUMP_AFTER static FILE *passes_f, *images_f; static int seq, finished, copied; // copied: passes and dispatches of the dump frame whose images were copied static int skip_runs, dump_ubo; // NX2NAVI_DUMP_SKIP_RUNS, NX2NAVI_DUMP_UBO (vec4 per block, 0 for none) @@ -221,8 +227,8 @@ static void write_images(const Job *j) { uint32_t pw = j->w * j->d, ph = j->h; uint64_t n = (uint64_t)pw * ph; float *px = malloc(n * 16); - uint8_t *out = malloc(n * 4); - if (!px || !out) rt_fatal("vk_dump: out of memory for %s", j->name); + uint8_t *out = malloc(n * 4), *isnan_ = calloc(n, 1); // isnan_: a pixel with a NaN channel, drawn magenta + if (!px || !out || !isnan_) rt_fatal("vk_dump: out of memory for %s", j->name); float mn[4] = {INFINITY, INFINITY, INFINITY, INFINITY}, mx[4] = {-INFINITY, -INFINITY, -INFINITY, -INFINITY}; uint64_t nans = 0; for (uint32_t z = 0; z < j->d; z++) @@ -231,7 +237,7 @@ static void write_images(const Job *j) { float *o = px + 4 * ((uint64_t)y * pw + (uint64_t)z * j->w + x); decode(j->f, j->map + (((uint64_t)z * j->h + y) * j->w + x) * j->f->bytes, o); for (int c = 0; c < 4; c++) { - if (isnan(o[c])) { nans++, o[c] = 0; continue; } + if (isnan(o[c])) { nans++, o[c] = 0, isnan_[(uint64_t)y * pw + (uint64_t)z * j->w + x] = 1; continue; } if (o[c] < mn[c]) mn[c] = o[c]; if (o[c] > mx[c]) mx[c] = o[c]; } @@ -255,6 +261,7 @@ static void write_images(const Job *j) { out[4 * i + c] = u8(x); } out[4 * i + 3] = 255; + if (isnan_[i]) out[4 * i] = 255, out[4 * i + 1] = 0, out[4 * i + 2] = 255; } char path[600]; snprintf(path, sizeof path, "%s/%s%s.png", dir, j->name, pass ? "_tm" : ""); @@ -270,6 +277,7 @@ static void write_images(const Job *j) { } plat_mutex_unlock(&img_lock); free(out); + free(isnan_); free(px); } @@ -527,9 +535,12 @@ static void flush_luts(VkExec *ex, int n) { VkFormat bf = block_format(l->format); snprintf(name, sizeof name, "pass_%04d_%s%d_%ux%ux%u_%d_%s%s", n, l->is2d ? "tex" : "lut", l->id, l->w, l->h, l->d, (int)l->format, fmt_name(l->format, fn), bf ? "_blocks" : ""); + // a sampled depth texture copies its depth aspect (the colour aspect of a depth image read as zeros) + VkImageAspectFlags asp = l->format >= VK_FORMAT_D16_UNORM && l->format <= VK_FORMAT_D32_SFLOAT_S8_UINT + ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT; const char *why = bf ? copy_region(ex, l->image, bf, VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 0, (VkExtent3D){l->w, l->h, l->d}, (l->w + 3) / 4, (l->h + 3) / 4, l->d, name) - : copy_out(ex, l->image, l->format, VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, l->w, l->h, l->d, name); + : copy_out(ex, l->image, l->format, asp, 0, 0, l->w, l->h, l->d, name); fprintf(passes_f, " %s tex %d -> %s.png%s%s\n", l->is2d ? "2D" : "3D", l->id, name, why ? " skipped: " : "", why ? why : ""); } @@ -572,8 +583,12 @@ static void describe_bindings(Str *b, const VkExec *ex, const VkPipeInfo *info) } if (st == 5) sp(b, " -> %#llx", (unsigned long long)vk_exec_cs_storage_addr(ex, r)); for (int k = 0; k < 16; k++) - if (ex->ssbo[st][k].addr) - sp(b, " %d=%#llx+%#llx", k, (unsigned long long)ex->ssbo[st][k].addr, (unsigned long long)ex->ssbo[st][k].size); + if (ex->ssbo[st][k].addr) { + VkDescriptorBufferInfo cb; // "(copy)": a binding that only reads it uses the submit-time copy (vk_usnap.c) + int copy = vk_usnap_find(ex->ssbo[st][k].addr, ex->ssbo[st][k].size, 1, &cb); + sp(b, " %d=%#llx+%#llx%s", k, (unsigned long long)ex->ssbo[st][k].addr, (unsigned long long)ex->ssbo[st][k].size, + copy ? "(copy)" : ""); + } sp(b, "\n"); // NX2NAVI_DUMP_UBO: also the small storage buffers' guest bytes (GPU results as of the last write-back); per-view // constants (exposure and such) live in such buffers. @@ -624,10 +639,11 @@ static void describe_bindings(Str *b, const VkExec *ex, const VkPipeInfo *info) const VkTexImage *t = vk_texture_by_id(NVN_HANDLE_TEX(h), &v, &hv); if (!t) { sp(b, "tex %d unresolved (dummy)\n", NVN_HANDLE_TEX(h)); continue; } static const char *vt[] = {"1D", "2D", "3D", "cube", "1DArray", "2DArray", "cubeArray"}; - sp(b, "tex %d %s(%d) %ux%ux%u levels %u layers %u image %s", NVN_HANDLE_TEX(h), fmt_name(t->format, fn), + sp(b, "tex %d %s(%d) %ux%ux%u levels %u layers %u image %s vkimage %p", NVN_HANDLE_TEX(h), fmt_name(t->format, fn), (int)t->format, t->width, t->height, t->depth, t->levels, t->layers, - t->view_type <= 6 ? vt[t->view_type] : "?"); + t->view_type <= 6 ? vt[t->view_type] : "?", (void *)t->image); if (hv && v.set_format) sp(b, " view-nvn-fmt %#x", v.format); + if (hv && v.set_target) sp(b, " view-target %u", v.target); if (hv && v.set_levels) sp(b, " view-levels %u+%u", v.base_level, v.nlevels); if (hv && v.set_layers) sp(b, " view-layers %u+%u", v.min_layer, v.nlayers); samp_par s; @@ -730,9 +746,9 @@ static void target_line(VkExec *ex, const VkExecTarget *t, const char *tag, int t->height, 1, name) : t->is3d ? "3D slice (the volume is dumped as lut where it is sampled)" : copy_out(ex, t->image, t->format, t->aspect, t->level, t->base_layer, t->width, t->height, 1, name); - fprintf(passes_f, " %s: nvn %#x %s(%d) %ux%u level %u layers %u+%u samples %u addr %#llx -> %s.png%s%s\n", tag, - t->nvn_format, fmt_name(t->format, fn), (int)t->format, t->width, t->height, t->level, t->base_layer, - t->layers, t->samples, (unsigned long long)(uintptr_t)t->tex->mem, name, why ? " skipped: " : "", + fprintf(passes_f, " %s: nvn %#x %s(%d) %ux%u level %u layers %u+%u samples %u addr %#llx vkimage %p -> %s.png%s%s\n", + tag, t->nvn_format, fmt_name(t->format, fn), (int)t->format, t->width, t->height, t->level, t->base_layer, + t->layers, t->samples, (unsigned long long)(uintptr_t)t->tex->mem, (void *)t->image, name, why ? " skipped: " : "", why ? why : ""); } @@ -755,6 +771,11 @@ static void pass_decide(int same_targets) { void vk_dump_pass_end(VkExec *ex) { plat_mutex_lock(&rec_lock); + if (dump_after && dump_frame == UINT64_MAX && ex->depth.tex && (uint64_t)(uintptr_t)ex->depth.tex->mem == dump_after) { + dump_frame = frame_now(); // this pass and the rest of its frame + rt_log("vk_dump: a pass drew into %#llx in frame %llu: it and the rest of the frame go to %s", + (unsigned long long)dump_after, (unsigned long long)dump_frame, dir); + } if (!active()) goto out; int n = seq++; uint64_t sig = targets_sig(ex); @@ -893,9 +914,34 @@ out: static void make_dirs(char *path) { plat_file_mkdir_p(path); } +// NX2NAVI_DUMP_WHITE=1 (diagnosis): no frame is named; from frame 1800 each present looks at the thumbnail of the frame +// three presents back (gfx_present.c vk_thumbs_*), and the first one whose picture left of the overlay and above the +// dialogue box is blank white makes the dump take the next frame (white flashes come in runs of several frames). +int vk_dump_white; // vk_backend.c asks for a thumbnail of every present while it is set +#define dump_white vk_dump_white + +// passes.txt and images.txt in dir (a new folder per NX2NAVI_DUMP_WHITE attempt) +static void open_files(void) { + make_dirs(dir); + char path[600]; + snprintf(path, sizeof path, "%s/passes.txt", dir); + passes_f = fopen(path, "w"); + snprintf(path, sizeof path, "%s/images.txt", dir); + images_f = fopen(path, "w"); + if (!passes_f || !images_f) rt_fatal("vk_dump: cannot write to %s", dir); +} + void vk_dump_init(void) { const char *e = cfg_get("NX2NAVI_DUMP"); - if (!e || !*e || !(dump_frame = strtoull(e, NULL, 10))) return; + dump_white = rt_knob("DUMP_WHITE", "0"); + // NX2NAVI_DUMP_AFTER=
: the first render pass whose depth target lives at that address and the rest of its frame + // (a target drawn once, at an area load) + const char *da = cfg_get("NX2NAVI_DUMP_AFTER"); + dump_after = da && *da ? strtoull(da, NULL, 0) : 0; + if (dump_white || dump_after) { + dump_frame = UINT64_MAX; + vk_thumbs_enable(16); + } else if (!e || !*e || !(dump_frame = strtoull(e, NULL, 10))) return; const char *lf = cfg_get("NX2NAVI_DUMP_LOG"); if (lf && *lf) log_from = strtoull(lf, NULL, 10); const char *sr = cfg_get("NX2NAVI_DUMP_SKIP_RUNS"); @@ -912,14 +958,10 @@ void vk_dump_init(void) { w[1] = *q == 'x' ? (uint32_t)strtoul(q + 1, &q, 10) : 0; } const char *d = cfg_get("NX2NAVI_DUMP_DIR"); - snprintf(dir, sizeof dir, "%s", d && *d ? d : "dump"); - make_dirs(dir); + snprintf(base_dir, sizeof base_dir, "%s", d && *d ? d : "dump"); + snprintf(dir, sizeof dir, "%s", base_dir); char path[600]; - snprintf(path, sizeof path, "%s/passes.txt", dir); - passes_f = fopen(path, "w"); - snprintf(path, sizeof path, "%s/images.txt", dir); - images_f = fopen(path, "w"); - if (!passes_f || !images_f) rt_fatal("vk_dump: cannot write to %s", dir); + if (!dump_white) open_files(); const char *bf = cfg_get("NX2NAVI_DUMP_BUF"); if (bf && *bf) { char *q; @@ -937,7 +979,43 @@ void vk_dump_init(void) { rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); } +// The thumbnail of frame f is blank white left of the overlay and above the dialogue box: -1 not available, 0 no, 1 yes. +static int thumb_white(uint64_t f) { + static uint8_t px[FLIGHT_THUMB_W * FLIGHT_THUMB_H * 4]; + if (!vk_thumbs_read(f, px)) return -1; + int n = 0, bright = 0; + for (int y = 0; y < 105; y++) + for (int x = 0; x < 250; x++, n++) { + const uint8_t *q = px + 4 * (y * FLIGHT_THUMB_W + x); + bright += (q[0] * 30 + q[1] * 59 + q[2] * 11) / 100 > 245; + } + return bright * 100 > n * 97; +} + +static void white_check(uint64_t frame) { + static int run; // consecutive white thumbnails: white runs are mostly 1-2 frames, too short for a dump 4 frames on + if (frame < 1800) return; + int w = thumb_white(frame - 3); + if (w < 0) return; + run = w ? run + 1 : 0; + if (run >= 3) { // frames frame-5..frame-3 white: the run is likely still on 4 frames later + run = 0; + dump_frame = frame + 1; + snprintf(dir, sizeof dir, "%s/f%llu", base_dir, (unsigned long long)dump_frame); + open_files(); + rt_log("vk_dump: frames %llu-%llu were white; frame %llu goes to %s", (unsigned long long)(frame - 5), + (unsigned long long)(frame - 3), (unsigned long long)dump_frame, dir); + } +} + void vk_dump_present(uint64_t frame) { + if (dump_white && white_last && frame == white_last + 3) { // was the dumped frame itself white? + int w = thumb_white(white_last); + rt_log("vk_dump: dumped frame %llu was %s", (unsigned long long)white_last, w > 0 ? "white: done" : "not white: trying again"); + if (w > 0 || ++white_tries >= 8) white_caught = 1; + white_last = 0; + } + if (dump_white && !white_caught && !white_last && dump_frame == UINT64_MAX) white_check(frame); if (frame != dump_frame) return; plat_mutex_lock(&rec_lock); finished = 1; @@ -952,4 +1030,13 @@ void vk_dump_present(uint64_t frame) { passes_f = NULL; rt_log("vk_dump: frame %llu: %d passes and dispatches, about %llu MB of PNGs in %s", (unsigned long long)frame, seq, (unsigned long long)(png_bytes >> 20), dir); + if (dump_white) { // ready for another attempt (vk_dump_present checks this one's thumbnail 3 presents on) + white_last = frame; + plat_mutex_lock(&rec_lock); + seq = copied = 0, last_targets = 0, pass_skip = NULL, png_bytes = 0; + nluts = nluts_done = nents = ents_over = pass_draws = 0; + dump_frame = UINT64_MAX; + finished = 0; + plat_mutex_unlock(&rec_lock); + } } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index e2af956b..ecaaf947 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -63,6 +63,7 @@ void vk_exec_frame_written(const tex_t *t) { } void vk_exec_frame_log(const tex_t *p, uint64_t frame) { + vk_exec_cond_frame_end(frame); // a frame boundary in replay order (present_image runs on the replay thread) if (!(rt_trace & RT_TRACE_GFX)) return; VkExecFrame *f = &vk_exec_frame; int hit = 0; @@ -743,6 +744,13 @@ static void clear_attachment(VkExec *ex, const VkClearAttachment *a) { static void clear_cond_log(const VkExec *ex, const char *what, int index) { static int on = -1, n; if (on < 0) on = rt_knob("LOG_RCOND", "0"); + if (on == 2 && index < 0 && ex->has_render_cond && ex->depth.width >= 1600) { // full-size depth clears, all of them + char why[256]; + vk_exec_cond_why(&ex->render_cond, why, sizeof why); + rt_log("SEQ depth clear %#llx %s; %s", ex->depth.img ? (unsigned long long)ex->depth.img->addr : 0ull, + ex->render_disabled ? "skipped" : "RUN", why); + return; + } if (!on || !ex->has_render_cond || n >= 400) return; n++; const VkExecTarget *t = index < 0 ? &ex->depth : index < (int)ex->pass_ncolor ? &ex->color[index] : NULL; @@ -754,6 +762,7 @@ static void clear_cond_log(const VkExec *ex, const char *what, int index) { static void clear_color(VkExec *ex, const p_clear_color *c) { vk_exec_frame.clears++; clear_cond_log(ex, "colour", c->index); + if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : vk_cond_clears_run++; if (ex->render_disabled) return; resolve_rt(ex); int i = c->index; @@ -785,6 +794,7 @@ static void clear_color(VkExec *ex, const p_clear_color *c) { static void clear_ds(VkExec *ex, const p_clear_ds *c) { clear_cond_log(ex, "depth", -1); + if (ex->has_render_cond) ex->render_disabled ? vk_cond_clears_skip++ : (vk_cond_clears_run++, vk_cond_depth_run++); if (ex->render_disabled) return; resolve_rt(ex); const VkExecTarget *t = &ex->depth; diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index ad437edd..e694760c 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -38,6 +38,10 @@ typedef struct { uint8_t scale; // VkTexImage.scale } VkExecTarget; +void vk_exec_cond_frame_end(uint64_t frame); // vk_exec_xfer.c: a present, in replay order (NX2NAVI_LOG_RCOND) +extern unsigned vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run; +void vk_exec_cond_why(const p_render_cond *c, char *buf, size_t cap); // NX2NAVI_LOG_RCOND=ab + // The replay state. One instance lives for the run: NVN state is queue state and carries over from one submission to // the next, as in MacNativeGames' and LAS Recomp's replays. The "per command buffer" fields are reset by vk_exec_record. typedef struct VkExec { diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index a8f3f2fa..a8d0893c 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -1526,13 +1526,33 @@ static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int sto // rewrites the entries whose slot changed (vk_texture_slot_token) and fills the kinds a new pipeline asked for; with // update-after-bind (descriptorIndexing) a write never invalidates a command buffer being recorded. Each submit's first // sync also looks every registered slot up once, the freshness check a bound texture gets (vk_texture_by_id_locked). -// The entries are not double-buffered: a slot registered again while a frame using it is in flight is read new by that -// frame, as on the Switch, where the pool is memory the GPU reads as it goes. +// The set is one of a ring of POOL_SETS, one per submission: on the Switch the GPU reads the pool as it goes, but it +// runs a submission right after nvnQueueSubmitCommands, so what it reads is the pool as of about the submit. Here the +// GPU runs a submission several milliseconds later, and Ocarina registers slots for its next submission right after +// submitting: with one set, an in-flight submission sampled a texture registered for the next one, the lit materials +// of distant characters read the wrong texture as their light or probe data, wrote NaN and bloom spread it (white +// dialogue frames, a fifth of them; none once each submission waited for its GPU work, NX2NAVI_SUBMIT_FINISH). A +// submission's first pool_sync takes the next set (pool_take_set), waits for the GPU only if that set's last +// submission is still running, and brings it up to date from the change log (every entry written since the set was +// last current; the whole set when the log has wrapped). Writes within a submission go to its own set at once, as +// before. volatile uint32_t vk_pool_kinds_used; +#define POOL_SETS 4 +#define POOL_LOG (1u << 20) // entries of the change log: binding * VK_POOL_TEX + id, one per write static struct { VkDescriptorSetLayout layout, empty; VkDescriptorPool dpool; - VkDescriptorSet set; + VkDescriptorSet set; // the current submission's (sets[cur]) + VkDescriptorSet sets[POOL_SETS]; + uint64_t set_fence[POOL_SETS]; // the timeline value of the last submission that bound it + uint64_t set_synced[POOL_SETS]; // the change-log position up to which it holds every write + int nsets, cur, all; // all: the first fill, written to every set + uint64_t cur_fence; // the submission sets[cur] serves + uint32_t *log; // POOL_LOG entries + uint64_t logn; // writes so far + uint64_t *last; // per entry: the log position of its newest write (catch-up writes it once, there) + uint64_t switches, waits, catchup, written, rewrites, catch_ns, sync_ns, n_slots, n_refill, n_fresh; + uint64_t dirty_pos; // vk_texture_dirty_slots_locked position (0: the next sync walks every slot) int ready, uab; uint32_t kinds, gen, epoch, sgen; int retry; // the last sync left slots to the creation budget @@ -1567,19 +1587,33 @@ static VkDescriptorSetLayout pool_layout_make(void) { if (vkCreateDescriptorSetLayout(c->device, &li, NULL, &l) != VK_SUCCESS) rt_fatal("vk: the texture pool layout failed"); VkDescriptorPoolSize ps[2] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX}, {VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}}; - VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0, - .maxSets = 1, .poolSizeCount = 2, .pPoolSizes = ps}; - if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) rt_fatal("vk: the texture pool descriptor pool failed"); - VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = PL.dpool, - .descriptorSetCount = 1, .pSetLayouts = &l}; - if (vkAllocateDescriptorSets(c->device, &ai, &PL.set) != VK_SUCCESS) rt_fatal("vk: the texture pool set failed"); + // POOL_SETS sets, fewer where the device has no room for them (one at the least, as before) + for (int n = POOL_SETS; n >= 1 && !PL.nsets; n--) { + VkDescriptorPoolSize psn[2] = {{ps[0].type, ps[0].descriptorCount * (uint32_t)n}, + {ps[1].type, ps[1].descriptorCount * (uint32_t)n}}; + VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, + .flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0, + .maxSets = (uint32_t)n, .poolSizeCount = 2, .pPoolSizes = psn}; + if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) continue; + VkDescriptorSetLayout ls[POOL_SETS]; + for (int i = 0; i < n; i++) ls[i] = l; + VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = PL.dpool, + .descriptorSetCount = (uint32_t)n, .pSetLayouts = ls}; + if (vkAllocateDescriptorSets(c->device, &ai, PL.sets) == VK_SUCCESS) PL.nsets = n; + else vkDestroyDescriptorPool(c->device, PL.dpool, NULL), PL.dpool = VK_NULL_HANDLE; + } + if (!PL.nsets) rt_fatal("vk: the texture pool set failed"); + PL.set = PL.sets[0]; + PL.log = malloc(sizeof *PL.log * POOL_LOG); + PL.last = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP, sizeof *PL.last); + if (!PL.log || !PL.last) rt_fatal("vk: out of memory"); PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok); PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view); PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp); if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory"); - rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s", VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, - PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)"); + rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s, %d sets (one per submission in turn)", + VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, + PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)", PL.nsets); return l; } @@ -1620,16 +1654,83 @@ static void pool_flush(void) { pool_nw = 0; } -static void pool_write(uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { +static void pool_write_to(VkDescriptorSet set, uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { if (pool_nw == POOL_WBATCH) pool_flush(); pool_ii[pool_nw] = (VkDescriptorImageInfo){samp, view, VK_IMAGE_LAYOUT_GENERAL}; - pool_w[pool_nw] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = PL.set, + pool_w[pool_nw] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = set, .dstBinding = binding, .dstArrayElement = id, .descriptorCount = 1, .descriptorType = view ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, .pImageInfo = &pool_ii[pool_nw]}; pool_nw++; } +// An entry whose wanted value PL.view / PL.samp already hold: into the current set, and into the log for the others +// (the first fill goes into every set). +static void pool_write(uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { + if (PL.all) { + for (int i = 0; i < PL.nsets; i++) pool_write_to(PL.sets[i], binding, id, view, samp); + return; + } + pool_write_to(PL.set, binding, id, view, samp); + uint32_t e = binding * VK_POOL_TEX + id; + PL.last[e] = PL.logn; + PL.log[PL.logn++ % POOL_LOG] = e; +} + +// Writes entry e (binding * VK_POOL_TEX + id) of set as PL.view / PL.samp hold it now. +static void pool_entry_to(VkDescriptorSet set, uint32_t e) { + uint32_t binding = e / VK_POOL_TEX, id = e % VK_POOL_TEX; + if (binding < VK_POOL_KINDS) { + VkImageView v = PL.view[(size_t)binding * VK_POOL_TEX + id]; + if (v) pool_write_to(set, binding, id, v, VK_NULL_HANDLE); + } else if (id < VK_POOL_SAMP && PL.samp[id]) { + pool_write_to(set, binding, id, VK_NULL_HANDLE, PL.samp[id]); + } +} + +// A submission's first pool use: the next set of the ring, once the GPU is done with it. 1 when it switched; *from is +// the log position the set holds writes up to (pool_catch_up brings it level once pool_sync has refreshed the entries: +// a view written into a set must be alive, and the refresh is what replaces the views of destroyed textures). +static int pool_take_set(VkExec *ex, uint64_t *from) { + if (PL.cur_fence == ex->fence) return 0; + int i = (PL.cur + 1) % PL.nsets; + if (PL.nsets > 1) { + const GfxBackend *b = gfx_backend(); + if (b->fence_done() < PL.set_fence[i]) PL.waits++, b->fence_wait(PL.set_fence[i], UINT64_MAX); + } + *from = PL.set_synced[i]; + PL.cur = i, PL.set = PL.sets[i], PL.cur_fence = ex->fence, PL.set_fence[i] = ex->fence; + return PL.nsets > 1; +} + +// The current set gets the entries written into other sets in [from, to) of the log, as PL.view / PL.samp hold them now. +static void pool_catch_up(uint64_t from, uint64_t to) { + VkDescriptorSet set = PL.set; + if (to - from > POOL_LOG) { // the log has wrapped since: every entry + PL.rewrites++; + for (uint32_t e = 0; e < VK_POOL_KINDS * VK_POOL_TEX + VK_POOL_SAMP; e++) pool_entry_to(set, e); + } else { + // each entry once, at its newest write: an entry written again later (also by this submission's own sync, + // which wrote it into this set already) is skipped at the older positions + PL.catchup += to - from; + for (uint64_t q = from; q < to; q++) { + uint32_t e = PL.log[q % POOL_LOG]; + if (PL.last[e] == q) pool_entry_to(set, e), PL.written++; + } + } + pool_flush(); + PL.switches++; + if (PL.switches % 3600 == 0) + rt_log("vk: texture pool sets: %llu switches, %llu waited for the GPU, %.1f log entries and %.1f writes per " + "catch-up, %llu whole rewrites; %.3f ms catch-up and %.3f ms entry refresh per switch (%.0f slots, %.0f " + "refilled, %.0f checked)", + (unsigned long long)PL.switches, (unsigned long long)PL.waits, (double)PL.catchup / (double)PL.switches, + (double)PL.written / (double)PL.switches, (unsigned long long)PL.rewrites, + PL.catch_ns / 1e6 / (double)PL.switches, PL.sync_ns / 1e6 / (double)PL.switches, + (double)PL.n_slots / (double)PL.switches, (double)PL.n_refill / (double)PL.switches, + (double)PL.n_fresh / (double)PL.switches); +} + // Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the // slot's texture was left to a later sync by the creation budget (pool_sync). // NX2NAVI_SKIP_SHADERS=,...: draws and dispatches whose shader's SPIR-V has one of these FNV-1a hashes (the "vs"/"fs"/ @@ -1684,8 +1785,38 @@ static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { return !vk_texture_create_deferred_locked(); } +static void pool_sync_entries(VkExec *ex); + static void pool_sync(VkExec *ex) { if (!PL.layout) vk_pool_set_layout(); + uint64_t from = 0, pre = PL.logn; + int switched = pool_take_set(ex, &from); // every submission its own set, before the shortcut + uint64_t t0 = kern_now_ns(); + pool_sync_entries(ex); + uint64_t t1 = kern_now_ns(); + PL.sync_ns += t1 - t0; + if (switched) { + vk_texture_lock(); + pool_catch_up(from, pre); // what the other sets got since; this sync's own writes went into it already + vk_texture_unlock(); + PL.catch_ns += kern_now_ns() - t1; + } + PL.set_synced[PL.cur] = PL.logn; +} + +typedef struct { VkExec *ex; uint32_t used; int n, retry; } PoolDirty; + +static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_locked: a slot whose token may have changed + PoolDirty *d = arg; + if (id >= d->n) return; + uint64_t t = vk_texture_slot_token(id); + if (t == PL.tok[id]) return; + PL.n_refill++; + if (pool_fill(d->ex, (uint32_t)id, d->used, t)) PL.tok[id] = t; + else d->retry = 1; +} + +static void pool_sync_entries(VkExec *ex) { uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED); uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); if (PL.ready && !PL.retry && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return; @@ -1694,24 +1825,37 @@ static void pool_sync(VkExec *ex) { int n = vk_texture_slot_count_locked(); if (n > VK_POOL_TEX) n = VK_POOL_TEX; if (!PL.ready) { // every entry a stand-in first: partially bound arrays are still read out of range by guests + PL.all = 1; for (uint32_t id = 0; id < VK_POOL_TEX; id++) pool_fill(ex, id, 0xffffu, 0); for (uint32_t id = 0; id < VK_POOL_SAMP; id++) PL.samp[id] = Z.samp, pool_write(VK_POOL_KINDS, id, VK_NULL_HANDLE, Z.samp); + pool_flush(); + PL.all = 0; PL.ready = 1, PL.sgen = 0; } int retry = 0; // slots whose texture the creation budget left: filled again by the next sync if (fresh || gen != PL.gen || ep != PL.epoch || PL.retry) { vk_texture_create_budget_locked(POOL_CREATE_BUDGET); - for (int id = 0; id < n; id++) { // slots past n were never registered: stand-ins since the first sync + // Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that + // changed (vk_texture_dirty_slots_locked), not the whole table + PoolDirty pd = {ex, used, n, 0}; + int walked = !fresh && ep == PL.epoch && !PL.retry && PL.dirty_pos && + vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, &pd); + retry = pd.retry; + if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(); + for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync uint64_t t = vk_texture_slot_token(id); if (t != PL.tok[id]) { + PL.n_refill++; if (pool_fill(ex, (uint32_t)id, used, t)) PL.tok[id] = t; else retry = 1; } else if (t && fresh) { pool_fill(ex, (uint32_t)id, fresh, t); } else if (t && ep != PL.epoch) { + PL.n_fresh++; vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check } } + if (!walked) PL.n_slots += (uint64_t)n; vk_texture_create_budget_locked(-1); } PL.retry = retry; @@ -2620,7 +2764,7 @@ static int draw_skipped(const VkExec *ex, int mesh, int why) { // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { - if (ex->render_disabled) return draw_skipped(ex, mesh, SK_DISABLED); + if (ex->render_disabled) return vk_cond_draws_skip += ex->has_render_cond, draw_skipped(ex, mesh, SK_DISABLED); cb_res(ex); dummies_init(); if (ex->pass_noatt) { // no attachment: the pass covers this draw's viewport or scissor, reopened when it needs more diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index c218288f..e6dce958 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -124,9 +124,29 @@ static void counter_cb(void *arg) { // report counts as pending until then, not until its fence passes: the callback runs after the fence, and in between // the record still held what the CPU wrote (Ocarina re-arms its condition records with 1 for the next frame), so a // condition read 1 where the GPU had written 0, a depth clear ran after the depth copy and frames showed only sky. -typedef struct { uint64_t addr, seq; volatile uint64_t written; int unknown, type; } Pending; +// frame: the present count (vk_exec_cond_frame_end, in replay order) when the report was replayed. The game arms its +// condition records from the CPU between frames, so within a frame a timestamp report decides the record's value even +// after it reached memory: Ocarina's CPU, running a frame ahead, had re-armed a record with 1 by the time a later depth +// clear of the frame read it, the clear ran after the depth copy and the frame showed only sky (white flashes). +typedef struct { uint64_t addr, seq, frame; volatile uint64_t written; int unknown, type; } Pending; static Pending pending[PENDING]; static uint64_t report_seq; +static uint64_t cond_frame; + +// Per-frame counts of conditional work (NX2NAVI_LOG_RCOND=1, vk_exec_cond_frame_end logs them): clears and +// draws issued under a render condition, run or skipped. +unsigned vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run; + +void vk_exec_cond_frame_end(uint64_t frame) { + static int on = -1; + if (on < 0) on = rt_knob("LOG_RCOND", "0"); + if (on && (vk_cond_clears_run || vk_cond_clears_skip || vk_cond_draws_skip)) + rt_log("vk: frame %llu conditions: clears run %u skipped %u, draws skipped %u, depth clears run %u", + (unsigned long long)frame, vk_cond_clears_run, vk_cond_clears_skip, vk_cond_draws_skip, vk_cond_depth_run); + vk_cond_clears_run = vk_cond_clears_skip = vk_cond_draws_skip = vk_cond_depth_run = 0; + if (on == 2) rt_log("SEQ present %llu", (unsigned long long)frame); + cond_frame++; +} static Pending *pending_slot(uint64_t addr, int insert); @@ -149,11 +169,12 @@ static Pending *pending_slot(uint64_t addr, int insert) { if (a == addr) return e; if (!insert) continue; if (!a) { if (!free_) free_ = e; continue; } - if (__atomic_load_n(&e->written, __ATOMIC_ACQUIRE) >= e->seq && (!victim || e->seq < victim->seq)) victim = e; + if (__atomic_load_n(&e->written, __ATOMIC_ACQUIRE) >= e->seq && e->frame != cond_frame && (!victim || e->seq < victim->seq)) + victim = e; // done, and not from this frame (whose reports still decide their records) } Pending *e = free_ ? free_ : victim; if (!e) return NULL; // every probed entry still has a report in flight - e->seq = 0, e->unknown = 0, e->type = -1; + e->seq = 0, e->frame = 0, e->unknown = 0, e->type = -1; __atomic_store_n(&e->written, 0, __ATOMIC_RELEASE); __atomic_store_n(&e->addr, addr, __ATOMIC_RELEASE); return e; @@ -342,7 +363,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { vk_completion_add(ex->fence, counter_cb, a); } Pending *e = pending_slot(p->addr, 1); - if (e) e->seq = report_seq, e->unknown = samples, e->type = (int)p->type; + if (e) e->seq = report_seq, e->frame = cond_frame, e->unknown = samples, e->type = (int)p->type; else LOG_ONCE("vk: a counter report's table neighbourhood is all in flight: conditions on %#llx read memory", (unsigned long long)p->addr); } @@ -352,6 +373,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { // 1: *v read, 0: not readable, 2: a samples-passed report not in memory yet (unknown) static int counter_word(uint64_t addr, uint64_t *v) { const Pending *e = pending_slot(addr, 0); // the last report to addr decides + if (e && !e->unknown && e->frame == cond_frame) return *v = 0, 1; // a timestamp/statistics report of this frame if (e && __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq) return *v = 0, e->unknown ? 2 : 1; VkMemSpan m; if (nvn_gpu_only(addr) || !vk_mem_resolve(addr, sizeof *v, &m)) return 0; @@ -366,6 +388,21 @@ static int last_type(uint64_t addr) { return e ? e->type : -1; } +// NX2NAVI_LOG_RCOND=ab (diagnosis): the words a condition compares and where each came from: "f" a report replayed +// this frame (0), "p" a pending report (0 or unknown), "m" memory, with the newest report's type and frame. +void vk_exec_cond_why(const p_render_cond *c, char *buf, size_t cap) { + int o = snprintf(buf, cap, "%#llx mode %d:", (unsigned long long)c->addr, c->mode); + for (int k = 0; k < 2 && o < (int)cap; k++) { + uint64_t a = c->addr + 16 * (uint64_t)k; + const Pending *e = pending_slot(a, 0); + const char *src = !e ? "m" : !e->unknown && e->frame == cond_frame ? "f" + : __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq ? "p" : "m"; + o += snprintf(buf + o, cap - (size_t)o, " %llu %s(type %d, frame %lld of %llu)", + (unsigned long long)*(const uint64_t *)(uintptr_t)a, src, e ? e->type : -1, + e ? (long long)e->frame : -1LL, (unsigned long long)cond_frame); + } +} + static int cond_enabled(const p_render_cond *c) { uint64_t x, y; int rx = counter_word(c->addr, &x), ry = counter_word(c->addr + 16, &y); @@ -546,6 +583,10 @@ static void copy_t2t(VkExec *ex, const p_copy_t2t *p) { } ex->recorded++; vk_texture_gpu_written(p->dst); + static int lg = -1; + if (lg < 0) lg = rt_knob("LOG_RCOND", "0"); + if (lg == 2 && (d.img->aspect & VK_IMAGE_ASPECT_DEPTH_BIT) && d.w >= 1600) + rt_log("SEQ depth copy %#llx -> %#llx", (unsigned long long)s.img->addr, (unsigned long long)d.img->addr); } // ---- buffer to buffer diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 4dc2d9cd..119960c6 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -126,11 +126,59 @@ static uint32_t reg_serial; static uint32_t agen_idx(uint64_t addr) { return (uint32_t)(addr * 0x9E3779B97F4A7C15ull >> 52); } +// Slots that may resolve differently since a position, for the texture pool descriptors (vk_exec_draw.c +// pool_sync_entries), which otherwise looked at every registered slot after each change: Ocarina registers slots all +// through a submission, and the refresh walked the whole table about twenty times per submission (3.5 ms). A ring of +// changes, each a slot id (registered) or an address bucket (an entry at that agen index changed: every slot registered +// over it, kept in per-bucket lists). Lock held for all of it. +#define DIRTY_N (1u << 16) +#define DIRTY_BUCKET 0x80000000u +static uint32_t dirty[DIRTY_N]; +static uint64_t ndirty; +static int32_t bucket_head[AGEN_N]; +static int32_t *slot_next, *slot_prev; // per slot id: its bucket list links (slot_prev -2: in no list) +static uint16_t *slot_bucket; + +static void dirty_push(uint32_t x) { dirty[ndirty++ % DIRTY_N] = x; } + static void tex_gen_bump(uint64_t addr) { if (!++tex_gen) tex_gen = 1; slots_gen++; - uint32_t *g = &agen[agen_idx(addr)]; + uint32_t b = agen_idx(addr); + uint32_t *g = &agen[b]; __atomic_store_n(g, *g + 1, __ATOMIC_RELEASE); + dirty_push(DIRTY_BUCKET | b); +} + +// Moves slot id into bucket b's list (the lists exist once a slot was registered: vk_texture_register). +static void slot_rebucket(int id, uint32_t b) { + if (slot_prev[id] != -2) { // out of its old list + if (slot_prev[id] >= 0) slot_next[slot_prev[id]] = slot_next[id]; + else bucket_head[slot_bucket[id]] = slot_next[id]; + if (slot_next[id] >= 0) slot_prev[slot_next[id]] = slot_prev[id]; + } + slot_bucket[id] = (uint16_t)b; + slot_prev[id] = -1; + slot_next[id] = bucket_head[b]; + if (bucket_head[b] >= 0) slot_prev[bucket_head[b]] = id; + bucket_head[b] = id; +} + +uint64_t vk_texture_dirty_pos_locked(void) { return ndirty; } + +int vk_texture_dirty_slots_locked(uint64_t *pos, void (*fn)(int id, void *arg), void *arg) { + if (!slot_next || ndirty - *pos > DIRTY_N) { + *pos = ndirty; + return 0; + } + for (uint64_t q = *pos; q < ndirty; q++) { + uint32_t x = dirty[q % DIRTY_N]; + if (x & DIRTY_BUCKET) + for (int32_t id = bucket_head[x & ~DIRTY_BUCKET]; id >= 0; id = slot_next[id]) fn(id, arg); + else fn((int)x, arg); + } + *pos = ndirty; + return 1; } static volatile int32_t tex_ready; @@ -1109,7 +1157,11 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { } if (!sinfo) { uint64_t *p = calloc(1 << 20, sizeof *p); - if (!p) rt_fatal("vk: out of memory for texture slot tokens"); + slot_next = malloc(sizeof *slot_next << 20), slot_prev = malloc(sizeof *slot_prev << 20); + slot_bucket = calloc(1 << 20, sizeof *slot_bucket); + if (!p || !slot_next || !slot_prev || !slot_bucket) rt_fatal("vk: out of memory for texture slot tokens"); + for (int i = 0; i < (1 << 20); i++) slot_prev[i] = -2, slot_next[i] = -1; + for (int i = 0; i < AGEN_N; i++) bucket_head[i] = -1; __atomic_store_n(&sinfo, p, __ATOMIC_RELEASE); } TexSlot *s = &tslots[id]; @@ -1119,6 +1171,8 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { slots_gen++; if (!++reg_serial) reg_serial = 1; __atomic_store_n(&sinfo[id], (uint64_t)reg_serial << 16 | agen_idx((uintptr_t)t->mem), __ATOMIC_RELEASE); + slot_rebucket(id, agen_idx((uintptr_t)t->mem)); + dirty_push((uint32_t)id); s->has_view = v != NULL; if (v) s->v = *v; else memset(&s->v, 0, sizeof s->v); @@ -1149,6 +1203,11 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { e = NULL; for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) { if (p->released) continue; + // Only an entry of the slot's target: Ocarina's volumetric fog registers a 3D two-channel volume (32x24x64) + // over the storage of the bloom pass's 480x270 2D target; followed, the fog computes and the fog pass got + // that 2D image, no 3D view can be made of it (vk_exec_draw.c get_view) and they read and wrote stand-ins. + // With the target kept they get the volume itself. + if (p->key.d.target != k.d.target) continue; if (!any) any = p; if (same_but_swizzle(&p->key, &k)) { e = p; break; } } diff --git a/tests/maxwell_warp_emu.py b/tests/maxwell_warp_emu.py new file mode 100644 index 00000000..52fb98e2 --- /dev/null +++ b/tests/maxwell_warp_emu.py @@ -0,0 +1,1306 @@ +"""A 32-lane Maxwell warp emulator for checking the SM86 tensor lowering on the CPU. + +tools/xlate/AmpereLowering.cs rewrites SM86 instructions as Maxwell (64-bit) instructions, and the patched Ryujinx shader +translator (build/xlate/ryujinx/src/Ryujinx.Graphics.Shader) turns those into SPIR-V. This file executes the Maxwell +words the way that translator does: fields are decoded exactly as Ryujinx's Decoders/InstDecoders.cs, opcodes are +matched with the bit patterns of Decoders/InstTable.cs, and each instruction does what Ryujinx's Instructions/*.cs +emit for it (not what a real Maxwell GPU does where the two differ; see "Ryujinx semantics" below). With it, the result +of one lowered SM86 instruction (HMMA, LDSM, MOVM, HFMA2.RELU, ...) can be compared with vectors captured on real +hardware without running the game. + +State per lane: R0-R254 (RZ = R255 reads 0, writes are dropped), P0-P6 (PT = P7 is true, writes dropped), the lane id, +local memory; per warp: shared memory. Every lane is active at the start; a guard (bits 16-19: predicate 16-18, inverted +19) masks the writes of one instruction, EXIT retires lanes. Branches must be uniform across the active lanes. + +Ryujinx semantics this follows (and real hardware may not): + * Half-precision arithmetic (HADD2, HMUL2, HFMA2) works in F32 on the unpacked halves and rounds to F16 once at the + end (Ryujinx InstEmitFloatArithmetic EmitHadd2Hmul2 / EmitHfma2, InstEmitHelper GetHalfPacked: PackHalf2x16). + HFMA2 is therefore an F32 fused multiply-add rounded to F32, then to F16. + * That rounding is skipped when the packed value goes straight into another unpack: Ryujinx Optimizer.PropagatePack + turns UnpackHalf2x16(PackHalf2x16(a, b)) into a (or b). So F2F.F16.F32 then F2F.F32.F16 of the same register, or + HFMA2 feeding HFMA2, passes the F32 value on unrounded; an integer instruction (SHFL, LOP, ISCADD, ...) in between + keeps the rounding, a MOV does not. Modelled per register (Warp.halves); a guarded write and a branch end it. + * FTZ, FMZ and rounding-mode fields of FFMA / FMUL / FADD / HFMA2 / HMUL2 are ignored, as Ryujinx ignores them. + Denormals are kept (IEEE), saturation is clamp(x, 0, 1) with NaN giving 0 (GLSL FClamp as min(max(x, 0), 1) on + a GPU whose min/max return the non-NaN operand). + * PRMT is not implemented in Ryujinx 1.3.2: InstEmit.PrmtR/PrmtI/PrmtC/PrmtRc only log "not implemented", so the + destination keeps its old value. That is the default here (prmt="ryujinx"); prmt="maxwell" gives the hardware + byte permute instead, to see what the lowering intended. + * SHFL follows Ryujinx's HelperFunctionManager GenerateShuffle* helpers (clamp and segment mask from operand C, the + lane's own value where the source lane is out of range). + * LDS / LDL read the 32-bit word at (address >> 2) and extract a sub-word at (address & 3) * 8 (InstEmitMemory + EmitLoad); STS of 8 and 16 bits stores those bytes at the address. + * Condition codes (.CC, .X) are not modelled: an instruction that writes or reads them raises. + * NaN results carry numpy's payload and sign. Neither is defined in SPIR-V, so result comparisons (same_value) let + a NaN match any NaN for float results; see "comparing results". + +Library use: + regs = run(words, regs_in, smem=None) # words: Maxwell 64-bit words; returns a (32, 256) uint32 array + warp = run_warp(words, regs_in, smem=None) # the whole Warp (regs, preds, smem, warnings) + +CLI (vectors written by the tensor ground-truth capture, see main()): + python tests/maxwell_warp_emu.py [--xlate EXE] [--prmt ryujinx|maxwell] [--case NAME] [--trace] + [--fill HEX] [--exact] +tests/tensor_lowering_check.py runs the full check of the tensor lowering on top of this (guards, untouched registers, +several initial fills). +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_XLATE = ROOT.parent.parent / "build" / "xlate" / ("nx2navi_xlate.exe" if os.name == "nt" else "nx2navi_xlate") + +LANES = 32 +RZ = 255 +PT = 7 +U32 = np.uint32 +F32 = np.float32 +F16 = np.float16 + + +class UnknownOpcode(Exception): + """A Maxwell word this emulator has no decoder entry for.""" + + +class Unsupported(Exception): + """A known instruction in a mode or form the emulator does not implement.""" + + +# ---- bit-level helpers -------------------------------------------------------------------------------------------- + +def u32(x) -> np.ndarray: + """A lane vector (or broadcast scalar) as uint32, wrapping Python ints to 32 bits.""" + if isinstance(x, (int, np.integer)): + return np.full(LANES, int(x) & 0xFFFFFFFF, dtype=U32) + return np.asarray(x).astype(U32, copy=True) + + +def sext(value: int, bits: int) -> int: + """Sign-extends a bits-wide field to a Python int.""" + value &= (1 << bits) - 1 + return value - (1 << bits) if value >> (bits - 1) else value + + +def as_f32(u: np.ndarray) -> np.ndarray: + return u32(u).view(F32) + + +def from_f32(f: np.ndarray) -> np.ndarray: + return np.asarray(f, dtype=F32).view(U32).copy() + + +def half_lo(u: np.ndarray) -> np.ndarray: + """UnpackHalf2x16Low: the low 16 bits as F16, widened to F32 (exact).""" + return (u32(u) & U32(0xFFFF)).astype(np.uint16).view(F16).astype(F32) + + +def half_hi(u: np.ndarray) -> np.ndarray: + """UnpackHalf2x16High.""" + return (u32(u) >> U32(16)).astype(np.uint16).view(F16).astype(F32) + + +def pack_half(lo: np.ndarray, hi: np.ndarray) -> np.ndarray: + """PackHalf2x16: each F32 rounded to F16 (round to nearest even, numpy's conversion), lo in bits 0-15.""" + l = np.asarray(lo, dtype=F32).astype(F16).view(np.uint16).astype(U32) + h = np.asarray(hi, dtype=F32).astype(F16).view(np.uint16).astype(U32) + return l | (h << U32(16)) + + +def fma_f32(a: np.ndarray, b: np.ndarray, c: np.ndarray) -> np.ndarray: + """F32 fused multiply-add, rounded once (FPFusedMultiplyAdd -> GLSL.std.450 Fma, fused on the target GPUs). + + a * b is exact in F64 (2 x 24 significant bits). The F64 sum s may be inexact; TwoSum gives its error e exactly + (s + e == a * b + c). Rounding s to F32 is then correct except when s lies exactly halfway between two F32 + values, where the sign of e decides the direction. F32 halfway points are F64 values, so no other case can + double-round.""" + a64 = np.asarray(a, dtype=F32).astype(np.float64) + b64 = np.asarray(b, dtype=F32).astype(np.float64) + c64 = np.asarray(c, dtype=F32).astype(np.float64) + p = a64 * b64 + s = p + c64 + bb = s - p + e = (p - (s - bb)) + (c64 - bb) + r = s.astype(F32) + # The F32 neighbour of r on the side of s. + toward = np.nextafter(r, np.where(s > r.astype(np.float64), F32(np.inf), F32(-np.inf)).astype(F32)) + mid = (r.astype(np.float64) + toward.astype(np.float64)) * 0.5 + fix = np.isfinite(s) & np.isfinite(r) & np.isfinite(toward) & (e != 0) & (s == mid) + if fix.any(): + # At a tie, the exact value lies on the side of e: take the neighbour when e points towards it. + up = (toward.astype(np.float64) > r.astype(np.float64)) + take = fix & (up == (e > 0)) + r = np.where(take, toward, r).astype(F32) + return r + + +def saturate(x: np.ndarray) -> np.ndarray: + """FPSaturate: clamp to [0, 1], NaN to 0 (fmax/fmin return the non-NaN operand).""" + return np.fmin(np.fmax(np.asarray(x, dtype=F32), F32(0)), F32(1)).astype(F32) + + +def abs_neg(x: np.ndarray, absolute: bool, negate: bool) -> np.ndarray: + """FPAbsNeg: abs first, then negate (sign-bit operations in SPIR-V; numpy's abs/negative act on the sign too).""" + x = np.asarray(x, dtype=F32) + if absolute: + x = np.abs(x) + if negate: + x = np.negative(x) + return x + + +class Ins: + """One Maxwell word with the field accessors InstDecoders.cs uses.""" + + def __init__(self, word: int, index: int): + self.w = word + self.index = index + + def bits(self, offset: int, count: int) -> int: + return (self.w >> offset) & ((1 << count) - 1) + + def bit(self, offset: int) -> bool: + return (self.w >> offset) & 1 == 1 + + # Fields shared by most decoders. + @property + def rd(self): return self.bits(0, 8) + @property + def ra(self): return self.bits(8, 8) + @property + def rb(self): return self.bits(20, 8) + @property + def rc(self): return self.bits(39, 8) + @property + def imm20(self): # InstDecoders Imm20: bits 20-38, bit 19 from bit 56 + return ((self.w >> 37) & 0x80000) | ((self.w >> 20) & 0x7FFFF) + @property + def imm20_sint(self): return sext(self.imm20, 20) # InstEmitHelper Imm20ToSInt + @property + def imm20_float(self): return (self.imm20 << 12) & 0xFFFFFFFF # Imm20ToFloat + @property + def imm32(self): return (self.w >> 20) & 0xFFFFFFFF + @property + def imm24_sint(self): return sext(self.bits(20, 24), 24) + + def __repr__(self): + return f"[{self.index}] {self.w:016x}" + + +# ---- warp state --------------------------------------------------------------------------------------------------- + +class Warp: + def __init__(self, smem: bytes | None = None, smem_size: int = 0x10000, local_size: int = 0x2000, + cbufs: dict | None = None, prmt: str = "ryujinx", fill: int = 0, trace: bool = False): + # Registers not given by the caller start as `fill` (SPIR-V locals are undefined unless the job zeroes them; + # a poison value such as 0x7fc0dead shows reads of registers the lowering never wrote). + self.r = np.full((LANES, 256), fill & 0xFFFFFFFF, dtype=U32) + self.r[:, RZ] = 0 + self.p = np.zeros((LANES, 8), dtype=bool) + self.p[:, PT] = True + self.smem = bytearray(smem) if smem is not None else bytearray(smem_size) + self.local = [bytearray(local_size) for _ in range(LANES)] + self.cbufs = {int(k): bytes(v) for k, v in (cbufs or {}).items()} + self.lane = np.arange(LANES, dtype=U32) + self.active = np.ones(LANES, dtype=bool) + if prmt not in ("ryujinx", "maxwell"): + raise ValueError(f"prmt mode {prmt!r}") + self.prmt = prmt + self.trace = trace + self.warnings: list[str] = [] + # Ryujinx Optimizer.PropagatePack turns an UnpackHalf2x16 of a PackHalf2x16 result into the pack's F32 + # operand, so that half is never rounded to F16. Per lane and register: the F32 halves of the pack that wrote + # the register (sh_ok), kept until the register is written otherwise. A write under a guard ends it in every + # lane: Ryujinx puts the instruction in an if-block and merges old and new value in a phi, which is no pack. + self.sh_lo = np.zeros((LANES, 256), dtype=F32) + self.sh_hi = np.zeros((LANES, 256), dtype=F32) + self.sh_ok = np.zeros((LANES, 256), dtype=bool) + self.unconditional = True # the current instruction's guard is PT (run_warp sets it) + + # Register file access: R(n) reads a lane vector, W(n, v, mask) writes the lanes in mask. halves: the F32 values + # a PackHalf2x16 rounded into v (see sh_ok); copy_from: v is a copy of that register (MOV), pack and all. + def R(self, n: int) -> np.ndarray: + if n == RZ: + return np.zeros(LANES, dtype=U32) + return self.r[:, n].copy() + + def W(self, n: int, v, mask: np.ndarray, halves=None, copy_from: int | None = None): + if n == RZ: + return + v = u32(v) + self.r[mask, n] = v[mask] + if not self.unconditional: + self.sh_ok[:, n] = False + elif halves is not None: + self.sh_lo[mask, n] = np.broadcast_to(np.asarray(halves[0], dtype=F32), (LANES,))[mask] + self.sh_hi[mask, n] = np.broadcast_to(np.asarray(halves[1], dtype=F32), (LANES,))[mask] + self.sh_ok[mask, n] = True + elif copy_from is not None and copy_from != RZ: + for arr in (self.sh_lo, self.sh_hi, self.sh_ok): + arr[mask, n] = arr[mask, copy_from] + else: + self.sh_ok[mask, n] = False + + def halves(self, n: int): + """UnpackHalf2x16Low / High of register n as F32, after PropagatePack (the pack's operand where one wrote n).""" + u = self.R(n) + lo, hi = half_lo(u), half_hi(u) + if n != RZ: + ok = self.sh_ok[:, n] + lo = np.where(ok, self.sh_lo[:, n], lo).astype(F32) + hi = np.where(ok, self.sh_hi[:, n], hi).astype(F32) + return lo, hi + + def P(self, n: int, inv: bool = False) -> np.ndarray: + v = np.ones(LANES, dtype=bool) if n == PT else self.p[:, n].copy() + return ~v if inv else v + + def WP(self, n: int, v, mask: np.ndarray): + if n == PT: + return + v = np.broadcast_to(np.asarray(v, dtype=bool), (LANES,)) + self.p[mask, n] = v[mask] + + def cbuf_word(self, slot: int, word_offset: int) -> np.ndarray: + """c[slot][word_offset * 4] (InstDecoders CbufOffset counts words).""" + data = self.cbufs.get(slot) + at = word_offset * 4 + if data is None or at + 4 > len(data): + raise Unsupported(f"constant buffer c[{slot}][0x{at:x}] not provided") + return u32(int.from_bytes(data[at:at + 4], "little")) + + +# ---- instruction semantics ---------------------------------------------------------------------------------------- +# Each handler takes (warp, ins, mask): mask = the active lanes whose guard holds. Lanes outside mask must not change. + +def _no_cc(i: Ins, cc_bit: int | None, x_bit: int | None, name: str): + if cc_bit is not None and i.bit(cc_bit): + raise Unsupported(f"{name}.CC (condition codes not modelled) {i}") + if x_bit is not None and i.bit(x_bit): + raise Unsupported(f"{name}.X (carry not modelled) {i}") + + +def op_nop(w, i, m): + pass + + +def op_mov_r(w, i, m): # InstEmitMove MovR: SrcA in bits 20-27; a Copy, which PropagateCopy sees through + w.W(i.rd, w.R(i.rb), m, copy_from=i.rb) + + +def op_mov_i(w, i, m): # MovI: GetSrcImm(op.Imm20), the raw 20-bit field (not shifted or sign-extended) + w.W(i.rd, u32(i.imm20), m) + + +def op_mov_c(w, i, m): + w.W(i.rd, w.cbuf_word(i.bits(34, 5), i.bits(20, 14)), m) + + +def op_mov32i(w, i, m): + w.W(i.rd, u32(i.imm32), m) + + +def _iadd(w, i, m, a, b, avg): # InstEmitIntegerArithmetic EmitIadd: AvgMode 1 negates B, 2 negates A + if avg == 2: + a = U32(0) - a + if avg == 1: + b = U32(0) - b + w.W(i.rd, a + b, m) + + +def op_iadd_r(w, i, m): + _no_cc(i, 47, 43, "IADD") + _iadd(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(48, 2)) + + +def op_iadd_i(w, i, m): + _no_cc(i, 47, 43, "IADD") + _iadd(w, i, m, w.R(i.ra), u32(i.imm20_sint), i.bits(48, 2)) + + +def op_iadd32i(w, i, m): + _no_cc(i, 52, 53, "IADD32I") + _iadd(w, i, m, w.R(i.ra), u32(i.imm32), i.bits(55, 2)) + + +def _iadd3(w, i, m, a, b, c, lrs, parts): + def extend(v, part): # EmitIadd3 Extend: HalfSelect 1 = low 16 bits, 2 = high 16 bits + if part == 1: + return v & U32(0xFFFF) + if part == 2: + return v >> U32(16) + return v + a, b, c = (extend(v, p) for v, p in zip((a, b, c), parts)) + if i.bit(51): + a = U32(0) - a + if i.bit(50): + b = U32(0) - b + if i.bit(49): + c = U32(0) - c + res = a + b + if lrs == 2: + res = res << U32(16) + elif lrs == 1: + res = res >> U32(16) + w.W(i.rd, res + c, m) + + +def op_iadd3_r(w, i, m): + _no_cc(i, 47, 48, "IADD3") + _iadd3(w, i, m, w.R(i.ra), w.R(i.rb), w.R(i.rc), i.bits(37, 2), (i.bits(35, 2), i.bits(33, 2), i.bits(31, 2))) + + +def op_iadd3_i(w, i, m): + _no_cc(i, 47, 48, "IADD3") + _iadd3(w, i, m, w.R(i.ra), u32(i.imm20_sint), w.R(i.rc), 0, (0, 0, 0)) + + +def _iscadd(w, i, m, a, b, shift, avg): # EmitIscadd: (A << shift), then the AvgMode negations, then A + B + a = a << U32(shift) + if avg == 2: + a = U32(0) - a + if avg == 1: + b = U32(0) - b + w.W(i.rd, a + b, m) + + +def op_iscadd_r(w, i, m): + _no_cc(i, 47, None, "ISCADD") + _iscadd(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(39, 5), i.bits(48, 2)) + + +def op_iscadd_i(w, i, m): + _no_cc(i, 47, None, "ISCADD") + _iscadd(w, i, m, w.R(i.ra), u32(i.imm20_sint), i.bits(39, 5), i.bits(48, 2)) + + +def op_iscadd32i(w, i, m): + _no_cc(i, 52, None, "ISCADD32I") + _iscadd(w, i, m, w.R(i.ra), u32(i.imm32), i.bits(53, 5), 0) + + +def _shl(w, i, m, a, b): # InstEmitShift EmitShl: .M masks the count to 5 bits, else counts >= 32 give 0 + _no_cc(i, 47, 43, "SHL") + if i.bit(39): + b = b & U32(31) + w.W(i.rd, a << b, m) + else: + w.W(i.rd, np.where(b < 32, a << (b & U32(31)), U32(0)), m) + + +def _shr(w, i, m, a, b): # EmitShr: .BREV (40) reverses A first, .S32 (48), .M (39) + _no_cc(i, 47, None, "SHR") + if i.bits(43, 2): + raise Unsupported(f"SHR.X {i}") + if i.bit(40): + a = u32([int(f"{int(v):032b}"[::-1], 2) for v in a]) + signed = i.bit(48) + if i.bit(39): + b = b & U32(31) + sh = b & U32(31) + res = (a.view(np.int32) >> sh.astype(np.int32)).view(U32) if signed else a >> sh + if not i.bit(39): + big = (a.view(np.int32) >> 31).view(U32) if signed else np.zeros(LANES, dtype=U32) + res = np.where(b < 32, res, big) + w.W(i.rd, res, m) + + +def op_shl_r(w, i, m): _shl(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_shl_i(w, i, m): _shl(w, i, m, w.R(i.ra), u32(i.imm20_sint)) +def op_shr_r(w, i, m): _shr(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_shr_i(w, i, m): _shr(w, i, m, w.R(i.ra), u32(i.imm20_sint)) + + +def _lop_pred(w, i, m, res, pred_op, dest_pred): # EmitLopPredWrite + if dest_pred == PT: + return + v = {0: np.zeros(LANES, bool), 1: np.ones(LANES, bool), 2: res == 0, 3: res != 0}[pred_op] + w.WP(dest_pred, v, m) + + +def _lop(w, i, m, a, b, logic, inv_a, inv_b, pred_op, dest_pred): # InstEmitIntegerLogical EmitLop + if inv_a: + a = ~a + if inv_b: + b = ~b + res = {0: a & b, 1: a | b, 2: a ^ b, 3: b}[logic] + _lop_pred(w, i, m, res, pred_op, dest_pred) + w.W(i.rd, res, m) + + +def op_lop_r(w, i, m): + _no_cc(i, 47, 43, "LOP") + _lop(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(41, 2), i.bit(39), i.bit(40), i.bits(44, 2), i.bits(48, 3)) + + +def op_lop_i(w, i, m): + _no_cc(i, 47, 43, "LOP") + _lop(w, i, m, w.R(i.ra), u32(i.imm20_sint), i.bits(41, 2), i.bit(39), i.bit(40), i.bits(44, 2), i.bits(48, 3)) + + +def op_lop32i(w, i, m): # Lop32i: no predicate output (DestPred PT) + _no_cc(i, 52, 57, "LOP32I") + _lop(w, i, m, w.R(i.ra), u32(i.imm32), i.bits(53, 2), i.bit(55), i.bit(56), 0, PT) + + +def _lop3(a, b, c, lut): + """The LOP3 truth table (Lop3Expression.GetFromTruthTable): bit (a << 2 | b << 1 | c) of lut, per bit.""" + res = np.zeros(LANES, dtype=U32) + for k in range(8): + if lut >> k & 1: + ta = a if k & 4 else ~a + tb = b if k & 2 else ~b + tc = c if k & 1 else ~c + res |= ta & tb & tc + return res + + +def op_lop3_r(w, i, m): + _no_cc(i, 47, 38, "LOP3") + res = _lop3(w.R(i.ra), w.R(i.rb), w.R(i.rc), i.bits(28, 8)) + _lop_pred(w, i, m, res, i.bits(36, 2), i.bits(48, 3)) + w.W(i.rd, res, m) + + +def op_lop3_i(w, i, m): + _no_cc(i, 47, 57, "LOP3") + w.W(i.rd, _lop3(w.R(i.ra), u32(i.imm20_sint), w.R(i.rc), i.bits(48, 8)), m) + + +def _prmt(w, i, m, a, sel, c): + if w.prmt == "ryujinx": + # Ryujinx InstEmit.PrmtR / PrmtI: "not implemented", nothing is emitted, the destination keeps its value. + w.warnings.append(f"PRMT at {i} is a no-op in Ryujinx 1.3.2 (destination R{i.rd} unchanged)") + return + if i.bits(48, 4): + raise Unsupported(f"PRMT mode {i.bits(48, 4)} {i}") + res = np.zeros(LANES, dtype=U32) + src = [(a >> U32(8 * k)) & U32(0xFF) for k in range(4)] + [(c >> U32(8 * k)) & U32(0xFF) for k in range(4)] + for byte in range(4): + nib = (sel >> U32(4 * byte)) & U32(0xF) + val = np.zeros(LANES, dtype=U32) + for k in range(8): + val = np.where((nib & U32(7)) == k, src[k], val) + val = np.where((nib & U32(8)) != 0, np.where(val & U32(0x80), U32(0xFF), U32(0)), val) + res |= val << U32(8 * byte) + w.W(i.rd, res, m) + + +def op_prmt_r(w, i, m): _prmt(w, i, m, w.R(i.ra), w.R(i.rb), w.R(i.rc)) +def op_prmt_i(w, i, m): _prmt(w, i, m, w.R(i.ra), u32(i.imm20_sint), w.R(i.rc)) + + +def op_sel_r(w, i, m): # InstEmitMove EmitSel: P ? A : B + p = w.P(i.bits(39, 3), i.bit(42)) + w.W(i.rd, np.where(p, w.R(i.ra), w.R(i.rb)), m) + + +def op_sel_i(w, i, m): + p = w.P(i.bits(39, 3), i.bit(42)) + w.W(i.rd, np.where(p, w.R(i.ra), u32(i.imm20_sint)), m) + + +def _isetp(w, i, m, a, b): # InstEmitIntegerComparison EmitIsetp (not extended) + if i.bit(43): + raise Unsupported(f"ISETP.X {i}") + cmp, signed = i.bits(49, 3), i.bit(48) + x, y = (a.view(np.int32), b.view(np.int32)) if signed else (a, b) + r = {0: np.zeros(LANES, bool), 1: x < y, 2: x == y, 3: x <= y, 4: x > y, 5: x != y, 6: x >= y, + 7: np.ones(LANES, bool)}[cmp] + pred = w.P(i.bits(39, 3), i.bit(42)) + op = i.bits(45, 2) + def comb(v): + if op == 0: + return v & pred + if op == 1: + return v | pred + if op == 2: + return v ^ pred + raise Unsupported(f"ISETP bool op {op} {i}") + w.WP(i.bits(3, 3), comb(r), m) + w.WP(i.bits(0, 3), comb(~r), m) + + +def op_isetp_r(w, i, m): _isetp(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_isetp_i(w, i, m): _isetp(w, i, m, w.R(i.ra), u32(i.imm20_sint)) + + +def _p2r(w, i, m, a, mask): # InstEmitPredicate EmitP2r + if i.bit(40): + raise Unsupported(f"P2R.CC {i}") + shift = i.bits(41, 2) * 8 + mask = (mask & U32(0xFF)) << U32(shift) + insert = np.zeros(LANES, dtype=U32) + for k in range(7): + insert |= np.where(w.p[:, k], U32(1 << (k + shift)), U32(0)) + w.W(i.rd, (a & ~mask) | (insert & mask), m) + + +def op_p2r_r(w, i, m): _p2r(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_p2r_i(w, i, m): _p2r(w, i, m, w.R(i.ra), u32(i.imm20)) # P2rI: the raw Imm20 as mask + + +def _r2p(w, i, m, value, mask): # InstEmitMove EmitR2p + if i.bit(40): + raise Unsupported(f"R2P.CC {i}") + shift = i.bits(41, 2) * 8 + for k in range(7): + sel = (mask >> U32(k)) & U32(1) != 0 + bit = (value >> U32(k + shift)) & U32(1) != 0 + w.WP(k, np.where(sel, bit, w.p[:, k]), m) + + +def op_r2p_r(w, i, m): _r2p(w, i, m, w.R(i.ra), w.R(i.rb)) +def op_r2p_i(w, i, m): _r2p(w, i, m, w.R(i.ra), u32(i.imm20_sint)) # R2pI: Imm20ToSInt + + +def op_s2r(w, i, m): # InstEmitMove S2r: SR_LANEID; one warp, so TID.X is the lane and TID.Y/Z are 0 + sreg = i.bits(20, 8) + if sreg in (0, 33): + v = w.lane + elif sreg in (34, 35): + v = np.zeros(LANES, dtype=U32) + elif sreg == 32: + v = w.lane + else: + raise Unsupported(f"S2R SR {sreg} {i}") + w.W(i.rd, v, m) + + +def op_shfl(w, i, m): # InstEmitWarp Shfl + HelperFunctionManager GenerateShuffle* (subgroup size 32) + a = w.R(i.ra) + b = u32(i.bits(20, 5)) if i.bit(28) else w.R(i.rb) + c = u32(i.bits(34, 13)) if i.bit(29) else w.R(i.rc) + lane = w.lane + clamp = c & U32(0x1F) + seg = (c >> U32(8)) & U32(0x1F) + min_t = lane & seg + max_t = (clamp & ~seg) | min_t + mode = i.bits(30, 2) + if mode == 0: # IDX + src = (b & ~seg) | min_t + valid = src <= max_t + elif mode == 1: # UP: signed compare (ICompareGreaterOrEqual) + src = lane - b + valid = src.view(np.int32) >= min_t.view(np.int32) + elif mode == 2: # DOWN + src = lane + b + valid = src <= max_t + else: # BFLY + src = lane ^ b + valid = src <= max_t + idx = np.where(valid, src, lane).astype(np.int64) + # A predicated SHFL becomes an if-block in SPIR-V: reading a lane outside it is undefined. + bad = m & valid & ~m[np.clip(idx, 0, LANES - 1)] + if bad.any(): + w.warnings.append(f"SHFL at {i} reads inactive lanes {sorted(set(idx[bad].tolist()))}") + res = np.where(valid, a[np.clip(idx, 0, LANES - 1)], a) + w.W(i.rd, res, m) + w.WP(i.bits(48, 3), valid, m) + + +def _mem_load(w, i, m, mem_of, size, base, offset, what): + """InstEmitMemory EmitLoad: per word, the word at (address >> 2), sub-words extracted at (address & 3) * 8.""" + if size > 6: + raise Unsupported(f"{what} size {size} {i}") + count = {5: 2, 6: 4}.get(size, 1) + for k in range(count): + if i.rd + k >= RZ: + break + vals = np.zeros(LANES, dtype=U32) + for lane in np.nonzero(m)[0]: + mem = mem_of(lane) + addr = (int(base[lane]) + offset + 4 * k) & 0xFFFFFFFF + at = (addr >> 2) * 4 + if at + 4 > len(mem): + raise IndexError(f"{what} lane {lane} address 0x{addr:x} outside {len(mem)} bytes {i}") + v = int.from_bytes(mem[at:at + 4], "little") >> ((addr & 3) * 8) + if size == 0: + v &= 0xFF + elif size == 1: + v = sext(v, 8) & 0xFFFFFFFF + elif size == 2: + v &= 0xFFFF + elif size == 3: + v = sext(v, 16) & 0xFFFFFFFF + vals[lane] = v + w.W(i.rd + k, vals, m) + + +def _mem_store(w, i, m, mem_of, size, base, offset, what, local): + """EmitStore: 32-bit words at (address >> 2); shared 8/16-bit stores write those bytes at the address, local + ones insert into the word at (address & 3) * 8. Lanes store in lane order (the last lane wins a conflict).""" + if size > 6: + raise Unsupported(f"{what} size {size} {i}") + count = {5: 2, 6: 4}.get(size, 1) + for k in range(count): + reg = i.rd if i.rd + k >= RZ else i.rd + k + vals = w.R(reg) + for lane in np.nonzero(m)[0]: + mem = mem_of(lane) + addr = (int(base[lane]) + offset + 4 * k) & 0xFFFFFFFF + v = int(vals[lane]) + if size < 4 and not local: + n = 1 if size < 2 else 2 + if addr + n > len(mem): + raise IndexError(f"{what} lane {lane} address 0x{addr:x} outside {len(mem)} bytes {i}") + mem[addr:addr + n] = (v & ((1 << (8 * n)) - 1)).to_bytes(n, "little") + continue + at = (addr >> 2) * 4 + if at + 4 > len(mem): + raise IndexError(f"{what} lane {lane} address 0x{addr:x} outside {len(mem)} bytes {i}") + if size < 4: + n = 8 if size < 2 else 16 + sh = (addr & 3) * 8 + old = int.from_bytes(mem[at:at + 4], "little") + fmask = ((1 << n) - 1) << sh + v = (old & ~fmask & 0xFFFFFFFF) | ((v << sh) & fmask) + mem[at:at + 4] = (v & 0xFFFFFFFF).to_bytes(4, "little") + + +def op_lds(w, i, m): + _mem_load(w, i, m, lambda lane: w.smem, i.bits(48, 3), w.R(i.ra), i.imm24_sint, "LDS") + + +def op_sts(w, i, m): + _mem_store(w, i, m, lambda lane: w.smem, i.bits(48, 3), w.R(i.ra), i.imm24_sint, "STS", False) + + +def op_ldl(w, i, m): + _mem_load(w, i, m, lambda lane: w.local[lane], i.bits(48, 3), w.R(i.ra), i.imm24_sint, "LDL") + + +def op_stl(w, i, m): + _mem_store(w, i, m, lambda lane: w.local[lane], i.bits(48, 3), w.R(i.ra), i.imm24_sint, "STL", True) + + +def op_f2f_r(w, i, m): # InstEmitConversion F2fR / EmitF2F + _no_cc(i, 47, None, "F2F") + dst, src = i.bits(8, 2), i.bits(10, 2) + if dst not in (1, 2) or src not in (1, 2): + raise Unsupported(f"F2F with F64 {i}") + x = w.halves(i.rb)[1 if i.bit(41) else 0] if src == 1 else as_f32(w.R(i.rb)) + x = abs_neg(x, i.bit(49), i.bit(45)) + rnd = (i.bits(42, 1) << 2) | i.bits(39, 2) # IntegerRound: 4 round, 5 floor, 6 ceil, 7 trunc + if src == dst: + x = {4: np.rint, 5: np.floor, 6: np.ceil, 7: np.trunc}.get(rnd, lambda v: v)(x).astype(F32) + if i.bit(50): + x = saturate(x) + if dst == 2: + w.W(i.rd, from_f32(x), m) + else: # WriteFP: PackHalf2x16(x, 0) + zero = np.zeros(LANES, F32) + w.W(i.rd, pack_half(x, zero), m, halves=(x, zero)) + + +def _ffma(w, i, m, a, b, c, neg_b, neg_c, sat): # EmitFfma (rounding mode and FMZ ignored) + r = fma_f32(as_f32(a), abs_neg(as_f32(b), False, neg_b), abs_neg(as_f32(c), False, neg_c)) + w.W(i.rd, from_f32(saturate(r) if sat else r), m) + + +def op_ffma_r(w, i, m): + _no_cc(i, 47, None, "FFMA") + _ffma(w, i, m, w.R(i.ra), w.R(i.rb), w.R(i.rc), i.bit(48), i.bit(49), i.bit(50)) + + +def op_ffma_i(w, i, m): + _no_cc(i, 47, None, "FFMA") + _ffma(w, i, m, w.R(i.ra), u32(i.imm20_float), w.R(i.rc), i.bit(48), i.bit(49), i.bit(50)) + + +def op_ffma32i(w, i, m): # Ffma32i: C is the destination register + _no_cc(i, 52, None, "FFMA32I") + _ffma(w, i, m, w.R(i.ra), u32(i.imm32), w.R(i.rd), i.bit(56), i.bit(57), i.bit(55)) + + +def _fmul(w, i, m, a, b, scale, neg_b, sat): # EmitFmul: A times the scale first, then A * B + a = as_f32(a) + if scale: + a = (a * F32({1: 0.5, 2: 0.25, 3: 0.125, 4: 8.0, 5: 4.0, 6: 2.0}.get(scale, 1.0))).astype(F32) + r = (a * abs_neg(as_f32(b), False, neg_b)).astype(F32) + w.W(i.rd, from_f32(saturate(r) if sat else r), m) + + +def op_fmul_r(w, i, m): + _no_cc(i, 47, None, "FMUL") + _fmul(w, i, m, w.R(i.ra), w.R(i.rb), i.bits(41, 3), i.bit(48), i.bit(50)) + + +def op_fmul_i(w, i, m): + _no_cc(i, 47, None, "FMUL") + _fmul(w, i, m, w.R(i.ra), u32(i.imm20_float), i.bits(41, 3), i.bit(48), i.bit(50)) + + +def op_fmul32i(w, i, m): + _no_cc(i, 52, None, "FMUL32I") + _fmul(w, i, m, w.R(i.ra), u32(i.imm32), 0, False, i.bit(55)) + + +def _fadd(w, i, m, a, b, neg_a, neg_b, abs_a, abs_b, sat): # EmitFadd + r = (abs_neg(as_f32(a), abs_a, neg_a) + abs_neg(as_f32(b), abs_b, neg_b)).astype(F32) + w.W(i.rd, from_f32(saturate(r) if sat else r), m) + + +def op_fadd_r(w, i, m): + _no_cc(i, 47, None, "FADD") + _fadd(w, i, m, w.R(i.ra), w.R(i.rb), i.bit(48), i.bit(45), i.bit(46), i.bit(49), i.bit(50)) + + +def op_fadd_i(w, i, m): + _no_cc(i, 47, None, "FADD") + _fadd(w, i, m, w.R(i.ra), u32(i.imm20_float), i.bit(48), i.bit(45), i.bit(46), i.bit(49), i.bit(50)) + + +def op_fadd32i(w, i, m): + _no_cc(i, 52, None, "FADD32I") + _fadd(w, i, m, w.R(i.ra), u32(i.imm32), i.bit(56), i.bit(53), i.bit(54), i.bit(57), False) + + +# Half precision: InstEmitHelper GetHalfSrc / GetHalfUnpacked / GetHalfPacked. +def half_src(w: Warp, reg: int, swizzle: int, neg: bool = False, absolute: bool = False): + """The two F32 values of a packed-half register operand: swizzle 0 (H1_H0), 1 (F32: the register as one F32 + twice), 2 (H0_H0), 3 (H1_H1); then abs, neg. The halves come through Warp.halves (PropagatePack).""" + if swizzle == 1: + x = as_f32(w.R(reg)) + pair = (x, x) + else: + lo, hi = w.halves(reg) + pair = {0: (lo, hi), 2: (lo, lo), 3: (hi, hi)}[swizzle] + return [abs_neg(x, absolute, neg) for x in pair] + + +def half_imm_pair(h0: int, h1: int): + """GetHalfSrc(immH0, immH1): each 10-bit field shifted left 6 as an F16.""" + return [np.full(LANES, np.array([(v << 6) & 0xFFFF], dtype=np.uint16).view(F16)[0], dtype=F32) for v in (h0, h1)] + + +def half_imm32(imm: int): + return [np.full(LANES, np.array([(imm >> s) & 0xFFFF], dtype=np.uint16).view(F16)[0], dtype=F32) for s in (0, 16)] + + +def half_pack(w: Warp, ofmt: int, res, rd: int): + """GetHalfPacked: 0 F16 pair, 1 F32 (lane 0's result), 2 MRG_H0 (high half kept), 3 MRG_H1 (low half kept). + Returns the register value and the F32 halves PackHalf2x16 rounded (None for the F32 format).""" + if ofmt == 1: + return from_f32(res[0]), None + if ofmt == 0: + lo, hi = res[0], res[1] + else: + old_lo, old_hi = w.halves(rd) # GetHalfDest: an UnpackHalf2x16 of Rd + lo, hi = (res[0], old_hi) if ofmt == 2 else (old_lo, res[1]) + return pack_half(lo, hi), (lo, hi) + + +def _hadd_hmul(w, i, m, a, b, add, ofmt, sat): # EmitHadd2Hmul2: F32 add/multiply per half, saturate, pack + res = [] + for k in range(2): + r = (a[k] + b[k]).astype(F32) if add else (a[k] * b[k]).astype(F32) + res.append(saturate(r) if sat else r) + v, halves = half_pack(w, ofmt, res, i.rd) + w.W(i.rd, v, m, halves=halves) + + +def _hfma(w, i, m, a, b, c, ofmt, sat): # EmitHfma2: F32 fused multiply-add per half, saturate, pack + res = [] + for k in range(2): + r = fma_f32(a[k], b[k], c[k]) + res.append(saturate(r) if sat else r) + v, halves = half_pack(w, ofmt, res, i.rd) + w.W(i.rd, v, m, halves=halves) + + +def op_hadd2_r(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2), i.bit(43), i.bit(44)) + b = half_src(w, i.rb, i.bits(28, 2), i.bit(31), i.bit(30)) + _hadd_hmul(w, i, m, a, b, True, i.bits(49, 2), i.bit(32)) + + +def op_hadd2_i(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2), i.bit(43), i.bit(44)) + b = half_imm_pair(i.bits(20, 10), ((i.w >> 47) & 0x200) | i.bits(30, 9)) + _hadd_hmul(w, i, m, a, b, True, i.bits(49, 2), i.bit(52)) + + +def op_hadd232i(w, i, m): + a = half_src(w, i.ra, i.bits(53, 2), i.bit(56), False) + _hadd_hmul(w, i, m, a, half_imm32(i.imm32), True, 0, i.bit(52)) + + +def op_hmul2_r(w, i, m): # Hmul2R: the NegA bit (31) negates B + a = half_src(w, i.ra, i.bits(47, 2), False, i.bit(44)) + b = half_src(w, i.rb, i.bits(28, 2), i.bit(31), i.bit(30)) + _hadd_hmul(w, i, m, a, b, False, i.bits(49, 2), i.bit(32)) + + +def op_hmul2_i(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2), i.bit(43), i.bit(44)) + b = half_imm_pair(i.bits(20, 10), ((i.w >> 47) & 0x200) | i.bits(30, 9)) + _hadd_hmul(w, i, m, a, b, False, i.bits(49, 2), i.bit(52)) + + +def op_hmul232i(w, i, m): + a = half_src(w, i.ra, i.bits(53, 2)) + _hadd_hmul(w, i, m, a, half_imm32(i.imm32), False, 0, i.bit(52)) + + +def op_hfma2_r(w, i, m): # Hfma2R: NegA (31) negates B, NegC (30) + a = half_src(w, i.ra, i.bits(47, 2)) + b = half_src(w, i.rb, i.bits(28, 2), i.bit(31)) + c = half_src(w, i.rc, i.bits(35, 2), i.bit(30)) + _hfma(w, i, m, a, b, c, i.bits(49, 2), i.bit(32)) + + +def op_hfma2_i(w, i, m): + a = half_src(w, i.ra, i.bits(47, 2)) + b = half_imm_pair(i.bits(20, 10), ((i.w >> 47) & 0x200) | i.bits(30, 9)) + c = half_src(w, i.rc, i.bits(53, 2), i.bit(51)) + _hfma(w, i, m, a, b, c, i.bits(49, 2), i.bit(52)) + + +def op_hfma232i(w, i, m): # Hfma232i: C is the destination register, H1_H0 + a = half_src(w, i.ra, i.bits(47, 2)) + c = half_src(w, i.rd, 0, i.bit(51)) + _hfma(w, i, m, a, half_imm32(i.imm32), c, 0, False) + + +# Control flow: handled by the executor (EXIT retires lanes, BRA moves the warp). +def op_exit(w, i, m): + if i.bits(0, 5) != 15: + raise Unsupported(f"EXIT with condition code test {i.bits(0, 5)} {i}") + w.active &= ~m + + +OP_BRA = "bra" + +# The decoder: patterns as Ryujinx Decoders/InstTable.cs writes them (bit 63 first, x = any). The most specific +# matching pattern wins, as InstTable's XBits ordering does. +TABLE = [ + ("0101000010110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "NOP", op_nop), + ("111000110000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "EXIT", op_exit), + ("111000100100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "BRA", OP_BRA), + ("0101110010011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV", op_mov_r), + ("0011100x10011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV.I", op_mov_i), + ("0100110010011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV.C", op_mov_c), + ("000000010000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "MOV32I", op_mov32i), + ("0101110000010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD", op_iadd_r), + ("0011100x00010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD.I", op_iadd_i), + ("0001110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD32I", op_iadd32i), + ("010111001100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD3", op_iadd3_r), + ("0011100x1100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "IADD3.I", op_iadd3_i), + ("0101110000011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISCADD", op_iscadd_r), + ("0011100x00011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISCADD.I", op_iscadd_i), + ("000101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISCADD32I", op_iscadd32i), + ("0101110001001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHL", op_shl_r), + ("0011100x01001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHL.I", op_shl_i), + ("0101110000101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHR", op_shr_r), + ("0011100x00101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHR.I", op_shr_i), + ("0101110001000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP", op_lop_r), + ("0011100x01000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP.I", op_lop_i), + ("000001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP32I", op_lop32i), + ("0101101111100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP3", op_lop3_r), + ("001111xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LOP3.I", op_lop3_i), + ("010110111100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "PRMT", op_prmt_r), + ("0011011x1100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "PRMT.I", op_prmt_i), + ("0101110010100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SEL", op_sel_r), + ("0011100x10100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SEL.I", op_sel_i), + ("010110110110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISETP", op_isetp_r), + ("0011011x0110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "ISETP.I", op_isetp_i), + ("0101110011101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "P2R", op_p2r_r), + ("0011100x11101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "P2R.I", op_p2r_i), + ("0101110011110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "R2P", op_r2p_r), + ("0011100x11110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "R2P.I", op_r2p_i), + ("1111000011001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "S2R", op_s2r), + ("1110111100010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "SHFL", op_shfl), + ("1110111101001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LDS", op_lds), + ("1110111101011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "STS", op_sts), + ("1110111101000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "LDL", op_ldl), + ("1110111101010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "STL", op_stl), + ("0101110010101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "F2F", op_f2f_r), + ("010110011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA", op_ffma_r), + ("0011001x1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA.I", op_ffma_i), + ("000011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FFMA32I", op_ffma32i), + ("0101110001101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FMUL", op_fmul_r), + ("0011100x01101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FMUL.I", op_fmul_i), + ("00011110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FMUL32I", op_fmul32i), + ("0101110001011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FADD", op_fadd_r), + ("0011100x01011xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FADD.I", op_fadd_i), + ("000010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "FADD32I", op_fadd32i), + ("0101110100010xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HADD2", op_hadd2_r), + ("0111101x0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HADD2.I", op_hadd2_i), + ("0010110xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HADD2_32I", op_hadd232i), + ("0101110100001xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HMUL2", op_hmul2_r), + ("0111100x0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HMUL2.I", op_hmul2_i), + ("0010101xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HMUL2_32I", op_hmul232i), + ("0101110100000xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HFMA2", op_hfma2_r), + ("01110xxx0xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HFMA2.I", op_hfma2_i), + ("0010100xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "HFMA2_32I", op_hfma232i), +] + + +def _compile_table(): + out = [] + for pattern, name, fn in TABLE: + assert len(pattern) == 64, name + mask = value = 0 + for k, ch in enumerate(pattern): + bit = 63 - k + if ch != "x": + mask |= 1 << bit + value |= (ch == "1") << bit + out.append((mask, value, bin(mask).count("1"), name, fn)) + return out + + +_DECODE = _compile_table() + + +def decode(word: int): + """(name, handler) of a Maxwell word, by the most specific matching InstTable pattern.""" + best = None + for mask, value, fixed, name, fn in _DECODE: + if word & mask == value and (best is None or fixed > best[0]): + best = (fixed, name, fn) + if best is None: + raise UnknownOpcode(f"unknown Maxwell opcode 0x{word:016x}") + return best[1], best[2] + + +def slot_address(k: int) -> int: + """Byte address of instruction k in a compute program as AmpereLowering.Lower lays it out (a scheduling word + first in every 32 bytes; --lower prints the instruction words only).""" + return 8 * (1 + k + k // 3) + + +def run_warp(words, regs_in=None, smem=None, max_steps: int = 200000, **opts) -> Warp: + """Executes the Maxwell words on one warp and returns its final state. regs_in in any form load_regs takes.""" + words = [w if isinstance(w, int) else int(str(w), 16) for w in words] + w = Warp(smem=parse_bytes(smem) if smem is not None else None, **opts) + for lane, regs in _normalised(regs_in or {}).items(): + for reg, val in regs.items(): + kind, n = reg + if kind == "R": + w.r[lane, n] = val & 0xFFFFFFFF if n != RZ else 0 + elif kind == "P" and n != PT: + w.p[lane, n] = bool(val) + addr_to_index = {slot_address(k): k for k in range(len(words) + 1)} + # Where control flow joins (branch targets) Ryujinx merges registers in phis, and an UnpackHalf2x16 of a phi is + # not folded: the PropagatePack state (Warp.sh_ok) is dropped there and at every BRA (conservative: a register no + # path writes needs no phi). + joins = set() + for k, word in enumerate(words): + if decode(word)[1] is OP_BRA: + joins.add(addr_to_index.get(slot_address(k) + 8 + Ins(word, k).imm24_sint)) + pc = steps = 0 + with np.errstate(all="ignore"): + while pc < len(words) and w.active.any(): + steps += 1 + if steps > max_steps: + raise RuntimeError(f"more than {max_steps} steps (loop?)") + ins = Ins(words[pc], pc) + name, fn = decode(ins.w) + guard = w.P(ins.bits(16, 3), ins.bit(19)) + mask = w.active & guard + w.unconditional = ins.bits(16, 3) == PT and not ins.bit(19) + if pc in joins or fn is OP_BRA: + w.sh_ok[:] = False + if w.trace: + print(f" {pc:4d} {ins.w:016x} {name:<10} lanes {int(mask.sum())}", file=sys.stderr) + if fn is OP_BRA: + if ins.bits(0, 5) != 15: + raise Unsupported(f"BRA with condition code test {ins.bits(0, 5)} {ins}") + if mask.any(): + if (mask != w.active).any(): + raise Unsupported(f"divergent BRA {ins}") + target = slot_address(pc) + 8 + ins.imm24_sint + if target not in addr_to_index: + raise Unsupported(f"BRA to 0x{target:x}, not an instruction slot {ins}") + pc = addr_to_index[target] + continue + else: + fn(w, ins, mask) + pc += 1 + return w + + +def run(words, regs_in=None, smem=None, **opts) -> np.ndarray: + """Executes the Maxwell words; returns the per-lane register file, a (32, 256) uint32 array (row = lane).""" + return run_warp(words, regs_in, smem, **opts).r + + +# ---- vector files and the --lower bridge -------------------------------------------------------------------------- + +def parse_int(v) -> int: + """An int, or a hex string (with or without 0x).""" + if isinstance(v, bool): + return int(v) + if isinstance(v, (int, np.integer)): + return int(v) + s = str(v).strip().lower() + return int(s[2:] if s.startswith("0x") else s, 16) + + +def parse_bytes(v) -> bytes: + if isinstance(v, (bytes, bytearray)): + return bytes(v) + if isinstance(v, list): + return bytes(parse_int(x) & 0xFF for x in v) + s = re.sub(r"\s+", "", str(v)) + return bytes.fromhex(s[2:] if s.lower().startswith("0x") else s) + + +def parse_reg(name, urbase: int | None = None): + """("R", n) or ("P", n) for a register name: an int, "R12", "12", "RZ", "UR3" (needs urbase), "P0", "PT".""" + if isinstance(name, (int, np.integer)): + return ("R", int(name)) + s = str(name).strip().upper() + if s in ("RZ", "R255"): + return ("R", RZ) + if s == "PT": + return ("P", PT) + if s.startswith("UR"): + if s == "URZ": + return ("R", RZ) + if urbase is None: + raise ValueError(f"{name}: uniform register without a urbase") + return ("R", urbase + int(s[2:])) + if s.startswith("P"): + return ("P", int(s[1:])) + if s.startswith("R"): + s = s[1:] + return ("R", int(s)) + + +def load_regs(spec, urbase: int | None = None) -> dict[int, dict]: + """Normalises register specs to {lane: {(kind, n): int}}. Accepts {lane: {reg: value}} with lane keys as ints or + strings (and "*" / "all" for every lane), or a list with one entry per lane; each lane entry a dict {reg: value} + or a list of values indexed by register number; values ints or hex strings.""" + out: dict[int, dict] = {lane: {} for lane in range(LANES)} + if isinstance(spec, list): + items = list(enumerate(spec)) + else: + items = list(spec.items()) + # "*" first so per-lane entries override it. + items.sort(key=lambda kv: 0 if str(kv[0]).strip().lower() in ("*", "all") else 1) + for lane_key, regs in items: + key = str(lane_key).strip().lower() + lanes = range(LANES) if key in ("*", "all") else [int(key)] + if isinstance(regs, list): + regs = {n: v for n, v in enumerate(regs) if v is not None} + for lane in lanes: + for reg, val in (regs or {}).items(): + out[lane][parse_reg(reg, urbase)] = parse_int(val) + return out + + +def _normalised(spec) -> dict[int, dict]: + """load_regs output passes through unchanged (keys (kind, n)); anything else is loaded.""" + if isinstance(spec, dict) and all(isinstance(k, int) for k in spec) and \ + all(isinstance(r, tuple) for regs in spec.values() if isinstance(regs, dict) for r in regs): + return spec + return load_regs(spec) + + +def lower(sm86, exe: Path = DEFAULT_XLATE): + """Runs nx2navi_xlate --lower on SM86 instructions (a list of [w0, w1, w2, w3] or one such list); returns + (Maxwell words, urbase, local bytes).""" + if sm86 and not isinstance(sm86[0], (list, tuple)): + sm86 = [sm86] + text = "".join(" ".join(f"{parse_int(x) & 0xFFFFFFFF:08x}" for x in ins) + "\n" for ins in sm86) + fd, path = tempfile.mkstemp(suffix=".sm86.txt") + try: + with os.fdopen(fd, "w") as f: + f.write(text) + p = subprocess.run([str(exe), "--lower", path], capture_output=True, text=True) + finally: + os.unlink(path) + if p.returncode != 0: + raise RuntimeError(f"--lower failed: {p.stderr.strip()}") + words, urbase, local = [], None, 0 + for line in p.stdout.splitlines(): + f = line.split() + if not f: + continue + if f[0] == "urbase": + urbase = int(f[1]) + elif f[0] == "local": + local = int(f[1]) + else: + words.append(int(f[0], 16)) + return words, urbase, local + + +def _fmt(v: int) -> str: + """A register value as hex plus its F32 and half-pair readings.""" + f = np.array([v], dtype=U32).view(F32)[0] + h = np.array([v & 0xFFFF, v >> 16], dtype=np.uint16).view(F16) + return f"{v:08x} (f32 {f:.6g}, h {float(h[0]):.5g}/{float(h[1]):.5g})" + + +# ---- comparing results ---------------------------------------------------------------------------------------------- +# A float result is right when its bits are, except that any NaN matches any NaN: SPIR-V leaves the payload and sign +# of a NaN to the host (Vulkan spec, "Precision and Operation of SPIR-V Instructions"), and this emulator's numpy +# arithmetic picks its own, while SM86 hardware writes 0x7fff for every F16 NaN result. Whether a value is NaN is the +# lowering's doing; which NaN is not. Moves (LDSM, MOVM) and integer results compare bit for bit. + +def result_kind(sm86) -> str: + """How the destination registers of an SM86 instruction compare: "f16x2" (two F16 values), "f32" or "bits". + sm86 is [w0, w1, w2, w3] or a list of them (the last one decides).""" + ins = sm86[-1] if isinstance(sm86[0], (list, tuple)) else sm86 + w0, w2 = parse_int(ins[0]), parse_int(ins[2]) + op = w0 & 0xFFF + if op == 0x23C: # HMMA: .F32 (bit 76) or .F16 results + return "f32" if (w2 >> 12) & 1 else "f16x2" + if (op & 0x1FF) in (0x030, 0x031, 0x032): # HADD2 / HFMA2 / HMUL2, F32 results with bit 78 + return "f32" if (w2 >> 14) & 1 else "f16x2" + return "bits" + + +def _nan16(h: int) -> bool: + return (h & 0x7FFF) > 0x7C00 + + +def _nan32(v: int) -> bool: + return (v & 0x7FFFFFFF) > 0x7F800000 + + +def same_value(got: int, want: int, kind: str) -> bool: + """got matches want: equal bits, or for float kinds NaN where want is NaN (see above).""" + got, want = got & 0xFFFFFFFF, want & 0xFFFFFFFF + if got == want: + return True + if kind == "f32": + return _nan32(got) and _nan32(want) + if kind == "f16x2": + return all(g == w or (_nan16(g) and _nan16(w)) + for g, w in ((got & 0xFFFF, want & 0xFFFF), (got >> 16, want >> 16))) + return False + + +def ulps(got: int, want: int, kind: str) -> int: + """Distance in units in the last place (the larger of the two halves for "f16x2"); 0 for matching NaNs, a huge + number for NaN against a number. +0 and -0 are 0 apart (same_value still tells them apart).""" + def order(v, sign_bit, nan): + if nan(v): + return None + mag = v & (sign_bit - 1) + return -mag if v & sign_bit else mag + def one(g, w, sign_bit, nan): + og, ow = order(g, sign_bit, nan), order(w, sign_bit, nan) + if og is None or ow is None: + return 0 if og is None and ow is None else 1 << 32 + return abs(og - ow) + got, want = got & 0xFFFFFFFF, want & 0xFFFFFFFF + if kind == "f32": + return one(got, want, 1 << 31, _nan32) + if kind == "f16x2": + return max(one(got & 0xFFFF, want & 0xFFFF, 1 << 15, _nan16), one(got >> 16, want >> 16, 1 << 15, _nan16)) + return 0 if got == want else 1 << 32 + + +def run_case(case: dict, exe: Path, prmt: str, trace: bool = False, fill: int = 0, exact: bool = False): + """Lowers, runs and compares one vector case; returns (ok, lines of report). Float results compare with + same_value (a NaN matches a NaN) unless exact.""" + kind = "bits" if exact else result_kind(case["sm86"]) + words, urbase, _ = lower(case["sm86"], exe) + regs_in = load_regs(case.get("regs_in", {}), urbase) + expected = load_regs(case.get("regs_out", {}), urbase) + # LDSM cases (work/tensor-gt/README.md): "lane_addresses" hold Ra per lane when regs_in does not, relative to + # "smem", which sits at shared address "smem_base". Shared memory is padded to 64 KiB so stray reads stay in range. + smem = None + if case.get("smem") is not None: + base = case.get("smem_base", 0) + base = base if isinstance(base, int) else parse_int(base) + data = parse_bytes(case["smem"]) + buf = bytearray(max(0x10000, base + len(data))) + buf[base:base + len(data)] = data + smem = bytes(buf) + sm86 = case["sm86"] if isinstance(case["sm86"][0], (list, tuple)) else [case["sm86"]] + if case.get("lane_addresses") is not None and len(sm86) == 1: + ra = (parse_int(sm86[0][0]) >> 24) & 0xFF + base = case.get("smem_base", 0) + base = base if isinstance(base, int) else parse_int(base) + for lane, a in enumerate(case["lane_addresses"]): + regs_in[lane].setdefault(("R", ra), (parse_int(a) + base) & 0xFFFFFFFF) + warp = run_warp(words, regs_in, smem, prmt=prmt, trace=trace, fill=fill) + diffs = [] + for lane in range(LANES): + for (reg_kind, n), want in sorted(expected[lane].items()): + if reg_kind == "R": + got = int(warp.r[lane, n]) + if not same_value(got, want, kind): + diffs.append((lane, reg_kind, n, got, want)) + elif int(warp.p[lane, n]) != int(bool(want)): + diffs.append((lane, reg_kind, n, int(warp.p[lane, n]), want)) + if "smem_out" in case: + want_mem = parse_bytes(case["smem_out"]) + if bytes(warp.smem[:len(want_mem)]) != want_mem: + diffs.append((-1, "smem", 0, 0, 0)) + report = [f" {len(words)} Maxwell words, urbase {urbase}"] + report += [f" warning: {m}" for m in dict.fromkeys(warp.warnings)] + if not diffs: + return True, report + lanes = sorted({d[0] for d in diffs}) + report.append(f" {len(diffs)} differing values in lanes {lanes[:16]}{' ...' if len(lanes) > 16 else ''}") + if kind != "bits": + worst = max(ulps(got, want, kind) for _, reg_kind, _, got, want in diffs if reg_kind == "R") + report.append(f" largest difference: {'NaN against a number' if worst >= 1 << 32 else f'{worst} ulp'} ({kind})") + for lane, reg_kind, n, got, want in diffs[:12]: + if reg_kind == "smem": + report.append(" shared memory differs from smem_out") + elif reg_kind == "P": + report.append(f" lane {lane:2d} P{n}: got {got} want {int(bool(want))}") + else: + report.append(f" lane {lane:2d} R{n}: got {_fmt(got)} want {_fmt(want & 0xFFFFFFFF)}") + return False, report + + +def load_cases(path: Path) -> list[dict]: + data = json.loads(path.read_text(encoding="utf-8")) + if isinstance(data, dict): + data = data.get("cases", data.get("vectors", list(data.values()) if all(isinstance(v, dict) for v in data.values()) else [])) + return data + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description="Lower SM86 vector cases with nx2navi_xlate --lower, run them on the " + "Maxwell warp emulator and compare with the captured results.") + ap.add_argument("vectors", type=Path, help="JSON list of cases: name, sm86 [w0..w3], regs_in, smem?, regs_out") + ap.add_argument("--xlate", type=Path, default=DEFAULT_XLATE, help="nx2navi_xlate executable") + ap.add_argument("--prmt", choices=("ryujinx", "maxwell"), default="ryujinx", + help="PRMT as the translator runs it (no-op) or as Maxwell hardware does") + ap.add_argument("--fill", default="0", help="initial value (hex) of registers regs_in does not set") + ap.add_argument("--case", action="append", help="run only cases whose name contains this (repeatable)") + ap.add_argument("--trace", action="store_true", help="print every executed Maxwell instruction") + ap.add_argument("--exact", action="store_true", help="compare float results bit for bit (NaN payloads too)") + args = ap.parse_args(argv) + cases = load_cases(args.vectors) + passed = failed = 0 + for idx, case in enumerate(cases): + name = case.get("name", f"case{idx}") + if args.case and not any(c in name for c in args.case): + continue + try: + ok, report = run_case(case, args.xlate, args.prmt, args.trace, parse_int(args.fill), args.exact) + except Exception as e: # one broken case does not stop the run + ok, report = False, [f" error: {type(e).__name__}: {e}"] + print(f"{'PASS' if ok else 'FAIL'} {name}") + if not ok or any("warning" in r for r in report): + print("\n".join(report)) + passed += ok + failed += not ok + print(f"{passed} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/maxwell_warp_emu_selftest.py b/tests/maxwell_warp_emu_selftest.py new file mode 100644 index 00000000..042f5463 --- /dev/null +++ b/tests/maxwell_warp_emu_selftest.py @@ -0,0 +1,351 @@ +"""Self-test of tests/maxwell_warp_emu.py: hand-encoded Maxwell sequences against hand-computed results. + +Part 1 runs Maxwell words encoded here from the field layouts of Ryujinx Decoders/InstDecoders.cs (integer and logic +ops, SHFL.IDX with clamp and segment masks, LDS/STS sizes, F2F, HFMA2, FFMA rounding, PRMT, predicates, EXIT, BRA). +Part 2, when build/xlate/nx2navi_xlate.exe exists, lowers SM86 HMMA.1688 / HMMA.16816, LDSM and MOVM with +nx2navi_xlate --lower and checks the emulated result against exactly representable reference values (small +integers, so every F16 rounding is exact and only the fragment layout can be wrong), and HFMA2.RELU as the translator +runs it (Ryujinx 1.3.2 drops PRMT, so AmpereLowering.cs Half must not use it). tests/tensor_lowering_check.py checks +the same lowerings against hardware results. + + python tests/maxwell_warp_emu_selftest.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import maxwell_warp_emu as emu # noqa: E402 + +PT = 7 << 16 # guard field: predicate PT +LANES = 32 +lane = np.arange(LANES, dtype=np.int64) +failures: list[str] = [] +checks = 0 + + +def check(name: str, got, want): + global checks + checks += 1 + got = np.asarray(got).astype(np.int64) + want = np.broadcast_to(np.asarray(want).astype(np.int64), got.shape) + if not np.array_equal(got, want): + bad = np.nonzero(got != want)[0][:6] + failures.append(f"{name}: " + ", ".join(f"lane {b} got {int(got[b]):#x} want {int(want[b]):#x}" for b in bad)) + + +# ---- Maxwell encoders (InstDecoders.cs field positions) ----------------------------------------------------------- + +def guard(p: int, inv: bool = False) -> int: + return (p | (8 if inv else 0)) << 16 + + +def iadd32i(d, a, imm, g=PT): return 0x1C00000000000000 | (imm & 0xFFFFFFFF) << 20 | a << 8 | d | g +def mov32i(d, imm, g=PT): return 0x0100000000000000 | 0xF << 12 | (imm & 0xFFFFFFFF) << 20 | d | g +def mov(d, s, g=PT): return 0x5C98000000000000 | 0xF << 39 | s << 20 | d | g +def s2r(d, sr=0): return 0xF0C8000000000000 | sr << 20 | d | PT +def lop32i(d, a, imm, op): return 0x0400000000000000 | op << 53 | (imm & 0xFFFFFFFF) << 20 | a << 8 | d | PT +def lop3(d, a, b, c, lut): return 0x5BE0000000000000 | 7 << 48 | c << 39 | lut << 28 | b << 20 | a << 8 | d | PT +def iscadd(d, a, b, sh): return 0x5C18000000000000 | sh << 39 | b << 20 | a << 8 | d | PT +def iadd3(d, a, b, c, neg_b=False): return 0x5CC0000000000000 | c << 39 | b << 20 | a << 8 | d | (1 << 50 if neg_b else 0) | PT +def shl_i(d, a, n): return 0x3848000000000000 | n << 20 | a << 8 | d | PT +def shr_i(d, a, n, signed=False): return 0x3828000000000000 | (1 << 48 if signed else 0) | n << 20 | a << 8 | d | PT +def shr_r(d, a, b): return 0x5C28000000000000 | b << 20 | a << 8 | d | PT +def prmt(d, a, sel, c): return 0x5BC0000000000000 | c << 39 | sel << 20 | a << 8 | d | PT +def isetp_i(pd, a, imm, cmp, signed=False): # ISETP.cmp Pd, PT, Ra, imm, PT (AND) + return 0x3660000000000000 | cmp << 49 | (1 << 48 if signed else 0) | 7 << 39 | (imm & 0x7FFFF) << 20 | a << 8 | pd << 3 | 7 | PT +def shfl(d, a, b, c, mode=0, b_imm=False, c_imm=False, pd=7): + return (0xEF10000000000000 | pd << 48 | (c << 34 if c_imm else c << 39) | (1 << 29 if c_imm else 0) + | (1 << 28 if b_imm else 0) | mode << 30 | b << 20 | a << 8 | d | PT) +def lds(d, a, off, size): return 0xEF48000000000000 | size << 48 | (off & 0xFFFFFF) << 20 | a << 8 | d | PT +def sts(v, a, off, size): return 0xEF58000000000000 | size << 48 | (off & 0xFFFFFF) << 20 | a << 8 | v | PT +def f2f(d, b, dst, src, high=False): return 0x5CA8000000000000 | dst << 8 | src << 10 | b << 20 | (1 << 41 if high else 0) | d | PT +def hfma2(d, a, b, c, neg_c=False, sat=False): + return 0x5D00000000000000 | c << 39 | (1 << 30 if neg_c else 0) | (1 << 32 if sat else 0) | b << 20 | a << 8 | d | PT +def ffma(d, a, b, c): return 0x5980000000000000 | c << 39 | b << 20 | a << 8 | d | PT +def bra(offset): return 0xE24000000000000F | (offset & 0xFFFFFF) << 20 | PT +EXIT = 0xE30000000007000F + + +def f32bits(x): return int(np.array([x], dtype=np.float32).view(np.uint32)[0]) +def h16bits(x): return int(np.array([x], dtype=np.float16).view(np.uint16)[0]) +def hpair(lo, hi): return h16bits(lo) | h16bits(hi) << 16 + + +def regs_all(**regs): + return {"*": {k: v for k, v in regs.items()}} + + +def per_lane(reg, values): + return {l: {reg: int(v) & 0xFFFFFFFF} for l, v in enumerate(values)} + + +def merge(*specs): + out: dict = {} + for s in specs: + for k, v in s.items(): + out.setdefault(k, {}).update(v) + return out + + +# ---- part 1: hand-encoded sequences ------------------------------------------------------------------------------- + +def part1(): + # IADD32I, MOV32I, IADD3 with a negated operand, ISCADD. + r = emu.run([mov32i(1, 0x7FFFFFFF), iadd32i(2, 0, 5), iadd32i(3, 1, 1), iadd32i(4, 0, -3), + iadd3(5, 0, 2, 1, neg_b=True), iscadd(6, 0, 2, 4), EXIT], per_lane("R0", lane * 3)) + check("IADD32I", r[:, 2], lane * 3 + 5) + check("IADD32I wraps", r[:, 3], 0x80000000) + check("IADD32I negative", r[:, 4], (lane.astype(np.int64) * 3 - 3) & 0xFFFFFFFF) + check("IADD3 -B", r[:, 5], (lane.astype(np.int64) * 3 - (lane * 3 + 5) + 0x7FFFFFFF) & 0xFFFFFFFF) + check("ISCADD", r[:, 6], ((lane.astype(np.int64) * 3) << 4) + lane * 3 + 5) + check("RZ stays 0", r[:, 255], 0) + + # S2R lane id, LOP32I, SHL/SHR, LOP3, SHR by register. + r = emu.run([s2r(1), lop32i(2, 1, 3, 0), shl_i(3, 1, 3), shr_i(4, 1, 2), lop32i(5, 1, 0xF0, 1), + lop32i(6, 1, 0x5, 2), lop3(7, 1, 2, 3, 0x96), mov32i(8, 0x80000000), shr_i(9, 8, 4, signed=True), + shr_r(10, 3, 2), EXIT], {}) + check("S2R SR_LANEID", r[:, 1], lane) + check("LOP32I.AND", r[:, 2], lane & 3) + check("SHL imm", r[:, 3], lane << 3) + check("SHR imm", r[:, 4], lane >> 2) + check("LOP32I.OR", r[:, 5], lane | 0xF0) + check("LOP32I.XOR", r[:, 6], lane ^ 5) + check("LOP3 0x96 (xor3)", r[:, 7], lane ^ (lane & 3) ^ (lane << 3)) + check("SHR.S32", r[:, 9], 0xF8000000) + check("SHR by register", r[:, 10], (lane << 3) >> (lane & 3)) + + # SHFL.IDX: the whole warp by a lane register; a segment of 4 lanes; out of range lanes keep their own value. + vals = lane * 0x101 + 7 + seq = [s2r(1), iadd32i(2, 1, 1), lop32i(2, 2, 31, 0), + shfl(3, 0, 2, 31, c_imm=True), # (lane + 1) % 32 + shfl(4, 0, 1, 0x1C03, b_imm=True, c_imm=True), # segments of 4: lane (t & ~3) | 1 + shfl(5, 0, 5, 0x1803, b_imm=True, c_imm=True, pd=0), # segments of 8, clamp 3: index 5 invalid + shfl(6, 0, 2, 0x1803, b_imm=True, c_imm=True, pd=1), # index 2 valid + shfl(7, 0, 1, 0x1F, mode=3, b_imm=True, c_imm=True), # BFLY 1 + shfl(8, 0, 2, 0x00, mode=2, b_imm=True, c_imm=True), # DOWN 2, clamp 0: lanes > 0... invalid + shfl(9, 0, 2, 0x1F, mode=2, b_imm=True, c_imm=True), # DOWN 2 + shfl(10, 0, 1, 0, mode=1, b_imm=True, c_imm=True), # UP 1 + EXIT] + w = emu.run_warp(seq, per_lane("R0", vals)) + r = w.r + check("SHFL.IDX lane register", r[:, 3], vals[(lane + 1) % 32]) + check("SHFL.IDX segment 4", r[:, 4], vals[(lane & ~3) | 1]) + check("SHFL.IDX clamp invalid keeps own value", r[:, 5], vals) + check("SHFL.IDX clamp invalid predicate", w.p[:, 0], False) + check("SHFL.IDX in clamp", r[:, 6], vals[(lane & ~7) | 2]) + check("SHFL.IDX valid predicate", w.p[:, 1], True) + check("SHFL.BFLY", r[:, 7], vals[lane ^ 1]) + check("SHFL.DOWN clamp 0", r[:, 8], vals) + check("SHFL.DOWN", r[:, 9], np.where(lane + 2 <= 31, vals[np.minimum(lane + 2, 31)], vals)) + check("SHFL.UP", r[:, 10], np.where(lane >= 1, vals[np.maximum(lane.astype(np.int64) - 1, 0)], vals)) + + # STS / LDS: 32, 16 and 8 bits, sign extension, 64 and 128 bits, the immediate offset. + seq = [s2r(1), shl_i(2, 1, 2), # R2 = 4 * lane + sts(1, 2, 0x100, 4), # word [0x100 + 4t] = t + lds(3, 2, 0x100, 4), + lds(4, 2, 0x100, 2), lds(5, 2, 0x102, 2), lds(6, 2, 0x101, 0), + mov32i(7, 0x8001F0F0), sts(7, 255, 0x40, 4), lds(8, 255, 0x40, 3), lds(9, 255, 0x42, 3), + lds(10, 255, 0x40, 1), lds(11, 255, 0x41, 0), + lds(12, 255, 0x100, 6), # R12-R15 = words 0..3 of the table + mov32i(16, 0xABCD), sts(16, 255, 0x82, 2), lds(17, 255, 0x80, 4), + EXIT] + r = emu.run(seq, {}) + check("LDS.32", r[:, 3], lane) + check("LDS.U16", r[:, 4], lane) + check("LDS.U16 high half", r[:, 5], 0) + check("LDS.U8 byte 1", r[:, 6], 0) + check("LDS.S16", r[:, 8], 0xFFFFF0F0) + check("LDS.S16 high", r[:, 9], 0xFFFF8001) + check("LDS.S8", r[:, 10], 0xFFFFFFF0) + check("LDS.U8", r[:, 11], 0xF0) + for k in range(4): + check(f"LDS.128 word {k}", r[:, 12 + k], k) + check("STS.U16 at +2", r[:, 17], 0xABCD0000) + + # F2F: F32 from either F16 half (exact), F16 from F32 (round to nearest even, high half zero). + src = hpair(1.5, -0.333251953125) + seq = [mov32i(1, src), f2f(2, 1, 2, 1), f2f(3, 1, 2, 1, high=True), + mov32i(4, f32bits(1 / 3)), f2f(5, 4, 1, 2), + mov32i(6, f32bits(1 + 2 ** -11)), f2f(7, 6, 1, 2), # tie: to even (1.0) + mov32i(8, f32bits(1 + 3 * 2 ** -11)), f2f(9, 8, 1, 2), # tie: to even (1 + 2^-9) + mov32i(10, f32bits(70000.0)), f2f(11, 10, 1, 2), # overflow: inf + EXIT] + r = emu.run(seq, {}) + check("F2F.F32.F16 H0", r[:, 2], f32bits(1.5)) + check("F2F.F32.F16 H1", r[:, 3], f32bits(-0.333251953125)) + check("F2F.F16.F32 1/3", r[:, 5], 0x3555) + check("F2F.F16.F32 tie down", r[:, 7], 0x3C00) + check("F2F.F16.F32 tie up", r[:, 9], 0x3C02) + check("F2F.F16.F32 overflow", r[:, 11], 0x7C00) + + # HFMA2: per half F32 FMA then F16; neg C; saturate. + seq = [mov32i(1, hpair(1.5, 2.0)), mov32i(2, hpair(2.0, -0.5)), mov32i(3, hpair(0.25, 1.0)), + hfma2(4, 1, 2, 3), hfma2(5, 1, 2, 3, neg_c=True), hfma2(6, 1, 2, 3, sat=True), + mov32i(7, hpair(1 + 2 ** -10, 1.0)), mov32i(8, hpair(1 + 2 ** -10, 2 ** -11)), mov32i(9, hpair(0.0, 1.0)), + hfma2(10, 7, 8, 9), EXIT] + r = emu.run(seq, {}) + check("HFMA2", r[:, 4], hpair(3.25, 0.0)) + check("HFMA2 -C", r[:, 5], hpair(2.75, -2.0)) + check("HFMA2.SAT", r[:, 6], hpair(1.0, 0.0)) + # (1 + 2^-10)^2 = 1 + 2^-9 + 2^-20 -> F16 1 + 2^-9; 1 + 2^-11 is an F16 tie -> 1.0. + check("HFMA2 rounding", r[:, 10], hpair(1 + 2 ** -9, 1.0)) + + # Ryujinx Optimizer.PropagatePack: an F16 result unpacked again without an integer operation in between is the + # unrounded F32 value (directly or through a MOV); LOP in between, or a guarded pack, keeps the F16 rounding. + third, third16 = f32bits(1 / 3), f32bits(float(np.float16(1 / 3))) + seq = [mov32i(1, third), f2f(2, 1, 1, 2), f2f(3, 2, 2, 1), # pack, unpack: folded + mov(4, 2), f2f(5, 4, 2, 1), # through a MOV: folded + lop32i(6, 2, 0xFFFFFFFF, 0), f2f(7, 6, 2, 1), # through LOP: rounded + s2r(8), isetp_i(0, 8, 32, 1), # P0 = lane < 32, true everywhere + f2f(9, 1, 1, 2) & ~(0xF << 16) | guard(0), f2f(10, 9, 2, 1), # guarded pack: rounded + mov32i(11, hpair(40000.0, 1.0)), mov32i(12, hpair(2.0, 2.0)), mov32i(13, 0), mov32i(16, hpair(0.5, 0.5)), + hfma2(14, 11, 12, 13), hfma2(15, 14, 16, 13), # 40000 * 2 = inf as F16, * 0.5 again + EXIT] + r = emu.run(seq, {}) + check("F2F.F16 then F2F.F32: unrounded (PropagatePack)", r[:, 3], third) + check("... through MOV: unrounded", r[:, 5], third) + check("... through LOP: rounded", r[:, 7], third16) + check("... guarded pack: rounded", r[:, 10], third16) + check("HFMA2 overflow stored as F16 inf", r[:, 14], hpair(np.inf, 2.0)) + check("HFMA2 into HFMA2: the unrounded 80000 halved", r[:, 15], hpair(40000.0, 1.0)) + + # FFMA: rounded once. Exact a*b + c lies just below an F32 halfway point; F64 alone would round to the tie and + # then to even (0x3f800002), the fused result is 0x3f800001. + seq = [mov32i(1, 0x39800001), mov32i(2, 0x397FFFFE), mov32i(3, 0x3F800001), ffma(4, 1, 2, 3), + mov32i(5, f32bits(3.0)), mov32i(6, f32bits(0.5)), ffma(7, 5, 6, 5), EXIT] + r = emu.run(seq, {}) + check("FFMA single rounding at a tie", r[:, 4], 0x3F800001) + check("FFMA", r[:, 7], f32bits(4.5)) + + # PRMT: a no-op as Ryujinx translates it, the Maxwell byte permute otherwise (0xbb99: sign of bytes 1 and 3). + seq = [mov32i(1, 0x7F80C000), mov32i(2, 0xBB99), mov32i(3, 0x11111111), prmt(3, 1, 2, 255), + mov32i(4, 0x3210), mov32i(5, 0x22222222), prmt(5, 1, 4, 255), EXIT] + r = emu.run(seq, {}, prmt="ryujinx") + check("PRMT (Ryujinx: no-op)", r[:, 3], 0x11111111) + r = emu.run(seq, {}, prmt="maxwell") + check("PRMT sign replicate (Maxwell)", r[:, 3], 0x0000FFFF) + check("PRMT identity (Maxwell)", r[:, 5], 0x7F80C000) + + # Predicates: ISETP, a guarded write, an inverted guard, EXIT of some lanes, a uniform BRA over one instruction. + seq = [s2r(1), isetp_i(0, 1, 16, 1), # P0 = lane < 16 + mov32i(2, 0xAAAA, g=guard(0)), mov32i(3, 0xBBBB, g=guard(0, inv=True)), + bra(emu.slot_address(6) - emu.slot_address(4) - 8), mov32i(4, 0xDEAD), # skipped + mov32i(5, 0x5555), + (EXIT & ~(0xF << 16)) | guard(0, inv=True), # lanes >= 16 exit + mov32i(6, 0x6666), EXIT] + w = emu.run_warp(seq, regs_all(R2=1, R3=1, R6=1)) + check("ISETP.LT", w.p[:, 0], lane < 16) + check("guarded write", w.r[:, 2], np.where(lane < 16, 0xAAAA, 1)) + check("inverted guard", w.r[:, 3], np.where(lane < 16, 1, 0xBBBB)) + check("BRA skips", w.r[:, 4], 0) + check("after BRA", w.r[:, 5], 0x5555) + check("guarded EXIT", w.r[:, 6], np.where(lane < 16, 0x6666, 1)) + + # Unknown opcodes raise with the word. + global checks + checks += 1 + try: + emu.run([0xFFFFFFFFFFFFFFFF], {}) + failures.append("unknown opcode did not raise") + except emu.UnknownOpcode as e: + if "0xffffffffffffffff" not in str(e): + failures.append(f"unknown opcode message: {e}") + + +# ---- part 2: lowered SM86 tensor instructions --------------------------------------------------------------------- + +def hmma_case(k16: bool, rng): + """HMMA.{1688,16816}.F16 R4, R8, R12, R16 with small integers; returns (sm86, regs_in, expected D regs).""" + K = 16 if k16 else 8 + A = rng.integers(-3, 4, size=(16, K)).astype(np.float64) + B = rng.integers(-3, 4, size=(K, 8)).astype(np.float64) + C = rng.integers(-8, 9, size=(16, 8)).astype(np.float64) + D = A @ B + C + regs_in, want = {}, {} + for t in range(LANES): + g, q = t // 4, t % 4 + regs = {} + for j in range(K // 8): + for r in range(2): + regs[f"R{8 + 2 * j + r}"] = hpair(A[g + 8 * r, 8 * j + 2 * q], A[g + 8 * r, 8 * j + 2 * q + 1]) + regs[f"R{12 + j}"] = hpair(B[8 * j + 2 * q, g], B[8 * j + 2 * q + 1, g]) + for r in range(2): + regs[f"R{16 + r}"] = hpair(C[g + 8 * r, 2 * q], C[g + 8 * r, 2 * q + 1]) + regs_in[t] = regs + want[t] = {4 + r: hpair(D[g + 8 * r, 2 * q], D[g + 8 * r, 2 * q + 1]) for r in range(2)} + w0 = 0x23C | 7 << 12 | 4 << 16 | 8 << 24 + sm86 = [w0, 12, 16 | (1 << 11 if k16 else 0), 0] + return sm86, regs_in, want + + +def part2(exe: Path): + rng = np.random.default_rng(86) + for k16 in (False, True): + sm86, regs_in, want = hmma_case(k16, rng) + words, urbase, _ = emu.lower(sm86, exe) + r = emu.run(words, regs_in) + for reg in (4, 5): + check(f"HMMA.{'16816' if k16 else '1688'} D R{reg} (lowered, {len(words)} words)", + r[:, reg], [want[t][reg] for t in range(LANES)]) + + # LDSM.16.M88.4 R4, [R1] and LDSM.16.MT88.1 R8, [R1]: matrix m row r at 0x200 * m + 0x20 * r (16 bytes used), + # element (m, row, col) = 0x1000 * m + 0x100 * row + col. + smem = bytearray(0x1000) + for m in range(4): + for row in range(8): + for col in range(8): + at = 0x200 * m + 0x20 * row + 2 * col + smem[at:at + 2] = (0x1000 * m + 0x100 * row + col).to_bytes(2, "little") + addr = {t: {"R1": 0x200 * (t // 8) + 0x20 * (t % 8)} for t in range(LANES)} + el = lambda m, row, col: 0x1000 * m + 0x100 * row + col + for transpose, size, dst in ((False, 2, 4), (True, 0, 8), (True, 2, 12), (False, 0, 16)): + sm86 = [0x83B | 7 << 12 | dst << 16 | 1 << 24, 0, size << 8 | (0x4000 if transpose else 0), 0] + words, _, _ = emu.lower(sm86, exe) + r = emu.run(words, addr, bytes(smem)) + for m in range(1 << size): + if transpose: # column t/4 of rows 2(t%4) (low) and 2(t%4)+1 (high) + want = [el(m, 2 * (t % 4), t // 4) | el(m, 2 * (t % 4) + 1, t // 4) << 16 for t in range(LANES)] + else: # row t/4, columns 2(t%4), 2(t%4)+1 + want = [el(m, t // 4, 2 * (t % 4)) | el(m, t // 4, 2 * (t % 4) + 1) << 16 for t in range(LANES)] + check(f"LDSM.16.{'MT88' if transpose else 'M88'}.{1 << size} matrix {m}", r[:, dst + m], want) + + # MOVM.16.MT88 R2, R1: element (row, col) = 8 * row + col at lane 4 row + col / 2, half col % 2. + regs_in = {t: {"R1": (8 * (t // 4) + 2 * (t % 4)) | (8 * (t // 4) + 2 * (t % 4) + 1) << 16} for t in range(LANES)} + words, _, _ = emu.lower([0x23A | 7 << 12 | 2 << 16 | 1 << 24, 0, 0, 0], exe) + r = emu.run(words, regs_in) + want = [(8 * (2 * (t % 4)) + t // 4) | (8 * (2 * (t % 4) + 1) + t // 4) << 16 for t in range(LANES)] + check("MOVM.16.MT88", r[:, 2], want) + + # HFMA2.RELU R0, R1, R2, R3 (PT) as the translator runs it (no PRMT: AmpereLowering.cs Half builds the mask with + # integer operations), whatever the scratch registers held before: negative sums become +0, positive ones stay. + sm86 = [0x231 | 7 << 12 | 0 << 16 | 1 << 24, 2, 3 | 0x8000 | 0x03800000, 0] + words, _, _ = emu.lower(sm86, exe) + for a, b, c, want in (((1.0, -2.0), (-3.0, 0.5), (1.0, 0.25), (0.0, 0.0)), # sums -2.0, -0.75 + ((1.0, 2.0), (3.0, -0.5), (1.0, 0.25), (4.0, 0.0))): # sums 4.0, -0.75 + regs_in = regs_all(R1=hpair(*a), R2=hpair(*b), R3=hpair(*c)) + for fill in (0, 0x7FC0DEAD): + w = emu.run_warp(words, regs_in, prmt="ryujinx", fill=fill) + check(f"HFMA2.RELU -> {want} as translated (fill {fill:#x})", w.r[:, 0], hpair(*want)) + check(f"HFMA2.RELU uses no PRMT (fill {fill:#x})", [len(w.warnings)], 0) + + +def main() -> int: + part1() + exe = emu.DEFAULT_XLATE + if exe.exists(): + part2(exe) + else: + print(f" skip: part 2 needs {exe}") + if failures: + print("\n".join("FAIL " + f for f in failures)) + print(f"{checks - len(failures)}/{checks} checks passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tensor_lowering_check.py b/tests/tensor_lowering_check.py new file mode 100644 index 00000000..7f21a20c --- /dev/null +++ b/tests/tensor_lowering_check.py @@ -0,0 +1,441 @@ +"""Checks the lowered SM86 tensor instructions against results captured on an NVIDIA GPU, in one command. + +tools/xlate/AmpereLowering.cs rewrites the SM86 instructions of the game's neural-network passes as Maxwell code: +Lowerer.Hmma (HMMA.1688 / HMMA.16816, .F16 and .F32), Lowerer.Ldsm (LDSM.16.M88 / MT88 .1/.2/.4), Lowerer.Movm +(MOVM.16.MT88) and Lowerer.Half (HFMA2, HFMA2.RELU). This script lowers every case of the vector file with +`nx2navi_xlate --lower`, runs the Maxwell words on tests/maxwell_warp_emu.py (which executes them as the Ryujinx +translator does) and compares with the registers the GPU wrote. Exit status 1 on any failure. The vector file names +the GPU that produced it ("device"). So far that is an RTX 5090 running the compute_86 PTX through the driver's JIT, +as no SM86 part was at hand: the results follow the PTX semantics but need not show SM86's internal precision. + + C:/Users/PC/AppData/Local/Programs/Python/Python312/python tests/tensor_lowering_check.py [vectors.json] + [--xlate EXE] [--case NAME] [-v] + +Defaults: vectors.json is /work/tensor-gt/vectors.json (/work/tensor-gt/gen_vectors.py writes +it: each case is one SM86 instruction in inline PTX, run on one warp), the translator +/build/xlate/nx2navi_xlate.exe (python tools/build_xlate.py builds it); is the directory two +levels above this repository. numpy is needed. + +Every case runs several times and must pass each run: + * with the registers the vectors do not set first 0, then 0x7fc0dead, so a read of a register the lowering never + wrote shows; + * then under @P0 (the guard field of the SM86 word set to P0) with P0 false in every third lane: lanes with P0 get + the GPU's result, the others keep their old destination values. The other lanes still supply their fragments, + rows and addresses, which works only if the shuffles run in every lane and just the destination writes take the + guard, the rule AmpereLowering.cs follows. +Each run checks: + * destination registers bit for bit, except that a NaN matches any NaN in a float result (maxwell_warp_emu + same_value: SPIR-V leaves NaN payloads to the host, the GPU writes 0x7fff). HMMA gets no tolerance: the GPU rounds + every element once (f16 or f32 of the exact A*B+C), and so does the lowering for these values, whose sums are + exact in F32; the largest difference in ulp is reported; + * every other register R0-R236 (guest and uniform registers; R237 up is the lowering's own) and P0-P6 keep their + values, shared memory is untouched, and the emulator has nothing to report (a shuffle reading a lane its guard + left out, a PRMT the translator drops); + * urbase, the register the lowering mapped UR0 to, is the highest register the instruction names plus two, rounded + down to even: AmpereLowering HighestGpr counted every register of the instruction's fragments and quads. +Read off the lowered words (both lowerings): every write to a guest register carries the instruction's guard and no +SHFL is guarded. +Derived cases, beyond the hardware vectors, for forms the game uses: reference models of HFMA2(.RELU) and LDSM, each +first checked against every hardware case of its instruction, give the expected values of HFMA2.RELU with an +immediate B (opcode 0x831, as in the game's HFMA2.RELU Rd, Ra, 1, 1, Rc) and LDSM .M88.4 / .MT88.4 with an offset. +""" + +from __future__ import annotations + +import argparse +import sys +from fractions import Fraction +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import maxwell_warp_emu as emu # noqa: E402 + +LANES = 32 +RZ = 255 +GUEST_REGS = 237 # R0-R236: guest and uniform registers; AmpereLowering.cs uses R237 up itself +FILLS = (0x00000000, 0x7FC0DEAD) +EXIT = 0xE30000000007000F # Maxwell EXIT: runs no code, gives the emulator's initial state +DEFAULT_VECTORS = emu.ROOT.parent.parent / "work" / "tensor-gt" / "vectors.json" + + +def guard_lanes() -> np.ndarray: + """P0 for the guarded run: false in lanes 1, 4, 7, ... so every quad and row group mixes both.""" + return np.arange(LANES) % 3 != 1 + + +# ---- the instructions of the vector file ---------------------------------------------------------------------- + +def words_of(sm86) -> list[int]: + return [emu.parse_int(x) & 0xFFFFFFFF for x in sm86] + + +def operands(sm86) -> tuple[list[int], list[int]]: + """(destination registers, source registers) one SM86 instruction names, from its fields (work/tensor-gt/README.md + "Encoding bits"; the same fields AmpereLowering.cs decodes).""" + w0, w1, w2, _ = words_of(sm86) + op = w0 & 0xFFF + rd, ra, rb, rc = (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF, w1 & 0xFF, w2 & 0xFF + + def regs(base, n): + return [] if base == RZ else list(range(base, base + n)) + if op == 0x23C: # HMMA: shape bit 75, .F32 bit 76 + k16, f32 = (w2 >> 11) & 1, (w2 >> 12) & 1 + cd = 4 if f32 else 2 + return regs(rd, cd), regs(ra, 4 if k16 else 2) + regs(rb, 2 if k16 else 1) + regs(rc, cd) + if op == 0x83B: # LDSM: count 1 << bits 72-73 + return regs(rd, 1 << ((w2 >> 8) & 3)), regs(ra, 1) + if op == 0x23A: # MOVM + return regs(rd, 1), regs(ra, 1) + if op in (0x231, 0x831): # HFMA2: B a register (0x231) or an immediate (0x831) + return regs(rd, 1), regs(ra, 1) + (regs(rb, 1) if op == 0x231 else []) + regs(rc, 1) + raise ValueError(f"no operand table for SM86 opcode 0x{op:03x}") + + +def expected_urbase(sm86) -> int: + dests, srcs = operands(sm86) + return (max(dests + srcs + [0]) + 2) & ~1 + + +def shared_memory(case: dict) -> bytes | None: + """The case's shared memory as maxwell_warp_emu.run_case places it: smem at smem_base, 64 KiB at least.""" + if case.get("smem") is None: + return None + base = emu.parse_int(case.get("smem_base", 0)) + data = emu.parse_bytes(case["smem"]) + buf = bytearray(max(0x10000, base + len(data))) + buf[base:base + len(data)] = data + return bytes(buf) + + +def consistency(case: dict) -> list[str]: + """Problems of the case itself: regs_out not exactly the destinations, LDSM addresses that disagree.""" + problems = [] + dests, _ = operands(case["sm86"]) + for lane, regs in emu.load_regs(case["regs_out"]).items(): + if sorted(n for kind, n in regs if kind == "R") != sorted(dests): + problems.append(f"lane {lane}: regs_out has {sorted(n for _, n in regs)}, the instruction writes {dests}") + break + if case.get("lane_addresses") is not None: + ra = (words_of(case["sm86"])[0] >> 24) & 0xFF + base = emu.parse_int(case.get("smem_base", 0)) + regs_in = emu.load_regs(case["regs_in"]) + for lane, a in enumerate(case["lane_addresses"]): + given = regs_in[lane].get(("R", ra)) + if given is not None and given != (emu.parse_int(a) + base) & 0xFFFFFFFF: + problems.append(f"lane {lane}: R{ra} = {given:#x} but lane_addresses says {emu.parse_int(a) + base:#x}") + break + return problems + + +# ---- running and comparing -------------------------------------------------------------------------------------- + +class Outcome: + def __init__(self): + self.problems: list[str] = [] + self.worst_ulp = 0 + self.nan_matches = 0 + self.runs = 0 + + +def check_run(out: Outcome, label: str, words, case: dict, fill: int, guard: np.ndarray | None, verbose: bool): + """One run of the lowered words; appends what is wrong to out.problems.""" + sm86 = case["sm86"] + dests, _ = operands(sm86) + kind = emu.result_kind(sm86) + regs_in = emu.load_regs(case["regs_in"]) + if case.get("lane_addresses") is not None: + ra = (words_of(sm86)[0] >> 24) & 0xFF + base = emu.parse_int(case.get("smem_base", 0)) + for lane, a in enumerate(case["lane_addresses"]): + regs_in[lane].setdefault(("R", ra), (emu.parse_int(a) + base) & 0xFFFFFFFF) + if guard is not None: + for lane in range(LANES): + regs_in[lane][("P", 0)] = bool(guard[lane]) + expected = emu.load_regs(case["regs_out"]) + smem = shared_memory(case) + start = emu.run_warp([EXIT], regs_in, smem, fill=fill) + warp = emu.run_warp(words, regs_in, smem, fill=fill) + out.runs += 1 + where = f"{label}, fill {fill:#010x}" + bad = [] + for lane in range(LANES): + on = guard is None or guard[lane] + for n in dests: + got = int(warp.r[lane, n]) + want = expected[lane][("R", n)] if on else int(start.r[lane, n]) + k = kind if on else "bits" + if k != "bits": + out.worst_ulp = max(out.worst_ulp, emu.ulps(got, want, k)) + if not emu.same_value(got, want, k): + bad.append(f"lane {lane:2d} R{n}: got {emu._fmt(got)} want {emu._fmt(want & 0xFFFFFFFF)}" + + ("" if on else " (guard false: old value)")) + elif got != want: + out.nan_matches += 1 + if bad: + out.problems.append(f"{where}: {len(bad)} wrong destination values") + out.problems += [" " + b for b in bad[:12 if verbose else 4]] + keep = [n for n in range(GUEST_REGS) if n not in dests] + lanes, cols = np.nonzero(warp.r[:, keep] != start.r[:, keep]) + if len(lanes): + regs = sorted({keep[c] for c in cols}) + out.problems.append(f"{where}: registers outside the destination changed: R{regs[:8]} in lanes {sorted(set(lanes.tolist()))[:8]}") + if (warp.p[:, :7] != start.p[:, :7]).any(): + out.problems.append(f"{where}: guest predicates P0-P6 changed") + if bytes(warp.smem) != bytes(start.smem): + out.problems.append(f"{where}: shared memory changed") + for m in dict.fromkeys(warp.warnings): + out.problems.append(f"{where}: emulator: {m}") + + +# Maxwell instructions of maxwell_warp_emu.TABLE without a destination register in bits 0-7. +NO_REGISTER_RESULT = {"EXIT", "BRA", "NOP", "STS", "STL", "ISETP", "ISETP.I", "R2P", "R2P.I"} + + +def guard_rule_problems(words, guard: int, label: str) -> list[str]: + """AmpereLowering.cs's guard rule read off the lowered words: a write to a guest register (below R237) carries the + SM86 instruction's guard (Maxwell guard field `guard`), a shuffle runs unguarded, in every lane.""" + problems = [] + for k, w in enumerate(words): + name, _ = emu.decode(w) + g = (w >> 16) & 0xF + if name == "SHFL" and g != 7: + problems.append(f"{label}: word {k} {w:016x}: SHFL under a guard") + if name not in NO_REGISTER_RESULT and (w & 0xFF) < GUEST_REGS and g != guard: + problems.append(f"{label}: word {k} {w:016x}: {name} writes guest register R{w & 0xFF} without the instruction's guard") + return problems + + +def check_case(case: dict, exe: Path, verbose: bool = False) -> Outcome: + out = Outcome() + out.problems += [f"vector file: {p}" for p in consistency(case)] + try: + words, urbase, _ = emu.lower(case["sm86"], exe) + except RuntimeError as e: + out.problems.append(str(e)) + return out + if urbase != expected_urbase(case["sm86"]): + out.problems.append(f"urbase {urbase}, expected {expected_urbase(case['sm86'])}: HighestGpr misses a register range") + out.problems += guard_rule_problems(words, (words_of(case["sm86"])[0] >> 12) & 0xF, "lowered words") + for fill in FILLS: + check_run(out, "all lanes", words, case, fill, None, verbose) + # The same instruction under @P0: SM86 guard bits 12-15 = 0. + w = words_of(case["sm86"]) + guarded = dict(case, sm86=[w[0] & ~0xF000] + w[1:]) + gwords, _, _ = emu.lower(guarded["sm86"], exe) + out.problems += guard_rule_problems(gwords, 0, "lowered words under @P0") + check_run(out, "@P0", gwords, guarded, FILLS[1], guard_lanes(), verbose) + return out + + +# ---- reference models for the derived cases ------------------------------------------------------------------------ + +def half_value(h: int): + """An F16 bit pattern as ("nan"), ("inf", sign) or ("num", Fraction, sign).""" + sign = h >> 15 + e, m = (h >> 10) & 0x1F, h & 0x3FF + if e == 0x1F: + return ("nan",) if m else ("inf", sign) + v = Fraction(m, 1 << 24) if e == 0 else Fraction(0x400 | m, 1 << 25) * (Fraction(2) ** e) + return ("num", -v if sign else v, sign) + + +def round_half(x: Fraction) -> int: + """x (not zero) rounded to F16, to nearest even; beyond the range infinity.""" + sign = 0x8000 if x < 0 else 0 + m = abs(x) + e = m.numerator.bit_length() - m.denominator.bit_length() + if Fraction(2) ** e > m: + e -= 1 + elif Fraction(2) ** (e + 1) <= m: + e += 1 + quantum = Fraction(2) ** (max(e, -14) - 10) + q = m / quantum + n = q.numerator // q.denominator + rest = q - n + if rest > Fraction(1, 2) or (rest == Fraction(1, 2) and n & 1): + n += 1 + value = n * quantum + if value >= 65536: + return sign | 0x7C00 + return sign | int(np.array([float(value)], dtype=np.float64).astype(np.float16).view(np.uint16)[0]) + + +def fma_half(a: int, b: int, c: int) -> int: + """F16 a * b + c rounded once (IEEE 754 fusedMultiplyAdd, round to nearest even); a NaN result is 0x7fff.""" + va, vb, vc = half_value(a), half_value(b), half_value(c) + if "nan" in (va[0], vb[0], vc[0]): + return 0x7FFF + zero = lambda v: v[0] == "num" and v[1] == 0 + if (va[0] == "inf" and zero(vb)) or (vb[0] == "inf" and zero(va)): + return 0x7FFF + psign = (a >> 15) ^ (b >> 15) + if va[0] == "inf" or vb[0] == "inf": + if vc[0] == "inf" and vc[1] != psign: + return 0x7FFF + return 0x7C00 | psign << 15 + if vc[0] == "inf": + return 0x7C00 | vc[1] << 15 + p = va[1] * vb[1] + s = p + vc[1] + if s == 0: # exact zero: -0 only when both terms are -0 + return 0x8000 if (p == 0 and psign and vc[2]) else 0 + return round_half(s) + + +def hfma2_model(a: int, b: int, c: int, relu: bool) -> int: + """HFMA2 (.RELU) on both halves (H1_H0 operands): RELU makes negative numbers and -0 +0, keeps NaN as 0x7fff.""" + res = 0 + for shift in (0, 16): + h = fma_half((a >> shift) & 0xFFFF, (b >> shift) & 0xFFFF, (c >> shift) & 0xFFFF) + if relu and h != 0x7FFF and h & 0x8000: + h = 0 + res |= h << shift + return res + + +def hfma2_operands(case: dict, lane: int, b_imm: int | None = None): + w0, w1, w2, _ = words_of(case["sm86"]) + regs = emu.load_regs(case["regs_in"])[lane] + get = lambda n: 0 if n == RZ else regs[("R", n)] + b = b_imm if b_imm is not None else get(w1 & 0xFF) + return get((w0 >> 24) & 0xFF), b, get(w2 & 0xFF), bool((w2 >> 15) & 1) + + +def ldsm_model(smem: bytes, addresses: list[int], count: int, transpose: bool, offset: int) -> list[list[int]]: + """LDSM per work/tensor-gt/README.md "Fragment layouts": out[lane][m] is lane's value of Rd + m.""" + out = [[0] * count for _ in range(LANES)] + rd16 = lambda at: int.from_bytes(smem[at:at + 2], "little") + for m in range(count): + rows = [addresses[8 * m + r] + offset for r in range(8)] + for t in range(LANES): + if transpose: + out[t][m] = rd16(rows[2 * (t % 4)] + 2 * (t // 4)) | rd16(rows[2 * (t % 4) + 1] + 2 * (t // 4)) << 16 + else: + at = rows[t // 4] + 4 * (t % 4) + out[t][m] = int.from_bytes(smem[at:at + 4], "little") + return out + + +def ldsm_fields(sm86): + w0, w1, w2, _ = words_of(sm86) + return (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF, 1 << ((w2 >> 8) & 3), bool((w2 >> 14) & 1), (w1 >> 8) & 0xFFFFFF + + +def model_problems(cases: list[dict]) -> dict[str, list[str]]: + """Each model against every hardware case of its instruction (exact bits: the models write 0x7fff for NaN).""" + found = {} + for case in cases: + w0 = words_of(case["sm86"])[0] + op = w0 & 0xFFF + expected = emu.load_regs(case["regs_out"]) + bad = [] + if op == 0x231: + rd = (w0 >> 16) & 0xFF + for lane in range(LANES): + want = expected[lane][("R", rd)] + got = hfma2_model(*hfma2_operands(case, lane)) + if got != want: + bad.append(f"lane {lane}: model {got:08x}, GPU {want:08x}") + elif op == 0x83B: + rd, ra, count, transpose, offset = ldsm_fields(case["sm86"]) + regs = emu.load_regs(case["regs_in"]) + model = ldsm_model(shared_memory(case), [regs[t][("R", ra)] for t in range(LANES)], count, transpose, offset) + for t in range(LANES): + for m in range(count): + if model[t][m] != expected[t][("R", rd + m)]: + bad.append(f"lane {t} R{rd + m}: model {model[t][m]:08x}, GPU {expected[t][('R', rd + m)]:08x}") + else: + continue + found[case["name"]] = bad + return found + + +def derived_cases(cases: list[dict]) -> list[dict]: + by_name = {c["name"]: c for c in cases} + out = [] + relu = by_name.get("hfma2_relu") + if relu is not None: + w0, _, w2, w3 = words_of(relu["sm86"]) + rb = words_of(relu["sm86"])[1] & 0xFF + for imm in (0x3C003C00, 0x4000BC00): # (1, 1) as the game's networks use it; (-1, 2) + regs_in = [{k: v for k, v in lane.items() if k != f"R{rb}"} for lane in relu["regs_in"]] + sm86 = [(w0 & ~0xFFF) | 0x831, imm, w2, w3] + rd = (w0 >> 16) & 0xFF + regs_out = [] + for lane in range(LANES): + a, b, c, r = hfma2_operands(relu, lane, imm) + regs_out.append({f"R{rd}": f"{hfma2_model(a, b, c, r):08x}"}) + out.append({"name": f"derived_hfma2_relu_imm_{imm:08x}", "sm86": [f"{x:08x}" for x in sm86], + "regs_in": regs_in, "regs_out": regs_out}) + src = by_name.get("ldsm_m88_x4") + if src is not None: + w0, _, w2, w3 = words_of(src["sm86"]) + regs = emu.load_regs(src["regs_in"]) + rd, ra = (w0 >> 16) & 0xFF, (w0 >> 24) & 0xFF + addresses = [regs[t][("R", ra)] for t in range(LANES)] + for transpose in (False, True): + sm86 = [w0, 0x40 << 8, (w2 & ~0x4000) | (0x4000 if transpose else 0), w3] + model = ldsm_model(shared_memory(src), addresses, 4, transpose, 0x40) + regs_out = [{f"R{rd + m}": f"{model[t][m]:08x}" for m in range(4)} for t in range(LANES)] + out.append(dict(src, name=f"derived_ldsm_{'mt88' if transpose else 'm88'}_x4_off40", + sm86=[f"{x:08x}" for x in sm86], regs_out=regs_out)) + return out + + +# ---- driver ------------------------------------------------------------------------------------------------------- + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("vectors", nargs="?", type=Path, default=DEFAULT_VECTORS, help=f"vector file (default {DEFAULT_VECTORS})") + ap.add_argument("--xlate", type=Path, default=emu.DEFAULT_XLATE, help="nx2navi_xlate executable") + ap.add_argument("--case", action="append", help="only cases whose name contains this (repeatable)") + ap.add_argument("-v", "--verbose", action="store_true", help="more differing values per failure") + args = ap.parse_args(argv) + if not args.vectors.exists(): + print(f"no vector file {args.vectors}") + return 1 + if not args.xlate.exists(): + print(f"no translator {args.xlate} (python tools/build_xlate.py)") + return 1 + cases = emu.load_cases(args.vectors) + failed = 0 + + models = model_problems(cases) + for name, bad in models.items(): + if bad: + failed += 1 + print(f"FAIL model vs GPU on {name}: " + "; ".join(bad[:4])) + model_ok = not any(models.values()) + print(f"reference models (HFMA2, LDSM) agree with the GPU on {sum(1 for b in models.values() if not b)}/{len(models)} cases") + + todo = cases + (derived_cases(cases) if model_ok else []) + passed = 0 + for case in todo: + name = case["name"] + if args.case and not any(c in name for c in args.case): + continue + try: + out = check_case(case, args.xlate, args.verbose) + except Exception as e: # one broken case does not stop the run + out = Outcome() + out.problems.append(f"error: {type(e).__name__}: {e}") + kind = emu.result_kind(case["sm86"]) + notes = [] + if kind != "bits": + notes.append("largest difference " + ("NaN against a number" if out.worst_ulp >= 1 << 32 else f"{out.worst_ulp} ulp")) + if out.nan_matches: + notes.append(f"{out.nan_matches} NaN values with another NaN encoding than the GPU's") + ok = not out.problems + passed += ok + failed += not ok + print(f"{'PASS' if ok else 'FAIL'} {name} ({out.runs} runs{', ' + ', '.join(notes) if notes else ''})") + for p in out.problems: + print(" " + p) + print(f"{passed} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/build_xlate.py b/tools/build_xlate.py index a6bcacf4..b1bbec6b 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 19: - sys.exit(f"expected 19 patches, found {len(patches)}") + if len(patches) != 20: + sys.exit(f"expected 20 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): diff --git a/tools/xlate/AmpereLowering.cs b/tools/xlate/AmpereLowering.cs index 0a4be334..4e347b22 100644 --- a/tools/xlate/AmpereLowering.cs +++ b/tools/xlate/AmpereLowering.cs @@ -130,6 +130,7 @@ static class AmpereLowering int highest = HighestGpr(code); if (spillAt < 0 && highest >= UrLimit) throw new RegisterPressureException($"guest registers up to R{highest}"); UrBase = spillAt >= 0 ? SpillFrom : (highest + 2) & ~1; + LastUrBase = UrBase; } // Spill mode: the temporary that holds guest register r for this instruction, loading its group on first use. @@ -168,13 +169,14 @@ static class AmpereLowering if (b is >= 0x140 and <= 0x15f) continue; // branches, calls, exits, barriers: no register fields see(i.Bits(16, 8)); see(i.Bits(24, 8)); // Register ranges the base fields do not show: LDSM writes Rd..Rd+count-1; TXD reads Ra..Ra+2 and Rb..Rb+3 - // and writes the second destination pair; HMMA reads A, B and C fragments and writes the D pair. + // and writes the second destination pair; HMMA reads A (K/4 registers), B (K/8) and C and writes D, where + // C and D are a pair (.F16) or four registers (.F32, bit 76). if (op == 0x83b && i.Bits(16, 8) != RZ) see(i.Bits(16, 8) + (1u << (int)i.Bits(72, 2)) - 1); if (op == 0xb6c) { see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 2); see(i.Bits(32, 8) + 3); see(i.Bits(64, 8) + 1); } if (op == 0x23c) { - uint k16 = i.Bits(75, 1); - see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + 1); + uint k16 = i.Bits(75, 1), cd = i.Bit(76) ? 3u : 1u; + see(i.Bits(16, 8) + cd); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + cd); } if (b < 0x080 || b is >= 0x160 and <= 0x16f) see(i.Bits(64, 8)); // C of ALU forms, TEX second destinations if (i.Bits(9, 3) == 1) see(i.Bits(32, 8)); @@ -1507,10 +1509,13 @@ static class AmpereLowering } // HADD2 / HMUL2 / HFMA2 with packed-half results: saturate 77, FTZ 80, bit 76 the FMZ precision of HMUL2/HFMA2. - // HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered: the sum - // goes to TexOut, PRMT replicates each lane's sign bit over that lane (selector 0xbb99) and LOP.AND with the - // inverted mask clears the negative lanes (negative NaN becomes 0 too). F32 results (78), the H0_NH1 source (86) - // and absolute values where Maxwell has none are not lowered. HADD2's second operand is B, or C for forms 2/3. + // HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered. Measured + // on an NVIDIA GPU (tests/tensor_lowering_check.py): a negative result or -0 becomes +0, +inf stays, -inf + // becomes 0, a NaN result stays NaN (0x7fff). The sum goes to TexOut and integer operations build the mask of + // the half lanes to clear. Maxwell PRMT would be the obvious tool, but the translator emits nothing for it + // (Ryujinx 1.3.2, Instructions/InstEmit.cs, PrmtR only logs "not implemented"). + // F32 results (78), the H0_NH1 source (86) and absolute values where Maxwell has none are not lowered. HADD2's + // second operand is B, or C for forms 2/3. void Half(ulong maxwell) { uint op = _i.Bits(0, 12) & 0x1ff; @@ -1548,11 +1553,20 @@ static class AmpereLowering | ((ulong)c.reg << 39); } if (!relu) { EmitG(w); return; } - Emit(Unguarded(w)); - Mov32i(TexOut + 1, 0xbb99); // PRMT selector: bytes 0-1 the sign of byte 1, bytes 2-3 the sign of byte 3 - Emit(Unguarded(0x5bc0000000000000UL | ((ulong)RZ << 39) | ((ulong)(TexOut + 1) << 20) | ((ulong)TexOut << 8) | (uint)(TexOut + 2))); - // LOP.AND Rd, sum, ~mask (InvB bit 40), no predicate output (PT at 48) - EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)(TexOut + 2) << 20) | ((ulong)TexOut << 8) | (uint)Gpr((uint)Dst)); + // RELU on both half lanes of the sum x at once: each half h becomes |h| (its sign bit cleared), or 0 when h is + // a negative number (sign set, not NaN). |h| > 0x7c00 means NaN, so |h| + 0x3ff carries into bit 15 of its + // half exactly for a NaN, and never into the other half (|h| + 0x3ff < 0x10000). A NaN keeps the payload the + // host computed, with the sign cleared (the hardware's is 0x7fff). Scratch writes unguarded, Rd guarded. + const int x = TexOut, y = TexOut + 1, t = TexOut + 2; + Emit(Unguarded(w)); // HFMA2 x + Lop32i(y, x, 0x7fff7fff, 0); // y = |h| per half + Emit(Unguarded(0x1c00000000000000UL | (0x03ff03ffUL << 20) | ((ulong)y << 8) | t)); // IADD32I t = y + 0x03ff03ff + Emit(Unguarded(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)x << 8) | t)); // LOP.AND t = x & ~t + Emit(Unguarded(0x3828000000000000UL | (15UL << 20) | ((ulong)t << 8) | t)); // SHR t >>= 15 + Lop32i(t, t, 0x00010001, 0); // bit 0 / 16: clear that half + Emit(Unguarded(0x5c18000000000000UL | (16UL << 39) | (1UL << 48) | ((ulong)t << 20) | ((ulong)t << 8) | t)); // ISCADD t = (t << 16) - t + // LOP.AND Rd = y & ~t (InvB bit 40), no predicate output (PT at 48) + EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)t << 20) | ((ulong)y << 8) | (uint)Gpr((uint)Dst)); } // MOVM.16.MT88 Rd, Ra (0x23a): an 8x8 matrix of 16-bit elements spread over the warp (lane t holds row t/4, @@ -1973,33 +1987,39 @@ static class AmpereLowering } } - // HMMA.1688.F16 / HMMA.16816.F16 Rd, Ra, Rb, Rc: a warp-wide matrix multiply-accumulate D(16x8) = A(16xK) * - // B(Kx8) + C(16x8) on F16 elements, K = 8, or 16 when bit 75 is set. Fields: D 16-23 (a register pair), A 24-31 - // (K/4 registers), B 32-39 (K/8 registers), C 64-71 (a pair; RZ means a zero accumulator). With g = lane / 4 and - // q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every register): + // HMMA.1688 / HMMA.16816 Rd, Ra, Rb, Rc with F16 inputs: a warp-wide matrix multiply-accumulate D(16x8) = + // A(16xK) * B(Kx8) + C(16x8), K = 8, or 16 when bit 75 is set. Bit 76 selects F32 accumulators and results + // (.F32: C and D are four registers each) instead of F16 ones (.F16: a register pair). Fields: D 16-23, A 24-31 + // (K/4 registers), B 32-39 (K/8 registers), C 64-71 (RZ means a zero accumulator). With g = lane / 4 and + // q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every packed register): // A register 2j + r: row g + 8r, k 8j + 2q and 8j + 2q + 1 (j is 0 only when K = 8) // B register j: k 8j + 2q and 8j + 2q + 1, column g - // C / D register r: row g + 8r, columns 2q and 2q + 1 + // C / D, .F16: register r holds row g + 8r, columns 2q and 2q + 1 + // C / D, .F32: register 2r + c holds row g + 8r, column 2q + c // Maxwell has no tensor instructions, so the product is expanded per lane. For each k pair (2p, 2p + 1) the lane // fetches the A word of its own rows from lane 4g + p % 4 and, for each of its two columns n = 2q + c, the B word - // from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA (an - // F16 x F16 product is exact in F32). The accumulators are rounded to F16 after every eight k (one 1688 step of - // the hardware) and widened again when more follow; the final F16 values are packed as F2fp in this file does - // (F2F.F16 writes the low half and zeroes the high half, ISCADD merges the column pair). - // The work runs unguarded because every lane must supply its fragments to the shuffles; only the two result - // writes take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest - // operands are never there (guest and uniform registers end below R237, spill temporaries are R204-R235). + // from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA in + // F32 (an F16 x F16 product is exact in F32, so the step rounds once whether or not the host fuses it). + // The .F16 form rounds to F16 once, after all K products, as the hardware does: it does not round between the + // two eight-k halves of HMMA.16816. Measured on an NVIDIA GPU (tests/tensor_lowering_check.py compares this + // lowering with those results): a first-half sum of 65536 that the second half cancels gives 0, which an F16 + // rounding after the first half turns into +inf, and 1 + 2^-11 + 2^-11 split over the halves gives 1 + 2^-10, + // which that rounding turns into 1. The F16 results are then packed as F2fp in this file does (F2F.F16 writes + // the low half and zeroes the high half, ISCADD merges the column pair). + // The work runs unguarded because every lane must supply its fragments to the shuffles; only the result writes + // take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest operands + // are never there (guest and uniform registers end below R237, spill temporaries are R188-R235). void Hmma() { - bool k16 = _i.Bit(75); + bool k16 = _i.Bit(75), f32 = _i.Bit(76); uint d = (uint)Dst, a = _i.Bits(24, 8), b = _i.Bits(32, 8), c = _i.Bits(64, 8); - int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1; + int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1, cdRegs = f32 ? 4 : 2; // A fragment base aligned to its size and the whole fragment below RZ (RZ itself reads as zeros). bool fits(uint reg, int n) => reg == RZ || (reg % (uint)n == 0 && reg + (uint)n <= RZ); - // Word 1 above B, word 2 other than C and the shape bit, and word 3 bits 96-104 / 126-127 select modes - // (F32 accumulators and others) that are not lowered. - if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x8ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0 - || !fits(d, 2) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, 2)) + // Word 1 above B, word 2 other than C, the shape (75) and the accumulator type (76), and word 3 bits 96-104 / + // 126-127 select modes (bit 74 another shape, HMMA.1684; other input types) that are not lowered. + if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x18ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0 + || !fits(d, cdRegs) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, cdRegs)) throw Fail("HMMA mode"); if (d == RZ) return; // no result and no other effect @@ -2027,13 +2047,14 @@ static class AmpereLowering return lane; } - // Accumulators from C (both halves of each row register), or zero. - for (int r = 0; r < 2; r++) - for (int col = 0; col < 2; col++) - { - if (c == RZ) Mov32i(acc + 2 * r + col, 0); - else widen(acc + 2 * r + col, part(c, r), col == 1); - } + // Accumulators from C, or zero: accumulator 2r + col is register 2r + col of an .F32 C, half col of register + // r of an .F16 C. + for (int k = 0; k < 4; k++) + { + if (c == RZ) Mov32i(acc + k, 0); + else if (f32) Mov(acc + k, part(c, k), false); + else widen(acc + k, part(c, k >> 1), (k & 1) == 1); + } // Lane bases: 8q first (it needs the raw lane id), then 4g. Put(0xf0c8000000000000UL | laneA, false); // S2R SR_LANEID @@ -2065,15 +2086,17 @@ static class AmpereLowering Emit(Unguarded(0x5980000000000000UL | ((ulong)t << 39) | ((ulong)(bw + h) << 20) | ((ulong)(aw + 2 * r + h) << 8) | (uint)t)); } } - if ((p & 3) == 3) // end of an eight-k step: round to F16, widen again if another step follows - for (int t = acc; t < acc + 4; t++) - { - narrow(t); - if (p + 1 < pairs) widen(t, t, false); - } } - // D register r = column 2q + 1 in the high half, column 2q in the low half (ISCADD hi << 16 + lo). + // .F32: D register 2r + col is accumulator 2r + col. + if (f32) + { + for (int k = 0; k < 4; k++) Mov(part(d, k), acc + k); + return; + } + // .F16: one rounding of each sum to F16, then D register r = column 2q + 1 in the high half, column 2q in the + // low half (ISCADD hi << 16 + lo). + for (int t = acc; t < acc + 4; t++) narrow(t); for (int r = 0; r < 2; r++) EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)(acc + 2 * r) << 20) | ((ulong)(acc + 2 * r + 1) << 8) | (uint)part(d, r)); } @@ -2328,6 +2351,10 @@ static class AmpereLowering // c1: the program's c1 constants (BRX jump tables); set before Lower by the caller. [ThreadStatic] public static byte[] C1Bytes; + // Diagnostics only (Program.cs LowerFile, the --lower mode): the register UR0 was mapped to by the last Lowerer + // built on this thread (Lowerer.UrBase; SpillFrom in spill mode). Lowering never reads it. + [ThreadStatic] public static int LastUrBase; + // localBytes: the local memory size the lowered program needs when that changed (spill mode), else 0. Graphics // programs carry it in their header already; compute and task/mesh programs need it from the translator's host // (job "local_mem"). diff --git a/tools/xlate/Program.cs b/tools/xlate/Program.cs index 3777a6a0..c375589c 100644 --- a/tools/xlate/Program.cs +++ b/tools/xlate/Program.cs @@ -6,6 +6,7 @@ // nx2navi_xlate --serve one request per stdin line: {"job": {...}, "out": ""}; one reply line per request: // "ok " or "fail "; a malformed line gets "fail ? " and the loop continues. // nx2navi_xlate batch mode: a JSON array of jobs, prints a summary, exit 1 if any failed. +// nx2navi_xlate --lower lowers SM86 instructions given as hex words and prints the Maxwell words (LowerFile). // job: { "key", "stage": "vs|ps|cs", "code": path (graphics = 0x50-byte SPH + instructions, compute = instructions), "c1": hex, // "sm86": bool + "size": bytes (an NVN2 program, lowered by AmpereLowering.cs), "nvn2_table": [start, textures, // images] (NVN2 compute binding table), "lowered_out": path, "mesh": {"stage": 6|7, "threads", "maxv", "maxp", @@ -231,8 +232,56 @@ static class SwshXlate catch (Exception e) { if (Environment.GetEnvironmentVariable("DMSH_STACK") != null) Console.Error.WriteLine(e.StackTrace); return $"{e.GetType().Name}: {e.Message.Split('\n')[0]}"; } } + // --lower : runs AmpereLowering on a few SM86 instructions, for executing single instructions on the CPU + // (tests/maxwell_warp_emu.py). Each line of the file is one instruction as four hex 32-bit words "w0 w1 w2 w3" + // (w0 = bits 0-31, optional 0x prefix; blank lines and '#' comments skipped). They are lowered as a compute program + // (AmpereLowering.Lower with graphics false), with an SM86 EXIT appended unless the last instruction is an + // unpredicated EXIT. Prints the lowered Maxwell instruction words in program order, one 16-digit hex word per line: + // the scheduling words Lower puts first in every 32 bytes are left out, the closing EXIT Lower appends is kept. Then + // "urbase N" (the register UR0 maps to, AmpereLowering.LastUrBase) and, in spill mode only, "local N" (the local + // memory bytes the program needs). Errors go to stderr with exit code 1. + static int LowerFile(string path) + { + try + { + var bytes = new List(); + uint[] last = null; + foreach (string raw in File.ReadAllLines(path)) + { + string line = raw.Split('#')[0].Trim(); + if (line.Length == 0) continue; + string[] f = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + if (f.Length != 4) throw new FormatException($"expected four hex words: \"{line}\""); + last = f.Select(s => Convert.ToUInt32(s.StartsWith("0x", StringComparison.OrdinalIgnoreCase) ? s[2..] : s, 16)).ToArray(); + foreach (uint w in last) bytes.AddRange(BitConverter.GetBytes(w)); + } + // SM86 EXIT: opcode 0x94d under PT (bits 12-15 = 7), second condition PT (bits 87-90 = 7); AmpereLowering Exit. + bool exits = last != null && (last[0] & 0xffff) == 0x794d && ((last[2] >> 23) & 15) == 7 && ((last[2] >> 20) & 3) == 0; + if (!exits) foreach (uint w in new uint[] { 0x0000794d, 0, 0x03800000, 0x000fea00 }) bytes.AddRange(BitConverter.GetBytes(w)); + byte[] code = bytes.ToArray(); + byte[] lowered = AmpereLowering.Lower(code, code.Length, false, out int local); + // Compute layout (AmpereLowering.Lower): a scheduling word at every 32-byte boundary, instructions in the other + // slots, zero padding after the closing EXIT. Instruction words are never 0, so the last nonzero one ends it. + var words = new List(); + int end = 0; + for (int a = 0; a + 8 <= lowered.Length; a += 8) + { + if ((a & 0x1f) == 0) continue; + ulong w = BitConverter.ToUInt64(lowered, a); + words.Add(w); + if (w != 0) end = words.Count; + } + foreach (ulong w in words.Take(end)) Console.WriteLine(w.ToString("x16")); + Console.WriteLine($"urbase {AmpereLowering.LastUrBase}"); + if (local != 0) Console.WriteLine($"local {local}"); + return 0; + } + catch (Exception e) { Console.Error.WriteLine($"{e.GetType().Name}: {e.Message}"); return 1; } + } + static int Main(string[] args) { + if (args.Length == 2 && args[0] == "--lower") return LowerFile(args[1]); if (args.Length == 1 && args[0] == "--serve") { // One request per line: {"job": {...}, "out": ""}; one reply line "ok " / "fail ". diff --git a/tools/xlate/patches/0021-prmt-byte-permute.patch b/tools/xlate/patches/0021-prmt-byte-permute.patch new file mode 100644 index 00000000..97b35aab --- /dev/null +++ b/tools/xlate/patches/0021-prmt-byte-permute.patch @@ -0,0 +1,161 @@ +diff --git a/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs b/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs +index d77621d..462c0ca 100644 +--- a/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs ++++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmit.cs +@@ -215,34 +215,6 @@ namespace Ryujinx.Graphics.Shader.Instructions + context.TranslatorContext.GpuAccessor.Log("Shader instruction Pret is not implemented."); + } + +- public static void PrmtR(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtR is not implemented."); +- } +- +- public static void PrmtI(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtI is not implemented."); +- } +- +- public static void PrmtC(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtC is not implemented."); +- } +- +- public static void PrmtRc(EmitterContext context) +- { +- context.GetOp(); +- +- context.TranslatorContext.GpuAccessor.Log("Shader instruction PrmtRc is not implemented."); +- } +- + public static void R2b(EmitterContext context) + { + context.GetOp(); +diff --git a/src/Ryujinx.Graphics.Shader/Instructions/InstEmitPermute.cs b/src/Ryujinx.Graphics.Shader/Instructions/InstEmitPermute.cs +new file mode 100644 +index 0000000..b79ad3e +--- /dev/null ++++ b/src/Ryujinx.Graphics.Shader/Instructions/InstEmitPermute.cs +@@ -0,0 +1,116 @@ ++using Ryujinx.Graphics.Shader.Decoders; ++using Ryujinx.Graphics.Shader.IntermediateRepresentation; ++using Ryujinx.Graphics.Shader.Translation; ++ ++using static Ryujinx.Graphics.Shader.Instructions.InstEmitHelper; ++using static Ryujinx.Graphics.Shader.IntermediateRepresentation.OperandHelper; ++ ++namespace Ryujinx.Graphics.Shader.Instructions ++{ ++ // NX2NAVI: PRMT (byte permute). Upstream Ryujinx only logged it as not implemented, so the destination kept its old ++ // value; the SM86 lowering (tools/xlate/AmpereLowering.cs Prmt, the HFMA2.RELU sign mask in Half) emits it for every ++ // byte permute of the Switch 2 shaders (82 pixel, 303 vertex and 277 compute programs of Ocarina), which lost them. ++ // Rd byte i is byte k of the 8 bytes {C, A} (A bytes 0-3, C bytes 4-7). In the default mode k is nibble i of B and ++ // its bit 3 replaces the byte with its sign (0x00 or 0xff); the other modes take k from B bits 0-1 alone, as the PTX ++ // prmt modes f4e, b4e, rc8, ecl, ecr and rc16 define. ++ static partial class InstEmit ++ { ++ public static void PrmtR(EmitterContext context) ++ { ++ InstPrmtR op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = GetSrcReg(context, op.SrcB); ++ Operand srcC = GetSrcReg(context, op.SrcC); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ public static void PrmtI(EmitterContext context) ++ { ++ InstPrmtI op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = Const(op.Imm20); ++ Operand srcC = GetSrcReg(context, op.SrcC); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ public static void PrmtC(EmitterContext context) ++ { ++ InstPrmtC op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = GetSrcCbuf(context, op.CbufSlot, op.CbufOffset); ++ Operand srcC = GetSrcReg(context, op.SrcC); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ public static void PrmtRc(EmitterContext context) ++ { ++ InstPrmtRc op = context.GetOp(); ++ ++ Operand srcA = GetSrcReg(context, op.SrcA); ++ Operand srcB = GetSrcReg(context, op.SrcC); ++ Operand srcC = GetSrcCbuf(context, op.CbufSlot, op.CbufOffset); ++ ++ context.Copy(GetDest(op.Dest), EmitPrmt(context, srcA, srcB, srcC, op.PMode)); ++ } ++ ++ private static Operand EmitPrmt(EmitterContext context, Operand srcA, Operand srcB, Operand srcC, PMode mode) ++ { ++ Operand result = Const(0); ++ Operand low = context.BitwiseAnd(srcB, Const(3)); // the start byte of the special modes ++ ++ for (int i = 0; i < 4; i++) ++ { ++ Operand index; // 0-7: the source byte of result byte i ++ Operand sign = null; // non-zero: replicate that byte's sign bit (default mode only) ++ ++ switch (mode) ++ { ++ case PMode.F4e: ++ index = context.BitwiseAnd(context.IAdd(low, Const(i)), Const(7)); ++ break; ++ case PMode.B4e: ++ index = context.BitwiseAnd(context.IAdd(low, Const(8 - i)), Const(7)); ++ break; ++ case PMode.Rc8: ++ index = low; ++ break; ++ case PMode.Ecl: ++ index = context.IMaximumU32(low, Const(i)); ++ break; ++ case PMode.Ecr: ++ index = context.IMinimumU32(low, Const(i)); ++ break; ++ case PMode.Rc16: ++ index = context.BitwiseOr(context.ShiftLeft(context.BitwiseAnd(low, Const(1)), Const(1)), Const(i & 1)); ++ break; ++ default: ++ Operand nibble = context.BitfieldExtractU32(srcB, Const(i * 4), Const(4)); ++ index = context.BitwiseAnd(nibble, Const(7)); ++ sign = context.BitwiseAnd(nibble, Const(8)); ++ break; ++ } ++ ++ Operand word = context.ConditionalSelect(context.ICompareLessUnsigned(index, Const(4)), srcA, srcC); ++ Operand shift = context.ShiftLeft(context.BitwiseAnd(index, Const(3)), Const(3)); ++ Operand value = context.BitfieldExtractU32(word, shift, Const(8)); ++ ++ if (sign != null) ++ { ++ Operand negative = context.ICompareNotEqual(context.BitwiseAnd(value, Const(0x80)), Const(0)); ++ Operand replicated = context.ConditionalSelect(negative, Const(0xff), Const(0)); ++ value = context.ConditionalSelect(context.ICompareNotEqual(sign, Const(0)), replicated, value); ++ } ++ ++ result = context.BitwiseOr(result, context.ShiftLeft(value, Const(i * 8))); ++ } ++ ++ return result; ++ } ++ } ++}