diff --git a/recomp/lifter.py b/recomp/lifter.py index 5dd506cd..e33e9871 100644 --- a/recomp/lifter.py +++ b/recomp/lifter.py @@ -296,7 +296,20 @@ class Lifter: return Insn('bcond', target=t, cond=f'(({X(rt)} >> {b}) & 1) {"!=" if nz else "=="} 0') if bits(w, 25, 7) == 0b1101011: # branch register opc, rn = bits(w, 21, 4), bits(w, 5, 5) - if bits(w, 16, 5) != 31 or bits(w, 10, 6) or bits(w, 0, 5): + op3 = bits(w, 10, 6) + if op3 in (0b000010, 0b000011): # BRAA/BLRAA/RETAA/RETAB (pointer auth, key A or B) + # The lift runs with no signing keys, so the authenticate step is a no-op: the branch + # itself is the ordinary one. RETAA/RETAB carry Rn = 11111 and return through X30. + if bits(w, 16, 5) != 31 or bits(w, 0, 5) not in (0, 31): + raise Unsupported('branch reg auth variant') + if opc == 0: + return Insn('br', target=X(rn)) + if opc == 1: + return Insn('blr', target=X(rn)) + if opc == 2: + return Insn('ret') + raise Unsupported('branch reg auth opc') + if bits(w, 16, 5) != 31 or op3 or bits(w, 0, 5): raise Unsupported('branch reg variant') if opc == 0: return Insn('br', target=X(rn)) @@ -462,6 +475,8 @@ class Lifter: p, v = f'({t} *)(uintptr_t)({XS(rn)})', f'({t}){X(rs)}' order = ('__ATOMIC_RELAXED', '__ATOMIC_RELEASE', '__ATOMIC_ACQUIRE', '__ATOMIC_ACQ_REL')[A << 1 | R] if o3: + if rs == 31 and opc in (1, 4): # LDAPR (byte/half/word/dword): load-acquire RCpc into Rt + return Insn('op', set_reg(size == 3, rt, f'__atomic_load_n({p}, __ATOMIC_ACQUIRE)')) if opc: raise Unsupported('ldapr') expr = f'__atomic_exchange_n({p}, {v}, {order})' @@ -673,6 +688,12 @@ class Lifter: if op2 == 6: opcode = bits(w, 10, 6) if bits(w, 30, 1): # 1-source + if bits(w, 16, 5) == 1 and sf and opcode <= 17: + # PACIA/PACIB/PACDA/PACDB/AUTIA/AUTIB/AUTDA/AUTDB (opcode 0-7), their Z forms (8-15) and + # XPACI/XPACD (16-17): pointer authentication. Lifted code never signs pointers, so signing, + # authenticating and stripping all leave Rd unchanged (as RETAB/BRAA are lifted as RET/BR). + # Switch 2's hardened _setjmp signs x30 with PACIB and checks its low bits afterwards. + return Insn('op', '/* pac/aut: no-op */') if bits(w, 16, 5): raise Unsupported('dp1 opcode2') a = R(sf, rn) @@ -700,6 +721,10 @@ class Lifter: 9: f'{a} >> ({b} & {size - 1})', 10: f'(uint{size}_t)((int{size}_t){a} >> ({b} & {size - 1}))', 11: f'ROR{size}({a}, {b} & {size - 1})', + # PACGA Xd, Xn, Xm|SP: a 32-bit code of (Xn, Xm) in the top half. Its only use is a checksum compared + # against a second PACGA of the same values (the SDK's _setjmp/longjmp guard), so a constant code + # keeps every such comparison equal. + 12: '0ull' if sf else None, }.get(opcode) if expr is None: raise Unsupported('dp2') diff --git a/recomp/nso.py b/recomp/nso.py index 03beab28..1222230e 100644 --- a/recomp/nso.py +++ b/recomp/nso.py @@ -24,7 +24,11 @@ class Nso: comp_size = struct.unpack_from('voices[i]; if (!v->in.in_use || v->state != SV_STARTED || !v->in.sample_rate || !v->valid[v->index]) continue; int cc = (int)v->in.channel_count; if (cc < 1 || cc > 6) { report_once(m, 2, "voice channel count unsupported", cc); continue; } + { /* each sample format a started voice uses, logged once: one the mixer cannot decode plays as silence */ + static volatile uint32_t seen; + unsigned f = v->in.sample_format & 31; + if (!(__atomic_fetch_or(&seen, 1u << f, __ATOMIC_RELAXED) & (1u << f))) + rt_log("audio: voice %d uses sample format %u (%s), %d channel(s) at %u Hz", i, v->in.sample_format, + v->in.sample_format == FMT_ADPCM || fmt_bytes(v->in.sample_format) ? "decoded" : "NOT decoded: silent", + cc, (unsigned)v->in.sample_rate); + } voice_render(r, v, vs, frames); for (int f = 0; f < 2; f++) if (v->in.biquads[f].enable) { const AudrenBiquad *b = &v->in.biquads[f]; @@ -886,6 +894,9 @@ void audio_mixer_render(AudioRenderer *r, double seconds) { if (!(m->reported & (1u << 6))) { m->reported |= 1u << 6; rt_log("audio: device sink has %u inputs, output %s", sink->input_count, oc == 6 ? "5.1" : "stereo"); + rt_log("audio: device sink inputs %d %d %d %d %d %d, downmix %s %.3f %.3f %.3f %.3f", sink->inputs[0], sink->inputs[1], + sink->inputs[2], sink->inputs[3], sink->inputs[4], sink->inputs[5], sink->downmix_enabled ? "on" : "off", + sink->downmix[0], sink->downmix[1], sink->downmix[2], sink->downmix[3]); } const float gain = host_volume() * atomic_load(&device_volume) * (r->mixes[0].in_use ? r->mixes[0].volume : 1.0f); const float *ch[6] = {0}; @@ -966,6 +977,122 @@ void audio_device_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) rs->copy[rs->ncopy++] = kern_event_readable(ev); return; } + // 17.0.0+: Acquire/Release Audio{Input,Output}DeviceNotification (15-18), 18.0.0+: SetAudioDeviceOutputVolume- + // AutoTuneEnabled (19) / IsAudioDeviceOutputVolumeAutoTuneEnabled (20), 21.0.0+: + // IsActiveOutputDeviceEstimatedLowLatency (21). Command ids from the system service/audio/audio_device.cpp. The + // device never changes and the host applies no auto-tuned volume, so the answers are "off" / "no". + case 15: case 17: { // Acquire...Notification -> a readable event that is never signalled + uint32_t ev = kern_event_new(); + rs->copy[rs->ncopy++] = kern_event_readable(ev); + return; + } + case 16: case 18: case 19: return; + case 20: case 21: { uint8_t no = 0; IPC_OUT(rs, no); return; } default: ipc_unimplemented(s, rq); } } + +// ---- auddmg:u: the Switch 2 audio device manager. Command 0 opens the virtual device manager. Its output-side +// commands and the 0x228-byte property block follow the system service/audio/nx2_audio_device_manager.h +// (INx2AudioVirtualDeviceManager); ids there are partly guessed. One virtual output device feeds the host mixer; +// there are no input devices, so a command not listed here (the input side the SDK's ListVirtualInputDevices +// reaches) is logged once and answered with success and a zero count. +static const char virtual_output[] = "AudioVirtualOutput0"; +#define AUD_R_NOT_SUPPORTED 0x40299u // audio module 153, description 513 +#define AUD_R_INVALID_ARGUMENT 0x70899u // audio module 153, description 900 + +static void put_block(const IpcRequest *rq, IpcResponse *rs, const void *src, size_t n) { + int pointer; + IpcBuffer b = out_buffer(rq, &pointer); + if (!b.size) return; + uint8_t *dst = ipc_guest(b.addr, b.size); + memset(dst, 0, b.size); + memcpy(dst, src, n < b.size ? n : b.size); + if (pointer) rs->xout[rs->nxout++] = (IpcBuffer){b.addr, b.size, b.index}; +} + +static void virtual_device_manager_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + (void)s; + switch (rq->cmd) { + case 0: { // ListVirtualOutputDevices -> count, names (0x100 each) + char name[0x100] = {0}; + memcpy(name, virtual_output, sizeof virtual_output); + int pointer; + IpcBuffer b = out_buffer(rq, &pointer); + uint32_t n = b.size >= sizeof name ? 1 : 0; + if (n) put_block(rq, rs, name, sizeof name); + IPC_OUT(rs, n); + return; + } + case 100: { // GetDefaultVirtualOutputDevice -> name + char name[0x100] = {0}; + memcpy(name, virtual_output, sizeof virtual_output); + put_block(rq, rs, name, sizeof name); + return; + } + case 300: { // IsVirtualOutputDeviceStreamFormatSupported(sample type, channels, rate, functionality) + uint32_t channels = IPC_ARG(rq, uint32_t, 4), rate = IPC_ARG(rq, uint32_t, 8), fn = IPC_ARG(rq, uint32_t, 12); + rs->result = fn == 1 && (channels == 1 || channels == 2 || channels == 6) && rate == 48000 ? 0 : AUD_R_NOT_SUPPORTED; + return; + } + case 401: { // GetVirtualOutputDeviceProperties(name) -> block + uint8_t block[0x228] = {0}; + memcpy(block, virtual_output, sizeof virtual_output); + uint32_t v[4] = {3, 2, 2, 48000}; // type, (2), channels, sample rate + memcpy(block + 0x100, v, sizeof v); + memcpy(block + 0x110, device_name, sizeof device_name); + uint64_t latency_ns = 10000000; + memcpy(block + 0x218, &latency_ns, sizeof latency_ns); + uint32_t id = 1; // nonzero: the SDK's output streams pass it on (the system: a zero breaks stream creation) + memcpy(block + 0x220, &id, sizeof id); + put_block(rq, rs, block, sizeof block); + return; + } + case 500: rs->copy[rs->ncopy++] = kern_event_readable(kern_event_new()); return; // state change event + case 700: case 800: return; // AttachStream / DetachStream + default: { + static volatile int32_t seen[64]; + if (rq->cmd >= 64 || !__atomic_exchange_n(&seen[rq->cmd], 1, __ATOMIC_RELAXED)) + rt_log("auddmg:u manager: command %u answered with success and a zero count (no input devices)", rq->cmd); + uint32_t zero = 0; + IPC_OUT(rs, zero); + return; + } + } +} + +void audio_dmg_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + if (rq->cmd == 0) { // OpenVirtualDeviceManager + rs->move[rs->nmove++] = ipc_session_new("auddmg:u manager", virtual_device_manager_handler, NULL, NULL); + return; + } + ipc_unimplemented(s, rq); +} + +// ---- audsmx:u: the Switch 2 sound mixer (sysmix). Command 0 opens a user session; the session's CreateStream (1) +// makes a stream of a virtual device (the system INx2SysmixUserSession: device 1 is the virtual output). This host +// renders through audren, has no sysmix output streams yet and no input devices: CreateStream of a device other +// than 1 fails with InvalidArgument (as the system does), and device 1 stops with its name so the first title that needs +// a sysmix output stream says so. Engage/DisengageStreams (2/3) succeed. +static void sysmix_session_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + (void)s; + switch (rq->cmd) { + case 1: { + uint32_t device = IPC_ARG(rq, uint32_t, 0), flags = IPC_ARG(rq, uint32_t, 4); + if (device == 1) rt_fatal("audsmx:u: CreateStream on the virtual output (flags %#x) is not implemented", flags); + rt_log("audsmx:u: CreateStream device %u flags %#x refused: no audio input devices", device, flags); + rs->result = AUD_R_INVALID_ARGUMENT; + return; + } + case 2: case 3: return; + default: ipc_unimplemented(s, rq); + } +} + +void audio_sysmix_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + if (rq->cmd == 0) { // OpenUserSession(process handle) + rs->move[rs->nmove++] = ipc_session_new("audsmx:u session", sysmix_session_handler, NULL, NULL); + return; + } + ipc_unimplemented(s, rq); +} diff --git a/runtime/audren.c b/runtime/audren.c index ba095051..9dd23e45 100644 --- a/runtime/audren.c +++ b/runtime/audren.c @@ -409,6 +409,8 @@ static IpcBuffer pick(const IpcBuffer *a, unsigned na, const IpcBuffer *b, unsig return x.size ? x : y; } +static uint32_t voice_drop_bits = 0x3f800000u; // SetVoiceDropParameter's float (1.0 until set) + static void renderer_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { AudioRenderer *r = s->state; switch (rq->cmd) { @@ -446,6 +448,20 @@ static void renderer_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *r case 7: rs->copy[rs->ncopy++] = kern_event_readable(r->event); return; // QuerySystemEvent case 8: r->time_limit = IPC_ARG(rq, uint32_t, 0); return; // SetRenderingTimeLimit case 9: { uint32_t v = r->time_limit ? r->time_limit : 100; IPC_OUT(rs, v); return; } // GetRenderingTimeLimit + // 15.0.0+ Set/GetVoiceDropParameter (the system service/audio/audio_renderer.cpp): the factor the DSP uses to drop + // voices when a frame overruns its time. The host mixer renders every voice, so the value is only kept. + case 12: { + float v = IPC_ARG(rq, float, 0); + __atomic_store_n(&voice_drop_bits, isfinite(v) ? *(uint32_t *)&v : 0x3f800000u, __ATOMIC_RELAXED); + return; + } + case 13: { + uint32_t bits = __atomic_load_n(&voice_drop_bits, __ATOMIC_RELAXED); + float v; + memcpy(&v, &bits, sizeof v); + IPC_OUT(rs, v); + return; + } default: ipc_unimplemented(s, rq); } } diff --git a/runtime/audren.h b/runtime/audren.h index 589cb141..030c18ba 100644 --- a/runtime/audren.h +++ b/runtime/audren.h @@ -187,3 +187,5 @@ int64_t audio_mixer_pace_ns(AudioRenderer *r, uint64_t now); uint64_t audio_voice_consumed(const AudioRenderer *r, ServerVoice *v, int frames); // source samples over one frame; advances v->fraction int audio_buffer_empty(const AudrenWaveBuffer *b); // no samples to decode: played as soon as it is reached void audio_device_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs); // IAudioDevice +void audio_dmg_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs); // auddmg:u (Switch 2) +void audio_sysmix_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs); // audsmx:u (Switch 2) diff --git a/runtime/boot.c b/runtime/boot.c index 905ff37c..0bf618b0 100644 --- a/runtime/boot.c +++ b/runtime/boot.c @@ -1,4 +1,4 @@ -// Adapted from MacNativeGames games//runtime/loader.c (lines 456-477) and services.c (lines 219-235) @ b75be8cf +// Adapted from MacNativeGames games//runtime/loader.c (lines 456-477) and services.c (lines 219-235) @ b75be8cf // (MIT, Copyright (c) 2026 MacNativeGames contributors) and LAS Recomp runtime/main.c (lines 16-35, 141-159) @ // da35b83 (MIT, Copyright (c) 2026 Peiikai). // The boot driver: the guest main thread runs the game's own nn::init::Start @@ -17,6 +17,8 @@ #include "cfg.h" #include "cheats.h" #include "fs.h" +#include "kernel.h" +#include "ipc.h" #include "gfx.h" #include "mods.h" #include "nvn_api.h" @@ -48,7 +50,9 @@ static const RtLoaded *module_named(const char *name) { rt_fatal("boot: module %s is not loaded", name); } -static uint32_t ctor_state(const RtLoaded *m) { return LD32(m->ctor_flag); } +// The constructor flag is a NX2navi symbol the loader resolves; a title without it (or with a different +// name) reports "done" rather than dereferencing a null address. +static uint32_t ctor_state(const RtLoaded *m) { return m->ctor_flag ? LD32(m->ctor_flag) : CTOR_DONE; } static const char *caller(const Cpu *c, char *buf, int n) { return ldr_describe(c->x[30] - 4, buf, n); } @@ -76,6 +80,14 @@ static void cb_loader_ready(Cpu *c) { // DT_INIT registers the module with musl (__nnmusl_init_dso) and runs its init_array; the constructor flag reads 2 // afterwards (MacNativeGames loader.c:458-462). static void cb_module_inits(Cpu *c) { + kern_seed_tls(); // the SDK clears the thread-local region during init; put the TLS back where it reads it + { // where the sdk's lock thunks actually land once the loader has resolved them + static const unsigned long long off[4] = {0xe8fd20ull, 0xe8dff8ull, 0xe8f538ull, 0xe91200ull}; + const unsigned char *sdkimg = (const unsigned char *)(uintptr_t)0x7005200000ull; + for (int k = 0; k < 4; k++) + rt_logf("boot: sdk GOT[%#llx] = %#llx\n", off[k], + *(const unsigned long long *)(const void *)(sdkimg + off[k])); + } if (plat_cas32(&inits_done, 0, 0)) rt_fatal("boot: module constructors requested twice"); for (unsigned k = 0; k < sizeof inits_121 / sizeof *inits_121; k++) { const RtLoaded *m = module_named(expected_inits[k].name); @@ -87,8 +99,8 @@ static void cb_module_inits(Cpu *c) { rt_logf("boot: DT_INIT %s state=%u (%llu)\n", m->name, state, (unsigned long long)entries); if (state != CTOR_DONE) rt_fatal("boot: %s DT_INIT left its constructor flag at %u, not 2", m->name, state); if (entries != expected_inits[k].entries) - rt_fatal("boot: %s has %llu init_array entries, this game version has %llu", m->name, (unsigned long long)entries, - (unsigned long long)expected_inits[k].entries); + rt_logf("boot: %s has %llu init_array entries (this title, not the %llu of the expected version)\n", + m->name, (unsigned long long)entries, (unsigned long long)expected_inits[k].entries); } plat_xchg32(&inits_done, 1); } @@ -351,6 +363,7 @@ void rt_boot(void) { cb_addr[0] = hostfn_register("", cb_loader_ready); cb_addr[1] = hostfn_register("", cb_module_inits); cb_addr[2] = hostfn_register("", cb_module_finis); + ipc_applet_event_init(); hostfn_seal(); uint64_t held = plat_now_ms(); if (warm) plat_thread_join(warm); diff --git a/runtime/cfg.c b/runtime/cfg.c index b7388778..f987a52d 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -30,6 +30,7 @@ static int save_ok = 1; // the last cfg_save_to wrote (o // Every name the runtime reads (and the patch sets in generated/), without the prefix. A file key outside this list is // logged as unknown. A new variable goes here too, or its ini key logs a harmless "unknown key" line. static const char *const known[] = { + "CORES", "FALLBACK_EXPOSURE", "TM_DISPATCH", "NVN2_CS_TABLE", "SKIP_SHADERS", "MOVIE_DEBAND", "MOVIE_SKIP", "MOVIE_OVERLAY", "MOVIES", "NPLN", "NVN2_PROGRAMS", "NVN_TRAP", "SHADER_DUMP", "SHOT_DIR", "SHOT_EVERY", // Switch 2 port: movie skip (romfs.c), NVN2 proc traps (nvn_procs.c) "AB", "AMIIBO_DIR", "ANISO", "APP_POOL", "ASPECT", "ASPECT_PATCH", "ASSUME_FOCUS", "ASTC", "ASTC_BC", "ASTC_TIME", "ASYNC_PREEMPT", "AUDIO", "AUDIO_CHANNELS", "AUDIO_OFF", "AUDIO_PREFILL_MS", "BACKGROUND_WINDOW", "BIND_A", "BIND_B", "BIND_DOWN", "BIND_L", "BIND_LCLICK", "BIND_LEFT", "BIND_LSTICK_DOWN", "BIND_LSTICK_LEFT", "BIND_LSTICK_RIGHT", "BIND_LSTICK_UP", "BIND_MINUS", @@ -39,7 +40,7 @@ static const char *const known[] = { "BIND_ZL", "BIND_ZR", "BLOCK_VIEWS_3D", "CLOTH_REALTIME", "COLOR_BRIGHTNESS", "COLOR_CONTRAST", "COLOR_GAMMA", "COLOR_PRESET", "COLOR_SATURATION", "CONTACT_DEBUG", "CONTACT_SHADOWS", "CORE_DIAG", "CRIT", "CRIT_OUT", "DESC_TEMPLATE", "DEVICE_PROFILE", "DISCORD_APP_ID", "DISCORD_IMAGE", "DISCORD_RPC", "DLAA", "DLAA_JITTER", "DLAA_MV", "DLAA_MASKDEBUG", "DLAA_MVDEBUG", "DLAA_OBJMV", "DLAA_SURVEY", "DLAA_TIMES", "DLSS", "DLSS_CHAR_MASK", "DLSS_PRESET", "DLSS_REACTIVE", "DUMP", "DUMP_BUF", "DUMP_DIR", - "DUMP_LOG", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", + "DUMP_LOG", "LOG_CS_BINDS", "LOG_TEX_FLAGS", "LOG_MESH", "MESH", "POOL", "NATIVE_TENSOR", "LOG_RCOND", "CS_FALLBACK", "EXEFS", "FEEDBACK", "FIX_C0", "FIX_ZERO", "FLIGHT", "FLIGHT_AT", "FLIGHT_DIR", "FLIGHT_FRAMES", "FLIGHT_SPIKE_MS", "FLIGHT_THUMBS", "FLUSH_SPAN", "FOV", "FOV_PATCH", "FPS60", "FPS_LIMIT", "FREEZE_IDLE", "FREE_CORES", "FULLSCREEN", "FXAA", "GPL", "GPL_DRAW_BUILD", "GPU", "GPU_INDEX", "GPU_NAME", "HDR", "HDR_GAME_NITS", "HDR_HUE", "HDR_PEAK_NITS", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", diff --git a/runtime/fallback_tonemap_spv.h b/runtime/fallback_tonemap_spv.h new file mode 100644 index 00000000..40037605 --- /dev/null +++ b/runtime/fallback_tonemap_spv.h @@ -0,0 +1,63 @@ +// Generated from runtime/shaders/fallback_tonemap.spvasm (spirv-as --target-env vulkan1.3); do not edit. +// source sha256 31e5bd112d259f6d65bd0d488b2bbe1dd0b2183ad1aca69f666c17e0589d447f +#pragma once +#include +static const uint32_t fallback_tonemap_spv[] = { + 0x07230203, 0x00010600, 0x00070000, 0x00000052, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, + 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, + 0x0009000f, 0x00000005, 0x00000002, 0x6e69616d, 0x00000000, 0x00000003, 0x00000004, 0x00000005, + 0x00000006, 0x00060010, 0x00000002, 0x00000011, 0x00000008, 0x00000008, 0x00000001, 0x00040047, + 0x00000003, 0x0000000b, 0x0000001c, 0x00040047, 0x00000004, 0x00000022, 0x00000000, 0x00040047, + 0x00000004, 0x00000021, 0x00000000, 0x00040047, 0x00000005, 0x00000022, 0x00000000, 0x00040047, + 0x00000005, 0x00000021, 0x00000001, 0x00030047, 0x00000005, 0x00000019, 0x00030047, 0x00000007, + 0x00000002, 0x00050048, 0x00000007, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x00000007, + 0x00000001, 0x00000023, 0x00000008, 0x00050048, 0x00000007, 0x00000002, 0x00000023, 0x0000000c, + 0x00050048, 0x00000007, 0x00000003, 0x00000023, 0x00000010, 0x00020013, 0x00000008, 0x00030021, + 0x00000009, 0x00000008, 0x00030016, 0x0000000a, 0x00000020, 0x00040015, 0x0000000b, 0x00000020, + 0x00000001, 0x00040015, 0x0000000c, 0x00000020, 0x00000000, 0x00020014, 0x0000000d, 0x00040017, + 0x0000000e, 0x0000000a, 0x00000002, 0x00040017, 0x0000000f, 0x0000000a, 0x00000003, 0x00040017, + 0x00000010, 0x0000000a, 0x00000004, 0x00040017, 0x00000011, 0x0000000b, 0x00000002, 0x00040017, + 0x00000012, 0x0000000c, 0x00000003, 0x00040017, 0x00000013, 0x0000000c, 0x00000002, 0x00040020, + 0x00000014, 0x00000001, 0x00000012, 0x0004003b, 0x00000014, 0x00000003, 0x00000001, 0x00090019, + 0x00000015, 0x0000000a, 0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000001, 0x00000000, + 0x0003001b, 0x00000016, 0x00000015, 0x00040020, 0x00000017, 0x00000000, 0x00000016, 0x0004003b, + 0x00000017, 0x00000004, 0x00000000, 0x00090019, 0x00000018, 0x0000000a, 0x00000001, 0x00000000, + 0x00000000, 0x00000000, 0x00000002, 0x00000002, 0x00040020, 0x00000019, 0x00000000, 0x00000018, + 0x0004003b, 0x00000019, 0x00000005, 0x00000000, 0x0006001e, 0x00000007, 0x0000000e, 0x0000000a, + 0x0000000a, 0x00000011, 0x00040020, 0x0000001a, 0x00000009, 0x00000007, 0x0004003b, 0x0000001a, + 0x00000006, 0x00000009, 0x00040020, 0x0000001b, 0x00000009, 0x0000000e, 0x00040020, 0x0000001c, + 0x00000009, 0x0000000a, 0x00040020, 0x0000001d, 0x00000009, 0x00000011, 0x0004002b, 0x0000000b, + 0x0000001e, 0x00000000, 0x0004002b, 0x0000000b, 0x0000001f, 0x00000001, 0x0004002b, 0x0000000b, + 0x00000020, 0x00000003, 0x0004002b, 0x0000000a, 0x00000021, 0x00000000, 0x0004002b, 0x0000000a, + 0x00000022, 0x3f800000, 0x0004002b, 0x0000000a, 0x00000023, 0x3f000000, 0x0004002b, 0x0000000a, + 0x00000024, 0x4020a3d7, 0x0004002b, 0x0000000a, 0x00000025, 0x3cf5c28f, 0x0004002b, 0x0000000a, + 0x00000026, 0x401b851f, 0x0004002b, 0x0000000a, 0x00000027, 0x3f170a3d, 0x0004002b, 0x0000000a, + 0x00000028, 0x3e0f5c29, 0x0005002c, 0x0000000e, 0x00000029, 0x00000023, 0x00000023, 0x0006002c, + 0x0000000f, 0x0000002a, 0x00000021, 0x00000021, 0x00000021, 0x0006002c, 0x0000000f, 0x0000002b, + 0x00000022, 0x00000022, 0x00000022, 0x0006002c, 0x0000000f, 0x0000002c, 0x00000024, 0x00000024, + 0x00000024, 0x0006002c, 0x0000000f, 0x0000002d, 0x00000025, 0x00000025, 0x00000025, 0x0006002c, + 0x0000000f, 0x0000002e, 0x00000026, 0x00000026, 0x00000026, 0x0006002c, 0x0000000f, 0x0000002f, + 0x00000027, 0x00000027, 0x00000027, 0x0006002c, 0x0000000f, 0x00000030, 0x00000028, 0x00000028, + 0x00000028, 0x00040017, 0x00000031, 0x0000000d, 0x00000002, 0x00050036, 0x00000008, 0x00000002, + 0x00000000, 0x00000009, 0x000200f8, 0x00000032, 0x0004003d, 0x00000012, 0x00000033, 0x00000003, + 0x0007004f, 0x00000013, 0x00000034, 0x00000033, 0x00000033, 0x00000000, 0x00000001, 0x0004007c, + 0x00000011, 0x00000035, 0x00000034, 0x00050041, 0x0000001d, 0x00000036, 0x00000006, 0x00000020, + 0x0004003d, 0x00000011, 0x00000037, 0x00000036, 0x000500af, 0x00000031, 0x00000038, 0x00000035, + 0x00000037, 0x0004009a, 0x0000000d, 0x00000039, 0x00000038, 0x000300f7, 0x0000003a, 0x00000000, + 0x000400fa, 0x00000039, 0x0000003b, 0x0000003a, 0x000200f8, 0x0000003b, 0x000100fd, 0x000200f8, + 0x0000003a, 0x0004006f, 0x0000000e, 0x0000003c, 0x00000035, 0x00050081, 0x0000000e, 0x0000003d, + 0x0000003c, 0x00000029, 0x00050041, 0x0000001b, 0x0000003e, 0x00000006, 0x0000001e, 0x0004003d, + 0x0000000e, 0x0000003f, 0x0000003e, 0x00050085, 0x0000000e, 0x00000040, 0x0000003d, 0x0000003f, + 0x0004003d, 0x00000016, 0x00000041, 0x00000004, 0x00070058, 0x00000010, 0x00000042, 0x00000041, + 0x00000040, 0x00000002, 0x00000021, 0x0008004f, 0x0000000f, 0x00000043, 0x00000042, 0x00000042, + 0x00000000, 0x00000001, 0x00000002, 0x00050041, 0x0000001c, 0x00000044, 0x00000006, 0x0000001f, + 0x0004003d, 0x0000000a, 0x00000045, 0x00000044, 0x0005008e, 0x0000000f, 0x00000046, 0x00000043, + 0x00000045, 0x00050085, 0x0000000f, 0x00000047, 0x00000046, 0x0000002c, 0x00050081, 0x0000000f, + 0x00000048, 0x00000047, 0x0000002d, 0x00050085, 0x0000000f, 0x00000049, 0x00000046, 0x00000048, + 0x00050085, 0x0000000f, 0x0000004a, 0x00000046, 0x0000002e, 0x00050081, 0x0000000f, 0x0000004b, + 0x0000004a, 0x0000002f, 0x00050085, 0x0000000f, 0x0000004c, 0x00000046, 0x0000004b, 0x00050081, + 0x0000000f, 0x0000004d, 0x0000004c, 0x00000030, 0x00050088, 0x0000000f, 0x0000004e, 0x00000049, + 0x0000004d, 0x0008000c, 0x0000000f, 0x0000004f, 0x00000001, 0x0000002b, 0x0000004e, 0x0000002a, + 0x0000002b, 0x00050050, 0x00000010, 0x00000050, 0x0000004f, 0x00000022, 0x0004003d, 0x00000018, + 0x00000051, 0x00000005, 0x00040063, 0x00000051, 0x00000035, 0x00000050, 0x000100fd, 0x00010038, +}; diff --git a/runtime/fs.h b/runtime/fs.h index cea5b57b..f25731a8 100644 --- a/runtime/fs.h +++ b/runtime/fs.h @@ -34,6 +34,9 @@ uint32_t romfs_read(uint64_t *out, uint64_t h, int64_t offset, void *buffer, uin uint32_t romfs_size(int64_t *out, uint64_t h); void romfs_close(uint64_t h); uint32_t romfs_entry_type(int *out, const char *path); +// The children of RomFS directory path: fn(name, is_dir, size, ctx) once per file and once per subdirectory, in +// index order; fn returns 0 to stop. FS_R_PATH_NOT_FOUND when path is not a directory. +uint32_t romfs_list(const char *path, int (*fn)(const char *name, int is_dir, uint64_t size, void *ctx), void *ctx); uint64_t romfs_file_count(void); // files in the index (fs_init first) const char *romfs_entry(uint64_t i, uint64_t *offset, uint64_t *size); // path, image offset and size of index row i const char *romfs_image_path(void); // the romfs.bin path fs_init opened diff --git a/runtime/fs_host.c b/runtime/fs_host.c index 65773cc4..d585e695 100644 --- a/runtime/fs_host.c +++ b/runtime/fs_host.c @@ -1059,16 +1059,28 @@ static uint32_t delete_recursive(const char *path) { return remove_tree(p) ? FS_R_INVALID_ARGUMENT : 0; } +static int romfs_listing_add(const char *name, int is_dir, uint64_t size, void *ctx) { + return listing_add(name, is_dir ? PLAT_FS_DIR : PLAT_FS_FILE, size, ctx); +} + static uint32_t open_directory(uint64_t *out, const char *path, int mode) { - if (romfs_owns(path)) rt_fatal("fs: OpenDirectory on the RomFS (%s) is not implemented", path); - char p[PATH_CAP]; - uint32_t r = host_path(path, p, sizeof p); - if (r) return r; - if (!out || mode < 1 || mode > 3) return FS_R_INVALID_ARGUMENT; - if (plat_fs_kind(p) != PLAT_FS_DIR) return FS_R_PATH_NOT_FOUND; - Listing l; - int e = list_dir(p, &l); - if (e) return fs_result(e); + Listing l = {0}; + if (romfs_owns(path)) { // Switch 2 titles list RomFS folders (content:/Foliage/...); NX2NAVI never did + if (!out || mode < 1 || mode > 3) return FS_R_INVALID_ARGUMENT; + uint32_t r = romfs_list(path, romfs_listing_add, &l); + if (r) { + listing_free(l.e, l.n); + return r; + } + } else { + char p[PATH_CAP]; + uint32_t r = host_path(path, p, sizeof p); + if (r) return r; + if (!out || mode < 1 || mode > 3) return FS_R_INVALID_ARGUMENT; + if (plat_fs_kind(p) != PLAT_FS_DIR) return FS_R_PATH_NOT_FOUND; + int e = list_dir(p, &l); + if (e) return fs_result(e); + } plat_mutex_lock(&lock); for (int i = 0; i < NDIRS; i++) if (!dirs[i].used) { @@ -1102,6 +1114,18 @@ static uint32_t read_directory(int64_t *count, DirectoryEntry *out, uint64_t h, return 0; } +// Result GetDirectoryEntryCount(int64_t *out, DirectoryHandle h): the entries ReadDirectory would still return +static uint32_t directory_entry_count(int64_t *count, uint64_t h) { + if (h < DIR_BASE || h >= DIR_BASE + NDIRS || !count) return FS_R_INVALID_ARGUMENT; + Dir *d = &dirs[h - DIR_BASE]; + if (!d->used) return FS_R_INVALID_ARGUMENT; + int64_t n = 0; + for (int i = d->next; i < d->n; i++) + if (strlen(d->e[i].name) < 769 && (d->mode & (d->e[i].kind == PLAT_FS_DIR ? 1 : 2))) n++; + *count = n; + return 0; +} + static void close_directory(uint64_t h) { if (h < DIR_BASE || h >= DIR_BASE + NDIRS || !dirs[h - DIR_BASE].used) rt_fatal("fs: close of invalid directory handle %#llx", (unsigned long long)h); @@ -1134,6 +1158,7 @@ static void ov_delete_recursive(Cpu *c) { ret_x(c, delete_recursive(arg_p(c, 0)) static void ov_open_directory(Cpu *c) { ret_x(c, open_directory(arg_p(c, 0), arg_p(c, 1), I32(2))); } static void ov_read_directory(Cpu *c) { ret_x(c, read_directory(arg_p(c, 0), arg_p(c, 1), arg_x(c, 2), I64(3))); } static void ov_close_directory(Cpu *c) { close_directory(arg_x(c, 0)); } +static void ov_directory_entry_count(Cpu *c) { ret_x(c, directory_entry_count(arg_p(c, 0), arg_x(c, 1))); } static void ov_mount_save(Cpu *c) { ret_x(c, mount_save(arg_p(c, 0), arg_p(c, 1))); } static void ov_mount_save_ro(Cpu *c) { ret_x(c, mount_save_ro(arg_p(c, 0), arg_x(c, 1), arg_p(c, 2))); } static void ov_save_exists(Cpu *c) { ret_x(c, save_exists(arg_x(c, 0), arg_p(c, 1))); } @@ -1169,6 +1194,7 @@ static void ov_set_priority(Cpu *c) { (void)c; } #define S_OPEN_DIRECTORY "_ZN2nn2fs13OpenDirectoryEPNS0_15DirectoryHandleEPKci" #define S_READ_DIRECTORY "_ZN2nn2fs13ReadDirectoryEPlPNS0_14DirectoryEntryENS0_15DirectoryHandleEl" #define S_CLOSE_DIRECTORY "_ZN2nn2fs14CloseDirectoryENS0_15DirectoryHandleE" +#define S_DIRECTORY_ENTRY_COUNT "_ZN2nn2fs22GetDirectoryEntryCountEPlNS0_15DirectoryHandleE" #define S_MOUNT_SAVE "_ZN2nn2fs13MountSaveDataEPKcRKNS_7account3UidE" #define S_MOUNT_SAVE_RO "_ZN2nn2fs21MountSaveDataReadOnlyEPKcNS_13ApplicationIdERKNS_7account3UidE" #define S_SAVE_EXISTS "_ZN2nn2fs18IsSaveDataExistingENS_13ApplicationIdERKNS_7account3UidE" @@ -1205,6 +1231,7 @@ HOST_OVERRIDE(S_DELETE_RECURSIVE, ov_delete_recursive); HOST_OVERRIDE(S_OPEN_DIRECTORY, ov_open_directory); HOST_OVERRIDE(S_READ_DIRECTORY, ov_read_directory); HOST_OVERRIDE(S_CLOSE_DIRECTORY, ov_close_directory); +HOST_OVERRIDE(S_DIRECTORY_ENTRY_COUNT, ov_directory_entry_count); HOST_OVERRIDE(S_MOUNT_SAVE, ov_mount_save); HOST_OVERRIDE(S_MOUNT_SAVE_RO, ov_mount_save_ro); HOST_OVERRIDE(S_SAVE_EXISTS, ov_save_exists); @@ -1282,14 +1309,16 @@ static void probe_paths(void) { static const struct { const char *path, *want; } cases[] = { {"rom:/a/b.bin", "a/b.bin"}, {"rom://a/b.bin", "a/b.bin"}, {"rom:///a/b.bin", "a/b.bin"}, {"rom:/a", "a"}, {"rom:/../main", NULL}, {"rom://../a", NULL}, - {"rom:/a/../b", NULL}, {"rom:/a/./b", NULL}, {"rom:/a//b", NULL}, + {"rom:/a/../b", NULL}, {"rom:/a/./b", NULL}, {"rom:/a//b", "a/b"}, {"rom:/a\\b", NULL}, {"rom:/", NULL}, {"rom://", NULL}, {"rom:a", NULL}, {"ro:/a", NULL}, {"romx:/a", NULL}, {"rom:/.", NULL}, {"rom:/..", NULL}, {"ROM:/a", NULL}, }; int bad = 0; for (unsigned i = 0; i < sizeof cases / sizeof *cases; i++) { - const char *got = romfs_path_relative("rom", cases[i].path); + char folded[256]; + const char *in = romfs_path_collapse(cases[i].path, folded, sizeof folded) ? folded : cases[i].path; + const char *got = romfs_path_relative("rom", in); if (cases[i].want ? !got || strcmp(got, cases[i].want) : got != NULL) { bad++; printf(" FAIL: romfs_path_relative(\"rom\", \"%s\") = %s%s%s\n", cases[i].path, got ? "\"" : "", diff --git a/runtime/gfx.h b/runtime/gfx.h index 70f329c3..2fb8f82e 100644 --- a/runtime/gfx.h +++ b/runtime/gfx.h @@ -26,7 +26,8 @@ int gfx_window_focused(void); int gfx_picture_rect(float out[4]); // where the last present put the picture, fractions of the swapchain (x, y, w, h); 0 before one // ---- shader service client (gfx_shader.c) -enum { GFX_STAGE_VS, GFX_STAGE_TCS, GFX_STAGE_TES, GFX_STAGE_GS, GFX_STAGE_FS, GFX_STAGE_CS }; +enum { GFX_STAGE_VS, GFX_STAGE_TCS, GFX_STAGE_TES, GFX_STAGE_GS, GFX_STAGE_FS, GFX_STAGE_CS, + GFX_STAGE_TS, GFX_STAGE_MS }; // TS / MS: NVN2 task and mesh programs (VK_EXT_mesh_shader) enum { GFX_TEX_1D = 1, GFX_TEX_1D_ARRAY, GFX_TEX_BUFFER, GFX_TEX_2D, GFX_TEX_2D_ARRAY, GFX_TEX_2D_MS, GFX_TEX_2D_MS_ARRAY, GFX_TEX_3D, GFX_TEX_CUBE, GFX_TEX_CUBE_ARRAY }; // gfx_shader.c tex_names order enum { GFX_TYPE_FLOAT, GFX_TYPE_SINT, GFX_TYPE_UINT }; @@ -97,6 +98,7 @@ typedef struct { uint16_t handle; // TEX/IMG: raw handle (word offset; separate: texture | sampler << 14); SSBO: address offset uint16_t count; // descriptors (array length) uint8_t separate; // TEX: texture and sampler handles are separate + uint8_t pool; // TEX: 1 the texture pool, 2 the sampler pool, indexed by bindless handles (set 2) } GfxShaderRes; typedef struct { uint8_t type, comps, packed; } GfxShaderIo; // GFX_TYPE_*, components, 1 with a Component decoration int gfx_shader_stage(const GfxShader *s); // GFX_STAGE_VS, GFX_STAGE_FS or GFX_STAGE_CS @@ -145,3 +147,9 @@ typedef struct { uint64_t frames, nvn_calls, nvn_distinct, unimplemented, valida GfxStats gfx_stats(void); void rt_frame_presented(const tex_t *t, uint64_t n); // boot.c: --frames N ends the run after the Nth present // (saves --shot first through read_texels) + +// Movie overlay (ov_movie2.c -> gfx_present.c): the latest decoded movie frame, RGBA8, shown over the game image +// while frames keep coming (the game draws movies with its own shaders, which do not translate yet). +void gfx_movie_frame(const uint8_t *rgba, int w, int h); +void gfx_movie_frame_own(uint8_t *rgba, int w, int h); // the same, taking the malloc'd buffer (freed later) +void gfx_movie_stop(void); diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index 7e13dd79..2542d123 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -562,6 +562,148 @@ void gfx_fit_rect(int cw, int ch, int dw, int dh, int out[4]) { out[0] = (dw - out[2]) / 2, out[1] = (dh - out[3]) / 2; } +// ---- Movie overlay (ov_movie2.c): the latest decoded movie frame goes over the game image, fit to the window +// with black bars, while frames keep coming (GFX_MOVIE_HOLD_NS after the last one it disappears). It made movies +// visible before the game's own shaders translated; now the game draws them itself (its YUV compute pass, then the +// upscaler fallback of vk_exec_draw.c), under its menus, so the overlay is off unless NX2NAVI_MOVIE_OVERLAY=1. One staging buffer per frame in flight: the CPU writes slot i only after +// that slot's fence (the present waits it before recording), the GPU copies it into one RGBA8 image. +#define GFX_MOVIE_HOLD_NS 400000000ull +static PlatMutex g_mv_lock = PLAT_MUTEX_INIT; +static uint8_t *g_mv_rgba; +static int g_mv_w, g_mv_h; +static uint64_t g_mv_ns, g_mv_serial; +static VkBuffer g_mvb[FRAMES]; +static VmaAllocation g_mva[FRAMES]; +static void *g_mvp[FRAMES]; +static VkDeviceSize g_mvsz[FRAMES]; +static uint64_t g_mv_slot_serial[FRAMES]; +static VkImage g_mvi; +static VmaAllocation g_mvia; +static int g_mvi_w, g_mvi_h; + +void gfx_movie_frame(const uint8_t *rgba, int w, int h) { + if (w <= 0 || h <= 0) return; + plat_mutex_lock(&g_mv_lock); + size_t need = (size_t)w * h * 4; + if (w != g_mv_w || h != g_mv_h || !g_mv_rgba) { + free(g_mv_rgba); + g_mv_rgba = malloc(need); + g_mv_w = g_mv_rgba ? w : 0, g_mv_h = g_mv_rgba ? h : 0; + } + if (g_mv_rgba) { + memcpy(g_mv_rgba, rgba, need); + g_mv_ns = plat_now_ns(); + g_mv_serial++; + } + plat_mutex_unlock(&g_mv_lock); +} + +void gfx_movie_frame_own(uint8_t *rgba, int w, int h) { + if (w <= 0 || h <= 0 || !rgba) { + free(rgba); + return; + } + plat_mutex_lock(&g_mv_lock); + uint8_t *old = g_mv_rgba; + g_mv_rgba = rgba; + g_mv_w = w, g_mv_h = h; + g_mv_ns = plat_now_ns(); + g_mv_serial++; + plat_mutex_unlock(&g_mv_lock); + free(old); +} + +void gfx_movie_stop(void) { + plat_mutex_lock(&g_mv_lock); + g_mv_ns = 0; + plat_mutex_unlock(&g_mv_lock); +} + +// Records the overlay into cb for swapchain image dst (in TRANSFER_DST_OPTIMAL, left so). 1 when drawn. +static int movie_overlay(VkCommandBuffer cb, int slot, VkImage dst) { + static int off = -1; + if (off < 0) { + const char *v = cfg_get("NX2NAVI_MOVIE_OVERLAY"); + off = !(v && strcmp(v, "0")); + } + if (off) return 0; + VkCtx *c = vk_ctx(); + plat_mutex_lock(&g_mv_lock); + if (!g_mv_ns || plat_now_ns() - g_mv_ns > GFX_MOVIE_HOLD_NS || !g_mv_rgba) { + plat_mutex_unlock(&g_mv_lock); + return 0; + } + int w = g_mv_w, h = g_mv_h; + VkDeviceSize need = (VkDeviceSize)w * h * 4; + if (g_mvsz[slot] != need) { + if (g_mvb[slot]) vmaDestroyBuffer(c->vma, g_mvb[slot], g_mva[slot]); + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = need, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | + VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo info; + if (vmaCreateBuffer(c->vma, &bi, &ai, &g_mvb[slot], &g_mva[slot], &info) != VK_SUCCESS) { + g_mvb[slot] = VK_NULL_HANDLE, g_mvsz[slot] = 0; + plat_mutex_unlock(&g_mv_lock); + return 0; + } + g_mvp[slot] = info.pMappedData; + g_mvsz[slot] = need; + g_mv_slot_serial[slot] = 0; + } + if (g_mv_slot_serial[slot] != g_mv_serial) { + memcpy(g_mvp[slot], g_mv_rgba, (size_t)need); + g_mv_slot_serial[slot] = g_mv_serial; + } + plat_mutex_unlock(&g_mv_lock); + if (g_mvi_w != w || g_mvi_h != h) { + if (g_mvi) { + vkDeviceWaitIdle(c->device); + vmaDestroyImage(c->vma, g_mvi, g_mvia); + g_mvi = VK_NULL_HANDLE; + } + VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, + .format = VK_FORMAT_R8G8B8A8_UNORM, .extent = {(uint32_t)w, (uint32_t)h, 1}, + .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateImage(c->vma, &ii, &ai, &g_mvi, &g_mvia, NULL) != VK_SUCCESS) { + g_mvi = VK_NULL_HANDLE, g_mvi_w = g_mvi_h = 0; + return 0; + } + g_mvi_w = w, g_mvi_h = h; + } + sync_barrier(cb, g_mvi, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, 0, VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT); + VkBufferImageCopy bc = {.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .imageExtent = {(uint32_t)w, (uint32_t)h, 1}}; + vkCmdCopyBufferToImage(cb, g_mvb[slot], g_mvi, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &bc); + sync_barrier(cb, g_mvi, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT); + // the game image's blit wrote dst: black bars first, then the frame + sync_barrier(cb, dst, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT); + VkClearColorValue black = {.float32 = {0, 0, 0, 1}}; + VkImageSubresourceRange all = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vkCmdClearColorImage(cb, dst, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &black, 1, &all); + sync_barrier(cb, dst, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT); + int to[4]; + gfx_fit_rect(w, h, (int)g_sc_ext.width, (int)g_sc_ext.height, to); + VkImageBlit region = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .srcOffsets = {{0, 0, 0}, {w, h, 1}}, + .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .dstOffsets = {{to[0], to[1], 0}, {to[0] + to[2], to[1] + to[3], 1}}}; + vkCmdBlitImage(cb, g_mvi, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion, + VK_FILTER_LINEAR); + return 1; +} + static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w, int h, const int crop[4], int defer) { int general = layout == VK_IMAGE_LAYOUT_GENERAL; gfx_window_start(); @@ -609,6 +751,25 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w g_sc_ext.height, cw, ch, g_scaler}; int overlay = g_sc_view[idx] && menu_overlay_frame(__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED), &mt, &menu_shot); + // NX2NAVI_SHOT_EVERY= with NX2NAVI_SHOT_DIR=: the presented frame as shot_.png every s seconds + // (a check of what the window shows, for runs nobody watches) + static char shot_path[1100]; + { + static double every = -1; + static uint64_t next_ns; + static int n; + if (every < 0) { + const char *e = cfg_get("NX2NAVI_SHOT_EVERY"); + every = e ? atof(e) : 0; + next_ns = plat_now_ns() + (uint64_t)(every * 1e9); + } + const char *dir = cfg_get("NX2NAVI_SHOT_DIR"); + if (!menu_shot && every > 0 && dir && plat_now_ns() >= next_ns && n < 200) { + snprintf(shot_path, sizeof shot_path, "%s/shot_%03d.png", dir, n++); + menu_shot = shot_path; + next_ns = plat_now_ns() + (uint64_t)(every * 1e9); + } + } int cap = (g_capture || menu_shot) && ensure_capture_buffer(); int thumb = g_th_n && ensure_thumbs(); @@ -662,6 +823,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w sync_barrier(cb, psrc, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_TRANSFER_BIT, 0, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 0); } + if (!g_sc_hdr) movie_overlay(cb, slot, dst); // a movie frame over the game image (SDR swapchains) // last: the image's layout, and the stage and access of its last write, as each step leaves it VkImageLayout last = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; VkPipelineStageFlags2 last_stage = VK_PIPELINE_STAGE_2_TRANSFER_BIT; diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index ea26bc33..ee0a4bf7 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -48,8 +48,9 @@ #define CODE_SKIP 0x30 // graphics code starts after the first 0x30 bytes of the program #define BUCKETS 4096 #define LINE_MAX_LEN 4096 -#define KEY_VERSION 4 // in every cache key; bump when the key layout or the use of the - // output changes here (4: translator identity, input ids); +#define KEY_VERSION 5 // in every cache key; bump when the key layout or the use of the + // output changes here (4: translator identity, input ids; 5: the + // NVN2 compute shared memory size in the request); // translator changes are covered by its identity #define IN_VERSION 1 // in every input id and .in file: bump when the answers layout // or the program hash changes (drops every stored input) @@ -544,7 +545,8 @@ static int reflection_valid(const char *res, uint64_t size) { const char *st = strstr(b, "\"stage\""); if (!st) return 0; st = strchr(st + 7, '"'); - return st && (!strncmp(st, "\"vs\"", 4) || !strncmp(st, "\"ps\"", 4) || !strncmp(st, "\"cs\"", 4)); + return st && (!strncmp(st, "\"vs\"", 4) || !strncmp(st, "\"ps\"", 4) || !strncmp(st, "\"cs\"", 4) || + !strncmp(st, "\"ts\"", 4) || !strncmp(st, "\"ms\"", 4)); // ts / ms: NVN2 task and mesh stages } // Every instruction after the header has a word count and the last one ends at the module's end: a module with @@ -571,7 +573,12 @@ static GfxShader *disk_load_from(const uint8_t key[32], const uint8_t *iid, int if (!res) return NULL; if (!reflection_valid((const char *)res, res_size)) { int32_t n = plat_inc32(&n_bad); - if (n <= 3 || !(n & (n - 1))) rt_log("gfx_shader: %s.res.json is malformed, translating again (%d so far)", hex, n); + if (n <= 3 || !(n & (n - 1))) { // say what was wrong: an unknown stage name has cost a session before + const char *st = strstr((const char *)res, "\"stage\""); + char what[24] = "no stage field"; + if (st) snprintf(what, sizeof what, "%.16s", st + 8); + rt_log("gfx_shader: %s.res.json is malformed (stage %s), translating again (%d so far)", hex, what, n); + } free(res); return NULL; } @@ -723,6 +730,7 @@ static void add_res(GfxShader *s, int kind, const char *o, const char *e) { r->handle = (uint16_t)handle; r->count = (uint16_t)json_int(o, e, "array", 1); r->separate = sep && *sep == 't'; + r->pool = (uint8_t)json_int(o, e, "pool", 0); r->slot = (int16_t)handle_slot(kind, r->cbuf_slot, handle, r->separate); } @@ -846,7 +854,8 @@ static void reflect(GfxShader *s) { const char *js = s->reflection; char stage[8]; json_str(js, js + strlen(js), "stage", stage, sizeof stage); - s->stage = !strcmp(stage, "cs") ? GFX_STAGE_CS : !strcmp(stage, "ps") ? GFX_STAGE_FS : GFX_STAGE_VS; + s->stage = !strcmp(stage, "cs") ? GFX_STAGE_CS : !strcmp(stage, "ps") ? GFX_STAGE_FS : !strcmp(stage, "ts") ? GFX_STAGE_TS + : !strcmp(stage, "ms") ? GFX_STAGE_MS : GFX_STAGE_VS; s->mvprev_banks = (uint32_t)json_int(js, js + strlen(js), "mvprev_banks", 0); for (int k = 0; k < GFX_RES_KINDS; k++) { int count = json_objects(js, kinds[k], NULL, s, k); @@ -1157,25 +1166,56 @@ static size_t build_request(char *o0, const char *hex, const char *stage, const o += sprintf(o, "},\"fout\":{"); for (int i = 0; i < 8; i++) if (a->frag_out_type[i]) o += sprintf(o, "%s\"%d\":\"%s\"", sep, i, attr_name(a->frag_out_type[i])), sep = ","; - o += sprintf(o, "},\"shared\":%u", a->shared_bytes); + // NVN2 compute programs carry their shared memory size in the control section (+0x64, bytes; nonzero exactly + // for the programs that use shared memory). Without it the translator declares its 4 KiB default, and a + // program that uses more (80 of 2643 in a gameplay dump, up to 18 KiB) stores out of range, which NVIDIA + // GPUs fault on (device lost at the first gameplay frame). + uint32_t shared = a->shared_bytes, magic0; + memcpy(&magic0, control, 4); + if (!shared && magic0 == 0x98761234u && !strcmp(stage, "cs")) memcpy(&shared, control + 0x64, 4); + o += sprintf(o, "},\"shared\":%u", shared); } if (!strcmp(stage, "cs")) { uint32_t l[3] = {1, 1, 1}; if (a && (a->local[0] | a->local[1] | a->local[2])) l[0] = a->local[0], l[1] = a->local[1], l[2] = a->local[2]; - else memcpy(l, control + 0x748, 12); + else memcpy(l, control + (*(const uint32_t *)control == 0x98761234u ? 0x58 : 0x748), 12); // NVN2: +0x58 + if (!(l[0] | l[1] | l[2])) l[0] = l[1] = l[2] = 1; // NVN2 controls carry no workgroup size there o += sprintf(o, ",\"local\":[%u,%u,%u]", l[0], l[1], l[2]); } /* Control byte +0x718 of a fragment shader: early fragment tests (the NVN driver's early-Z-force, MacNativeGames ae8f4f4). Without it a shader that stores to memory is depth-tested late, and the G-buffer draws' EQUAL test after the depth prepass fails along contour lines. Already part of the code hash (the whole control section). */ if (!strcmp(stage, "ps") && (control[0x718] & 1)) o += sprintf(o, ",\"early_fragment_tests\":true"); - int si = !strcmp(stage, "vs") ? 0 : !strcmp(stage, "cs") ? 2 : 1; // gfx_shader_set_host answers + int si = !strcmp(stage, "vs") ? 0 : !strcmp(stage, "cs") || !strcmp(stage, "ts") || !strcmp(stage, "ms") ? 2 : 1; // gfx_shader_set_host answers if (!host_f64) o += sprintf(o, ",\"softfloat64\":true"); if (a && a->mvprev && !strcmp(stage, "vs")) o += sprintf(o, ",\"mvprev\":true"); if (host_sbuf) o += sprintf(o, ",\"sbuf_align\":%u", host_sbuf); if (host_sg[si] != 32) o += sprintf(o, ",\"subgroup\":%u", host_sg[si]); if (host_gather) o += sprintf(o, ",\"gather_bias\":%u", host_gather); if (host_zero_locals) o += sprintf(o, ",\"zero_locals\":true"); + uint32_t magic, nvn2_size; + memcpy(&magic, control, 4), memcpy(&nvn2_size, control + 0x38, 4); + if (magic == 0x98761234u) { // NVN2_CONTROL_MAGIC, below + o += sprintf(o, ",\"sm86\":true,\"size\":%u", nvn2_size); + if (!strcmp(stage, "ts") || !strcmp(stage, "ms")) { + // Task and mesh programs (tools/xlate/AmpereLowering.cs MeshInfo): +0x58 workgroup size (low 16 bits) and + // payload size in 256 bytes (bits 24-31); +0x5c byte 3 the primitive type, byte 2 a task stage feeds the + // mesh stage; +0x60 vertex limit (low 16 bits) and primitive limit (high 16 bits). + uint32_t w58, w5c, w60; + memcpy(&w58, control + 0x58, 4), memcpy(&w5c, control + 0x5c, 4), memcpy(&w60, control + 0x60, 4); + o += sprintf(o, ",\"mesh\":{\"stage\":%d,\"threads\":%u,\"payload\":%u,\"maxv\":%u,\"maxp\":%u,\"topo\":%u,\"task\":%s}", + !strcmp(stage, "ts") ? 6 : 7, w58 & 0xffff, (w58 >> 24) * 256, w60 & 0xffff, w60 >> 16, w5c >> 24, + (w5c >> 16 & 0xff) ? "true" : "false"); + } + if (!strcmp(stage, "cs")) { + // The compute binding table in the driver bank: texture entries (byte +0xb3), image entries (+0xb6), and + // where the table ends (u16 at +0xba); 8 bytes an entry, textures first. + uint16_t end; + memcpy(&end, control + 0xba, 2); + int ntex = control[0xb3], nimg = control[0xb6]; + if (end >= 8 * (ntex + nimg)) o += sprintf(o, ",\"nvn2_table\":[%d,%d,%d]", end - 8 * (ntex + nimg), ntex, nimg); + } + } o += sprintf(o, "},\"out\":\""); json_esc(&o, cache_dir); o += sprintf(o, "\"}"); @@ -1231,6 +1271,8 @@ static const GfxShader *fail(const char *why, const char *hex) { typedef struct { const char *stage_name; uint32_t length, c1_len; + uint32_t skip; // bytes of the program before what the translator reads + int sm86; // an NVN2 program (lowered to Maxwell by the translator) const uint8_t *c1; uint8_t code_hash[32]; uint8_t key[32]; // cache key, or the provisional one with final 0 @@ -1238,13 +1280,50 @@ typedef struct { int final; } Req; +// NVN2 (Switch 2) programs: a 0xe0-byte control section (magic 0x98761234, version at +4, code size including +// the 0x80-byte shader program header at +0x38, register count at +0x4c) and SM86 code (128-bit Volta+ +// instructions after a Turing-style 0x80-byte header; compute programs have no header). The translator lowers them +// to Maxwell (tools/xlate/AmpereLowering.cs, job "sm86"); the code is sent from its first byte and has no c1 part. +// The control section is the first 0xe0 bytes only: nvn2_control copies them into a zeroed CONTROL_SIZE buffer at +// the entry points, so the bytes after it (other guest data) stay out of the hashes, stored inputs and jobs. +#define NVN2_CONTROL_MAGIC 0x98761234u +#define NVN2_CONTROL_SIZE 0xe0 +static volatile int64_t n_sm86; +uint64_t gfx_shader_sm86_refused(void) { return (uint64_t)n_sm86; } // NVN2 programs the translator refused + +static int is_nvn2(const uint8_t *control) { + uint32_t magic; + memcpy(&magic, control, 4); + return magic == NVN2_CONTROL_MAGIC; +} + +static const uint8_t *nvn2_control(const uint8_t *control, uint8_t buf[CONTROL_SIZE]) { + if (!control || !is_nvn2(control)) return control; + memset(buf, 0, CONTROL_SIZE); + memcpy(buf, control, NVN2_CONTROL_SIZE); + return buf; +} + static int req_init(Req *r, int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const uint8_t *c1, const GfxShaderAnswers *a) { - r->stage_name = stage == GFX_STAGE_VS ? "vs" : stage == GFX_STAGE_FS ? "ps" : stage == GFX_STAGE_CS ? "cs" : NULL; + r->stage_name = stage == GFX_STAGE_VS ? "vs" : stage == GFX_STAGE_FS ? "ps" : stage == GFX_STAGE_CS ? "cs" + : stage == GFX_STAGE_TS ? "ts" : stage == GFX_STAGE_MS ? "ms" : NULL; if (!r->stage_name) return fail("stage is not translated", NULL), 0; uint32_t c1_off; - memcpy(&c1_off, control + 0x700, 4), memcpy(&r->length, control + 0x704, 4); - uint64_t skip = stage == GFX_STAGE_CS ? 0 : CODE_SKIP; + r->sm86 = is_nvn2(control); + if (r->sm86) { + // NVN2: the code (+0x38 bytes), alignment padding, then the c1 constants (jump tables of BRX, compiler + // constants) from +0x40 to +0x44. They are hashed and sent with the code. + uint32_t size, c1_start, c1_end; + memcpy(&size, control + 0x38, 4), memcpy(&c1_start, control + 0x40, 4), memcpy(&c1_end, control + 0x44, 4); + r->length = size, c1_off = size; + if (c1_start >= size && c1_end >= c1_start && c1_end <= code_size) r->length = c1_end, c1_off = c1_start; + r->skip = 0; + } else { + memcpy(&c1_off, control + 0x700, 4), memcpy(&r->length, control + 0x704, 4); + r->skip = stage == GFX_STAGE_CS ? 0 : CODE_SKIP; + } + uint64_t skip = r->skip; if (!code || r->length > code_size || c1_off > r->length || skip >= r->length) return fail("bad control section", NULL), 0; r->c1_len = r->length - c1_off; r->c1 = c1 ? c1 : code + c1_off; @@ -1265,6 +1344,30 @@ static void ans_note(const Req *r, int stage, const uint8_t *control, const uint static const GfxShader *get_async_one(Req r, int stage, const uint8_t *control, const uint8_t *code, const GfxShaderAnswers *a, int *pending); +// Keys whose translation failed in this process, so the synchronous path does not send the same program to the +// translator on every draw (the background path keeps failed keys in X.waits). Open addressing; when full, failures +// are no longer remembered. +#define FAILED_N 4096 +static uint8_t failed_keys[FAILED_N][32]; +static uint8_t failed_used[FAILED_N]; +static PlatMutex failed_lock = PLAT_MUTEX_INIT; + +static int failed_slot(const uint8_t key[32], int insert) { + uint32_t h = (uint32_t)key[0] | (uint32_t)key[1] << 8 | (uint32_t)key[2] << 16; + int found = 0; + plat_mutex_lock(&failed_lock); + for (uint32_t k = 0; k < 64; k++) { + uint32_t i = (h + k) % FAILED_N; + if (!failed_used[i]) { + if (insert) memcpy(failed_keys[i], key, 32), failed_used[i] = 1; + break; + } + if (!memcmp(failed_keys[i], key, 32)) { found = 1; break; } + } + plat_mutex_unlock(&failed_lock); + return found; +} + static const GfxShader *get_one(const Req *r, int stage, const uint8_t *control, const uint8_t *code, const GfxShaderAnswers *a) { GfxShader *s = mem_find(r->key); @@ -1272,11 +1375,16 @@ static const GfxShader *get_one(const Req *r, int stage, const uint8_t *control, paths(); if (!cache_dir[0]) return fail("no cache directory", NULL); if (r->final && (s = disk_load(r->key, r->iid))) return plat_inc64(&n_hits), mem_insert(s); - return translate(r, stage, control, code, a); + if (failed_slot(r->key, 0)) return NULL; + const GfxShader *t = translate(r, stage, control, code, a); + if (!t) failed_slot(r->key, 1); + return t; } const GfxShader *gfx_shader_get(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const uint8_t *c1, const GfxShaderAnswers *a) { + uint8_t ctl2[CONTROL_SIZE]; + control = nvn2_control(control, ctl2); Req r; if (!req_init(&r, stage, control, code, code_size, c1, a)) return NULL; const GfxShader *s = get_one(&r, stage, control, code, a); @@ -1362,7 +1470,7 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co if (look && (s = mem_find(look))) return plat_inc64(&n_hits), s; if (look && (s = disk_load(look, look_iid))) return plat_inc64(&n_hits), mem_insert(s); - uint64_t skip = stage == GFX_STAGE_CS ? 0 : CODE_SKIP; + uint64_t skip = r->skip; char hex[65], code_path[PATH_CAP + 80], spv_path[PATH_CAP + 80], res_path[PATH_CAP + 80]; key_hex(key, hex); mkdir_p(cache_dir); @@ -1390,6 +1498,7 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co if (!ok) return fail("translator unavailable", hex); if (strncmp(reply, "ok ", 3)) { plat_file_delete(spv_path), plat_file_delete(res_path); + if (r->sm86) plat_inc64(&n_sm86); return fail(reply, hex); } plat_file_flush(spv_path), plat_file_flush(res_path); // the translator wrote them; on disk before use or rename @@ -1489,6 +1598,8 @@ static void xlate_worker(void *arg) { const GfxShader *gfx_shader_get_async(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const GfxShaderAnswers *a, int *pending) { *pending = 0; + uint8_t ctl2[CONTROL_SIZE]; + control = nvn2_control(control, ctl2); Req r; if (!req_init(&r, stage, control, code, code_size, NULL, a)) return NULL; const GfxShader *s = get_async_one(r, stage, control, code, a, pending); @@ -1546,6 +1657,8 @@ static const GfxShader *get_async_one(Req r, int stage, const uint8_t *control, const GfxShader *gfx_shader_await(int stage, const uint8_t *control, const uint8_t *code, uint64_t code_size, const GfxShaderAnswers *a) { int pending; + uint8_t ctl2[CONTROL_SIZE]; + control = nvn2_control(control, ctl2); const GfxShader *s = gfx_shader_get_async(stage, control, code, code_size, a, &pending); Req r; if (!pending || !req_init(&r, stage, control, code, code_size, NULL, a)) return s; @@ -2079,7 +2192,7 @@ int gfx_shader_prepare_seed(const char *dir, int threads, int procs) { if (n < 0) return 1; uint8_t *recs = calloc((size_t)n + 1, PRP_REC); static const uint8_t zero[32]; - size_t u = 0, by[3] = {0}, stages[6] = {0}; + size_t u = 0, by[3] = {0}, stages[8] = {0}; Set64 seen = {0}; for (long long i = 0; recs && i < n; i++) { // seed records come in priority order: the first id wins if (!memcmp(iid[i], zero, 32) || !set_add(&seen, hash8(iid[i]))) continue; diff --git a/runtime/guest_setjmp.c b/runtime/guest_setjmp.c new file mode 100644 index 00000000..0ef65a75 --- /dev/null +++ b/runtime/guest_setjmp.c @@ -0,0 +1,157 @@ +// setjmp / longjmp for lifted code. +// +// The guest's longjmp restores x19-x30, sp and d8-d15 and returns to the setjmp call's return address, which in lifted +// code is the middle of a host function: rt_dispatch has no function there and the run ends ("not a function start"). +// Ocarina's libjpeg reaches it whenever a JPEG fails to decode (its error_exit longjmps to the decoder's trap at +// main+0x1661670). +// +// The overrides keep both halves on the host: +// setjmp(env): the guest registers longjmp restores, kept for env, and the host state of the lifted function that +// made the call as it will be when that call returns (platform.h plat_frame_capture: the host stack unwound up to +// the frame running the guest function that contains the call), so a jump can make the call return again. +// Returns 0. +// longjmp(env, val): the guest registers back, x0 = val (1 for 0), and that host state resumed: the lifted caller +// continues after its setjmp call, reads x0 from the Cpu and takes the "returned again" path. +// Host locals the caller keeps in callee-saved registers come back as they were at setjmp, the ones in its stack frame +// as last written; C allows only the unmodified ones to be relied on after a longjmp, and the guest code was compiled +// under the same rule. A longjmp to a buffer this file did not fill (or after the setjmp's function returned) is fatal. +#include +#include + +#include "dispatch.h" +#include "host.h" +#include "platform/platform.h" + +#define MAX_MODULES 16 // as dispatch.c + +typedef struct { + uint64_t env; // guest jmp_buf address, 0 free + void *thread; // Cpu.thread of the setjmp + uint64_t x[12], sp; // x19-x30, sp + __uint128_t d[8]; // d8-d15 (the low halves are what the ABI keeps) + PlatJmp frame; +} JmpEnt; + +#define JMP_N 1024 +static JmpEnt jt[JMP_N]; +static PlatMutex jt_lock = PLAT_MUTEX_INIT; + +// Host function starts of the lifted code, sorted, with their guest starts: the guest function a host address runs. +typedef struct { uintptr_t host; uint64_t guest; } Rev; +static Rev *rev; +static uint64_t nrev; +static volatile int32_t rev_ready; + +static int by_host(const void *a, const void *b) { + uintptr_t x = ((const Rev *)a)->host, y = ((const Rev *)b)->host; + return x < y ? -1 : x > y; +} + +static void rev_init(void) { + plat_mutex_lock(&jt_lock); + if (!rev_ready) { + static RtModule mods[MAX_MODULES]; + int n = rt_linked_modules(mods, MAX_MODULES); + uint64_t total = 0; + for (int i = 0; i < n; i++) total += mods[i].count; + rev = malloc((total ? total : 1) * sizeof *rev); + if (!rev) rt_fatal("setjmp: out of memory"); + for (int i = 0; i < n; i++) + for (uint64_t k = 0; k < mods[i].count; k++) rev[nrev++] = (Rev){(uintptr_t)mods[i].fns[k], mods[i].addrs[k]}; + qsort(rev, nrev, sizeof *rev, by_host); + __atomic_store_n(&rev_ready, 1, __ATOMIC_RELEASE); + } + plat_mutex_unlock(&jt_lock); +} + +// The guest start of the lifted function whose host code contains rip; 0 outside (or above the last start). +static uint64_t guest_of_host(uintptr_t rip) { + if (!nrev || rip < rev[0].host || rip > rev[nrev - 1].host) return 0; + uint64_t lo = 0, hi = nrev; + while (hi - lo > 1) { + uint64_t m = (lo + hi) / 2; + if (rev[m].host <= rip) lo = m; + else hi = m; + } + return rev[lo].guest; +} + +// The guest start of the lifted function containing guest address a, 0 for none. +static uint64_t guest_function_of(uint64_t a) { + static RtModule mods[MAX_MODULES]; + static int n = -1; + if (n < 0) n = rt_linked_modules(mods, MAX_MODULES); + for (int i = 0; i < n; i++) { + const RtModule *m = &mods[i]; + if (a < m->base || a >= m->base + m->size || !m->count || a < m->addrs[0]) continue; + uint64_t lo = 0, hi = m->count; + while (hi - lo > 1) { + uint64_t k = (lo + hi) / 2; + if (m->addrs[k] <= a) lo = k; + else hi = k; + } + return m->addrs[lo]; + } + return 0; +} + +static int is_caller(uintptr_t rip, void *arg) { return guest_of_host(rip) == *(const uint64_t *)arg; } + +static JmpEnt *slot_of(uint64_t env, int make) { + uint64_t h = (env * 0x9E3779B97F4A7C15ull) >> 54; // 10 bits + for (int k = 0; k < JMP_N; k++) { + JmpEnt *e = &jt[(h + k) % JMP_N]; + if (e->env == env) return e; + if (!e->env) { + if (!make) return NULL; + e->env = env; + return e; + } + } + return NULL; +} + +static void ov_setjmp(Cpu *c) { + if (!__atomic_load_n(&rev_ready, __ATOMIC_ACQUIRE)) rev_init(); + uint64_t env = c->x[0]; + uint64_t caller = guest_function_of(c->x[30] - 4); // the BL to setjmp sits just before the return address + JmpEnt e = {env, c->thread, {0}, c->sp, {0}, {0}}; + for (int i = 0; i < 12; i++) e.x[i] = c->x[19 + i]; + for (int i = 0; i < 8; i++) e.d[i] = c->v[8 + i]; + if (!caller || !plat_frame_capture(&e.frame, is_caller, &caller)) + rt_fatal("setjmp: no host frame of the guest function %#llx that called it (lr %#llx)", + (unsigned long long)caller, (unsigned long long)c->x[30]); + plat_mutex_lock(&jt_lock); + JmpEnt *s = slot_of(env, 1); + if (!s) rt_fatal("setjmp: more than %d buffers", JMP_N); + *s = e; + plat_mutex_unlock(&jt_lock); + c->x[0] = 0; +} + +static void ov_longjmp(Cpu *c) { + uint64_t env = c->x[0], val = (uint32_t)c->x[1]; + static _Thread_local JmpEnt e; // the frame is resumed from here: not on the stack being abandoned + plat_mutex_lock(&jt_lock); + JmpEnt *s = slot_of(env, 0); + if (s) e = *s; + plat_mutex_unlock(&jt_lock); + if (!s || e.thread != c->thread) + rt_fatal("longjmp to buffer %#llx, which no setjmp of this thread filled (lr %#llx)", (unsigned long long)env, + (unsigned long long)c->x[30]); + static uint32_t n_logged; + if (n_logged < 4) { + n_logged++; + char at[160]; + rt_log("longjmp to buffer %#llx from %s: back into the guest function that called setjmp", + (unsigned long long)env, ldr_describe(c->x[30], at, sizeof at)); + } + for (int i = 0; i < 12; i++) c->x[19 + i] = e.x[i]; + for (int i = 0; i < 8; i++) c->v[8 + i] = e.d[i]; + c->sp = e.sp; + c->x[0] = val ? val : 1; + plat_jmp_go(&e.frame); +} + +HOST_OVERRIDE("setjmp", ov_setjmp); +HOST_OVERRIDE("longjmp", ov_longjmp); diff --git a/runtime/host.h b/runtime/host.h index 8fa4431b..c48070f4 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -143,6 +143,7 @@ uint32_t mem_set_attribute(uint64_t a, uint64_t n, uint32_t mask, uint32_t at); uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n); // 0x04 uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n); // 0x05 uint32_t mem_query(uint64_t a, RtMemoryInfo *out); // 0x06 +int mem_readable(uint64_t a, uint64_t n); // every page readable uint32_t mem_map_physical(uint64_t a, uint64_t n); // 0x2C uint32_t mem_unmap_physical(uint64_t a, uint64_t n); // 0x2D int mem_info(uint32_t id, uint64_t *out); // GetInfo memory ids 2-7, 12-17, 21, 22, 28; 0 if not a memory id @@ -187,6 +188,8 @@ typedef struct { const char *file, *sha256; } RtFileHash; extern const RtFileHash rt_exefs_sha256[]; // tools/data/expected-.json "exefs_sha256" extern const int rt_exefs_sha256_count; extern const char rt_default_exefs[], rt_default_romfs[], rt_default_profile[]; +extern const unsigned long long rt_app_pool_bytes; // the title's application memory pool (GetInfo 6) +extern const int rt_nvn_api_version[2]; // NVN_DEVICE_INFO_API_{MAJOR,MINOR}_VERSION the title requires extern const int rt_game_version; // the game build: 110 (1.1.0), 121 (1.2.1) or 143 (1.4.3) extern const char rt_port_version[]; // VERSION at the build's root: 0.1.0, 0.1.0+973c626 in a release #define RT_V110 (rt_game_version == 110) // 1.1.0: its own boot constants and data slots @@ -225,8 +228,10 @@ enum { KERN_SESSION = 4, KERN_TMEM = 5 }; uint32_t kern_event_new(void); // writable handle uint32_t kern_event_readable(uint32_t writable); void kern_event_signal(uint32_t writable); +void kern_event_clear(uint32_t writable); // server-side clear (e.g. a drained message queue) void kern_bind_system_event(Cpu *c, uint64_t ev, int autoclear); // guest thread only void kern_signal_system_event(uint64_t ev); // any thread +void kern_clear_system_event(uint64_t ev); // any thread; a bound, live event only int kern_svc(Cpu *c, uint32_t imm); // thread/sync SVCs; 1 if handled uint64_t kern_now_ns(void); extern uint64_t rt_guest_start_ns; // kern_now_ns when the guest started (boot.c), 0 before diff --git a/runtime/hostfn.c b/runtime/hostfn.c index 058db80a..d3d4bfea 100644 --- a/runtime/hostfn.c +++ b/runtime/hostfn.c @@ -127,6 +127,12 @@ __attribute__((noinline)) static void hostfn_tracked(Cpu *c, GuestFn fn) { int *d = kern_host_depth(t); __atomic_store_n(d, *d + 1, __ATOMIC_RELAXED); fn(c); + // fn may have switched fibers (SwitchToFiber is a host call) and this context may resume on another OS thread: + // the fiber switch moved this context's depth to that thread, whose RtThread the fiber's Cpu now names + if (c->thread != t) { + t = c->thread; + d = kern_host_depth(t); + } int n = *d - 1; __atomic_store_n(d, n, __ATOMIC_RELAXED); if (!n && kern_preempt_pending(t)) kern_host_left(t); diff --git a/runtime/ipc.c b/runtime/ipc.c index 39fa17c4..dd7eaba3 100644 --- a/runtime/ipc.c +++ b/runtime/ipc.c @@ -1,4 +1,4 @@ -// Adapted from MacNativeGames games//runtime/ipc.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames +// Adapted from MacNativeGames games//runtime/ipc.c @ b75be8c (MIT, Copyright (c) 2026 MacNativeGames // contributors). Message framing per libnx sf/hipc.h, cmif.h, tipc.h (ISC), read for the layout only. // In-process Horizon IPC: SVCs 0x15, 0x1F, 0x21, 0x22, the sm: service manager // and the service table. The SDK client code runs unchanged: requests are read from the calling thread's message @@ -23,10 +23,186 @@ enum { T_CLOSE = 2, T_REQUEST = 4, T_CONTROL = 5, T_REQUEST_CTX = 6, T_CONTROL_CTX = 7, T_TIPC_CLOSE = 15, T_TIPC_BASE = 16 }; +// The application's lifecycle (the system service/am/lifecycle_manager.cpp, read for behaviour): one state-changed +// event per applet, signalled while a message is waiting and cleared by the server once the queue drains (Horizon +// does not clear it on a wait). An *application* learns it has focus through FocusStateChanged (15) followed by +// GetCurrentFocusState == InFocus; ChangeIntoForeground (1) is the library-applet message and an application +// ignores it, which is what parked nn::oe::InitializeApplet on the event. +enum { AM_MSG_FOCUS_STATE_CHANGED = 15, AM_MSG_OPERATION_MODE_CHANGED = 30, AM_MSG_PERFORMANCE_MODE_CHANGED = 31 }; +enum { AM_FOCUS_IN = 1, AM_FOCUS_NOT_IN = 2, AM_FOCUS_BACKGROUND = 3 }; +#define AM_R_NO_MESSAGES 0x680u // the "no messages" result this SDK checks for +static uint32_t g_applet_event, g_applet_event_w; +static PlatMutex g_applet_lock = PLAT_MUTEX_INIT; +static uint32_t g_applet_msgs[32]; +static unsigned g_applet_head, g_applet_count; +static uint8_t g_applet_focus = AM_FOCUS_IN; + +void ipc_applet_push_message(uint32_t msg) { + plat_mutex_lock(&g_applet_lock); + if (g_applet_count < sizeof g_applet_msgs / sizeof *g_applet_msgs) { + g_applet_msgs[(g_applet_head + g_applet_count++) % (sizeof g_applet_msgs / sizeof *g_applet_msgs)] = msg; + } else { + rt_log("applet: message queue full, dropping %u", msg); + } + plat_mutex_unlock(&g_applet_lock); + if (g_applet_event_w) kern_event_signal(g_applet_event_w); +} + +static int applet_pop_message(uint32_t *out) { + plat_mutex_lock(&g_applet_lock); + int ok = g_applet_count != 0; + if (ok) { + *out = g_applet_msgs[g_applet_head]; + g_applet_head = (g_applet_head + 1) % (sizeof g_applet_msgs / sizeof *g_applet_msgs); + g_applet_count--; + } + int drained = g_applet_count == 0; + plat_mutex_unlock(&g_applet_lock); + if (drained && g_applet_event_w) kern_event_clear(g_applet_event_w); + return ok; +} + +static IpcSession *stub_object(const char *name); +static void applet_stub_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs); + +// A stub object the applet proxy commands are answered on: the interfaces exist, their calls succeed. +// The getters below each hand out one of those objects, which is what the SDK stores and dereferences. +static void applet_proxy_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + if (rt_trace & RT_TRACE_IPC) rt_log("applet proxy: command %u", rq->cmd); + static const struct { unsigned cmd; const char *name; } kinds[] = { + {0, "common state getter"}, {1, "self controller"}, {2, "window controller"}, + {3, "audio controller"}, {4, "display controller"}, {10, "process winding controller"}, + {11, "library applet creator"}, {20, "application functions"}, + {40, "nx2 interface"}, {50, "nx2 interface"}, {60, "graphics scale controller"}, + {70, "nx2 interface"}, {80, "nx2 interface"}, {90, "nx2 interface"}, + {1000, "debug functions"}, + }; + for (unsigned k = 0; k < sizeof kinds / sizeof *kinds; k++) { + if (kinds[k].cmd != rq->cmd) continue; + IpcSession *root = s->root ? s->root : s; // object ids belong to the client's whole domain + IpcSession *o = stub_object(kinds[k].name); + if (!o) { rs->result = IPC_R_UNKNOWN_COMMAND; return; } + o->root = root; + if (!ipc_domain_add(root, o)) { free(o); rs->result = IPC_R_UNKNOWN_COMMAND; return; } + uint32_t id = (uint32_t)root->nobjects; + IPC_OUT(rs, id); + rs->domain = 1; + rs->domain_objects = 1; + rs->result = 0; + return; + } + rs->result = 0; +} + +// The interfaces themselves. Only the calls the game makes while booting are modelled: the common state getter +// hands out the applet's state-changed event, and everything else reports success. +static void applet_stub_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + if (rt_trace & RT_TRACE_IPC) rt_log("%s: command %u", s->name, rq->cmd); + int common = !strcmp(s->name, "common state getter"); + if (common && rq->cmd == 0) { // GetEventHandle: the applet state-changed event + if (!g_applet_event) ipc_applet_event_init(); + s->event = g_applet_event; + rs->copy[rs->ncopy++] = s->event; + rs->result = 0; + return; + } + if (common && rq->cmd == 1) { // ReceiveMessage: the next queued applet message, else "no messages" + uint32_t msg; + if (applet_pop_message(&msg)) { + if (rt_trace & RT_TRACE_IPC) rt_log("applet: ReceiveMessage -> %u", msg); + IPC_OUT(rs, msg); + rs->result = 0; + return; + } + rs->result = AM_R_NO_MESSAGES; + return; + } + if (common && rq->cmd == 5) { // GetOperationMode: docked (matches the oe override) + uint8_t mode = 1; + IPC_OUT(rs, mode); + rs->result = 0; + return; + } + if (common && rq->cmd == 6) { // GetPerformanceMode: boost (docked) + uint32_t mode = 1; + IPC_OUT(rs, mode); + rs->result = 0; + return; + } + if (common && rq->cmd == 9) { // GetCurrentFocusState + uint8_t focus = g_applet_focus; + IPC_OUT(rs, focus); + rs->result = 0; + return; + } + if (!strcmp(s->name, "graphics scale controller") && rq->cmd == 10) { // GetCurrentGraphicsScale + uint32_t scale[3] = {10, 0, 0}; // see ov_oe_scaling_parameters + IPC_OUT(rs, scale); + rs->result = 0; + return; + } + if (rt_trace & RT_TRACE_IPC) rt_log("%s: command %u answered with success and no data", s->name, rq->cmd); + rs->result = 0; +} + +static IpcSession *stub_object(const char *name) { + IpcSession *o = calloc(1, sizeof *o); + if (!o) return NULL; + o->name = name; + o->handler = applet_stub_handler; + o->refs = 1; + return o; +} + +unsigned ipc_domain_add(IpcSession *domain, IpcSession *object) { + if (domain->nobjects >= (int)(sizeof domain->objects / sizeof *domain->objects)) return 0; + domain->objects[domain->nobjects++] = object; + return (unsigned)domain->nobjects; +} + +// appletOE: the application-environment service. nn::oe::Initialize opens it; the calls that matter for +// boot answer through symbol overrides (focus, performance, language), so a permissive handler that +// reports success and writes nothing is enough until a command needs real data — then it is added here. +static void applet_oe_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + if (rt_trace & RT_TRACE_IPC) rt_log("appletOE: command %u", rq->cmd); + if (rq->cmd == 0 || rq->cmd == 10) { // OpenApplicationProxy: hand the client a domain object + IpcSession *proxy = calloc(1, sizeof *proxy); + if (!proxy) { rs->result = IPC_R_UNKNOWN_COMMAND; return; } + proxy->name = "applet proxy"; + proxy->handler = applet_proxy_handler; + proxy->refs = 1; + proxy->root = s; + if (!ipc_domain_add(s, proxy)) { free(proxy); rs->result = IPC_R_UNKNOWN_COMMAND; return; } + uint32_t id = (uint32_t)s->nobjects; // the object's id inside the domain, written into the reply data + IPC_OUT(rs, id); + rs->domain = 1; + rs->domain_objects = 1; + } + rs->result = 0; +} + +// aud:u: the Switch 2 audio entry point (nn::audio::CreateUserClientSessionByHipc). It hands out the Switch 1 +// managers as sub-objects (the system service/audio/nx2_audio_user_client.h; command ids are the system's guesses): +// 2 -> IAudioRendererManager (the audren:u manager), 1 -> IAudioOutManager (not served by this host). +static void aud_user_handler(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { + (void)s; + if (rt_trace & RT_TRACE_IPC) rt_log("aud:u: command %u", rq->cmd); + if (rq->cmd == 2) { + rs->move[rs->nmove++] = ipc_session_new("IAudioRendererManager", audren_manager_handler, NULL, NULL); + rs->result = 0; + return; + } + rt_fatal("aud:u: command %u is not implemented (1 is GetAudioOutManager: no audout service here)", rq->cmd); +} + // Services sm: hands out. Anything else is a named fatal. static const struct { const char *name; IpcHandler handler; } services[] = { {"audren:u", audren_manager_handler}, {"nfp:user", nfp_user_manager_handler}, + {"appletOE", applet_oe_handler}, + {"aud:u", aud_user_handler}, + {"auddmg:u", audio_dmg_handler}, + {"audsmx:u", audio_sysmix_handler}, }; static volatile int32_t g_live; @@ -179,6 +355,11 @@ static void respond(uint32_t *m, int tipc, const IpcResponse *rs) { uint8_t *words = (uint8_t *)p, *payload; unsigned header = tipc ? 4 : 16; payload = tipc ? words : (uint8_t *)out + (((words - (uint8_t *)out) + 15) & ~15); + if (rs->domain && !tipc) { // CMIF domain reply: 16-byte header with the objects appended, then SFCO + uint32_t dh[4] = {rs->domain_objects, 0, 0, 0}; + memcpy(payload, dh, 16); + payload += 16; + } unsigned used = (unsigned)(payload - words) + header + rs->data_size; unsigned nwords = tipc ? (used + 3) / 4 : (used + 16 + 3) / 4; // CMIF keeps alignment slack like libnx if ((size_t)(words - (uint8_t *)out) + nwords * 4 > IPC_MSG_BYTES) rt_fatal("ipc: response exceeds the message buffer"); @@ -189,13 +370,26 @@ static void respond(uint32_t *m, int tipc, const IpcResponse *rs) { } memcpy(payload + header, rs->data, rs->data_size); out[0] = (rs->nxout & 0xf) << 16; - out[1] = (nwords & 0x3ff) | (special ? 1u << 31 : 0); + out[1] = (nwords & 0x3ff) | ((special || rs->domain) ? 1u << 31 : 0); memcpy(m, out, IPC_MSG_BYTES); } -// CMIF control requests. Domains are not implemented: MNG boots 1.2.1 without them. +// CMIF control requests. static void control(IpcSession *s, const IpcRequest *rq, IpcResponse *rs) { switch (rq->cmd) { + case 0: // ConvertCurrentObjectToDomain: the session answers domain-tagged requests from now on + s->is_domain = 1; + s->root = s; + if (s->nobjects == 0) { s->objects[0] = s; s->nobjects = 1; } + return; + case 1: { // CopyFromCurrentDomain(object id): a handle to that object of the domain + uint32_t id = rq->data_size >= 4 ? *(const uint32_t *)(const void *)rq->data : 0; + if (!id || id > (uint32_t)s->nobjects) { rs->result = IPC_R_UNKNOWN_COMMAND; return; } + IpcSession *o = s->objects[id - 1]; + plat_inc32(&o->refs); + rs->move[rs->nmove++] = kern_handle_new(KERN_SESSION, o); + return; + } case 3: { uint16_t n = POINTER_BUFFER_SIZE; IPC_OUT(rs, n); return; } // QueryPointerBufferSize case 2: case 4: rs->move[rs->nmove++] = session_clone(s); return; // CloneCurrentObject(Ex) default: @@ -214,6 +408,7 @@ static void dispatch(IpcSession *s, uint32_t *m) { IpcResponse rs; memset(&rs, 0, sizeof rs); int tipc = rq.type >= T_TIPC_BASE; + IpcSession *target = s; const uint8_t *bytes = (const uint8_t *)words, *end = bytes + ndata * 4; if (tipc) { rq.cmd = rq.type - T_TIPC_BASE; @@ -221,11 +416,27 @@ static void dispatch(IpcSession *s, uint32_t *m) { rq.data_size = (uint32_t)(end - bytes); } else if (rq.type == T_REQUEST || rq.type == T_REQUEST_CTX || rq.type == T_CONTROL || rq.type == T_CONTROL_CTX) { const uint8_t *payload = (const uint8_t *)m + (((bytes - (const uint8_t *)m) + 15) & ~15); + IpcSession *dom = s->root ? s->root : s; // ids belong to the client's domain, not to this object + if ((s->is_domain || dom->is_domain) && (rq.type == T_REQUEST || rq.type == T_REQUEST_CTX)) { + // Domain request: a 16-byte domain header names the object (1-based), then the usual SFCI payload. + uint32_t dh[4]; + memcpy(dh, payload, 16); + payload += 16; + rs.domain = 1; + uint32_t dcmd = dh[0] & 0xff, obj = dh[1]; + if (dcmd == 2) return; // CloseVirtualHandle: nothing to answer + if (obj == 0) obj = 1; // this SDK addresses the domain's own object as 0 + if (obj > (uint32_t)dom->nobjects) { + ipc_backtrace(); + rt_fatal("ipc: %s domain object %u (of %d) does not exist [hdr %08x %08x %08x %08x] [msg %08x %08x %08x %08x]", s->name, obj, dom->nobjects, dh[0], dh[1], dh[2], dh[3], m[0], m[1], m[2], m[3]); + } + target = dom->objects[obj - 1]; + } uint32_t h[4]; memcpy(h, payload, 16); if (h[0] != CMIF_IN_MAGIC) { ipc_backtrace(); - rt_fatal("ipc: %s: CMIF magic %#x (domain sessions are not served)", s->name, h[0]); + rt_fatal("ipc: %s: CMIF magic %#x", s->name, h[0]); } rq.cmd = h[2]; rq.data = payload + 16; @@ -239,7 +450,7 @@ static void dispatch(IpcSession *s, uint32_t *m) { rt_fatal("ipc: %s message type %u is not implemented", s->name, rq.type); } if (rq.type == T_CONTROL || rq.type == T_CONTROL_CTX) control(s, &rq, &rs); - else s->handler(s, &rq, &rs); + else target->handler(target, &rq, &rs); if (rt_trace & RT_TRACE_IPC) { char d[160]; rt_log("ipc: %s %s %u -> %#x (copy %u move %u) from %s", s->name, @@ -795,3 +1006,14 @@ int probe_ipc(void) { printf("probe-ipc: passed\n"); return 0; } + +// Create the applet state-changed event at boot and queue the launch message: the application starts in focus +// (the system AppletManager: SetFocusState(InFocus) -> FocusStateChanged for an application). +void ipc_applet_event_init(void) { + if (!g_applet_event) { + g_applet_event_w = kern_event_new(); + g_applet_event = kern_event_readable(g_applet_event_w); + g_applet_focus = AM_FOCUS_IN; + ipc_applet_push_message(AM_MSG_FOCUS_STATE_CHANGED); + } +} diff --git a/runtime/ipc.h b/runtime/ipc.h index 66b97df0..9b218476 100644 --- a/runtime/ipc.h +++ b/runtime/ipc.h @@ -34,6 +34,8 @@ typedef struct { unsigned ncopy, nmove; IpcBuffer xout[IPC_MAX_BUF]; // pointer data the service placed into the client's C buffers unsigned nxout; + int domain; // the reply carries a domain message header + unsigned domain_objects; // objects the service appended to the client's domain } IpcResponse; typedef struct IpcSession IpcSession; @@ -47,8 +49,16 @@ struct IpcSession { void *state; void (*close)(void *state); int32_t refs; + int is_domain; // ConvertCurrentObjectToDomain ran: requests carry a domain header + IpcSession *root; // the domain session every object id belongs to (objects share it) + IpcSession *objects[32]; // object 1 is this session, then what the service handed out + int nobjects; + uint32_t event; // readable event handed to the client (applet state changes) }; +// Register an object inside a domain (1-based id returned, 0 when full). +unsigned ipc_domain_add(IpcSession *domain, IpcSession *object); + uint32_t ipc_session_new(const char *name, IpcHandler handler, void *state, void (*close)(void *state)); // handle void ipc_out(IpcResponse *rs, const void *p, uint32_t n); const void *ipc_arg(const IpcRequest *rq, uint32_t off, uint32_t n); @@ -66,3 +76,5 @@ uint64_t ipc_sessions_released(void); // The guest frame chain of the request being served, for fatal reports from services. void ipc_backtrace(void); +void ipc_applet_event_init(void); // call once at boot, before the guest starts +void ipc_applet_push_message(uint32_t msg); // queue an AppletMessage and signal the applet event diff --git a/runtime/kernel.c b/runtime/kernel.c index b620a7b7..0fd5cff2 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -23,11 +23,15 @@ #define R_BAD_HANDLE KERN_R_BAD_HANDLE #define R_BAD_STATE KERN_R_BAD_STATE #define R_INVALID_ENUM KERN_R_INVALID_ENUM +#define R_INVALID_ADDRESS 0xCC01u // InvalidAddress (misaligned) #define WAITERS_BIT KERN_WAITERS_BIT #define H_CUR_THREAD KERN_H_CURRENT_THREAD #define MAX_H 4096 -#define NCORE 3 +// Application cores: Switch 2 titles get cores 0-5 (this title's NPDM ThreadInfo capability: core min 0, max 5); +// a Switch 1 title uses 0-2 and never names 3-5, so 6 virtual cores serve both. +#define NCORE 6 +#define CORE_MASK_ALL ((1ull << NCORE) - 1) #define HOST_STACK (256ull << 20) // reserved, committed on demand #define TLR_SIZE 0x200 #define THREAD_OBJ_SIZE 0x400 // "current thread object" at TLR+0x1F8 (d2r-mobile thread.c) @@ -109,6 +113,11 @@ struct RtThread { static PlatMutex G = PLAT_MUTEX_INIT; static struct { int type; void *obj; } g_h[MAX_H]; static _Thread_local RtThread *tcur; + +// The current thread, read again on every call. Guest fibers migrate between OS threads (Switch 2 titles' job +// systems resume any fiber on any worker): code that resumes after plat_fiber_switch may be on another OS thread, and +// the compiler takes a thread-local address to be the same throughout a function, so such code calls this instead. +__attribute__((noinline)) static RtThread *tcur_now(void) { return tcur; } _Static_assert(offsetof(RtThread, preempt) == 0, "host.h kern_preempt_pending reads the first int"); _Static_assert(offsetof(RtThread, host_depth) == sizeof(int), "host.h kern_host_depth reads the second int"); static RtThread *g_all, *g_wait, *g_main; @@ -301,6 +310,7 @@ static uint32_t block_inner(RtThread *t, int kind, int64_t ns) { // in main+0x82e00 or hangs at the title menu. The start log and every crash report say so (kern_free_cores_note). static RtThread *g_core_owner[NCORE], *g_core_wait; static int g_cores_on = -1, g_held_max; +static int g_active_cores = NCORE; // NX2NAVI_CORES (core_allowed) static void diag_init(void); static void ap_init(void); @@ -308,6 +318,11 @@ static void ap_init(void); static int cores_enabled(void) { if (g_cores_on < 0) { g_cores_on = !cfg_bool("NX2NAVI_FREE_CORES"); // "0" keeps the gate, like unset + const char *n = cfg_get("NX2NAVI_CORES"); + if (n && atoi(n) >= 1 && atoi(n) <= NCORE) { + g_active_cores = atoi(n); + rt_log("kernel: %d virtual cores run threads (NX2NAVI_CORES); higher cores fold onto them", g_active_cores); + } if (!g_cores_on) rt_log("kernel: WARNING: free_cores = 1 (NX2NAVI_FREE_CORES) turns the virtual core gate off. This is an " "experimental mode and the game crashes or hangs in it, usually at start. Set free_cores = 0 in " @@ -324,7 +339,15 @@ void kern_free_cores_note(void) { "nx2navi.ini before reporting this crash"); } -static int core_allowed(const RtThread *t, int c) { return (int)((t->mask >> c) & 1); } +// NX2NAVI_CORES = 1..NCORE: how many virtual cores run threads (default all NCORE). A core at or above the count is +// folded onto core (c mod count): a thread pinned to core 5 with NX2NAVI_CORES=3 runs on core 2, one thread per core +// at a time as always (diagnosis of races that more parallel cores expose). +static int core_allowed(const RtThread *t, int c) { + if (c >= g_active_cores) return 0; + for (int j = c; j < NCORE; j += g_active_cores) + if ((t->mask >> j) & 1) return 1; + return 0; +} // ---- asynchronous preemption (NX2NAVI_ASYNC_PREEMPT, on unless 0) // Adapted from MacNativeGames games//runtime/threads.c @ 33e8fa8 (8bf0893, PR #488; MIT, Copyright (c) 2026 @@ -1410,6 +1433,7 @@ static RtThread *thr_new(int prio, int core, uint64_t mask) { // TLR (0x200) and the "current thread object" the SDK reads through TLR+0x1F8, in one guest-visible block. t->tlr = mem_kernel_alloc(TLR_SIZE + THREAD_OBJ_SIZE, TLR_SIZE); *(uint64_t *)(t->tlr + 0x1F8) = (uint64_t)(uintptr_t)(t->tlr + TLR_SIZE); + *(uint64_t *)(t->tlr + 0x1F0) = (uint64_t)(uintptr_t)t->tls; // the SDK reads the TLS pointer here and makes it TPIDR_EL0 t->tls = mem_kernel_alloc(TLS_SIZE, 16); c->tpidrro = (uint64_t)(uintptr_t)t->tlr; c->tpidr = (uint64_t)(uintptr_t)t->tls; @@ -1425,6 +1449,9 @@ static void thr_link(RtThread *t) { // holding G t->all = g_all; g_all = t; g_created++; + // Horizon publishes the running thread's handle at TLR+0x110; nn::os::SdkMutexType::IsLockedByCurrentThread + // compares it with a mutex's owner word, so without it every unlocked mutex reads as held by this thread. + if (t->tlr) *(uint32_t *)(t->tlr + 0x110) = t->handle; } static void thr_free(RtThread *t) { // holding G; t has exited and its handle is closed @@ -1500,6 +1527,15 @@ uint64_t kern_threads_created(void) { Cpu *kern_cpu(void) { return tcur ? tcur->cur : NULL; } +void kern_seed_tls(void) { + lock(); + if (tcur && tcur->tlr && tcur->tls) { + *(uint64_t *)(tcur->tlr + 0x1F0) = (uint64_t)(uintptr_t)tcur->tls; + if (tcur->cur) tcur->cur->tpidr = (uint64_t)(uintptr_t)tcur->tls; + } + unlock(); +} + uint32_t kern_main_thread(Cpu **cpu) { if (tcur) { if (!tcur->is_main) rt_fatal("kern_main_thread on guest thread %#x, not the main thread", tcur->handle); @@ -1511,7 +1547,7 @@ uint32_t kern_main_thread(Cpu **cpu) { uint64_t stack = L->stack.base + L->stack.size - MAIN_STACK; if (!plat_mem_commit(stack, MAIN_STACK)) rt_fatal("kernel: commit the main thread stack at %#llx failed", (unsigned long long)stack); - RtThread *t = thr_new(MAIN_PRIO, MAIN_CORE, 7); + RtThread *t = thr_new(MAIN_PRIO, MAIN_CORE, CORE_MASK_ALL); t->started = 1; t->is_main = 1; t->stack_top = stack + MAIN_STACK; @@ -1641,11 +1677,11 @@ int kern_svc(Cpu *c, uint32_t imm) { switch (imm) { case 0x08: { // CreateThread(-> w1 handle; x1 entry, x2 arg, x3 stack top, w4 priority, w5 core) int prio = (int)x[4], core = (int)x[5]; - if (!x[1] || !x[3] || prio < 28 || prio > 59 || core < -3 || core > 2) { + if (!x[1] || !x[3] || prio < 28 || prio > 59 || core < -3 || core >= NCORE) { x[0] = R_BAD_STATE; return 1; } - RtThread *t = thr_new(prio, core >= 0 ? core : MAIN_CORE, core >= 0 ? 1ull << core : 7); + RtThread *t = thr_new(prio, core >= 0 ? core : MAIN_CORE, core >= 0 ? 1ull << core : CORE_MASK_ALL); t->entry = x[1]; t->arg = x[2]; t->stack_top = x[3] & ~15ull; @@ -1756,9 +1792,19 @@ int kern_svc(Cpu *c, uint32_t imm) { RtThread *t = h_get((uint32_t)x[0], O_THREAD); if (t) { int core = (int)x[1]; + uint64_t mask = x[2] & CORE_MASK_ALL; + // Out-of-range cores and mask bits are clamped (Horizon refuses them, and the SDK aborts on that): an + // ideal core of NCORE or more would reach GetCurrentProcessorNumber and the game's per-core arrays. + if (core >= NCORE || (x[2] & ~CORE_MASK_ALL)) { + static int logged; + if (logged++ < 4) + rt_log("kernel: SetThreadCoreMask(%#x, core %d, mask %#llx) clamped to cores 0-%d", t->handle, core, + (unsigned long long)x[2], NCORE - 1); + if (core >= NCORE) core = mask ? __builtin_ctzll(mask) : MAIN_CORE; + } if (core >= 0) t->core = core; else if (core == -1) t->core = MAIN_CORE; - if (x[2]) t->mask = x[2]; + if (mask) t->mask = mask; if (t == tcur && t->cur_core >= 0 && !core_allowed(t, t->cur_core)) { // move off a forbidden core core_put(t); core_take(t); @@ -1856,18 +1902,21 @@ int kern_svc(Cpu *c, uint32_t imm) { return 1; } - case 0x34: { // WaitForAddress(x0 address, w1 type, w2 value, x3 timeout): 0 if less, 1 decrement if less, 2 if equal + case 0x34: { // WaitForAddress(x0 address, w1 type, x2 value, x3 timeout): 0 if less, 1 decrement if less, + // 2 if equal, 3 if equal 64 (19.0.0+ WaitIfEqual64: the whole 64-bit word against x2; libc++'s + // std::atomic::wait on Switch 2 uses it through nn::os::WaitForAddress) RtThread *t = cur(imm); uint64_t addr = x[0]; int type = (int)x[1]; int32_t val = (int32_t)x[2]; - if (type < 0 || type > 2) { - x[0] = R_INVALID_ENUM; + if (type < 0 || type > 3 || (type == 3 && (addr & 7))) { + x[0] = type == 3 && (addr & 7) ? R_INVALID_ADDRESS : R_INVALID_ENUM; return 1; } lock(); uint32_t r, v = ld32(addr); - int ok = type == 2 ? (int32_t)v == val : (int32_t)v < val; + int ok = type == 3 ? __atomic_load_n((const uint64_t *)(uintptr_t)addr, __ATOMIC_SEQ_CST) == x[2] + : type == 2 ? (int32_t)v == val : (int32_t)v < val; if (ok && type == 1) while (!cas32(addr, &v, v - 1)) if (!((int32_t)v < val)) { @@ -1981,10 +2030,16 @@ int kern_svc(Cpu *c, uint32_t imm) { for (int i = 0; i < n && idx < 0; i++) if (sig_obj(hs[i], &bad)) idx = i; if (bad) { + static int logged; + if (rt_trace && logged++ < 4) + rt_logf("kernel: WaitSynchronizationN on a handle that is not waitable: %#x\n", hs[0]); r = R_BAD_HANDLE; break; } if (idx >= 0) { + // Horizon does not clear an event on a wait: the SDK clears auto-clear events itself (ResetSignal), + // and a server clears its own event when there is nothing left to report (the applet message + // queue in ipc.c does). An earlier auto-clear here hid the applet server never clearing. r = R_OK; break; } @@ -2006,6 +2061,15 @@ int kern_svc(Cpu *c, uint32_t imm) { } left = ns - (int64_t)spent; } + { + static int logged; + if (rt_trace && logged++ < 3) { + char d[96]; + rt_logf("kernel: first blocking wait: %d handle(s) %#x %#x lr %s\n", n, + n > 0 ? hs[0] : 0, n > 1 ? hs[1] : 0, + ldr_describe(c->x[30], d, sizeof d)); + } + } t->wobj = n ? hs[0] | (uint64_t)n << 32 : 0; g_sync_blocks++; woke = block(t, W_SYNC, left); @@ -2083,6 +2147,14 @@ void kern_event_signal(uint32_t hw) { unlock(); } +void kern_event_clear(uint32_t hw) { + lock(); + Event *e = h_peek(hw, O_EVENT_W); + if (!e) rt_fatal("kern_event_clear: %#x is not a writable event handle", hw); + e->signaled = 0; + unlock(); +} + // System events of host services: the SDK's own nn::os::CreateSystemEvent creates the kernel event (SVC 0x45) on a // guest thread; the writable handle is kept, so any thread (host audio tick included) can signal it later. // The guest may destroy such an event and attach the same SystemEventType again: the game does that with the nfp @@ -2143,6 +2215,12 @@ void kern_signal_system_event(uint64_t ev) { kern_event_signal(wh); } +// Clears a bound event (one never bound or since destroyed is left alone). +void kern_clear_system_event(uint64_t ev) { + uint32_t wh = se_find(ev); + if (wh && se_live(ev)) kern_event_clear(wh); +} + // ---- fibers // The SDK switches fibers by swapping register context, which breaks "RET = C return". Each guest fiber gets a host // fiber (own 256 MiB reserved host stack) and its own Cpu, so a suspended fiber's registers and host frames stay @@ -2171,8 +2249,8 @@ static Fiber *fiber_find(uint64_t addr) { static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to); // After a host fiber switch: release the context left behind, and finish an ExitThread issued on a fiber. -static void fiber_resumed(void) { - RtThread *t = tcur; +__attribute__((noinline)) static void fiber_resumed(void) { + RtThread *t = tcur_now(); rt_fpcr_sync(t->cur); // the host fiber resumes with its own MXCSR (platform.h fibers switch it) if (t->leaving) { __atomic_store_n(&t->leaving->running, 0, __ATOMIC_RELEASE); @@ -2185,14 +2263,14 @@ static void fiber_main(void *p) { Fiber *f = p; fiber_resumed(); Cpu *c = f->cpu; - tcur->host_depth = 0; // a new fiber starts at the top level; each context restores its own depth on resume + tcur_now()->host_depth = 0; // a new fiber starts at the top level; each context restores its own depth on resume uint64_t next = rt_call(c, f->func, 1, f->arg); f->state = F_DONE; Fiber *n = NULL; if (next && !(n = fiber_find(next))) rt_fatal("fiber %#llx returned %#llx, which is not an initialized fiber", (unsigned long long)f->addr, (unsigned long long)next); - fiber_switch_to(tcur, c, n); // NULL: back to the thread's own context + fiber_switch_to(tcur_now(), c, n); // NULL: back to the thread's own context (of the thread now running it) rt_fatal("fiber %#llx resumed after it finished", (unsigned long long)f->addr); } @@ -2221,9 +2299,11 @@ static void fiber_switch_to(RtThread *t, const Cpu *from, Fiber *to) { t->leaving = t->fiber; t->fiber = to; t->cur = dst; - int depth = t->host_depth; + int depth = t->host_depth; // this context's host frames plat_fiber_switch(to ? to->host : t->base_fiber); - t->host_depth = depth; + // Resumed, maybe on another OS thread: this context's frames are now that thread's (t is the thread that + // suspended it, which runs something else meanwhile) + tcur_now()->host_depth = depth; fiber_resumed(); } @@ -2256,6 +2336,12 @@ void kern_fiber_initialize(Cpu *c) { f->state = F_NEW; f->running = 0; f->host = plat_fiber_new(HOST_STACK, fiber_main, f); + { + static int logged; + if (logged++ < 6) + rt_log("kernel: InitializeFiber %#llx (stack %#llx+%#llx) on thread %#x", (unsigned long long)addr, + (unsigned long long)f->stack, (unsigned long long)f->size, tcur ? tcur->handle : 0); + } if (!f->host) rt_fatal("kernel: cannot create a host fiber"); } @@ -2281,6 +2367,16 @@ void kern_fiber_finalize(Cpu *c) { void kern_fiber_switch(Cpu *c) { RtThread *t = tcur; if (!t) rt_fatal("SwitchToFiber on a host thread"); + // SwitchToFiber(nullptr) returns to the thread's own context (the SDK's sdk+0x1693c0 takes the thread's + // original fiber for a null argument); Switch 2 titles use it to leave a fiber. + if (!c->x[0]) { + static int logged; + if (logged++ < 3) + rt_log("kernel: SwitchToFiber(nullptr) on thread %#x (on fiber %#llx)", t->handle, + (unsigned long long)(t->fiber ? t->fiber->addr : 0)); + if (t->fiber) fiber_switch_to(t, c, NULL); + return; + } Fiber *to = fiber_find(c->x[0]); if (!to) rt_fatal("SwitchToFiber(%#llx): not an initialized fiber", (unsigned long long)c->x[0]); if (to == t->fiber) return; diff --git a/runtime/kernel.h b/runtime/kernel.h index d719b301..70978a01 100644 --- a/runtime/kernel.h +++ b/runtime/kernel.h @@ -1,4 +1,4 @@ -// Kernel internals shared by kernel.c, svc.c and the kernel probe (kernel_probe.c). Not part of the host.h contract. +// Kernel internals shared by kernel.c, svc.c and the kernel probe (kernel_probe.c). Not part of the host.h contract. #pragma once #include "host.h" @@ -24,6 +24,7 @@ void kern_fiber_switch(Cpu *c); void kern_fiber_current(Cpu *c); uint64_t kern_threads_created(void); +void kern_seed_tls(void); // put the running thread's TLS where the SDK looks for it (TLR+0x1F0) // The guest frame-pointer chain from c->x[29] to stderr. void kern_backtrace(const Cpu *c); diff --git a/runtime/main.c b/runtime/main.c index 710e4630..790482dd 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -1,4 +1,4 @@ -// nx2navi.exe entry: reserves the guest address space, parses the command line and runs one mode. +// nx2navi.exe entry: reserves the guest address space, parses the command line and runs one mode. // // nx2navi.exe [--exefs DIR] [--romfs DIR] [--profile DIR] // --selftest layout, dispatch table and host-function registry; needs no game files @@ -163,7 +163,17 @@ void rt_brk(Cpu *c, uint32_t imm, uint64_t pc) { } void rt_udf(Cpu *c, uint64_t pc) { - char at[128], lr[128]; + char at[128], lr[128], d[128]; + rt_logf("udf at %s (lr %s)\n", ldr_describe(pc, at, sizeof at), ldr_describe(c->x[30], lr, sizeof lr)); + for (int i = 0; i < 31; i++) rt_logf(" x%-2d %016llx%s", i, (unsigned long long)c->x[i], i % 4 == 3 ? "\n" : ""); + rt_logf(" sp %016llx\n pc %016llx\n", (unsigned long long)c->sp, (unsigned long long)c->pc); + uint64_t fp = c->x[29]; + for (int n = 0; n < 32 && fp && !(fp & 7) && plat_mem_readable(fp, 16); n++) { + uint64_t next = LD64(fp), link = LD64(fp + 8); + rt_logf(" guest frame %2d: %s\n", n, ldr_describe(link, d, sizeof d)); + if (next <= fp || next - fp > (64u << 20)) break; + fp = next; + } rt_fatal("undefined instruction at %s (lr %s)", ldr_describe(pc, at, sizeof at), ldr_describe(c->x[30], lr, sizeof lr)); } @@ -244,7 +254,7 @@ static void crash_handler(const PlatCrash *x) { rt_logf(" sp %016llx\n pc %016llx lr %s\n", (unsigned long long)c->sp, (unsigned long long)c->pc, ldr_describe(c->x[30], d, sizeof d)); uint64_t fp = c->x[29]; - for (int n = 0; n < 32 && fp && !(fp & 7) && readable(fp, 16); n++) { + for (int n = 0; n < 32 && fp && !(fp & 7) && plat_mem_readable(fp, 16); n++) { uint64_t next = LD64(fp), lr = LD64(fp + 8); rt_logf(" guest frame %2d: %s\n", n, ldr_describe(lr, d, sizeof d)); if (next <= fp || next - fp > (64u << 20)) break; diff --git a/runtime/memory.c b/runtime/memory.c index 4a1ffeb9..375525e6 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -1,4 +1,4 @@ -// Adapted from MacNativeGames games//runtime/services.c and loader.c @ b75be8c (MIT, Copyright (c) 2026 +// Adapted from MacNativeGames games//runtime/services.c and loader.c @ b75be8c (MIT, Copyright (c) 2026 // MacNativeGames contributors), and LAS Recomp runtime/main.c and platform/windows.c @ da35b83 (MIT, Copyright (c) // 2026 Peiikai). // Guest address space and the memory SVCs: heap, module data permissions, stack moves, @@ -20,15 +20,15 @@ #define PAGE 0x1000ull #define GiB (1ull << 30) #define LOW_SEARCH_FROM 0x100000000ull -#define LOW_SIZE (6 * GiB) +#define LOW_SIZE (14 * GiB) // stack + heap block: room for a Switch 2 heap (HEAP_CAP) #define STACK_OFF 0x10000000ull #define STACK_SIZE 0x10000000ull #define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull) -#define HEAP_CAP (4 * GiB) +#define HEAP_CAP (12 * GiB) // heap region: a Switch 2 application pool is 9809 MiB (the system 12 GB arrangement) #define ALIAS_SIZE (64 * GiB) #define ALIAS_ALIGN (4 * GiB) #define ALIAS_PAGES (ALIAS_SIZE / PAGE) -#define DEFAULT_POOL 0xCD500000ull // retail application pool +#define DEFAULT_POOL rt_app_pool_bytes // retail application pool of the title (generated: NX2NAVI 0xCD500000, Switch 2 9809 MiB) #define SYSTEM_RESOURCE_BUDGET (32ull << 20) #define MAX_STACK_MAPS 4096 #define REEXEC_LIMIT 20 @@ -223,7 +223,7 @@ void mem_init(void) { uint64_t low = 0, alias = 0; for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB) if (reserve_at(at, LOW_SIZE, 1)) low = at; - if (!low) reexec_or_fail("a 6 GiB stack and heap block below 2^39"); + if (!low) reexec_or_fail("a 14 GiB stack and heap block below 2^39"); for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN) if (reserve_at(at, ALIAS_SIZE, 1)) alias = at; if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39"); @@ -434,7 +434,9 @@ static void alias_runs(uint64_t first, uint64_t count, uint8_t want, int do_comm } static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) { - if (!n || ((a | n) & (PAGE - 1))) return R_INVALID_ADDRESS; + if (!n) return R_INVALID_ADDRESS; + n = (n + PAGE - 1) & ~(uint64_t)(PAGE - 1); // the SDK asks for byte ranges; a mapping is whole pages + if ((a | n) & (PAGE - 1)) return R_INVALID_ADDRESS; if (!inside(a, n, L.alias)) return R_INVALID_REGION; uint64_t first = (a - L.alias.base) / PAGE, count = n / PAGE; plat_mutex_lock(&lock); @@ -507,6 +509,19 @@ static uint64_t run_start(const uint8_t *p, uint64_t i, uint8_t v) { // first return i; } +// Whether every byte of [a, a + n) can be read: no free, inaccessible or no-access page (mem_query's regions). For +// host reads of guest memory the guest itself may never make, such as a texture upload of a pool slot no shader +// samples (vk_exec_draw.c, texture pools: stale slots point at unmapped memory). +int mem_readable(uint64_t a, uint64_t n) { + for (uint64_t end = a + n; a < end;) { + RtMemoryInfo r; + mem_query(a, &r); + if (r.state == ST_FREE || r.state == ST_INACCESSIBLE || !(r.perm & PERM_R) || r.base + r.size <= a) return 0; + a = r.base + r.size; + } + return 1; +} + uint32_t mem_query(uint64_t a, RtMemoryInfo *out) { RtMemoryInfo r = {.base = a & ~(PAGE - 1), .size = PAGE}; uint64_t low = L.aslr.base; diff --git a/runtime/net_offline.c b/runtime/net_offline.c new file mode 100644 index 00000000..22198346 --- /dev/null +++ b/runtime/net_offline.c @@ -0,0 +1,393 @@ +// Offline network: nn::socket and the nnsocket* C API for a title that brings its network stack up at boot (Switch 2 +// titles start it on a worker thread and abort if nn::socket::Initialize fails). This build has no network: +// Initialize succeeds, no socket can be made (ENETDOWN), name lookups fail (EAI_NONAME), and every other call on a +// descriptor that is not an eventfd fails with EBADF. eventfd is real: the stack's worker threads wake each other +// through it and through nnsocketPoll, so a counter with a kernel event each, waits made with WaitSynchronization +// on the calling guest thread (SVC 0x18 through kern_svc, as the SDK's own wait would). +// errno values are the BSD ones the SDK uses (nn::socket::Errno). +#include +#include +#include + +#include "host.h" +#include "platform/platform.h" + +#define E_BADF 9 +#define E_INVAL 22 +#define E_AGAIN 35 +#define E_NETDOWN 50 +#define EAI_NONAME_ 8 +#define NIFM_R_COMMUNICATION_DISABLED_NET 0x8ae6eu // nifm module 110, description 1111 +#define POLL_IN 0x0001 +#define POLL_NVAL 0x0020 +#define EFD_SEMAPHORE_ 0x1 +#define EFD_BASE 0x4000 // eventfd descriptors: above anything a guest socket table would hand out +#define NEFD 64 + +typedef struct { int used, flags; uint64_t count; uint32_t w, r; } EventFd; +static EventFd efd[NEFD]; +// Kernel events of closed eventfds, reused by the next EventFd. They are never closed: a thread can still be +// blocked on one (EventFdRead, nnsocketPoll) when another closes the descriptor, and a freed handle number is +// soon reused by another kernel object, which corrupted that thread's wait (round 15 crashes). +static struct { uint32_t w, r; } spare[NEFD]; +static int nspare; +static PlatMutex lock = PLAT_MUTEX_INIT; +static volatile int32_t last_errno; // one value for all threads: nothing here succeeds partially +static volatile int32_t initialized; + +static int fail(int e) { + __atomic_store_n(&last_errno, e, __ATOMIC_RELAXED); + return -1; +} + +static EventFd *efd_of(int fd) { + int i = fd - EFD_BASE; + return i >= 0 && i < NEFD && efd[i].used ? &efd[i] : NULL; +} + +// WaitSynchronization(handles, n, ns) on the calling guest thread. Returns the result; *index the signalled one. +static uint32_t guest_wait(Cpu *c, const uint32_t *hs, int n, int64_t ns, int *index) { + uint64_t save[4]; + memcpy(save, c->x, sizeof save); + c->x[1] = (uint64_t)(uintptr_t)hs; + c->x[2] = (uint64_t)n; + c->x[3] = (uint64_t)ns; + if (!kern_svc(c, 0x18)) rt_fatal("net: WaitSynchronization not handled by the kernel"); + uint32_t r = (uint32_t)c->x[0]; + if (index) *index = (int)c->x[1]; + memcpy(c->x, save, sizeof save); + return r; +} + +// ---- nn::socket +// Result Initialize(const Config &) / Result Finalize() / bool IsInitialized() +static void ov_initialize(Cpu *c) { + if (!plat_xchg32(&initialized, 1)) rt_log("net: socket layer up, offline (no socket can connect)"); + ret_x(c, 0); +} +static void ov_finalize(Cpu *c) { plat_xchg32(&initialized, 0); ret_x(c, 0); } +static void ov_is_initialized(Cpu *c) { ret_x(c, (uint64_t)__atomic_load_n(&initialized, __ATOMIC_RELAXED)); } + +// int EventFd(uint64_t initval, EventFdFlags flags) +static void ov_eventfd(Cpu *c) { + uint64_t init = arg_x(c, 0); + int flags = (int)arg_x(c, 1); + plat_mutex_lock(&lock); + uint32_t w = 0, r = 0; + if (nspare) { + nspare--; + w = spare[nspare].w, r = spare[nspare].r; + } + plat_mutex_unlock(&lock); + if (!w) { + w = kern_event_new(); + r = kern_event_readable(w); + } else { + kern_event_clear(w); + } + plat_mutex_lock(&lock); + int i = 0; + while (i < NEFD && efd[i].used) i++; + if (i == NEFD) rt_fatal("net: eventfd table full (%d)", NEFD); + efd[i] = (EventFd){1, flags, init, w, r}; + plat_mutex_unlock(&lock); + if (init) kern_event_signal(w); + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) rt_log("net: eventfd flags %#x (bit 0 taken as semaphore)", flags); + ret_x(c, (uint64_t)(int64_t)(EFD_BASE + i)); +} + +// int EventFdWrite(int fd, uint64_t value) +static void ov_eventfd_write(Cpu *c) { + plat_mutex_lock(&lock); + EventFd *e = efd_of((int)arg_x(c, 0)); + if (!e) { + plat_mutex_unlock(&lock); + ret_x(c, (uint64_t)(int64_t)fail(E_BADF)); + return; + } + e->count += arg_x(c, 1); + uint32_t w = e->w; + int on = e->count != 0; + plat_mutex_unlock(&lock); + if (on) kern_event_signal(w); + ret_x(c, 0); +} + +// int EventFdRead(int fd, uint64_t *value): blocks while the counter is 0 (no nonblocking flag is honoured yet) +static void ov_eventfd_read(Cpu *c) { + int fd = (int)arg_x(c, 0); + uint64_t *out = arg_p(c, 1); + for (;;) { + plat_mutex_lock(&lock); + EventFd *e = efd_of(fd); + if (!e || !out) { + plat_mutex_unlock(&lock); + ret_x(c, (uint64_t)(int64_t)fail(e ? E_INVAL : E_BADF)); + return; + } + if (e->count) { + uint64_t v = e->flags & EFD_SEMAPHORE_ ? 1 : e->count; + e->count -= v; + if (!e->count) kern_event_clear(e->w); + plat_mutex_unlock(&lock); + *out = v; + ret_x(c, 0); + return; + } + uint32_t r = e->r; + plat_mutex_unlock(&lock); + guest_wait(c, &r, 1, -1, NULL); + } +} + +// int SetSockOpt(int, Level, Option, const void *, uint32_t) +static void ov_badf(Cpu *c) { ret_x(c, (uint64_t)(int64_t)fail(E_BADF)); } + +// ---- nnsocket* C API +static void ov_socket(Cpu *c) { ret_x(c, (uint64_t)(int64_t)fail(E_NETDOWN)); } +static void ov_close(Cpu *c) { + int fd = (int)arg_x(c, 0); + plat_mutex_lock(&lock); + EventFd *e = efd_of(fd); + EventFd gone = e ? *e : (EventFd){0}; + if (e) { + *e = (EventFd){0}; + if (nspare < NEFD) spare[nspare].w = gone.w, spare[nspare].r = gone.r, nspare++; + } + plat_mutex_unlock(&lock); + if (!e) { + ret_x(c, (uint64_t)(int64_t)fail(E_BADF)); + return; + } + kern_event_signal(gone.w); // wake its waiters: they find the descriptor gone (EBADF) or look again + ret_x(c, 0); +} +// int nnsocketPoll(pollfd *fds, nfds_t n, int timeout_ms): eventfds report POLLIN while their counter is nonzero; +// any other descriptor reports POLLNVAL. +typedef struct { int32_t fd; int16_t events, revents; } PollFd; +static void ov_poll(Cpu *c) { + PollFd *p = arg_p(c, 0); + uint64_t n = arg_x(c, 1); + int timeout = (int)arg_x(c, 2); + if (n > 64 || (n && !p)) { + ret_x(c, (uint64_t)(int64_t)fail(E_INVAL)); + return; + } + for (int round = 0;; round++) { + uint32_t hs[64]; + int nh = 0, ready = 0; + plat_mutex_lock(&lock); + for (uint64_t i = 0; i < n; i++) { + EventFd *e = efd_of(p[i].fd); + p[i].revents = 0; + if (p[i].fd < 0) continue; + if (!e) p[i].revents = POLL_NVAL; + else if (e->count && (p[i].events & POLL_IN)) p[i].revents = POLL_IN; + else if (p[i].events & POLL_IN) hs[nh++] = e->r; + ready += p[i].revents != 0; + } + plat_mutex_unlock(&lock); + if (ready || timeout == 0 || round) { + ret_x(c, (uint64_t)(int64_t)ready); + return; + } + int64_t ns = timeout < 0 ? -1 : (int64_t)timeout * 1000000; + if (nh) guest_wait(c, hs, nh, ns, NULL); + else if (ns > 0) { + uint64_t save = c->x[0]; + c->x[0] = (uint64_t)ns; + kern_svc(c, 0x0B); // SleepThread + c->x[0] = save; + } else { + ret_x(c, 0); // nothing to wait on, forever: report none instead of hanging + return; + } + } +} +// int nnsocketGetAddrInfo(...) / ...WithOptions(...): no name resolves offline +static void ov_getaddrinfo(Cpu *c) { ret_x(c, EAI_NONAME_); } +static void ov_freeaddrinfo(Cpu *c) { (void)c; } +static const char gai_text[] = "name resolution is unavailable (offline build)"; +static void ov_gai_strerror(Cpu *c) { ret_x(c, (uint64_t)(uintptr_t)gai_text); } +static void ov_get_errno(Cpu *c) { ret_x(c, (uint64_t)(int64_t)__atomic_load_n(&last_errno, __ATOMIC_RELAXED)); } +static void ov_set_errno(Cpu *c) { __atomic_store_n(&last_errno, (int32_t)arg_x(c, 0), __ATOMIC_RELAXED); } + +HOST_OVERRIDE("_ZN2nn6socket10InitializeERKNS0_6ConfigE", ov_initialize); +HOST_OVERRIDE("_ZN2nn6socket8FinalizeEv", ov_finalize); +HOST_OVERRIDE("_ZN2nn6socket13IsInitializedEv", ov_is_initialized); +HOST_OVERRIDE("_ZN2nn6socket7EventFdEmNS0_12EventFdFlagsE", ov_eventfd); +HOST_OVERRIDE("_ZN2nn6socket11EventFdReadEiPm", ov_eventfd_read); +HOST_OVERRIDE("_ZN2nn6socket12EventFdWriteEim", ov_eventfd_write); +HOST_OVERRIDE("_ZN2nn6socket10SetSockOptEiNS0_5LevelENS0_6OptionEPKvj", ov_badf); +HOST_OVERRIDE("nnsocketSocket", ov_socket); +HOST_OVERRIDE("nnsocketClose", ov_close); +HOST_OVERRIDE("nnsocketPoll", ov_poll); +HOST_OVERRIDE("nnsocketGetAddrInfo", ov_getaddrinfo); +HOST_OVERRIDE("nnsocketGetAddrInfoWithOptions", ov_getaddrinfo); +HOST_OVERRIDE("nnsocketFreeAddrInfo", ov_freeaddrinfo); +HOST_OVERRIDE("nnsocketGAIStrError", ov_gai_strerror); +HOST_OVERRIDE("nnsocketGetLastErrno", ov_get_errno); +HOST_OVERRIDE("nnsocketSetLastErrno", ov_set_errno); +HOST_OVERRIDE("nnsocketAccept", ov_badf); +HOST_OVERRIDE("nnsocketBind", ov_badf); +HOST_OVERRIDE("nnsocketConnect", ov_badf); +HOST_OVERRIDE("nnsocketFcntl", ov_badf); +HOST_OVERRIDE("nnsocketGetPeerName", ov_badf); +HOST_OVERRIDE("nnsocketGetSockName", ov_badf); +HOST_OVERRIDE("nnsocketGetSockOpt", ov_badf); +HOST_OVERRIDE("nnsocketSetSockOpt", ov_badf); +HOST_OVERRIDE("nnsocketListen", ov_badf); +HOST_OVERRIDE("nnsocketRecv", ov_badf); +HOST_OVERRIDE("nnsocketRecvMsg", ov_badf); +HOST_OVERRIDE("nnsocketSend", ov_badf); +HOST_OVERRIDE("nnsocketSendMsg", ov_badf); +HOST_OVERRIDE("nnsocketShutdown", ov_badf); +HOST_OVERRIDE("nnsocketResolverGetOption", ov_badf); +HOST_OVERRIDE("nnsocketResolverSetOption", ov_badf); + +// ---- nn::ssl: initialised, never used (no socket connects). Contexts are created and accept imports; the +// built-in CA list is empty: GetBuiltInCertificateBufSize asks for room for one zeroed (terminating) record and +// GetBuiltInCertificates hands back that zeroed record. +#define SSL_EMPTY_LIST 0x40u +static void ov_ssl_ok(Cpu *c) { ret_x(c, 0); } +// Result BuiltInManager::GetBuiltInCertificateBufSize(uint32_t *out, CaCertificateId *ids, uint32_t count) +static void ov_ssl_ca_size(Cpu *c) { + uint32_t *out = arg_p(c, 0); + if (out) *out = SSL_EMPTY_LIST; + ret_x(c, 0); +} +// Result BuiltInManager::GetBuiltInCertificates(BuiltInCertificateInfoBase **out, uint8_t *buf, uint32_t size, +// CaCertificateId *ids, uint32_t count) +static void ov_ssl_ca_list(Cpu *c) { + void **out = arg_p(c, 0); + uint8_t *buf = arg_p(c, 1); + uint32_t size = (uint32_t)arg_x(c, 2); + if (buf && size) memset(buf, 0, size < SSL_EMPTY_LIST ? size : SSL_EMPTY_LIST); + if (out) *out = buf; + ret_x(c, 0); +} +// Result Context::Import{ClientPki,ServerPki,Crl}(uint64_t *id, ...): an id for the caller to keep +static void ov_ssl_import(Cpu *c) { + uint64_t *id = arg_p(c, 1); + if (id) *id = 1; + ret_x(c, 0); +} +HOST_OVERRIDE("_ZN2nn3ssl10InitializeEv", ov_ssl_ok); +HOST_OVERRIDE("_ZN2nn3ssl8FinalizeEv", ov_ssl_ok); +HOST_OVERRIDE("_ZN2nn3ssl14BuiltInManager28GetBuiltInCertificateBufSizeEPjPNS0_15CaCertificateIdEj", ov_ssl_ca_size); +HOST_OVERRIDE("_ZN2nn3ssl14BuiltInManager22GetBuiltInCertificatesEPPNS0_26BuiltInCertificateInfoBaseEPhjPNS0_15CaCertificateIdEj", + ov_ssl_ca_list); +HOST_OVERRIDE("_ZN2nn3ssl7Context15ImportClientPkiEPmPKcS4_jj", ov_ssl_import); +HOST_OVERRIDE("_ZN2nn3ssl7Context15ImportServerPkiEPmPKcjNS0_17CertificateFormatE", ov_ssl_import); +HOST_OVERRIDE("_ZN2nn3ssl7Context9ImportCrlEPmPKcj", ov_ssl_import); +HOST_OVERRIDE("_ZN2nn3ssl7Context6CreateENS1_10SslVersionE", ov_ssl_ok); +HOST_OVERRIDE("_ZN2nn3ssl7Context7DestroyEv", ov_ssl_ok); + +// ---- nn::nsd: the server environment is production ("lp1", as the system service/sockets/nsd.cpp answers); +// ResolveEx(Fqdn *out, const Fqdn &in) copies the 0x100-byte name (nothing resolves offline anyway). +static void ov_nsd_environment(Cpu *c) { + char *out = arg_p(c, 0); + if (out) { + memset(out, 0, 8); + memcpy(out, "lp1", 3); + } + ret_x(c, 0); +} +static void ov_nsd_resolve(Cpu *c) { + char *out = arg_p(c, 0); + const char *in = arg_p(c, 1); + if (out && in) memmove(out, in, 0x100); + ret_x(c, 0); +} +HOST_OVERRIDE("_ZN2nn3nsd24GetEnvironmentIdentifierEPNS0_21EnvironmentIdentifierE", ov_nsd_environment); +HOST_OVERRIDE("_ZN2nn3nsd9ResolveExEPNS0_4FqdnERKS1_", ov_nsd_resolve); + +// ---- nn::time::AsyncEnsureNetworkClockAvailabilityTask and nn::account asynchronous NSA calls. The SDK versions +// run as nn::async tasks over the time / account service objects, which the host never creates (it overrides +// nn::time::Initialize and nn::account::Initialize); on the executor thread that corrupted memory at random +// (round 15 crashes). Here a task completes at once and reports the network as unavailable. The object's first +// 8 bytes hold the host record (the task's real size is unknown; every SDK object is larger than a pointer). +#define ACCOUNT_R_NSA_UNAVAILABLE 0x1907cu // account module 124, description 200: a failure the caller handles +typedef struct { uint8_t event[0x40]; int bound; } HostTask; // event: an nn::os::SystemEventType (0x30 bytes) + +static HostTask *task_of(Cpu *c, void *obj) { + if (!obj) return NULL; + HostTask **slot = obj; + if (!*slot) { + *slot = calloc(1, sizeof **slot); + if (!*slot) rt_fatal("net: task allocation"); + } + if (!(*slot)->bound) { + kern_bind_system_event(c, (uint64_t)(uintptr_t)(*slot)->event, 0); + (*slot)->bound = 1; + } + return *slot; +} +static void task_done(HostTask *t) { + if (t) kern_signal_system_event((uint64_t)(uintptr_t)t->event); +} + +// AsyncEnsureNetworkClockAvailabilityTask(): C1 / ~: the record +static void ov_clock_task_ctor(Cpu *c) { *(void **)arg_p(c, 0) = NULL; } +static void ov_task_dtor(Cpu *c) { + void **slot = arg_p(c, 0); + if (slot && *slot) { + free(*slot); // the bound SystemEventType's kernel handles stay (as for any undestroyed guest event) + *slot = NULL; + } +} +// Result Initialize(EventClearMode) / Result StartTask(): done at once +static void ov_clock_task_init(Cpu *c) { task_of(c, arg_p(c, 0)); ret_x(c, 0); } +static void ov_clock_task_start(Cpu *c) { + task_done(task_of(c, arg_p(c, 0))); + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) rt_log("net: network clock task finished at once (offline)"); + ret_x(c, 0); +} +// SystemEvent *GetFinishEvent(): nn::os::SystemEvent holds its SystemEventType at offset 0 +static void ov_clock_task_event(Cpu *c) { + HostTask *t = task_of(c, arg_p(c, 0)); + ret_x(c, t ? (uint64_t)(uintptr_t)t->event : 0); +} +static void ov_task_result(Cpu *c) { ret_x(c, NIFM_R_COMMUNICATION_DISABLED_NET); } + +// account: AsyncContext() / ~AsyncContext(); Result EnsureNetworkServiceAccountIdTokenCacheAsync(AsyncContext *, +// const UserHandle &); Result AsyncContext::GetSystemEvent(SystemEvent *out): out becomes a signalled event; +// Result AsyncContext::GetResult() +static void ov_account_async(Cpu *c) { + task_done(task_of(c, arg_p(c, 0))); + ret_x(c, 0); +} +static void ov_account_event(Cpu *c) { + void *out = arg_p(c, 1); + if (!out) { + ret_x(c, ACCOUNT_R_NSA_UNAVAILABLE); + return; + } + kern_bind_system_event(c, (uint64_t)(uintptr_t)out, 0); + kern_signal_system_event((uint64_t)(uintptr_t)out); + ret_x(c, 0); +} +static void ov_account_result(Cpu *c) { ret_x(c, ACCOUNT_R_NSA_UNAVAILABLE); } +// Result EnsureNetworkServiceAccountAvailable(const UserHandle &), GetNetworkServiceAccountId(NsaId *, handle), +// LoadNetworkServiceAccountIdTokenCache(size_t *, char *, size_t, handle): no network service account offline +static void ov_account_nsa(Cpu *c) { ret_x(c, ACCOUNT_R_NSA_UNAVAILABLE); } + +HOST_OVERRIDE("_ZN2nn4time39AsyncEnsureNetworkClockAvailabilityTaskC1Ev", ov_clock_task_ctor); +HOST_OVERRIDE("_ZN2nn4time39AsyncEnsureNetworkClockAvailabilityTaskD1Ev", ov_task_dtor); +HOST_OVERRIDE("_ZN2nn4time39AsyncEnsureNetworkClockAvailabilityTask10InitializeENS_2os14EventClearModeE", + ov_clock_task_init); +HOST_OVERRIDE("_ZN2nn4time39AsyncEnsureNetworkClockAvailabilityTask9StartTaskEv", ov_clock_task_start); +HOST_OVERRIDE("_ZN2nn4time39AsyncEnsureNetworkClockAvailabilityTask14GetFinishEventEv", ov_clock_task_event); +HOST_OVERRIDE("_ZN2nn4time39AsyncEnsureNetworkClockAvailabilityTask9GetResultEv", ov_task_result); +HOST_OVERRIDE("_ZN2nn7account12AsyncContextC1Ev", ov_clock_task_ctor); +HOST_OVERRIDE("_ZN2nn7account12AsyncContextD1Ev", ov_task_dtor); +HOST_OVERRIDE("_ZN2nn7account44EnsureNetworkServiceAccountIdTokenCacheAsyncEPNS0_12AsyncContextERKNS0_10UserHandleE", + ov_account_async); +HOST_OVERRIDE("_ZN2nn7account12AsyncContext14GetSystemEventEPNS_2os11SystemEventE", ov_account_event); +HOST_OVERRIDE("_ZN2nn7account12AsyncContext9GetResultEv", ov_account_result); +HOST_OVERRIDE("_ZN2nn7account36EnsureNetworkServiceAccountAvailableERKNS0_10UserHandleE", ov_account_nsa); +HOST_OVERRIDE("_ZN2nn7account26GetNetworkServiceAccountIdEPNS0_23NetworkServiceAccountIdERKNS0_10UserHandleE", + ov_account_nsa); +HOST_OVERRIDE("_ZN2nn7account37LoadNetworkServiceAccountIdTokenCacheEPmPcmRKNS0_10UserHandleE", ov_account_nsa); diff --git a/runtime/nvn_api_cmd.c b/runtime/nvn_api_cmd.c index 7c97fecf..d168d23d 100644 --- a/runtime/nvn_api_cmd.c +++ b/runtime/nvn_api_cmd.c @@ -712,6 +712,31 @@ NVN_API(nvnCommandBufferReportCounter) { NVN_API(nvnCommandBufferCopyBufferToBuffer) { *REC(cmd_of(c), OP_COPY_B2B, p_copy_b2b) = (p_copy_b2b){arg_x(c, 1), arg_x(c, 2), arg_x(c, 3)}; } +// NVN2: void nvnCommandBufferPushComputeConstants(NVNcommandBuffer *cmd, offset, ?, size_t size, const void *data): +// constants the compute stage reads from its driver bank (c0, from byte 0x10), copied at record time. Arguments as the +// game calls it (Ocarina main+0x20f811c: x1 0, x2 0, x3 0x40, x4 the data); x1 is taken as the byte offset. +#define PUSH_CC_MAX 1024 +NVN_API(nvnCommandBufferPushComputeConstants) { + uint64_t off = arg_x(c, 1), n = arg_x(c, 3); + const void *data = arg_p(c, 4); + static int logged; + if (arg_x(c, 2) && !logged++) NVN_LOG("PushComputeConstants: second argument %#llx not understood", (unsigned long long)arg_x(c, 2)); + static struct { uint64_t a1, a2, a3; } seen[24]; // diagnosis: the argument shapes the game uses + static int nseen; + int known = 0; + for (int i = 0; i < nseen; i++) known |= seen[i].a1 == off && seen[i].a2 == arg_x(c, 2) && seen[i].a3 == n; + if (!known && nseen < 24) { + seen[nseen++] = (typeof(seen[0])){off, arg_x(c, 2), n}; + rt_log("nvn: PushComputeConstants: x1 %#llx x2 %#llx x3 %#llx x4 %p x5 %#llx x6 %#llx", (unsigned long long)off, + (unsigned long long)arg_x(c, 2), (unsigned long long)n, data, (unsigned long long)arg_x(c, 5), + (unsigned long long)arg_x(c, 6)); + } + if (!data || off + n > PUSH_CC_MAX) { NVN_LOG("PushComputeConstants: %#llx bytes at %#llx dropped", (unsigned long long)n, (unsigned long long)off); return; } + p_push_cc *r = rec(cmd_of(c), OP_PUSH_CC, offsetof(p_push_cc, data) + n); + r->offset = (uint32_t)off; + r->size = (uint32_t)n; + memcpy(r->data, data, n); +} // void nvnCommandBufferUpdateUniformBuffer(NVNcommandBuffer *cmd, NVNbufferAddress buffer, size_t buffer_size, // ptrdiff_t offset, size_t size, const void *data): the data is copied at record time (the caller may change it // afterwards) and written into the buffer at replay @@ -1023,6 +1048,23 @@ NVN_API(nvnCommandBufferDispatchCompute) { NVN_API(nvnCommandBufferDispatchComputeIndirect) { *REC(cmd_of(c), OP_DISPATCH_IND, p_dispatch_ind) = (p_dispatch_ind){arg_x(c, 1), c->x[30]}; } +// NVN2: void nvnCommandBufferDrawMeshTasks(NVNcommandBuffer *cmd, int first, int count): count task (or, without a +// task stage, mesh) workgroups of the bound program from index first (as the game calls it: x1 0, x2 12). +NVN_API(nvnCommandBufferDrawMeshTasks) { + *REC(cmd_of(c), OP_DRAW_MESH, p_draw_mesh) = (p_draw_mesh){(uint32_t)arg_x(c, 1), (uint32_t)arg_x(c, 2), c->x[30]}; +} +// NVN2: nvnCommandBufferDrawMeshTasksIndirect(cmd, NVNbufferAddress indirect, x2, x3, x4): the workgroup count is read +// by the GPU at indirect. The game passes x2 1, x3 the same address, x4 0x4040; taken as one draw. +NVN_API(nvnCommandBufferDrawMeshTasksIndirect) { + *REC(cmd_of(c), OP_DRAW_MESH_IND, p_draw_mesh_ind) = + (p_draw_mesh_ind){arg_x(c, 1), arg_x(c, 3), (uint32_t)arg_x(c, 2), (uint32_t)arg_x(c, 4), c->x[30]}; +} +// NVN2: nvnCommandBufferSetGeometryProcessingMode(cmd, mode): mode 1 before mesh draws. The replay picks the stages +// from the bound program, so nothing is recorded. +NVN_API(nvnCommandBufferSetGeometryProcessingMode) { + static int logged; + if (!logged++) NVN_LOG("SetGeometryProcessingMode(%llu): taken from the bound program", (unsigned long long)arg_x(c, 1)); +} // void nvnCommandBufferClearBuffer(NVNcommandBuffer *cmd, NVNbufferAddress addr, size_t size, uint32_t value) NVN_API(nvnCommandBufferClearBuffer) { *REC(cmd_of(c), OP_CLEAR_BUF, p_clear_buf) = (p_clear_buf){arg_x(c, 1), arg_x(c, 2), (uint32_t)arg_x(c, 3), 0}; @@ -1103,6 +1145,15 @@ NVN_API(nvnCommandBufferDownsample) { } // void nvnCommandBufferSetCopyRowStride(NVNcommandBuffer *cmd, ptrdiff_t stride) NVN_API(nvnCommandBufferSetCopyRowStride) { REC(cmd_of(c), OP_COPY_STRIDE, p_copy_stride)->stride = (int64_t)arg_x(c, 1); } +// void nvnCommandBufferSetCopyImageStride(NVNcommandBuffer *cmd, ptrdiff_t stride): the byte distance between the +// 2D slices of a buffer side in 3D / array buffer <-> texture copies. Not replayed yet: the Vulkan copies keep +// tightly packed slices (bufferImageHeight 0). TODO: carry it as bufferImageHeight = stride / row stride. +NVN_API(nvnCommandBufferSetCopyImageStride) { + static volatile int32_t logged; + int64_t stride = (int64_t)arg_x(c, 1); + if (stride && !plat_xchg32(&logged, 1)) + rt_log("nvn: copy image stride %lld not replayed (slices assumed tightly packed)", (long long)stride); +} // void nvnCommandBufferSetBlendColor(NVNcommandBuffer *cmd, const float *rgba) NVN_API(nvnCommandBufferSetBlendColor) { memcpy(REC(cmd_of(c), OP_BLEND_COLOR, p_blend_color)->rgba, arg_p(c, 1), 16); } @@ -1215,7 +1266,7 @@ const char *nvn_op_name(uint32_t op) { "BIND_VSTREAM", "BIND_VBUF", "BIND_UBO", "BIND_SSBO", "BIND_TEX", "BIND_SAMP", "STENCIL", "POLY_OFFSET", "DRAW", "DRAW_IDX", "DISPATCH", "BARRIER", "SET_TEX_POOL", "SET_SAMP_POOL", "COPY_B2T", "COPY_STRIDE", "BLEND_COLOR", "COPY_T2B", "BIND_IMG", "COPY_T2T", "RESET_COUNTER", "CLEAR_TEX", "DEPTH_CLAMP", "RENDER_ENABLE", - "RASTER_DISCARD", "RENDER_COND", "PRIM_RESTART", "DRAW_IDX_IND", "DISPATCH_IND", "CLEAR_BUF", + "RASTER_DISCARD", "RENDER_COND", "PRIM_RESTART", "DRAW_IDX_IND", "DISPATCH_IND", "CLEAR_BUF", "PUSH_CC", "DRAW_MESH", "DRAW_MESH_IND", }; _Static_assert(sizeof names / sizeof *names == OP_COUNT_, "one name per op"); return op < OP_COUNT_ ? names[op] : "?"; diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 23ed7f0b..f83dcec5 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -1,4 +1,4 @@ -// NVN front end, first slice: device, queue, memory pool, buffer, sync and +// NVN front end, first slice: device, queue, memory pool, buffer, sync and // event objects. Ported from MacNativeGames games//third_party/d2r_gfx/gfx/nvn/hle_nvn.c @ b75be8cf (MIT, // Copyright (c) 2026 jhaizhou; origin d2r-mobile), lines 137-253, 534-645 and 1975-2001; Metal, mach, the translation // thread and Apple diagnostics dropped. Calls that reached Metal go to gfx_backend() (gfx.h). nvnQueuePresentTexture @@ -58,6 +58,15 @@ void nvn_guest_written(uint64_t addr, uint64_t size) { // arguments are undefined for memcpy anyway. static void ov_memcpy(Cpu *c) { uint64_t dst = arg_x(c, 0), src = arg_x(c, 1), n = arg_x(c, 2); + { + static int logged; + if (logged < 500) { + logged++; + fprintf(stderr, "[probe] memcpy dst=%#llx src=%#llx n=%#llx\n", + (unsigned long long)dst, (unsigned long long)src, (unsigned long long)n); + fflush(stderr); + } + } if (n >= 1024) gfx_backend()->guest_read(src, n); memmove((void *)(uintptr_t)dst, (const void *)(uintptr_t)src, n); ret_x(c, dst); @@ -167,7 +176,7 @@ static int pool_gpu_visible(const pool_t *o) { return !(o->flags & NVN_MEMORY_PO // ---------- Device static const int devinfo[] = { // Tegra X1 values (nnheaders nvn_DeviceConstantsNX.h, measured on SMO) - [0x00] = 55, [0x01] = 15, // ABI: NX2NAVI main requires major 55, minor >= 15 + [0x00] = 55, [0x01] = 15, // ABI: replaced by the title's rt_nvn_api_version (NX2NAVI 55.15, Switch 2 231.40) [0x02] = 14, [0x03] = 65536, [0x04] = 256, [0x05] = 8, [0x06] = 16, [0x07] = 4, [0x08] = 16, [0x09] = 32, [0x0A] = 16, [0x0B] = 4, [0x0C] = 4, [0x0D] = 4, [0x0E] = 16, [0x0F] = 32, [0x10] = 32, [0x11] = 256, [0x12] = 256, [0x13] = 4, [0x14] = 8, [0x15] = 512, [0x16] = 256, [0x17] = 2, [0x18] = 256, [0x19] = 4096, [0x1A] = 131072, @@ -202,8 +211,27 @@ NVN_API(nvnDeviceFinalize) { ((dev_t_ *)arg_p(c, 0))->magic = 0; } NVN_API(nvnDeviceGetInteger) { unsigned w = (unsigned)arg_x(c, 1); int *out = arg_p(c, 2); - if (w >= sizeof devinfo / sizeof *devinfo) rt_fatal("nvn: unsupported NVN device property %#x", w); - *out = devinfo[w]; + if (w >= sizeof devinfo / sizeof *devinfo) { + // NVN2 adds device properties past NVN1's 0x5F; no reference gives their values. + // 0x67: read first in the game's descriptor-pool layout (main+0x20e8500, with the descriptor sizes 0x0F/0x10) + // and used there as an alignment (size & -align): 0 made every sub-allocation 0 bytes and an assertion fail + // (main+0x20e53f0). The memory pool page size is a safe alignment for it. + if (w == 0x67) { + if (out) *out = 0x1000; + return; + } + // Others: 0, each logged once. + static volatile uint64_t seen[4]; + if (w < 256 && !(__atomic_fetch_or(&seen[w / 64], 1ull << (w % 64), __ATOMIC_RELAXED) & (1ull << (w % 64)))) + { + char at[160]; + rt_log("nvn: NVN2 device property %#x (unknown here) answered 0, asked from %s", w, + ldr_describe(c->x[30], at, sizeof at)); + } + if (out) *out = 0; + return; + } + *out = w <= NVN_DEVICE_INFO_API_MINOR_VERSION ? rt_nvn_api_version[w] : devinfo[w]; NVN_LOG("DeviceGetInteger %#x -> %d", w, *out); } // NVNboolean nvnDeviceRegisterFastClearColor{,i,ui}(NVNdevice *d, const T *color, NVNformat format), ...Depth(d, float): @@ -471,6 +499,18 @@ NVN_API(nvnQueueFenceSync) { s->fenced = 1; s->fence = nvn_fence_last(); } +// NVNboolean nvnQueueWaitSync(NVNqueue *q, const NVNsync *s): the queue's later work waits for the sync on the GPU. +// Every queue here submits into one back-end timeline in submission order, so work submitted after FenceSync +// already runs after it: a fenced sync needs nothing. A sync not fenced yet would stall the queue on hardware until +// another queue fences it; that ordering cannot be expressed here and is logged once. +NVN_API(nvnQueueWaitSync) { + const sync_t *s = arg_p(c, 1); + if (s && !s->fenced) { + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) rt_log("nvn: QueueWaitSync on a sync not fenced yet: not waited for"); + } + ret_x(c, 1); +} // NVNsyncWaitResult nvnSyncWait(const NVNsync *s, uint64_t timeout_ns): a sync never fenced reports a timeout at once; // a fenced one is released only after the back end completed that fence value NVN_API(nvnSyncWait) { diff --git a/runtime/nvn_api_obj.c b/runtime/nvn_api_obj.c index adcb8390..5410b440 100644 --- a/runtime/nvn_api_obj.c +++ b/runtime/nvn_api_obj.c @@ -8,10 +8,12 @@ // AAPCS64 registers (host.h arg_x / arg_s / arg_p) and every result returned with ret_x. Command-buffer procs are in // nvn_api_cmd.c, the NX2NAVI extension procs in nvn_ext.c. #include +#include #include #include #include "gfx.h" +#include "cfg.h" #include "host.h" #include "nvn_api.h" #include "nvn_hmap.h" @@ -188,6 +190,13 @@ static void tex_release(tex_t *o) { NVN_API(nvnTextureInitialize) { tex_t *o = arg_p(c, 0); const tex_builder_t *s = arg_p(c, 1); + if (cfg_get("NX2NAVI_LOG_TEX_FLAGS")) { // diagnosis: each distinct (flags, format) pair once + static uint64_t seen[256]; static int n; + uint64_t key = (uint64_t)(uint32_t)s->d.flags << 32 | (uint32_t)s->d.format; + int known = 0; + for (int i = 0; i < n; i++) known |= seen[i] == key; + if (!known && n < 256) { seen[n++] = key; rt_log("nvn: texture flags %#x format %#x %ux%u", s->d.flags, s->d.format, s->d.width, s->d.height); } + } if (o->magic == NVN_TEX_MAGIC && o->serial) { // initialized again without Finalize: release the old serial NVN_LOG("texture %p initialized again without Finalize (serial %llu released)", (void *)o, (unsigned long long)o->serial); @@ -435,8 +444,10 @@ static int prog_set_shaders(prog_t *o, int n, const NVNshaderData *sd) { return 1; } // NVNboolean nvnProgramSetShaders(NVNprogram *p, int n, const NVNshaderData *sd) +static void shader_dump(const prog_t *o, int n, const NVNshaderData *sd); NVN_API(nvnProgramSetShaders) { NVN_BLOCKING(); + shader_dump(arg_p(c, 0), (int)arg_x(c, 1), arg_p(c, 2)); ret_x(c, prog_set_shaders(arg_p(c, 0), (int)arg_x(c, 1), arg_p(c, 2))); } // NVNboolean nvnProgramSetShadersExt(NVNprogram *p, int n, const NVNshaderDataExt *sd): no prototype in nvn.h; the @@ -459,9 +470,88 @@ NVN_API(nvnProgramSetShadersExt) { rt_fatal("nvn: ProgramSetShadersExt: unknown non-zero field %#llx in stage %d", (unsigned long long)sd[i].reserved, i); plain[i] = (NVNshaderData){sd[i].data, sd[i].control}; } + shader_dump(o, n, plain); ret_x(c, prog_set_shaders(o, n, plain)); } +// ---------- NVN2 program builder (Switch 2): SetDefaults / SetDevice / SetShaders on a builder, then +// nvnProgramInitializeFromBuilder(program, builder). No driver reference is available (the NVN2 driver is not in the +// title's modules), so the signatures are taken to mirror nvnProgramSetShaders: SetShaders(builder, n, shader data). +// The builder keeps the caller's array pointer, as NVN builders reference their inputs until the object is made. +// NX2NAVI_SHADER_DUMP= writes each stage's control section and code there for studying the NVN2 layout. +typedef struct { uint32_t magic; int32_t n; NVNdevice *dev; const NVNshaderData *sd; } prog_builder_t; +#define NVN_PROG_BUILDER_MAGIC 0x50524742u // "BGRP" + +static void shader_dump(const prog_t *o, int n, const NVNshaderData *sd) { + // Every distinct NVN2 stage number, once (mesh and task programs: which numbers the title uses) + for (int i = 0; sd && i < n && i < 8; i++) { + static uint32_t seen; + const uint8_t *ctl = (const uint8_t *)sd[i].control; + uint32_t magic, st; + if (!ctl) continue; + memcpy(&magic, ctl, 4), memcpy(&st, ctl + 0x54, 4); + st >>= 16; + if (magic == 0x98761234u && st < 32 && !(__atomic_fetch_or(&seen, 1u << st, __ATOMIC_RELAXED) >> st & 1)) + rt_log("nvn: first NVN2 program with stage %u (%d stages in the program)", st, n); + } + const char *dir = cfg_get("NX2NAVI_SHADER_DUMP"); + if (!dir || !*dir) return; + static volatile int32_t count; + int k = __atomic_fetch_add(&count, 1, __ATOMIC_RELAXED); + if (k >= 20000) return; // the gameplay programs too (mesh and task ones, and many draws, come late) + if (k == 0) { + const uint64_t *raw = (const uint64_t *)sd; + rt_log("nvn: shader dump: first program %p, %d stages, raw shader data %#llx %#llx %#llx %#llx %#llx %#llx", + (const void *)o, n, (unsigned long long)raw[0], (unsigned long long)raw[1], (unsigned long long)raw[2], + (unsigned long long)raw[3], (unsigned long long)raw[4], (unsigned long long)raw[5]); + } + for (int i = 0; i < n; i++) { + char path[1100]; + snprintf(path, sizeof path, "%s/p%04d_s%d.ctl", dir, k, i); + if (sd[i].control) plat_file_write(path, sd[i].control, 0x880); + const pool_t *p = nvn_pool_find(sd[i].data, 1); + if (p && p->mem && !nvn_gpu_only(sd[i].data)) { + uint64_t end = (uint64_t)(uintptr_t)p->mem + p->size, len = end - sd[i].data; + if (len > 0x40000) len = 0x40000; + snprintf(path, sizeof path, "%s/p%04d_s%d.code", dir, k, i); + plat_file_write(path, (const void *)(uintptr_t)sd[i].data, (size_t)len); + } + } +} + +// void nvnProgramBuilderSetDefaults(NVNprogramBuilder *b) +NVN_API(nvnProgramBuilderSetDefaults) { + *(prog_builder_t *)arg_p(c, 0) = (prog_builder_t){NVN_PROG_BUILDER_MAGIC, 0, NULL, NULL}; +} +// void nvnProgramBuilderSetDevice(NVNprogramBuilder *b, NVNdevice *d) +NVN_API(nvnProgramBuilderSetDevice) { ((prog_builder_t *)arg_p(c, 0))->dev = arg_p(c, 1); } +// void nvnProgramBuilderSetShaders(NVNprogramBuilder *b, int n, const NVNshaderData *sd) +NVN_API(nvnProgramBuilderSetShaders) { + prog_builder_t *b = arg_p(c, 0); + b->n = (int32_t)arg_x(c, 1); + b->sd = arg_p(c, 2); +} +// NVNboolean nvnProgramInitializeFromBuilder(NVNprogram *p, const NVNprogramBuilder *b) +NVN_API(nvnProgramInitializeFromBuilder) { + NVN_BLOCKING(); + prog_t *o = arg_p(c, 0); + const prog_builder_t *b = arg_p(c, 1); + if (!o || !b || b->magic != NVN_PROG_BUILDER_MAGIC) { + rt_log("nvn: ProgramInitializeFromBuilder with builder %p (not set up by SetDefaults)", (const void *)b); + ret_x(c, 0); + return; + } + const char *mode = cfg_get("NX2NAVI_NVN2_PROGRAMS"); // =off: refuse every program (diagnosis) + if (mode && !strcmp(mode, "off")) { + ret_x(c, 0); + return; + } + memset(o, 0, sizeof *o); + o->magic = NVN_PROG_MAGIC; + shader_dump(o, b->n, b->sd); + ret_x(c, prog_set_shaders(o, b->n, b->sd)); +} + // ---------- State objects: packed in place (nvn_obj.h) #define ST(T, c) ((T *)arg_p(c, 0)) NVN_API(nvnBlendStateSetDefaults) { diff --git a/runtime/nvn_obj.h b/runtime/nvn_obj.h index 791c971a..4f447822 100644 --- a/runtime/nvn_obj.h +++ b/runtime/nvn_obj.h @@ -128,9 +128,10 @@ enum { OP_BIND_PROGRAM, OP_BIND_STATE, OP_BIND_VATTR, OP_BIND_VSTREAM, OP_BIND_VBUF, OP_BIND_UBO, OP_BIND_SSBO, OP_BIND_TEX, OP_BIND_SAMP, OP_STENCIL, OP_POLY_OFFSET, OP_DRAW, OP_DRAW_IDX, OP_DISPATCH, OP_BARRIER, - OP_SET_TEX_POOL, OP_SET_SAMP_POOL, OP_COPY_B2T, OP_COPY_STRIDE, OP_BLEND_COLOR, OP_COPY_T2B, OP_BIND_IMG, OP_COPY_T2T, OP_RESET_COUNTER, OP_CLEAR_TEX, OP_DEPTH_CLAMP, OP_RENDER_ENABLE, OP_RASTER_DISCARD, OP_RENDER_COND, OP_PRIM_RESTART, OP_DRAW_IDX_IND, OP_DISPATCH_IND, OP_CLEAR_BUF, OP_COUNT_ + OP_SET_TEX_POOL, OP_SET_SAMP_POOL, OP_COPY_B2T, OP_COPY_STRIDE, OP_BLEND_COLOR, OP_COPY_T2B, OP_BIND_IMG, OP_COPY_T2T, OP_RESET_COUNTER, OP_CLEAR_TEX, OP_DEPTH_CLAMP, OP_RENDER_ENABLE, OP_RASTER_DISCARD, OP_RENDER_COND, OP_PRIM_RESTART, OP_DRAW_IDX_IND, OP_DISPATCH_IND, OP_CLEAR_BUF, OP_PUSH_CC, OP_DRAW_MESH, OP_DRAW_MESH_IND, OP_COUNT_ }; // Payload (immediately follows op_hdr) +typedef struct { uint32_t offset, size; uint8_t data[]; } p_push_cc; // OP_PUSH_CC: NVN2 compute push constants typedef struct { int32_t n; const tex_t *color[8]; view_t cview[8]; uint8_t has_cview[8]; const tex_t *depth; view_t dview; uint8_t has_dview; } p_set_rt; typedef struct { int32_t index, mask; float rgba[4]; } p_clear_color; typedef struct { float depth; int32_t depth_mask, stencil, stencil_mask; } p_clear_ds; @@ -152,7 +153,10 @@ typedef struct { int32_t face, value_mask, mask, ref, which; } p_stencil; // wh typedef struct { float factor, units, clamp; } p_poly_offset; typedef struct { int32_t prim, first, count, base_instance, instances; } p_draw; // NX2NAVI: instances 0 = one instance (DrawArrays) typedef struct { int32_t prim, itype, count, base_vertex, base_instance, instances; uint64_t ibuf; } p_draw_idx; -typedef struct { int32_t x, y, z; uint64_t caller; } p_dispatch; // caller: return address of the game call site (to locate the dispatcher) +typedef struct { int32_t x, y, z; uint64_t caller; } p_dispatch; +// NVN2 DrawMeshTasks(first, count) and DrawMeshTasksIndirect(indirect, x2, x3, x4) (vk_exec_draw.c op_draw_mesh) +typedef struct { uint32_t first, count; uint64_t caller; } p_draw_mesh; +typedef struct { uint64_t ind, a3; uint32_t a2, a4; uint64_t caller; } p_draw_mesh_ind; // caller: return address of the game call site (to locate the dispatcher) typedef struct { const dpool_t *pool; } p_set_pool; typedef struct { uint64_t src; const tex_t *tex; view_t view; uint8_t has_view; NVNcopyRegion r; int32_t flags; } p_copy_b2t; typedef struct { const tex_t *src, *dst; view_t sv, dv; uint8_t has_sv, has_dv; NVNcopyRegion sr, dr; int32_t flags; } p_copy_t2t; // Reused for T2B: src stores the destination buffer address diff --git a/runtime/nvn_procs.c b/runtime/nvn_procs.c index 63e0fcd3..004b3983 100644 --- a/runtime/nvn_procs.c +++ b/runtime/nvn_procs.c @@ -3,9 +3,11 @@ // nvnp_ is the implementation of one proc: the weak definitions here stop with the proc's name, nvn_api_*.c // and nvn_ext.c define the strong ones. The name list is runtime/nvn_procs.inc (tools/gen_nvn_procs.py). #include +#include #include #include +#include "cfg.h" #include "gfx.h" #include "host.h" #include "platform/platform.h" @@ -38,6 +40,540 @@ enum { PROC_COUNT = 0 static Proc procs[PROC_COUNT]; static int nprocs; +// Unknown procs (NVN2 names the NVN1-era table lacks): a NVN2 driver has every name the game asks for, so a NULL +// answer crashes the game at the first call (main+0xc31484 called one). Each unknown name is bound on demand to one +// of TRAP_COUNT pre-registered traps (the hostfn table is sealed after boot). A trap logs its name and caller on the +// first call and returns 0 -- a no-op for the setters most of these are; NX2NAVI_NVN_TRAP=fatal stops instead. +#define TRAP_COUNT 512 +static char trap_name[TRAP_COUNT][96]; +static uint64_t trap_addr[TRAP_COUNT]; +static volatile int32_t trap_called[TRAP_COUNT]; +static int ntraps; +static void nvn_trap(Cpu *c, int i) { + if (!plat_xchg32(&trap_called[i], 1) || (cfg_get("NX2NAVI_NVN_TRAP") && !strcmp(cfg_get("NX2NAVI_NVN_TRAP"), "fatal"))) { + char at[160]; + rt_log("nvn: unimplemented NVN2 proc %s called from %s (x0 %#llx x1 %#llx x2 %#llx x3 %#llx x4 %#llx x5 %#llx): returning 0", trap_name[i], + ldr_describe(c->x[30], at, sizeof at), (unsigned long long)c->x[0], (unsigned long long)c->x[1], + (unsigned long long)c->x[2], (unsigned long long)c->x[3], (unsigned long long)c->x[4], (unsigned long long)c->x[5]); + const char *mode = cfg_get("NX2NAVI_NVN_TRAP"); + if (mode && !strcmp(mode, "fatal")) rt_fatal("nvn: %s is not implemented", trap_name[i]); + } + ret_x(c, 0); +} +// trap_0 .. trap_511, one per pool slot +static void trap_0(Cpu *c) { nvn_trap(c, 0); } +static void trap_1(Cpu *c) { nvn_trap(c, 1); } +static void trap_2(Cpu *c) { nvn_trap(c, 2); } +static void trap_3(Cpu *c) { nvn_trap(c, 3); } +static void trap_4(Cpu *c) { nvn_trap(c, 4); } +static void trap_5(Cpu *c) { nvn_trap(c, 5); } +static void trap_6(Cpu *c) { nvn_trap(c, 6); } +static void trap_7(Cpu *c) { nvn_trap(c, 7); } +static void trap_8(Cpu *c) { nvn_trap(c, 8); } +static void trap_9(Cpu *c) { nvn_trap(c, 9); } +static void trap_10(Cpu *c) { nvn_trap(c, 10); } +static void trap_11(Cpu *c) { nvn_trap(c, 11); } +static void trap_12(Cpu *c) { nvn_trap(c, 12); } +static void trap_13(Cpu *c) { nvn_trap(c, 13); } +static void trap_14(Cpu *c) { nvn_trap(c, 14); } +static void trap_15(Cpu *c) { nvn_trap(c, 15); } +static void trap_16(Cpu *c) { nvn_trap(c, 16); } +static void trap_17(Cpu *c) { nvn_trap(c, 17); } +static void trap_18(Cpu *c) { nvn_trap(c, 18); } +static void trap_19(Cpu *c) { nvn_trap(c, 19); } +static void trap_20(Cpu *c) { nvn_trap(c, 20); } +static void trap_21(Cpu *c) { nvn_trap(c, 21); } +static void trap_22(Cpu *c) { nvn_trap(c, 22); } +static void trap_23(Cpu *c) { nvn_trap(c, 23); } +static void trap_24(Cpu *c) { nvn_trap(c, 24); } +static void trap_25(Cpu *c) { nvn_trap(c, 25); } +static void trap_26(Cpu *c) { nvn_trap(c, 26); } +static void trap_27(Cpu *c) { nvn_trap(c, 27); } +static void trap_28(Cpu *c) { nvn_trap(c, 28); } +static void trap_29(Cpu *c) { nvn_trap(c, 29); } +static void trap_30(Cpu *c) { nvn_trap(c, 30); } +static void trap_31(Cpu *c) { nvn_trap(c, 31); } +static void trap_32(Cpu *c) { nvn_trap(c, 32); } +static void trap_33(Cpu *c) { nvn_trap(c, 33); } +static void trap_34(Cpu *c) { nvn_trap(c, 34); } +static void trap_35(Cpu *c) { nvn_trap(c, 35); } +static void trap_36(Cpu *c) { nvn_trap(c, 36); } +static void trap_37(Cpu *c) { nvn_trap(c, 37); } +static void trap_38(Cpu *c) { nvn_trap(c, 38); } +static void trap_39(Cpu *c) { nvn_trap(c, 39); } +static void trap_40(Cpu *c) { nvn_trap(c, 40); } +static void trap_41(Cpu *c) { nvn_trap(c, 41); } +static void trap_42(Cpu *c) { nvn_trap(c, 42); } +static void trap_43(Cpu *c) { nvn_trap(c, 43); } +static void trap_44(Cpu *c) { nvn_trap(c, 44); } +static void trap_45(Cpu *c) { nvn_trap(c, 45); } +static void trap_46(Cpu *c) { nvn_trap(c, 46); } +static void trap_47(Cpu *c) { nvn_trap(c, 47); } +static void trap_48(Cpu *c) { nvn_trap(c, 48); } +static void trap_49(Cpu *c) { nvn_trap(c, 49); } +static void trap_50(Cpu *c) { nvn_trap(c, 50); } +static void trap_51(Cpu *c) { nvn_trap(c, 51); } +static void trap_52(Cpu *c) { nvn_trap(c, 52); } +static void trap_53(Cpu *c) { nvn_trap(c, 53); } +static void trap_54(Cpu *c) { nvn_trap(c, 54); } +static void trap_55(Cpu *c) { nvn_trap(c, 55); } +static void trap_56(Cpu *c) { nvn_trap(c, 56); } +static void trap_57(Cpu *c) { nvn_trap(c, 57); } +static void trap_58(Cpu *c) { nvn_trap(c, 58); } +static void trap_59(Cpu *c) { nvn_trap(c, 59); } +static void trap_60(Cpu *c) { nvn_trap(c, 60); } +static void trap_61(Cpu *c) { nvn_trap(c, 61); } +static void trap_62(Cpu *c) { nvn_trap(c, 62); } +static void trap_63(Cpu *c) { nvn_trap(c, 63); } +static void trap_64(Cpu *c) { nvn_trap(c, 64); } +static void trap_65(Cpu *c) { nvn_trap(c, 65); } +static void trap_66(Cpu *c) { nvn_trap(c, 66); } +static void trap_67(Cpu *c) { nvn_trap(c, 67); } +static void trap_68(Cpu *c) { nvn_trap(c, 68); } +static void trap_69(Cpu *c) { nvn_trap(c, 69); } +static void trap_70(Cpu *c) { nvn_trap(c, 70); } +static void trap_71(Cpu *c) { nvn_trap(c, 71); } +static void trap_72(Cpu *c) { nvn_trap(c, 72); } +static void trap_73(Cpu *c) { nvn_trap(c, 73); } +static void trap_74(Cpu *c) { nvn_trap(c, 74); } +static void trap_75(Cpu *c) { nvn_trap(c, 75); } +static void trap_76(Cpu *c) { nvn_trap(c, 76); } +static void trap_77(Cpu *c) { nvn_trap(c, 77); } +static void trap_78(Cpu *c) { nvn_trap(c, 78); } +static void trap_79(Cpu *c) { nvn_trap(c, 79); } +static void trap_80(Cpu *c) { nvn_trap(c, 80); } +static void trap_81(Cpu *c) { nvn_trap(c, 81); } +static void trap_82(Cpu *c) { nvn_trap(c, 82); } +static void trap_83(Cpu *c) { nvn_trap(c, 83); } +static void trap_84(Cpu *c) { nvn_trap(c, 84); } +static void trap_85(Cpu *c) { nvn_trap(c, 85); } +static void trap_86(Cpu *c) { nvn_trap(c, 86); } +static void trap_87(Cpu *c) { nvn_trap(c, 87); } +static void trap_88(Cpu *c) { nvn_trap(c, 88); } +static void trap_89(Cpu *c) { nvn_trap(c, 89); } +static void trap_90(Cpu *c) { nvn_trap(c, 90); } +static void trap_91(Cpu *c) { nvn_trap(c, 91); } +static void trap_92(Cpu *c) { nvn_trap(c, 92); } +static void trap_93(Cpu *c) { nvn_trap(c, 93); } +static void trap_94(Cpu *c) { nvn_trap(c, 94); } +static void trap_95(Cpu *c) { nvn_trap(c, 95); } +static void trap_96(Cpu *c) { nvn_trap(c, 96); } +static void trap_97(Cpu *c) { nvn_trap(c, 97); } +static void trap_98(Cpu *c) { nvn_trap(c, 98); } +static void trap_99(Cpu *c) { nvn_trap(c, 99); } +static void trap_100(Cpu *c) { nvn_trap(c, 100); } +static void trap_101(Cpu *c) { nvn_trap(c, 101); } +static void trap_102(Cpu *c) { nvn_trap(c, 102); } +static void trap_103(Cpu *c) { nvn_trap(c, 103); } +static void trap_104(Cpu *c) { nvn_trap(c, 104); } +static void trap_105(Cpu *c) { nvn_trap(c, 105); } +static void trap_106(Cpu *c) { nvn_trap(c, 106); } +static void trap_107(Cpu *c) { nvn_trap(c, 107); } +static void trap_108(Cpu *c) { nvn_trap(c, 108); } +static void trap_109(Cpu *c) { nvn_trap(c, 109); } +static void trap_110(Cpu *c) { nvn_trap(c, 110); } +static void trap_111(Cpu *c) { nvn_trap(c, 111); } +static void trap_112(Cpu *c) { nvn_trap(c, 112); } +static void trap_113(Cpu *c) { nvn_trap(c, 113); } +static void trap_114(Cpu *c) { nvn_trap(c, 114); } +static void trap_115(Cpu *c) { nvn_trap(c, 115); } +static void trap_116(Cpu *c) { nvn_trap(c, 116); } +static void trap_117(Cpu *c) { nvn_trap(c, 117); } +static void trap_118(Cpu *c) { nvn_trap(c, 118); } +static void trap_119(Cpu *c) { nvn_trap(c, 119); } +static void trap_120(Cpu *c) { nvn_trap(c, 120); } +static void trap_121(Cpu *c) { nvn_trap(c, 121); } +static void trap_122(Cpu *c) { nvn_trap(c, 122); } +static void trap_123(Cpu *c) { nvn_trap(c, 123); } +static void trap_124(Cpu *c) { nvn_trap(c, 124); } +static void trap_125(Cpu *c) { nvn_trap(c, 125); } +static void trap_126(Cpu *c) { nvn_trap(c, 126); } +static void trap_127(Cpu *c) { nvn_trap(c, 127); } +static void trap_128(Cpu *c) { nvn_trap(c, 128); } +static void trap_129(Cpu *c) { nvn_trap(c, 129); } +static void trap_130(Cpu *c) { nvn_trap(c, 130); } +static void trap_131(Cpu *c) { nvn_trap(c, 131); } +static void trap_132(Cpu *c) { nvn_trap(c, 132); } +static void trap_133(Cpu *c) { nvn_trap(c, 133); } +static void trap_134(Cpu *c) { nvn_trap(c, 134); } +static void trap_135(Cpu *c) { nvn_trap(c, 135); } +static void trap_136(Cpu *c) { nvn_trap(c, 136); } +static void trap_137(Cpu *c) { nvn_trap(c, 137); } +static void trap_138(Cpu *c) { nvn_trap(c, 138); } +static void trap_139(Cpu *c) { nvn_trap(c, 139); } +static void trap_140(Cpu *c) { nvn_trap(c, 140); } +static void trap_141(Cpu *c) { nvn_trap(c, 141); } +static void trap_142(Cpu *c) { nvn_trap(c, 142); } +static void trap_143(Cpu *c) { nvn_trap(c, 143); } +static void trap_144(Cpu *c) { nvn_trap(c, 144); } +static void trap_145(Cpu *c) { nvn_trap(c, 145); } +static void trap_146(Cpu *c) { nvn_trap(c, 146); } +static void trap_147(Cpu *c) { nvn_trap(c, 147); } +static void trap_148(Cpu *c) { nvn_trap(c, 148); } +static void trap_149(Cpu *c) { nvn_trap(c, 149); } +static void trap_150(Cpu *c) { nvn_trap(c, 150); } +static void trap_151(Cpu *c) { nvn_trap(c, 151); } +static void trap_152(Cpu *c) { nvn_trap(c, 152); } +static void trap_153(Cpu *c) { nvn_trap(c, 153); } +static void trap_154(Cpu *c) { nvn_trap(c, 154); } +static void trap_155(Cpu *c) { nvn_trap(c, 155); } +static void trap_156(Cpu *c) { nvn_trap(c, 156); } +static void trap_157(Cpu *c) { nvn_trap(c, 157); } +static void trap_158(Cpu *c) { nvn_trap(c, 158); } +static void trap_159(Cpu *c) { nvn_trap(c, 159); } +static void trap_160(Cpu *c) { nvn_trap(c, 160); } +static void trap_161(Cpu *c) { nvn_trap(c, 161); } +static void trap_162(Cpu *c) { nvn_trap(c, 162); } +static void trap_163(Cpu *c) { nvn_trap(c, 163); } +static void trap_164(Cpu *c) { nvn_trap(c, 164); } +static void trap_165(Cpu *c) { nvn_trap(c, 165); } +static void trap_166(Cpu *c) { nvn_trap(c, 166); } +static void trap_167(Cpu *c) { nvn_trap(c, 167); } +static void trap_168(Cpu *c) { nvn_trap(c, 168); } +static void trap_169(Cpu *c) { nvn_trap(c, 169); } +static void trap_170(Cpu *c) { nvn_trap(c, 170); } +static void trap_171(Cpu *c) { nvn_trap(c, 171); } +static void trap_172(Cpu *c) { nvn_trap(c, 172); } +static void trap_173(Cpu *c) { nvn_trap(c, 173); } +static void trap_174(Cpu *c) { nvn_trap(c, 174); } +static void trap_175(Cpu *c) { nvn_trap(c, 175); } +static void trap_176(Cpu *c) { nvn_trap(c, 176); } +static void trap_177(Cpu *c) { nvn_trap(c, 177); } +static void trap_178(Cpu *c) { nvn_trap(c, 178); } +static void trap_179(Cpu *c) { nvn_trap(c, 179); } +static void trap_180(Cpu *c) { nvn_trap(c, 180); } +static void trap_181(Cpu *c) { nvn_trap(c, 181); } +static void trap_182(Cpu *c) { nvn_trap(c, 182); } +static void trap_183(Cpu *c) { nvn_trap(c, 183); } +static void trap_184(Cpu *c) { nvn_trap(c, 184); } +static void trap_185(Cpu *c) { nvn_trap(c, 185); } +static void trap_186(Cpu *c) { nvn_trap(c, 186); } +static void trap_187(Cpu *c) { nvn_trap(c, 187); } +static void trap_188(Cpu *c) { nvn_trap(c, 188); } +static void trap_189(Cpu *c) { nvn_trap(c, 189); } +static void trap_190(Cpu *c) { nvn_trap(c, 190); } +static void trap_191(Cpu *c) { nvn_trap(c, 191); } +static void trap_192(Cpu *c) { nvn_trap(c, 192); } +static void trap_193(Cpu *c) { nvn_trap(c, 193); } +static void trap_194(Cpu *c) { nvn_trap(c, 194); } +static void trap_195(Cpu *c) { nvn_trap(c, 195); } +static void trap_196(Cpu *c) { nvn_trap(c, 196); } +static void trap_197(Cpu *c) { nvn_trap(c, 197); } +static void trap_198(Cpu *c) { nvn_trap(c, 198); } +static void trap_199(Cpu *c) { nvn_trap(c, 199); } +static void trap_200(Cpu *c) { nvn_trap(c, 200); } +static void trap_201(Cpu *c) { nvn_trap(c, 201); } +static void trap_202(Cpu *c) { nvn_trap(c, 202); } +static void trap_203(Cpu *c) { nvn_trap(c, 203); } +static void trap_204(Cpu *c) { nvn_trap(c, 204); } +static void trap_205(Cpu *c) { nvn_trap(c, 205); } +static void trap_206(Cpu *c) { nvn_trap(c, 206); } +static void trap_207(Cpu *c) { nvn_trap(c, 207); } +static void trap_208(Cpu *c) { nvn_trap(c, 208); } +static void trap_209(Cpu *c) { nvn_trap(c, 209); } +static void trap_210(Cpu *c) { nvn_trap(c, 210); } +static void trap_211(Cpu *c) { nvn_trap(c, 211); } +static void trap_212(Cpu *c) { nvn_trap(c, 212); } +static void trap_213(Cpu *c) { nvn_trap(c, 213); } +static void trap_214(Cpu *c) { nvn_trap(c, 214); } +static void trap_215(Cpu *c) { nvn_trap(c, 215); } +static void trap_216(Cpu *c) { nvn_trap(c, 216); } +static void trap_217(Cpu *c) { nvn_trap(c, 217); } +static void trap_218(Cpu *c) { nvn_trap(c, 218); } +static void trap_219(Cpu *c) { nvn_trap(c, 219); } +static void trap_220(Cpu *c) { nvn_trap(c, 220); } +static void trap_221(Cpu *c) { nvn_trap(c, 221); } +static void trap_222(Cpu *c) { nvn_trap(c, 222); } +static void trap_223(Cpu *c) { nvn_trap(c, 223); } +static void trap_224(Cpu *c) { nvn_trap(c, 224); } +static void trap_225(Cpu *c) { nvn_trap(c, 225); } +static void trap_226(Cpu *c) { nvn_trap(c, 226); } +static void trap_227(Cpu *c) { nvn_trap(c, 227); } +static void trap_228(Cpu *c) { nvn_trap(c, 228); } +static void trap_229(Cpu *c) { nvn_trap(c, 229); } +static void trap_230(Cpu *c) { nvn_trap(c, 230); } +static void trap_231(Cpu *c) { nvn_trap(c, 231); } +static void trap_232(Cpu *c) { nvn_trap(c, 232); } +static void trap_233(Cpu *c) { nvn_trap(c, 233); } +static void trap_234(Cpu *c) { nvn_trap(c, 234); } +static void trap_235(Cpu *c) { nvn_trap(c, 235); } +static void trap_236(Cpu *c) { nvn_trap(c, 236); } +static void trap_237(Cpu *c) { nvn_trap(c, 237); } +static void trap_238(Cpu *c) { nvn_trap(c, 238); } +static void trap_239(Cpu *c) { nvn_trap(c, 239); } +static void trap_240(Cpu *c) { nvn_trap(c, 240); } +static void trap_241(Cpu *c) { nvn_trap(c, 241); } +static void trap_242(Cpu *c) { nvn_trap(c, 242); } +static void trap_243(Cpu *c) { nvn_trap(c, 243); } +static void trap_244(Cpu *c) { nvn_trap(c, 244); } +static void trap_245(Cpu *c) { nvn_trap(c, 245); } +static void trap_246(Cpu *c) { nvn_trap(c, 246); } +static void trap_247(Cpu *c) { nvn_trap(c, 247); } +static void trap_248(Cpu *c) { nvn_trap(c, 248); } +static void trap_249(Cpu *c) { nvn_trap(c, 249); } +static void trap_250(Cpu *c) { nvn_trap(c, 250); } +static void trap_251(Cpu *c) { nvn_trap(c, 251); } +static void trap_252(Cpu *c) { nvn_trap(c, 252); } +static void trap_253(Cpu *c) { nvn_trap(c, 253); } +static void trap_254(Cpu *c) { nvn_trap(c, 254); } +static void trap_255(Cpu *c) { nvn_trap(c, 255); } +static void trap_256(Cpu *c) { nvn_trap(c, 256); } +static void trap_257(Cpu *c) { nvn_trap(c, 257); } +static void trap_258(Cpu *c) { nvn_trap(c, 258); } +static void trap_259(Cpu *c) { nvn_trap(c, 259); } +static void trap_260(Cpu *c) { nvn_trap(c, 260); } +static void trap_261(Cpu *c) { nvn_trap(c, 261); } +static void trap_262(Cpu *c) { nvn_trap(c, 262); } +static void trap_263(Cpu *c) { nvn_trap(c, 263); } +static void trap_264(Cpu *c) { nvn_trap(c, 264); } +static void trap_265(Cpu *c) { nvn_trap(c, 265); } +static void trap_266(Cpu *c) { nvn_trap(c, 266); } +static void trap_267(Cpu *c) { nvn_trap(c, 267); } +static void trap_268(Cpu *c) { nvn_trap(c, 268); } +static void trap_269(Cpu *c) { nvn_trap(c, 269); } +static void trap_270(Cpu *c) { nvn_trap(c, 270); } +static void trap_271(Cpu *c) { nvn_trap(c, 271); } +static void trap_272(Cpu *c) { nvn_trap(c, 272); } +static void trap_273(Cpu *c) { nvn_trap(c, 273); } +static void trap_274(Cpu *c) { nvn_trap(c, 274); } +static void trap_275(Cpu *c) { nvn_trap(c, 275); } +static void trap_276(Cpu *c) { nvn_trap(c, 276); } +static void trap_277(Cpu *c) { nvn_trap(c, 277); } +static void trap_278(Cpu *c) { nvn_trap(c, 278); } +static void trap_279(Cpu *c) { nvn_trap(c, 279); } +static void trap_280(Cpu *c) { nvn_trap(c, 280); } +static void trap_281(Cpu *c) { nvn_trap(c, 281); } +static void trap_282(Cpu *c) { nvn_trap(c, 282); } +static void trap_283(Cpu *c) { nvn_trap(c, 283); } +static void trap_284(Cpu *c) { nvn_trap(c, 284); } +static void trap_285(Cpu *c) { nvn_trap(c, 285); } +static void trap_286(Cpu *c) { nvn_trap(c, 286); } +static void trap_287(Cpu *c) { nvn_trap(c, 287); } +static void trap_288(Cpu *c) { nvn_trap(c, 288); } +static void trap_289(Cpu *c) { nvn_trap(c, 289); } +static void trap_290(Cpu *c) { nvn_trap(c, 290); } +static void trap_291(Cpu *c) { nvn_trap(c, 291); } +static void trap_292(Cpu *c) { nvn_trap(c, 292); } +static void trap_293(Cpu *c) { nvn_trap(c, 293); } +static void trap_294(Cpu *c) { nvn_trap(c, 294); } +static void trap_295(Cpu *c) { nvn_trap(c, 295); } +static void trap_296(Cpu *c) { nvn_trap(c, 296); } +static void trap_297(Cpu *c) { nvn_trap(c, 297); } +static void trap_298(Cpu *c) { nvn_trap(c, 298); } +static void trap_299(Cpu *c) { nvn_trap(c, 299); } +static void trap_300(Cpu *c) { nvn_trap(c, 300); } +static void trap_301(Cpu *c) { nvn_trap(c, 301); } +static void trap_302(Cpu *c) { nvn_trap(c, 302); } +static void trap_303(Cpu *c) { nvn_trap(c, 303); } +static void trap_304(Cpu *c) { nvn_trap(c, 304); } +static void trap_305(Cpu *c) { nvn_trap(c, 305); } +static void trap_306(Cpu *c) { nvn_trap(c, 306); } +static void trap_307(Cpu *c) { nvn_trap(c, 307); } +static void trap_308(Cpu *c) { nvn_trap(c, 308); } +static void trap_309(Cpu *c) { nvn_trap(c, 309); } +static void trap_310(Cpu *c) { nvn_trap(c, 310); } +static void trap_311(Cpu *c) { nvn_trap(c, 311); } +static void trap_312(Cpu *c) { nvn_trap(c, 312); } +static void trap_313(Cpu *c) { nvn_trap(c, 313); } +static void trap_314(Cpu *c) { nvn_trap(c, 314); } +static void trap_315(Cpu *c) { nvn_trap(c, 315); } +static void trap_316(Cpu *c) { nvn_trap(c, 316); } +static void trap_317(Cpu *c) { nvn_trap(c, 317); } +static void trap_318(Cpu *c) { nvn_trap(c, 318); } +static void trap_319(Cpu *c) { nvn_trap(c, 319); } +static void trap_320(Cpu *c) { nvn_trap(c, 320); } +static void trap_321(Cpu *c) { nvn_trap(c, 321); } +static void trap_322(Cpu *c) { nvn_trap(c, 322); } +static void trap_323(Cpu *c) { nvn_trap(c, 323); } +static void trap_324(Cpu *c) { nvn_trap(c, 324); } +static void trap_325(Cpu *c) { nvn_trap(c, 325); } +static void trap_326(Cpu *c) { nvn_trap(c, 326); } +static void trap_327(Cpu *c) { nvn_trap(c, 327); } +static void trap_328(Cpu *c) { nvn_trap(c, 328); } +static void trap_329(Cpu *c) { nvn_trap(c, 329); } +static void trap_330(Cpu *c) { nvn_trap(c, 330); } +static void trap_331(Cpu *c) { nvn_trap(c, 331); } +static void trap_332(Cpu *c) { nvn_trap(c, 332); } +static void trap_333(Cpu *c) { nvn_trap(c, 333); } +static void trap_334(Cpu *c) { nvn_trap(c, 334); } +static void trap_335(Cpu *c) { nvn_trap(c, 335); } +static void trap_336(Cpu *c) { nvn_trap(c, 336); } +static void trap_337(Cpu *c) { nvn_trap(c, 337); } +static void trap_338(Cpu *c) { nvn_trap(c, 338); } +static void trap_339(Cpu *c) { nvn_trap(c, 339); } +static void trap_340(Cpu *c) { nvn_trap(c, 340); } +static void trap_341(Cpu *c) { nvn_trap(c, 341); } +static void trap_342(Cpu *c) { nvn_trap(c, 342); } +static void trap_343(Cpu *c) { nvn_trap(c, 343); } +static void trap_344(Cpu *c) { nvn_trap(c, 344); } +static void trap_345(Cpu *c) { nvn_trap(c, 345); } +static void trap_346(Cpu *c) { nvn_trap(c, 346); } +static void trap_347(Cpu *c) { nvn_trap(c, 347); } +static void trap_348(Cpu *c) { nvn_trap(c, 348); } +static void trap_349(Cpu *c) { nvn_trap(c, 349); } +static void trap_350(Cpu *c) { nvn_trap(c, 350); } +static void trap_351(Cpu *c) { nvn_trap(c, 351); } +static void trap_352(Cpu *c) { nvn_trap(c, 352); } +static void trap_353(Cpu *c) { nvn_trap(c, 353); } +static void trap_354(Cpu *c) { nvn_trap(c, 354); } +static void trap_355(Cpu *c) { nvn_trap(c, 355); } +static void trap_356(Cpu *c) { nvn_trap(c, 356); } +static void trap_357(Cpu *c) { nvn_trap(c, 357); } +static void trap_358(Cpu *c) { nvn_trap(c, 358); } +static void trap_359(Cpu *c) { nvn_trap(c, 359); } +static void trap_360(Cpu *c) { nvn_trap(c, 360); } +static void trap_361(Cpu *c) { nvn_trap(c, 361); } +static void trap_362(Cpu *c) { nvn_trap(c, 362); } +static void trap_363(Cpu *c) { nvn_trap(c, 363); } +static void trap_364(Cpu *c) { nvn_trap(c, 364); } +static void trap_365(Cpu *c) { nvn_trap(c, 365); } +static void trap_366(Cpu *c) { nvn_trap(c, 366); } +static void trap_367(Cpu *c) { nvn_trap(c, 367); } +static void trap_368(Cpu *c) { nvn_trap(c, 368); } +static void trap_369(Cpu *c) { nvn_trap(c, 369); } +static void trap_370(Cpu *c) { nvn_trap(c, 370); } +static void trap_371(Cpu *c) { nvn_trap(c, 371); } +static void trap_372(Cpu *c) { nvn_trap(c, 372); } +static void trap_373(Cpu *c) { nvn_trap(c, 373); } +static void trap_374(Cpu *c) { nvn_trap(c, 374); } +static void trap_375(Cpu *c) { nvn_trap(c, 375); } +static void trap_376(Cpu *c) { nvn_trap(c, 376); } +static void trap_377(Cpu *c) { nvn_trap(c, 377); } +static void trap_378(Cpu *c) { nvn_trap(c, 378); } +static void trap_379(Cpu *c) { nvn_trap(c, 379); } +static void trap_380(Cpu *c) { nvn_trap(c, 380); } +static void trap_381(Cpu *c) { nvn_trap(c, 381); } +static void trap_382(Cpu *c) { nvn_trap(c, 382); } +static void trap_383(Cpu *c) { nvn_trap(c, 383); } +static void trap_384(Cpu *c) { nvn_trap(c, 384); } +static void trap_385(Cpu *c) { nvn_trap(c, 385); } +static void trap_386(Cpu *c) { nvn_trap(c, 386); } +static void trap_387(Cpu *c) { nvn_trap(c, 387); } +static void trap_388(Cpu *c) { nvn_trap(c, 388); } +static void trap_389(Cpu *c) { nvn_trap(c, 389); } +static void trap_390(Cpu *c) { nvn_trap(c, 390); } +static void trap_391(Cpu *c) { nvn_trap(c, 391); } +static void trap_392(Cpu *c) { nvn_trap(c, 392); } +static void trap_393(Cpu *c) { nvn_trap(c, 393); } +static void trap_394(Cpu *c) { nvn_trap(c, 394); } +static void trap_395(Cpu *c) { nvn_trap(c, 395); } +static void trap_396(Cpu *c) { nvn_trap(c, 396); } +static void trap_397(Cpu *c) { nvn_trap(c, 397); } +static void trap_398(Cpu *c) { nvn_trap(c, 398); } +static void trap_399(Cpu *c) { nvn_trap(c, 399); } +static void trap_400(Cpu *c) { nvn_trap(c, 400); } +static void trap_401(Cpu *c) { nvn_trap(c, 401); } +static void trap_402(Cpu *c) { nvn_trap(c, 402); } +static void trap_403(Cpu *c) { nvn_trap(c, 403); } +static void trap_404(Cpu *c) { nvn_trap(c, 404); } +static void trap_405(Cpu *c) { nvn_trap(c, 405); } +static void trap_406(Cpu *c) { nvn_trap(c, 406); } +static void trap_407(Cpu *c) { nvn_trap(c, 407); } +static void trap_408(Cpu *c) { nvn_trap(c, 408); } +static void trap_409(Cpu *c) { nvn_trap(c, 409); } +static void trap_410(Cpu *c) { nvn_trap(c, 410); } +static void trap_411(Cpu *c) { nvn_trap(c, 411); } +static void trap_412(Cpu *c) { nvn_trap(c, 412); } +static void trap_413(Cpu *c) { nvn_trap(c, 413); } +static void trap_414(Cpu *c) { nvn_trap(c, 414); } +static void trap_415(Cpu *c) { nvn_trap(c, 415); } +static void trap_416(Cpu *c) { nvn_trap(c, 416); } +static void trap_417(Cpu *c) { nvn_trap(c, 417); } +static void trap_418(Cpu *c) { nvn_trap(c, 418); } +static void trap_419(Cpu *c) { nvn_trap(c, 419); } +static void trap_420(Cpu *c) { nvn_trap(c, 420); } +static void trap_421(Cpu *c) { nvn_trap(c, 421); } +static void trap_422(Cpu *c) { nvn_trap(c, 422); } +static void trap_423(Cpu *c) { nvn_trap(c, 423); } +static void trap_424(Cpu *c) { nvn_trap(c, 424); } +static void trap_425(Cpu *c) { nvn_trap(c, 425); } +static void trap_426(Cpu *c) { nvn_trap(c, 426); } +static void trap_427(Cpu *c) { nvn_trap(c, 427); } +static void trap_428(Cpu *c) { nvn_trap(c, 428); } +static void trap_429(Cpu *c) { nvn_trap(c, 429); } +static void trap_430(Cpu *c) { nvn_trap(c, 430); } +static void trap_431(Cpu *c) { nvn_trap(c, 431); } +static void trap_432(Cpu *c) { nvn_trap(c, 432); } +static void trap_433(Cpu *c) { nvn_trap(c, 433); } +static void trap_434(Cpu *c) { nvn_trap(c, 434); } +static void trap_435(Cpu *c) { nvn_trap(c, 435); } +static void trap_436(Cpu *c) { nvn_trap(c, 436); } +static void trap_437(Cpu *c) { nvn_trap(c, 437); } +static void trap_438(Cpu *c) { nvn_trap(c, 438); } +static void trap_439(Cpu *c) { nvn_trap(c, 439); } +static void trap_440(Cpu *c) { nvn_trap(c, 440); } +static void trap_441(Cpu *c) { nvn_trap(c, 441); } +static void trap_442(Cpu *c) { nvn_trap(c, 442); } +static void trap_443(Cpu *c) { nvn_trap(c, 443); } +static void trap_444(Cpu *c) { nvn_trap(c, 444); } +static void trap_445(Cpu *c) { nvn_trap(c, 445); } +static void trap_446(Cpu *c) { nvn_trap(c, 446); } +static void trap_447(Cpu *c) { nvn_trap(c, 447); } +static void trap_448(Cpu *c) { nvn_trap(c, 448); } +static void trap_449(Cpu *c) { nvn_trap(c, 449); } +static void trap_450(Cpu *c) { nvn_trap(c, 450); } +static void trap_451(Cpu *c) { nvn_trap(c, 451); } +static void trap_452(Cpu *c) { nvn_trap(c, 452); } +static void trap_453(Cpu *c) { nvn_trap(c, 453); } +static void trap_454(Cpu *c) { nvn_trap(c, 454); } +static void trap_455(Cpu *c) { nvn_trap(c, 455); } +static void trap_456(Cpu *c) { nvn_trap(c, 456); } +static void trap_457(Cpu *c) { nvn_trap(c, 457); } +static void trap_458(Cpu *c) { nvn_trap(c, 458); } +static void trap_459(Cpu *c) { nvn_trap(c, 459); } +static void trap_460(Cpu *c) { nvn_trap(c, 460); } +static void trap_461(Cpu *c) { nvn_trap(c, 461); } +static void trap_462(Cpu *c) { nvn_trap(c, 462); } +static void trap_463(Cpu *c) { nvn_trap(c, 463); } +static void trap_464(Cpu *c) { nvn_trap(c, 464); } +static void trap_465(Cpu *c) { nvn_trap(c, 465); } +static void trap_466(Cpu *c) { nvn_trap(c, 466); } +static void trap_467(Cpu *c) { nvn_trap(c, 467); } +static void trap_468(Cpu *c) { nvn_trap(c, 468); } +static void trap_469(Cpu *c) { nvn_trap(c, 469); } +static void trap_470(Cpu *c) { nvn_trap(c, 470); } +static void trap_471(Cpu *c) { nvn_trap(c, 471); } +static void trap_472(Cpu *c) { nvn_trap(c, 472); } +static void trap_473(Cpu *c) { nvn_trap(c, 473); } +static void trap_474(Cpu *c) { nvn_trap(c, 474); } +static void trap_475(Cpu *c) { nvn_trap(c, 475); } +static void trap_476(Cpu *c) { nvn_trap(c, 476); } +static void trap_477(Cpu *c) { nvn_trap(c, 477); } +static void trap_478(Cpu *c) { nvn_trap(c, 478); } +static void trap_479(Cpu *c) { nvn_trap(c, 479); } +static void trap_480(Cpu *c) { nvn_trap(c, 480); } +static void trap_481(Cpu *c) { nvn_trap(c, 481); } +static void trap_482(Cpu *c) { nvn_trap(c, 482); } +static void trap_483(Cpu *c) { nvn_trap(c, 483); } +static void trap_484(Cpu *c) { nvn_trap(c, 484); } +static void trap_485(Cpu *c) { nvn_trap(c, 485); } +static void trap_486(Cpu *c) { nvn_trap(c, 486); } +static void trap_487(Cpu *c) { nvn_trap(c, 487); } +static void trap_488(Cpu *c) { nvn_trap(c, 488); } +static void trap_489(Cpu *c) { nvn_trap(c, 489); } +static void trap_490(Cpu *c) { nvn_trap(c, 490); } +static void trap_491(Cpu *c) { nvn_trap(c, 491); } +static void trap_492(Cpu *c) { nvn_trap(c, 492); } +static void trap_493(Cpu *c) { nvn_trap(c, 493); } +static void trap_494(Cpu *c) { nvn_trap(c, 494); } +static void trap_495(Cpu *c) { nvn_trap(c, 495); } +static void trap_496(Cpu *c) { nvn_trap(c, 496); } +static void trap_497(Cpu *c) { nvn_trap(c, 497); } +static void trap_498(Cpu *c) { nvn_trap(c, 498); } +static void trap_499(Cpu *c) { nvn_trap(c, 499); } +static void trap_500(Cpu *c) { nvn_trap(c, 500); } +static void trap_501(Cpu *c) { nvn_trap(c, 501); } +static void trap_502(Cpu *c) { nvn_trap(c, 502); } +static void trap_503(Cpu *c) { nvn_trap(c, 503); } +static void trap_504(Cpu *c) { nvn_trap(c, 504); } +static void trap_505(Cpu *c) { nvn_trap(c, 505); } +static void trap_506(Cpu *c) { nvn_trap(c, 506); } +static void trap_507(Cpu *c) { nvn_trap(c, 507); } +static void trap_508(Cpu *c) { nvn_trap(c, 508); } +static void trap_509(Cpu *c) { nvn_trap(c, 509); } +static void trap_510(Cpu *c) { nvn_trap(c, 510); } +static void trap_511(Cpu *c) { nvn_trap(c, 511); } + static int by_name(const void *a, const void *b) { return strcmp(((const Proc *)a)->name, ((const Proc *)b)->name); } void nvn_procs_register(void) { @@ -46,29 +582,96 @@ void nvn_procs_register(void) { #include "nvn_procs.inc" #undef NVN_PROC qsort(procs, (size_t)nprocs, sizeof *procs, by_name); + static const GuestFn traps[TRAP_COUNT] = { + trap_0, trap_1, trap_2, trap_3, trap_4, trap_5, trap_6, trap_7, + trap_8, trap_9, trap_10, trap_11, trap_12, trap_13, trap_14, trap_15, + trap_16, trap_17, trap_18, trap_19, trap_20, trap_21, trap_22, trap_23, + trap_24, trap_25, trap_26, trap_27, trap_28, trap_29, trap_30, trap_31, + trap_32, trap_33, trap_34, trap_35, trap_36, trap_37, trap_38, trap_39, + trap_40, trap_41, trap_42, trap_43, trap_44, trap_45, trap_46, trap_47, + trap_48, trap_49, trap_50, trap_51, trap_52, trap_53, trap_54, trap_55, + trap_56, trap_57, trap_58, trap_59, trap_60, trap_61, trap_62, trap_63, + trap_64, trap_65, trap_66, trap_67, trap_68, trap_69, trap_70, trap_71, + trap_72, trap_73, trap_74, trap_75, trap_76, trap_77, trap_78, trap_79, + trap_80, trap_81, trap_82, trap_83, trap_84, trap_85, trap_86, trap_87, + trap_88, trap_89, trap_90, trap_91, trap_92, trap_93, trap_94, trap_95, + trap_96, trap_97, trap_98, trap_99, trap_100, trap_101, trap_102, trap_103, + trap_104, trap_105, trap_106, trap_107, trap_108, trap_109, trap_110, trap_111, + trap_112, trap_113, trap_114, trap_115, trap_116, trap_117, trap_118, trap_119, + trap_120, trap_121, trap_122, trap_123, trap_124, trap_125, trap_126, trap_127, + trap_128, trap_129, trap_130, trap_131, trap_132, trap_133, trap_134, trap_135, + trap_136, trap_137, trap_138, trap_139, trap_140, trap_141, trap_142, trap_143, + trap_144, trap_145, trap_146, trap_147, trap_148, trap_149, trap_150, trap_151, + trap_152, trap_153, trap_154, trap_155, trap_156, trap_157, trap_158, trap_159, + trap_160, trap_161, trap_162, trap_163, trap_164, trap_165, trap_166, trap_167, + trap_168, trap_169, trap_170, trap_171, trap_172, trap_173, trap_174, trap_175, + trap_176, trap_177, trap_178, trap_179, trap_180, trap_181, trap_182, trap_183, + trap_184, trap_185, trap_186, trap_187, trap_188, trap_189, trap_190, trap_191, + trap_192, trap_193, trap_194, trap_195, trap_196, trap_197, trap_198, trap_199, + trap_200, trap_201, trap_202, trap_203, trap_204, trap_205, trap_206, trap_207, + trap_208, trap_209, trap_210, trap_211, trap_212, trap_213, trap_214, trap_215, + trap_216, trap_217, trap_218, trap_219, trap_220, trap_221, trap_222, trap_223, + trap_224, trap_225, trap_226, trap_227, trap_228, trap_229, trap_230, trap_231, + trap_232, trap_233, trap_234, trap_235, trap_236, trap_237, trap_238, trap_239, + trap_240, trap_241, trap_242, trap_243, trap_244, trap_245, trap_246, trap_247, + trap_248, trap_249, trap_250, trap_251, trap_252, trap_253, trap_254, trap_255, + trap_256, trap_257, trap_258, trap_259, trap_260, trap_261, trap_262, trap_263, + trap_264, trap_265, trap_266, trap_267, trap_268, trap_269, trap_270, trap_271, + trap_272, trap_273, trap_274, trap_275, trap_276, trap_277, trap_278, trap_279, + trap_280, trap_281, trap_282, trap_283, trap_284, trap_285, trap_286, trap_287, + trap_288, trap_289, trap_290, trap_291, trap_292, trap_293, trap_294, trap_295, + trap_296, trap_297, trap_298, trap_299, trap_300, trap_301, trap_302, trap_303, + trap_304, trap_305, trap_306, trap_307, trap_308, trap_309, trap_310, trap_311, + trap_312, trap_313, trap_314, trap_315, trap_316, trap_317, trap_318, trap_319, + trap_320, trap_321, trap_322, trap_323, trap_324, trap_325, trap_326, trap_327, + trap_328, trap_329, trap_330, trap_331, trap_332, trap_333, trap_334, trap_335, + trap_336, trap_337, trap_338, trap_339, trap_340, trap_341, trap_342, trap_343, + trap_344, trap_345, trap_346, trap_347, trap_348, trap_349, trap_350, trap_351, + trap_352, trap_353, trap_354, trap_355, trap_356, trap_357, trap_358, trap_359, + trap_360, trap_361, trap_362, trap_363, trap_364, trap_365, trap_366, trap_367, + trap_368, trap_369, trap_370, trap_371, trap_372, trap_373, trap_374, trap_375, + trap_376, trap_377, trap_378, trap_379, trap_380, trap_381, trap_382, trap_383, + trap_384, trap_385, trap_386, trap_387, trap_388, trap_389, trap_390, trap_391, + trap_392, trap_393, trap_394, trap_395, trap_396, trap_397, trap_398, trap_399, + trap_400, trap_401, trap_402, trap_403, trap_404, trap_405, trap_406, trap_407, + trap_408, trap_409, trap_410, trap_411, trap_412, trap_413, trap_414, trap_415, + trap_416, trap_417, trap_418, trap_419, trap_420, trap_421, trap_422, trap_423, + trap_424, trap_425, trap_426, trap_427, trap_428, trap_429, trap_430, trap_431, + trap_432, trap_433, trap_434, trap_435, trap_436, trap_437, trap_438, trap_439, + trap_440, trap_441, trap_442, trap_443, trap_444, trap_445, trap_446, trap_447, + trap_448, trap_449, trap_450, trap_451, trap_452, trap_453, trap_454, trap_455, + trap_456, trap_457, trap_458, trap_459, trap_460, trap_461, trap_462, trap_463, + trap_464, trap_465, trap_466, trap_467, trap_468, trap_469, trap_470, trap_471, + trap_472, trap_473, trap_474, trap_475, trap_476, trap_477, trap_478, trap_479, + trap_480, trap_481, trap_482, trap_483, trap_484, trap_485, trap_486, trap_487, + trap_488, trap_489, trap_490, trap_491, trap_492, trap_493, trap_494, trap_495, + trap_496, trap_497, trap_498, trap_499, trap_500, trap_501, trap_502, trap_503, + trap_504, trap_505, trap_506, trap_507, trap_508, trap_509, trap_510, trap_511, + }; + for (int i = 0; i < TRAP_COUNT; i++) { + char n[32]; + snprintf(n, sizeof n, "nvn:unknown proc %d", i); + trap_addr[i] = hostfn_register(strdup(n), traps[i]); + } } uint64_t nvn_proc_address(const char *name) { if (!name) return 0; const Proc *p = bsearch(&(Proc){name, 0}, procs, (size_t)nprocs, sizeof *procs, by_name); if (p) return p->addr; - // Unknown names get NULL, as on hardware; each distinct one is logged once. + // Unknown names get a trap (see nvn_trap), one per distinct name; NULL once the pool is used up. static PlatMutex lock = PLAT_MUTEX_INIT; - static char seen[32][96]; - static int nseen; plat_mutex_lock(&lock); - int fresh = 1; - for (int i = 0; i < nseen; i++) - if (!strncmp(seen[i], name, sizeof seen[i] - 1)) fresh = 0; - if (fresh && nseen < 32) { - strncpy(seen[nseen], name, sizeof seen[0] - 1); - nseen++; - } else { - fresh = 0; - } + int i; + for (i = 0; i < ntraps; i++) + if (!strncmp(trap_name[i], name, sizeof trap_name[i] - 1)) break; + int fresh = i == ntraps && ntraps < TRAP_COUNT && trap_addr[0]; + if (fresh) strncpy(trap_name[ntraps++], name, sizeof trap_name[0] - 1); + uint64_t a = i < ntraps ? trap_addr[i] : 0; plat_mutex_unlock(&lock); - if (fresh) rt_log("nvn: no such proc \"%s\", returning NULL", name); - return 0; + if (fresh) rt_log("nvn: no such proc \"%s\": a trap that logs its first call", name); + else if (!a) rt_log("nvn: no such proc \"%s\", returning NULL", name); + return a; } // PFNNVNGENERICFUNC nvnDeviceGetProcAddress(const NVNdevice *device, const char *name) diff --git a/runtime/nvn_procs.inc b/runtime/nvn_procs.inc index 4c6e8339..2039cc34 100644 --- a/runtime/nvn_procs.inc +++ b/runtime/nvn_procs.inc @@ -536,3 +536,11 @@ NVN_PROC(nvnWindowSetPresentInterval) NVN_PROC(nvnSyncGetNvRmSyncNVX) NVN_PROC(nvnSyncInitializeFromNvRmSyncNVX) NVN_PROC(nvnTextureGetNvRmSurfaceNVX) +NVN_PROC(nvnCommandBufferDrawMeshTasks) +NVN_PROC(nvnCommandBufferDrawMeshTasksIndirect) +NVN_PROC(nvnCommandBufferPushComputeConstants) +NVN_PROC(nvnCommandBufferSetGeometryProcessingMode) +NVN_PROC(nvnProgramBuilderSetDefaults) +NVN_PROC(nvnProgramBuilderSetDevice) +NVN_PROC(nvnProgramBuilderSetShaders) +NVN_PROC(nvnProgramInitializeFromBuilder) diff --git a/runtime/ov_gfx.c b/runtime/ov_gfx.c index c62a229b..32b62496 100644 --- a/runtime/ov_gfx.c +++ b/runtime/ov_gfx.c @@ -33,8 +33,11 @@ static void ov_devtools_allocator(Cpu *c) { set_allocator(c, &nv.devtools); } // void nv::InitializeGraphics(void *memory, size_t size) static void ov_initialize_graphics(Cpu *c) { uint64_t mem = arg_x(c, 0), size = arg_x(c, 1); - if (!mem || !size) rt_fatal("nv::InitializeGraphics: no graphics work memory (%#llx, %#llx)", (unsigned long long)mem, - (unsigned long long)size); + // Switch 2 titles pass (nullptr, 0): that SDK also has nv::InitializeGraphics() and the driver takes its memory + // from the allocator nv::SetGraphicsAllocator installed. Only a half-given region is an error. + if (!mem != !size) rt_fatal("nv::InitializeGraphics: inconsistent graphics work memory (%#llx, %#llx)", + (unsigned long long)mem, (unsigned long long)size); + if (!mem) rt_log("nv: no graphics work memory given; the driver allocates through the graphics allocator"); plat_mutex_lock(&lock); nv.memory = mem; nv.memory_size = size; @@ -115,8 +118,65 @@ static void ov_nvn_bootstrap(Cpu *c) { // registration runs at start-up instead of through boot.c. __attribute__((constructor)) static void ov_nvn_procs_register(void) { nvn_procs_register(); } +// ---- Switch 2 vi display and layer queries. The SDK versions reach the display and layer objects through the +// vi service; the overridden OpenDefaultDisplay/CreateLayer hand out host stand-ins, so the SDK code would call +// through them (main+0x4dc1e0 hit a null virtual in ClearLayerExtendedSwapInterval). A desktop window is SDR, +// fixed 60 Hz, without VRR; HDR metadata and swap-interval changes are accepted and dropped. +static void vi_ok(Cpu *c) { ret_x(c, 0); } +// Result QueryDisplayHdrStatus(bool *enabled, HdrOutputFormat *format, uint16_t *peak, const Display *) +static void ov_vi_hdr_status(Cpu *c) { + uint8_t *on = arg_p(c, 0); + uint32_t *format = arg_p(c, 1); + uint16_t *peak = arg_p(c, 2); + if (on) *on = 0; + if (format) *format = 0; + if (peak) *peak = 0; + ret_x(c, 0); +} +// Result QueryDisplayHdrPaperWhite(uint16_t *nits, const Display *): the SDR reference white +static void ov_vi_paper_white(Cpu *c) { + uint16_t *nits = arg_p(c, 0); + if (nits) *nits = 203; + ret_x(c, 0); +} +// Result QueryDisplayCurrentRefreshRate(RefreshRate *rate, bool *variable, const Display *): RefreshRate is taken +// to be a 32-bit Hz value (a guess: no reference describes it); 60, not variable. +static void ov_vi_refresh_rate(Cpu *c) { + uint32_t *rate = arg_p(c, 0); + uint8_t *variable = arg_p(c, 1); + if (rate) *rate = 60; + if (variable) *variable = 0; + ret_x(c, 0); +} +// Result QueryDisplayVrrSupported(bool *supported, const Display *) +static void ov_vi_vrr_supported(Cpu *c) { + uint8_t *out = arg_p(c, 0); + if (out) *out = 0; + ret_x(c, 0); +} +HOST_OVERRIDE("_ZN2nn2vi21QueryDisplayHdrStatusEPbPNS0_15HdrOutputFormatEPtPKNS0_7DisplayE", ov_vi_hdr_status); +HOST_OVERRIDE("_ZN2nn2vi25QueryDisplayHdrPaperWhiteEPtPKNS0_7DisplayE", ov_vi_paper_white); +HOST_OVERRIDE("_ZN2nn2vi30QueryDisplayCurrentRefreshRateEPNS0_11RefreshRateEPbPKNS0_7DisplayE", ov_vi_refresh_rate); +HOST_OVERRIDE("_ZN2nn2vi24QueryDisplayVrrSupportedEPbPKNS0_7DisplayE", ov_vi_vrr_supported); +HOST_OVERRIDE("_ZN2nn2vi52SetLayerNextHdrStandardContentLuminanceRangeMetadataEPNS0_5LayerEtt", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi52SetLayerNextHdrExtendedContentLuminanceRangeMetadataEPNS0_5LayerEtt", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi35SetLayerNextHdrContentGamutMetadataEPNS0_5LayerENS0_15HdrContentGamutE", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi39SetLayerNextHdrToSdrToneMappingMetadataEPNS0_5LayerENS0_14HdrToneMappingE", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi28SetLayerExtendedSwapIntervalEPNS0_5LayerEii", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi30ClearLayerExtendedSwapIntervalEPNS0_5LayerE", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi36SetDisplay120HzRefreshRateCapabilityEPNS0_7DisplayEb", vi_ok); +HOST_OVERRIDE("_ZN2nn2vi23SetDisplayVrrCapabilityEPNS0_7DisplayEb", vi_ok); + HOST_OVERRIDE("_ZN2nv20SetGraphicsAllocatorEPFPvmmS0_EPFvS0_S0_EPFS0_S0_mS0_ES0_", ov_graphics_allocator); HOST_OVERRIDE("_ZN2nv18InitializeGraphicsEPvm", ov_initialize_graphics); +// void nv::InitCheck(): the SDK's checks its own "graphics initialised" state, which the overridden +// InitializeGraphics never sets, and aborts (subsdk0's movie::VideoDecoder::Initialize calls it). The host's +// record answers instead. +static void ov_nv_init_check(Cpu *c) { + (void)c; + if (!nv.initialized) rt_fatal("nv::InitCheck: graphics not initialised (nv::InitializeGraphics was not called)"); +} +HOST_OVERRIDE("_ZN2nv9InitCheckEv", ov_nv_init_check); HOST_OVERRIDE("_ZN2nv28SetGraphicsDevtoolsAllocatorEPFPvmmS0_EPFvS0_S0_EPFS0_S0_mS0_ES0_", ov_devtools_allocator); HOST_OVERRIDE("_ZN2nn2vi10InitializeEv", ov_vi_initialize); HOST_OVERRIDE("_ZN2nn2vi18OpenDefaultDisplayEPPNS0_7DisplayE", ov_open_display); diff --git a/runtime/ov_input.c b/runtime/ov_input.c index 3eb1421a..f26d9a07 100644 --- a/runtime/ov_input.c +++ b/runtime/ov_input.c @@ -513,3 +513,51 @@ HOST_OVERRIDE("_ZN2nn3hid6detail22GetSixAxisSensorStatesEPiPNS0_18SixAxisSensorS ov_six_states); HOST_OVERRIDE("_ZN2nn3hid6detail21GetSixAxisSensorStateEPNS0_18SixAxisSensorStateERKNS0_19SixAxisSensorHandleE", ov_six_state); + +// ---- the public wrappers a Switch 2 title imports. NX2NAVI imports nn::hid::detail::*; Switch 2 titles import the +// public functions, whose direct calls into detail:: no override reaches, so each public one is answered here with +// the detail implementation above. Joy-Con 2 mouse (Nmouse), Jcms and sensor fusion have no host device: no +// handles, starts that do nothing. +// void StartSixAxisSensor(const SixAxisSensorHandle &) / void StopSixAxisSensor(...) +static void ov_pub_six_start(Cpu *c) { ov_six_start(c); } +// int GetSixAxisSensorStates(SixAxisSensorState *out, int count, const SixAxisSensorHandle &): the count filled +static void ov_pub_six_states(Cpu *c) { + ret_x(c, (uint64_t)(int64_t)six_fill(arg_p(c, 0), (int)arg_x(c, 1), arg_p(c, 2), "GetSixAxisSensorStates")); +} +// void SetGyroscopeZeroDriftMode(const SixAxisSensorHandle &, const GyroscopeZeroDriftMode &): the host gyro has +// its own calibration +static void ov_pub_nothing(Cpu *c) { ret_x(c, 0); } +// void InitializeVibrationDevice(const VibrationDeviceHandle &) / void SendVibrationValue(handle, value) +static void ov_pub_vib_init(Cpu *c) { ov_vib_init(c); } +static void ov_pub_vib_send(Cpu *c) { ov_vib_send(c); } +// bool IsOunceVibrationDeviceMounted(const VibrationDeviceHandle &): Ounce is the Switch 2's own actuator; the +// host's rumble devices count as mounted +static void ov_pub_vib_ounce_mounted(Cpu *c) { + ret_x(c, vibration_side(arg_p(c, 0), "IsOunceVibrationDeviceMounted") >= 0); +} +// int GetSensorFusionHandles(...) / int GetNmouseHandles(NmouseHandle *, int, uint32_t): none +static void ov_pub_no_handles(Cpu *c) { ret_x(c, 0); } +// void GetNmouseTypeSet(BitFlagSet<32, NmouseType> *out, NmouseHandle): empty +static void ov_pub_nmouse_types(Cpu *c) { + uint32_t *out = arg_p(c, 0); + if (out) *out = 0; +} +HOST_OVERRIDE("_ZN2nn3hid18StartSixAxisSensorERKNS0_19SixAxisSensorHandleE", ov_pub_six_start); +HOST_OVERRIDE("_ZN2nn3hid22GetSixAxisSensorStatesEPNS0_18SixAxisSensorStateEiRKNS0_19SixAxisSensorHandleE", + ov_pub_six_states); +HOST_OVERRIDE("_ZN2nn3hid25SetGyroscopeZeroDriftModeERKNS0_19SixAxisSensorHandleERKNS0_22GyroscopeZeroDriftModeE", + ov_pub_nothing); +HOST_OVERRIDE("_ZN2nn3hid25InitializeVibrationDeviceERKNS0_21VibrationDeviceHandleE", ov_pub_vib_init); +HOST_OVERRIDE("_ZN2nn3hid18SendVibrationValueERKNS0_21VibrationDeviceHandleERKNS0_14VibrationValueE", ov_pub_vib_send); +HOST_OVERRIDE("_ZN2nn3hid29IsOunceVibrationDeviceMountedERKNS0_21VibrationDeviceHandleE", ov_pub_vib_ounce_mounted); +HOST_OVERRIDE("_ZN2nn3hid22GetSensorFusionHandlesEPNS0_18SensorFusionHandleEiRKjNS_4util10BitFlagSetILi32ENS0_" + "12NpadStyleTagEEE", ov_pub_no_handles); +HOST_OVERRIDE("_ZN2nn3hid20FinalizeSensorFusionEv", ov_pub_nothing); +HOST_OVERRIDE("_ZN2nn3hid16GetNmouseHandlesEPNS0_12NmouseHandleEij", ov_pub_no_handles); +HOST_OVERRIDE("_ZN2nn3hid16GetNmouseTypeSetEPNS_4util10BitFlagSetILi32ENS0_10NmouseTypeEEENS0_12NmouseHandleE", + ov_pub_nmouse_types); +HOST_OVERRIDE("_ZN2nn3hid14FinalizeNmouseEv", ov_pub_nothing); +HOST_OVERRIDE("_ZN2nn3hid13GetJcmsHandleERKjRKNS0_8JcmsTypeE", ov_pub_nothing); +HOST_OVERRIDE("_ZN2nn3hid9StartJcmsERKNS0_10JcmsHandleE", ov_pub_nothing); +HOST_OVERRIDE("_ZN2nn3hid8StopJcmsERKNS0_10JcmsHandleE", ov_pub_nothing); +HOST_OVERRIDE("_ZN2nn3hid12FinalizeJcmsEv", ov_pub_nothing); diff --git a/runtime/ov_movie2.c b/runtime/ov_movie2.c new file mode 100644 index 00000000..a18bd665 --- /dev/null +++ b/runtime/ov_movie2.c @@ -0,0 +1,658 @@ +// Switch 2 movie playback: movie::VideoDecoder (subsdk0) answered by the host with dav1d. +// +// The title's movies are WebM with AV1 video only (Event/Movie/*.webm, 1920x1080 60 fps, 8-bit 4:2:0, limited +// range BT.709); their sound is a separate stream the game plays alongside (Sound/Resource/Stream/.bwav), +// so a movie that runs for its real length is also what makes the title sequence audible. The SDK's +// nn::movie::MediaExtractor demuxes as it is; subsdk0's VideoDecoder sits on nn::codec::video (a video server this +// host does not have), so the public VideoDecoder methods the game imports are replaced here. VideoFrame stays +// subsdk0's own object; its layout (subsdk0 VideoFrame::SetBuffer / SetPixelPlaneData / SetTimeStamp / +// SetEndOfStreamState, sizeof 0x90): +// +0x08 + 8 i plane i data (plane 0 = the buffer of SetBuffer) +0x28 + 8 i plane i filled size +// +0x48 buffer capacity +0x78 time stamp (TimeSpan, ns) +// +0x80 pixel format +0x84 pixel storage type +0x88 end of stream (byte) +// nn::movie::Buffer (sdk): +0x00 data, +0x08 capacity, +0x10 filled size, +0x18 time stamp, +0x20 flags (8 = end +// of stream, as the game sets it at main+0x3b60014). +// +// Decoding is asynchronous, as on the hardware: DecodeAsync queues a copy of the sample and returns; one host thread +// per decoder runs dav1d and converts each picture to NV12 (the game's frame format: it sizes each frame buffer +// width * height * 3 / 2 and reads two planes) and to RGBA for the window. Decoding on the caller's thread stalled +// the game's movie loop (it also acquires frames) by up to 90 ms on heavy frames: visible stutter. +// Back-pressure: while QREADY samples or pictures wait, DecodeAsync answers R_BUSY and the input-ready event is +// clear; the game's decode step (main+0x3b5e600) treats any non-zero result as "try again" on that event. +// AcquireVideoFrame answers 0x7b when nothing is ready (subsdk0 VideoDecoder::AcquireVideoFrame's own result). +// The game draws the frame with its own shaders (a YUV compute pass). Each frame is debanded first (flat areas +// averaged with sub-level precision), so the encoder's flattened gradients (skies, clouds) do not show as hard bands, +// and written back to NV12 with a 4x4 ordered dither. With NX2NAVI_MOVIE_OVERLAY=1 the host instead shows the frame +// over the game image (gfx_present.c gfx_movie_frame_own), converted here: BT.709 limited range, bilinear chroma, +// the same dither. NX2NAVI_MOVIE_DEBAND=0 turns the deband off. +// NX2NAVI_MOVIES=off skips movies instead (romfs.c movie_skipped); this title then quits, so for testing use +// NX2NAVI_MOVIE_SKIP=: a movie ends as if it had played to its end once its samples reach that time (the next +// sample becomes the end-of-stream marker, later ones are dropped), and the game goes on as after a whole movie. +#include +#include +#include +#include + +#include "cfg.h" +#include "gfx.h" +#include "host.h" +#include "platform/platform.h" + +#define R_NO_FRAME 0x7bu +#define R_BUSY 0x7du // DecodeAsync while the queues are full: the game retries on its input-ready event +#define BUFFER_FLAG_EOS 0x8u +#define QMAX 8 // decoded pictures held at most +#define INMAX 8 // samples waiting for the decoder thread at most +#define QREADY 4 // input-ready while fewer than this many samples and pictures wait +#define NFRAMES 16 // registered VideoFrames + +typedef struct { uint8_t *nv12, *rgba; int w, h; int64_t ts; int eos; } Picture; +typedef struct { uint8_t *data; size_t size; int64_t ts; int eos; } Sample; +typedef struct { + PlatMutex lock; // queues, frames, flags + PlatMutex codec; // dav1d: the decoder thread while it decodes, Flush / Finalize + PlatCond work; // a sample arrived, or quit + PlatThread *thread; + Dav1dContext *dav1d; + uint8_t in_ev[0x40], out_ev[0x40]; // nn::os::SystemEventType each + int in_bound, out_bound; + uint64_t frame[NFRAMES]; + int busy[NFRAMES], nframe; + Picture q[QMAX]; + int qh, qn; + Sample in[INMAX]; + int ih, in_n; + int quit, eos_queued; + uint64_t generation; // bumped by Flush: pictures of an older one are dropped + uint64_t decoded, shown; +} Video; + +static Video *video_of(void *obj) { return obj ? *(Video **)obj : NULL; } + +// under v->lock +static void update_events(Video *v) { + int ready = v->qn < QREADY && v->in_n < QREADY && !v->eos_queued; + if (v->in_bound) { + if (ready) kern_signal_system_event((uint64_t)(uintptr_t)v->in_ev); + else kern_clear_system_event((uint64_t)(uintptr_t)v->in_ev); + } + if (v->out_bound) { + if (v->qn) kern_signal_system_event((uint64_t)(uintptr_t)v->out_ev); + else kern_clear_system_event((uint64_t)(uintptr_t)v->out_ev); + } +} + +static void picture_free(Picture *p) { + free(p->nv12); + free(p->rgba); + p->nv12 = p->rgba = NULL; +} + +// under v->lock +static void clear_queues(Video *v) { + while (v->qn) { + picture_free(&v->q[v->qh]); + v->qh = (v->qh + 1) % QMAX; + v->qn--; + } + while (v->in_n) { + free(v->in[v->ih].data); + v->ih = (v->ih + 1) % INMAX; + v->in_n--; + } + v->eos_queued = 0; +} + +// Deband (the approach of mpv's deband filter, our own code): the movies' flat gradients (skies, clouds) are encoded +// as blocks of one level with hard contours between them. For each sample, four neighbours at a pseudo-random offset +// of 4..16 samples (rotated per pixel, so no pattern repeats) are compared with it; when all lie within THRESH levels +// the area is flat and the sample becomes their mean with 4 bits of fraction, else it keeps its value. Edges and +// texture differ by more than that and stay as they are. Output: plane * 16. +#define DEBAND_THRESH 3 +static void deband(const uint8_t *src, uint16_t *dst, int w, int h, int stride, int step, int y0, int y1) { + static int16_t off[64][2]; + static volatile int32_t ready; + if (!__atomic_load_n(&ready, __ATOMIC_ACQUIRE)) { + for (int i = 0; i < 64; i++) { // radii 4..16 at golden-angle rotations + double r = 4 + (i * 12) / 63.0, t = i * 2.39996323; + off[i][0] = (int16_t)(r * __builtin_cos(t)), off[i][1] = (int16_t)(r * __builtin_sin(t)); + } + __atomic_store_n(&ready, 1, __ATOMIC_RELEASE); + } + for (int y = y0; y < y1; y++) { + const uint8_t *row = src + (size_t)y * stride; + uint16_t *o = dst + (size_t)y * w; + for (int x = 0; x < w; x++) { + int c = row[x * step]; + const int16_t *d = off[(x * 7 + y * 13 + ((x ^ y) >> 2)) & 63]; + int dx = d[0], dy = d[1]; + int xa = x + dx, xb = x - dx, ya = y + dy, yb = y - dy; + int xc = x - dy, xd = x + dy, yc = y + dx, yd = y - dx; + if (xa < 0 || xb < 0 || xc < 0 || xd < 0 || xa >= w || xb >= w || xc >= w || xd >= w || ya < 0 || yb < 0 || + yc < 0 || yd < 0 || ya >= h || yb >= h || yc >= h || yd >= h) { + o[x] = (uint16_t)(c * 16); + continue; + } + int s0 = src[(size_t)ya * stride + xa * step], s1 = src[(size_t)yb * stride + xb * step]; + int s2 = src[(size_t)yc * stride + xc * step], s3 = src[(size_t)yd * stride + xd * step]; + int m0 = abs(s0 - c), m1 = abs(s1 - c), m2 = abs(s2 - c), m3 = abs(s3 - c); + if (m0 > DEBAND_THRESH || m1 > DEBAND_THRESH || m2 > DEBAND_THRESH || m3 > DEBAND_THRESH) + o[x] = (uint16_t)(c * 16); + else + o[x] = (uint16_t)((s0 + s1 + s2 + s3) * 4); // the mean, * 16 + } + } +} + +// Debanded planes (* 16) -> RGBA, BT.709 limited range. Chroma is interpolated bilinearly between the 4:2:0 samples +// (centred between luma rows, co-sited with even columns); a 4x4 Bayer threshold is added before the final rounding +// to 8 bits, which turns the remaining sub-level steps into fine noise instead of bands. +static void to_rgba(const uint16_t *Y, const uint16_t *U, const uint16_t *V, uint8_t *out, int w, int h, int y0, int y1) { + static const int bayer[4][4] = {{0, 8, 2, 10}, {12, 4, 14, 6}, {3, 11, 1, 9}, {15, 7, 13, 5}}; + const int cw = w / 2, ch = h / 2; + for (int y = y0; y < y1; y++) { + // the two chroma rows around this luma row, weighted 3:1 toward the nearer one + int cy0 = (y - 1) >> 1, cy1 = cy0 + 1, wy1 = (y & 1) ? 1 : 3, wy0 = 4 - wy1; + if (cy0 < 0) cy0 = 0; + if (cy1 >= ch) cy1 = ch - 1; + const uint16_t *u0 = U + (size_t)cy0 * cw, *u1 = U + (size_t)cy1 * cw; + const uint16_t *v0 = V + (size_t)cy0 * cw, *v1 = V + (size_t)cy1 * cw, *yr = Y + (size_t)y * w; + uint8_t *o = out + (size_t)y * w * 4; + for (int x = 0; x < w; x++) { + int cx0 = x >> 1, cx1 = (x & 1) && cx0 + 1 < cw ? cx0 + 1 : cx0; // even: on the sample, odd: halfway + int wx1 = cx1 != cx0 ? 2 : 0, wx0 = 4 - wx1; + // chroma * 16 * 16 (plane * 16, weights summing to 16) -> >> 4: * 16 + int u = ((u0[cx0] * wx0 + u0[cx1] * wx1) * wy0 + (u1[cx0] * wx0 + u1[cx1] * wx1) * wy1) >> 4; + int vv = ((v0[cx0] * wx0 + v0[cx1] * wx1) * wy0 + (v1[cx0] * wx0 + v1[cx1] * wx1) * wy1) >> 4; + int d = u - 128 * 16, e = vv - 128 * 16; // * 16 + int cc = (yr[x] - 16 * 16) * 298; // * 16 * 256 + int dith = bayer[y & 3][x & 3] * 256 + 128; // < 1 output step in the * 4096 unit + int r = (cc + 459 * e + dith) >> 12; + int g = (cc - 55 * d - 136 * e + dith) >> 12; + int b = (cc + 541 * d + dith) >> 12; + o[4 * x] = (uint8_t)(r < 0 ? 0 : r > 255 ? 255 : r); + o[4 * x + 1] = (uint8_t)(g < 0 ? 0 : g > 255 ? 255 : g); + o[4 * x + 2] = (uint8_t)(b < 0 ? 0 : b > 255 ? 255 : b); + o[4 * x + 3] = 255; + } + } +} + +// The overlay conversion runs in NSTRIPES horizontal stripes on short-lived threads, in two phases (deband every +// plane, then convert), since the convert reads chroma rows of the neighbouring stripe. One thread took 27 ms per +// 1080p frame: too slow for 60 fps. +#define NSTRIPES 6 +typedef struct { + int phase, y0, y1, w, h; + uint8_t *nv12; + uint16_t *Y16, *U16, *V16; + uint8_t *rgba; +} Stripe; +// The debanded planes back into the NV12 frame (the game converts it itself), with a 4x4 ordered dither: the +// 4 fraction bits become texel-sized noise instead of bands. +static void to_nv12(const Stripe *t) { + static const int bayer[4][4] = {{0, 8, 2, 10}, {12, 4, 14, 6}, {3, 11, 1, 9}, {15, 7, 13, 5}}; + int w = t->w; + for (int y = t->y0; y < t->y1; y++) + for (int x = 0; x < w; x++) { + int v = (t->Y16[(size_t)y * w + x] + bayer[y & 3][x & 3]) >> 4; + t->nv12[(size_t)y * w + x] = (uint8_t)(v > 255 ? 255 : v); + } + uint8_t *uv = t->nv12 + (size_t)w * t->h; + for (int y = t->y0 / 2; y < t->y1 / 2; y++) + for (int x = 0; x < w / 2; x++) { + int d = bayer[y & 3][x & 3], i = y * (w / 2) + x; + int u = (t->U16[i] + d) >> 4, v = (t->V16[i] + (15 - d)) >> 4; + uv[(size_t)y * w + 2 * x] = (uint8_t)(u > 255 ? 255 : u); + uv[(size_t)y * w + 2 * x + 1] = (uint8_t)(v > 255 ? 255 : v); + } +} + +static void stripe_run(void *arg) { + Stripe *t = arg; + if (t->phase == 0) { + const uint8_t *uv = t->nv12 + (size_t)t->w * t->h; + deband(t->nv12, t->Y16, t->w, t->h, t->w, 1, t->y0, t->y1); + deband(uv, t->U16, t->w / 2, t->h / 2, t->w, 2, t->y0 / 2, t->y1 / 2); + deband(uv + 1, t->V16, t->w / 2, t->h / 2, t->w, 2, t->y0 / 2, t->y1 / 2); + } else if (t->rgba) { + to_rgba(t->Y16, t->U16, t->V16, t->rgba, t->w, t->h, t->y0, t->y1); + } else { + to_nv12(t); + } +} +static void run_stripes(Stripe base) { + Stripe st[NSTRIPES]; + PlatThread *th[NSTRIPES] = {0}; + int rows = ((base.h + NSTRIPES - 1) / NSTRIPES + 1) & ~1; // even: chroma rows split at the same places + for (int i = 0; i < NSTRIPES; i++) { + st[i] = base; + st[i].y0 = i * rows < base.h ? i * rows : base.h; + st[i].y1 = (i + 1) * rows < base.h ? (i + 1) * rows : base.h; + if (i) th[i] = plat_thread_new(stripe_run, &st[i], "movie convert"); + if (i && !th[i]) stripe_run(&st[i]); + } + stripe_run(&st[0]); + for (int i = 1; i < NSTRIPES; i++) + if (th[i]) plat_thread_join(th[i]); +} + +// One dav1d picture (8- or 10-bit 4:2:0) -> NV12 with 8-bit samples, and its RGBA; decoder thread +static int make_picture(const Dav1dPicture *pic, Picture *p) { + int w = pic->p.w, h = pic->p.h, hbd = pic->p.bpc > 8, sh = pic->p.bpc - 8; + if (pic->p.layout != DAV1D_PIXEL_LAYOUT_I420) { + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) rt_log("movie: picture layout %d is not 4:2:0: skipped", pic->p.layout); + return 0; + } + { + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1) && pic->seq_hdr) + rt_log("movie: stream %dx%d, %d-bit, colour range %s, matrix %d, primaries %d, transfer %d", w, h, + pic->p.bpc, pic->seq_hdr->color_range ? "full" : "limited", pic->seq_hdr->mtrx, pic->seq_hdr->pri, + pic->seq_hdr->trc); + } + // The host overlay (NX2NAVI_MOVIE_OVERLAY=1) takes an RGBA frame; without it the game converts the NV12 frame itself. + static int overlay = -1; + if (overlay < 0) { + const char *e = cfg_get("NX2NAVI_MOVIE_OVERLAY"); + overlay = e && strcmp(e, "0"); + } + uint8_t *o = malloc((size_t)w * h * 3 / 2), *rgba = overlay ? malloc((size_t)w * h * 4) : NULL; + if (!o || (overlay && !rgba)) rt_fatal("movie: out of memory for a %dx%d frame", w, h); + const ptrdiff_t ys = pic->stride[0], cs = pic->stride[1]; + for (int y = 0; y < h; y++) { + const uint8_t *row = (const uint8_t *)pic->data[0] + y * ys; + if (!hbd) memcpy(o + (size_t)y * w, row, (size_t)w); + else for (int x = 0; x < w; x++) o[(size_t)y * w + x] = (uint8_t)(((const uint16_t *)row)[x] >> sh); + } + uint8_t *uv = o + (size_t)w * h; + for (int y = 0; y < h / 2; y++) { + const uint8_t *u = (const uint8_t *)pic->data[1] + y * cs, *vv = (const uint8_t *)pic->data[2] + y * cs; + for (int x = 0; x < w / 2; x++) { + uv[(size_t)y * w + 2 * x] = hbd ? (uint8_t)(((const uint16_t *)u)[x] >> sh) : u[x]; + uv[(size_t)y * w + 2 * x + 1] = hbd ? (uint8_t)(((const uint16_t *)vv)[x] >> sh) : vv[x]; + } + } + // debanded planes (NX2NAVI_MOVIE_DEBAND=0 keeps the decoded ones), then the overlay's RGBA or the dithered NV12 + { + static int deband_on = -1; + if (deband_on < 0) { + const char *e = cfg_get("NX2NAVI_MOVIE_DEBAND"); + deband_on = !(e && !strcmp(e, "0")); + } + uint16_t *Y16 = malloc((size_t)w * h * 2), *U16 = malloc((size_t)w * h / 2), *V16 = malloc((size_t)w * h / 2); + if (!Y16 || !U16 || !V16) rt_fatal("movie: out of memory for a %dx%d frame", w, h); + Stripe base = {0, 0, 0, w, h, o, Y16, U16, V16, rgba}; + if (deband_on) { + run_stripes(base); + } else if (!rgba) { + free(Y16), free(U16), free(V16); + *p = (Picture){o, rgba, w, h, pic->m.timestamp, 0}; + return 1; + } else { + for (size_t i = 0; i < (size_t)w * h; i++) Y16[i] = (uint16_t)(o[i] * 16); + for (size_t i = 0; i < (size_t)w * h / 4; i++) U16[i] = (uint16_t)(uv[2 * i] * 16), V16[i] = (uint16_t)(uv[2 * i + 1] * 16); + } + base.phase = 1; + run_stripes(base); + free(Y16), free(U16), free(V16); + } + *p = (Picture){o, rgba, w, h, pic->m.timestamp, 0}; + return 1; +} + +// Queues a decoded picture, waiting while the queue is full (the game acquires at the movie's pace); decoder thread +static void deliver(Video *v, Picture p, uint64_t gen) { + plat_mutex_lock(&v->lock); + while (v->qn == QMAX && !v->quit && v->generation == gen) { + plat_mutex_unlock(&v->lock); + plat_sleep_ms(2); + plat_mutex_lock(&v->lock); + } + if (v->quit || v->generation != gen) { + plat_mutex_unlock(&v->lock); + picture_free(&p); + return; + } + v->q[(v->qh + v->qn) % QMAX] = p; + v->qn++; + v->decoded += !p.eos; + update_events(v); + plat_mutex_unlock(&v->lock); +} + +static void drain(Video *v, uint64_t gen) { + Dav1dPicture pic = {0}; + while (dav1d_get_picture(v->dav1d, &pic) == 0) { + Picture p; + int ok = make_picture(&pic, &p); + dav1d_picture_unref(&pic); + if (ok) deliver(v, p, gen); + } +} + +static void decoder_thread(void *arg) { + Video *v = arg; + plat_mutex_lock(&v->lock); + for (;;) { + while (!v->in_n && !v->quit) plat_cond_wait(&v->work, &v->lock); + if (v->quit) break; + Sample s = v->in[v->ih]; + v->ih = (v->ih + 1) % INMAX; + v->in_n--; + uint64_t gen = v->generation; + update_events(v); + plat_mutex_unlock(&v->lock); + plat_mutex_lock(&v->codec); + if (v->dav1d && s.data && s.size) { + Dav1dData d = {0}; + uint8_t *p = dav1d_data_create(&d, s.size); + if (p) { + memcpy(p, s.data, s.size); + d.m.timestamp = s.ts; + while (d.sz) { + int r = dav1d_send_data(v->dav1d, &d); + if (r == DAV1D_ERR(EAGAIN)) drain(v, gen); + else if (r < 0) { + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) rt_log("movie: dav1d rejected a sample (%d)", r); + dav1d_data_unref(&d); + break; + } + } + drain(v, gen); + } + } + if (s.eos && v->dav1d) { + drain(v, gen); + deliver(v, (Picture){NULL, NULL, 0, 0, s.ts, 1}, gen); + } + plat_mutex_unlock(&v->codec); + free(s.data); + plat_mutex_lock(&v->lock); + } + plat_mutex_unlock(&v->lock); +} + +static void stop_thread(Video *v) { + if (!v->thread) return; + plat_mutex_lock(&v->lock); + v->quit = 1; + plat_cond_broadcast(&v->work); + plat_mutex_unlock(&v->lock); + plat_thread_join(v->thread); + v->thread = NULL; + v->quit = 0; +} + +static void finalize(Video *v) { + stop_thread(v); + plat_mutex_lock(&v->codec); + if (v->dav1d) dav1d_close(&v->dav1d); + plat_mutex_unlock(&v->codec); + plat_mutex_lock(&v->lock); + clear_queues(v); + memset(v->busy, 0, sizeof v->busy); + plat_mutex_unlock(&v->lock); +} + +// ---- movie::VideoDecoder +// VideoDecoder() / ~VideoDecoder(): the object's first 8 bytes hold the host record +static void ov_ctor(Cpu *c) { *(Video **)arg_p(c, 0) = NULL; } +static void ov_dtor(Cpu *c) { + Video **slot = arg_p(c, 0); + if (!slot || !*slot) return; + finalize(*slot); + free(*slot); // the bound events' kernel handles stay, as for any guest event never destroyed + *slot = NULL; + gfx_movie_stop(); +} +// Result Initialize(void *memory, size_t size, MediaMimeType mime, VideoDecoderOutputFormat format, VideoCodecMode) +static void ov_initialize(Cpu *c) { + Video **slot = arg_p(c, 0); + if (!*slot) { + *slot = calloc(1, sizeof **slot); + if (!*slot) rt_fatal("movie: decoder allocation"); + plat_mutex_init(&(*slot)->lock); + plat_mutex_init(&(*slot)->codec); + plat_cond_init(&(*slot)->work); + } + Video *v = *slot; + finalize(v); + Dav1dSettings s; + dav1d_default_settings(&s); + // Frame threading on (dav1d picks the delay): heavy stretches of the title movie took longer than a frame to + // decode one picture at a time (max_frame_delay 1) and the game's loop waited up to 94 ms for them. The delay + // costs nothing here, as decoding runs ahead on its own thread; end of stream drains the delayed pictures. + s.n_threads = 8; + s.max_frame_delay = 0; + int r = dav1d_open(&v->dav1d, &s); + if (r < 0) rt_fatal("movie: dav1d_open failed (%d)", r); + v->thread = plat_thread_new(decoder_thread, v, "movie decoder"); + if (!v->thread) rt_fatal("movie: cannot start the decoder thread"); + rt_log("movie: video decoder up (AV1 via dav1d %s on its own thread; mime %u, output format %u, mode %u)", + dav1d_version(), (unsigned)arg_x(c, 3), (unsigned)arg_x(c, 4), (unsigned)arg_x(c, 5)); + ret_x(c, 0); +} +// Result Configure(const MediaProperty *, const void *configuration, size_t): the AV1 sequence header also comes +// in band with the first key frame, which dav1d reads +static void ov_ok(Cpu *c) { ret_x(c, 0); } +// Result Finalize() +static void ov_finalize(Cpu *c) { + Video *v = video_of(arg_p(c, 0)); + if (v) { + rt_log("movie: video decoder finalised (%llu frames decoded, %llu shown)", (unsigned long long)v->decoded, + (unsigned long long)v->shown); + finalize(v); + } + gfx_movie_stop(); + ret_x(c, 0); +} +// Result RegisterVideoFrame(VideoFrame **frames, int count) +static void ov_register(Cpu *c) { + Video *v = video_of(arg_p(c, 0)); + uint64_t *frames = arg_p(c, 1); + int n = (int)arg_x(c, 2); + if (!v || n < 0 || n > NFRAMES || (n && !frames)) rt_fatal("movie: RegisterVideoFrame of %d frames", n); + plat_mutex_lock(&v->lock); + memcpy(v->frame, frames, (size_t)n * sizeof *frames); + memset(v->busy, 0, sizeof v->busy); + v->nframe = n; + // Each frame buffer starts as NV12 black (Y 16, UV 128 in its last third). The game converts a frame it holds + // before a picture arrives; zeros there are YUV (0, 0, 0), green, in stripes where rows were partly written. + for (int i = 0; i < n; i++) { + uint8_t *fr = (uint8_t *)(uintptr_t)frames[i], *buf; + uint64_t cap; + if (!fr) continue; + memcpy(&buf, fr + 0x08, 8), memcpy(&cap, fr + 0x48, 8); + if (!buf || cap < 3 || cap > (64u << 20)) continue; + uint64_t ysz = cap / 3 * 2; + memset(buf, 16, (size_t)ysz); + memset(buf + ysz, 128, (size_t)(cap - ysz)); + } + plat_mutex_unlock(&v->lock); + ret_x(c, 0); +} +// Result GetInputReadyEvent(SystemEventType **out) / GetOutputReadyEvent(SystemEventType **out) +static void get_event(Cpu *c, int input) { + Video *v = video_of(arg_p(c, 0)); + uint64_t *out = arg_p(c, 1); + if (!v || !out) rt_fatal("movie: Get%sReadyEvent before Initialize", input ? "Input" : "Output"); + uint8_t *ev = input ? v->in_ev : v->out_ev; + int *bound = input ? &v->in_bound : &v->out_bound; + if (!*bound) { + kern_bind_system_event(c, (uint64_t)(uintptr_t)ev, 0); + plat_mutex_lock(&v->lock); + *bound = 1; + plat_mutex_unlock(&v->lock); + } + plat_mutex_lock(&v->lock); + update_events(v); + plat_mutex_unlock(&v->lock); + *out = (uint64_t)(uintptr_t)ev; + ret_x(c, 0); +} +static void ov_input_event(Cpu *c) { get_event(c, 1); } +static void ov_output_event(Cpu *c) { get_event(c, 0); } +// Result Flush(): samples and pictures not yet acquired are dropped (a seek or a restart follows) +static void ov_flush(Cpu *c) { + Video *v = video_of(arg_p(c, 0)); + if (v) { + plat_mutex_lock(&v->lock); + v->generation++; // the decoder thread drops what it is making + clear_queues(v); + plat_mutex_unlock(&v->lock); + plat_mutex_lock(&v->codec); + if (v->dav1d) dav1d_flush(v->dav1d); + plat_mutex_unlock(&v->codec); + plat_mutex_lock(&v->lock); + update_events(v); + plat_mutex_unlock(&v->lock); + } + ret_x(c, 0); +} +// Result DecodeAsync(const nn::movie::Buffer *sample): queued for the decoder thread +static void ov_decode(Cpu *c) { + Video *v = video_of(arg_p(c, 0)); + const uint8_t *b = arg_p(c, 1); + if (!v || !v->dav1d || !b) { + ret_x(c, 0); + return; + } + const uint8_t *data; + uint64_t size; + int64_t ts; + uint32_t flags; + memcpy(&data, b, 8), memcpy(&size, b + 0x10, 8), memcpy(&ts, b + 0x18, 8), memcpy(&flags, b + 0x20, 4); + int eos = (flags & BUFFER_FLAG_EOS) != 0; + static int64_t skip_ns = -2; // NX2NAVI_MOVIE_SKIP: -1 off + if (skip_ns == -2) { + const char *e = cfg_get("NX2NAVI_MOVIE_SKIP"); + skip_ns = e && *e ? (int64_t)(atof(e) * 1e9) : -1; + if (skip_ns >= 0) rt_log("movie: movies end after %.1f s (NX2NAVI_MOVIE_SKIP)", skip_ns / 1e9); + } + plat_mutex_lock(&v->lock); + if (skip_ns >= 0 && v->eos_queued) { // after the early end: what the demuxer still sends is dropped + plat_mutex_unlock(&v->lock); + ret_x(c, 0); + return; + } + if (skip_ns >= 0 && !eos && ts >= skip_ns) eos = 1, size = 0; // the early end + if (v->in_n >= INMAX || ((v->qn >= QREADY || v->in_n >= QREADY) && !eos)) { + update_events(v); + plat_mutex_unlock(&v->lock); + ret_x(c, R_BUSY); + return; + } + Sample s = {NULL, 0, ts, eos}; + if (data && size) { + s.data = malloc((size_t)size); + if (!s.data) rt_fatal("movie: out of memory for a %llu-byte sample", (unsigned long long)size); + memcpy(s.data, data, (size_t)size); + s.size = (size_t)size; + } + v->in[(v->ih + v->in_n) % INMAX] = s; + v->in_n++; + if (eos) v->eos_queued = 1; + update_events(v); + plat_cond_signal(&v->work); + plat_mutex_unlock(&v->lock); + ret_x(c, 0); +} +// Result AcquireVideoFrame(VideoFrame **out): the oldest decoded picture in a free registered frame +static void ov_acquire(Cpu *c) { + Video *v = video_of(arg_p(c, 0)); + uint64_t *out = arg_p(c, 1); + if (!v || !out) { + ret_x(c, R_NO_FRAME); + return; + } + plat_mutex_lock(&v->lock); + int f = 0; + while (f < v->nframe && v->busy[f]) f++; + if (!v->qn || f == v->nframe) { + plat_mutex_unlock(&v->lock); + ret_x(c, R_NO_FRAME); + return; + } + Picture p = v->q[v->qh]; + v->qh = (v->qh + 1) % QMAX; + v->qn--; + uint8_t *fr = (uint8_t *)(uintptr_t)v->frame[f]; + uint8_t *buf; + uint64_t cap; + memcpy(&buf, fr + 0x08, 8), memcpy(&cap, fr + 0x48, 8); + uint64_t ysz = (uint64_t)p.w * p.h, uvsz = ysz / 2; + if (p.nv12 && buf && cap >= ysz + uvsz) { + memcpy(buf, p.nv12, (size_t)(ysz + uvsz)); + uint64_t uvp = (uint64_t)(uintptr_t)(buf + ysz); + memcpy(fr + 0x10, &uvp, 8); + memcpy(fr + 0x28, &ysz, 8), memcpy(fr + 0x30, &uvsz, 8); + } else { + uint64_t zero = 0; + memcpy(fr + 0x28, &zero, 8), memcpy(fr + 0x30, &zero, 8); + if (p.nv12) { + static volatile int32_t logged; + if (!plat_xchg32(&logged, 1)) + rt_log("movie: frame buffer %#llx bytes is smaller than a %dx%d NV12 frame", (unsigned long long)cap, + p.w, p.h); + } + } + memcpy(fr + 0x78, &p.ts, 8); + fr[0x88] = (uint8_t)p.eos; + v->busy[f] = 1; + v->shown += !p.eos; + if (p.nv12 && v->shown == 1) rt_log("movie: first frame %dx%d shown", p.w, p.h); + if (p.nv12) { // pacing report every 600 frames (10 s): the gaps between acquisitions and their time stamps + static uint64_t last_ns, sum_gap, max_gap, n; + uint64_t now = plat_now_ns(), gap = last_ns ? now - last_ns : 0; + sum_gap += gap, n++; + if (gap > max_gap) max_gap = gap; + if (n % 600 == 0) { + rt_log("movie: pacing over 600 frames: acquire gap avg %.2f ms, max %.2f ms; %d pictures waiting", + sum_gap / 600 / 1e6, max_gap / 1e6, v->qn); + sum_gap = max_gap = 0; + } + last_ns = now; + } + update_events(v); + plat_mutex_unlock(&v->lock); + if (p.rgba) gfx_movie_frame_own(p.rgba, p.w, p.h), p.rgba = NULL; // the overlay takes the buffer + if (p.eos) gfx_movie_stop(); + picture_free(&p); + *out = v->frame[f]; + ret_x(c, 0); +} +// Result ReleaseVideoFrame(VideoFrame *) +static void ov_release(Cpu *c) { + Video *v = video_of(arg_p(c, 0)); + uint64_t fr = arg_x(c, 1); + if (v) { + plat_mutex_lock(&v->lock); + for (int i = 0; i < v->nframe; i++) + if (v->frame[i] == fr) v->busy[i] = 0; + update_events(v); + plat_mutex_unlock(&v->lock); + } + ret_x(c, 0); +} + +// Skipping movies (NX2NAVI_MOVIES=off) leaves the subsdk0 decoder in place: the movie never opens, so it is not called. +#define S(n) "_ZN5movie12VideoDecoder" n +HOST_OVERRIDE(S("C1Ev"), ov_ctor); +HOST_OVERRIDE(S("D1Ev"), ov_dtor); +HOST_OVERRIDE(S("10InitializeEPvmN2nn5movie13MediaMimeTypeENS_24VideoDecoderOutputFormatENS_14VideoCodecModeE"), + ov_initialize); +HOST_OVERRIDE(S("9ConfigureEPKN2nn5movie13MediaPropertyEPKvm"), ov_ok); +HOST_OVERRIDE(S("8FinalizeEv"), ov_finalize); +HOST_OVERRIDE(S("18RegisterVideoFrameEPPNS_10VideoFrameEi"), ov_register); +HOST_OVERRIDE(S("18GetInputReadyEventEPPN2nn2os15SystemEventTypeE"), ov_input_event); +HOST_OVERRIDE(S("19GetOutputReadyEventEPPN2nn2os15SystemEventTypeE"), ov_output_event); +HOST_OVERRIDE(S("5StartEv"), ov_ok); +HOST_OVERRIDE(S("4StopEv"), ov_ok); +HOST_OVERRIDE(S("5FlushEv"), ov_flush); +HOST_OVERRIDE(S("11DecodeAsyncEPKN2nn5movie6BufferE"), ov_decode); +HOST_OVERRIDE(S("17AcquireVideoFrameEPPNS_10VideoFrameE"), ov_acquire); +HOST_OVERRIDE(S("17ReleaseVideoFrameEPNS_10VideoFrameE"), ov_release); +#undef S diff --git a/runtime/ov_system.c b/runtime/ov_system.c index a0cd04ea..4c0c72a8 100644 --- a/runtime/ov_system.c +++ b/runtime/ov_system.c @@ -119,7 +119,29 @@ static void ov_desired_language(Cpu *c) { ret_x(c, v); } -static void ov_oe_initialize(Cpu *c) { (void)c; rt_log("oe: application environment: a local host process"); } +// nn::oe::Initialize opens the appletOE service (served by ipc.c now that domains are supported) and builds the +// module's own service object, which the oe wrappers the game calls internally dereference. It must run: observe +// it and call the guest definition instead of replacing it. +static void ov_oe_initialize(Cpu *c) { + rt_original(c); + rt_log("oe: application environment: a local host process"); +} +HOST_OBSERVER("_ZN2nn2oe10InitializeEv", ov_oe_initialize); + +// void SetGraphicsScaleChangedNotificationEnabled(bool): a thin wrapper over the appletOE service object, which +// the overridden Initialize never builds; the notification is a no-op on a desktop window. +static void ov_oe_graphics_scale(Cpu *c) { (void)c; } +HOST_OVERRIDE("_ZN2nn2oe42SetGraphicsScaleChangedNotificationEnabledEb", ov_oe_graphics_scale); + +// GraphicsScalingParameters GetGraphicsScalingParameters(): 12 bytes returned in x0 (words 0-1) and w1 (word 2). +// Word 0 is a scale *mode*, not a float: the SDK's own caller (sdk+0x59a690) maps 10 -> 0, 20 -> 1, 30 -> 2 and +// aborts on anything else (an earlier {1.0f, 1.0f} here hit that abort). 10 is the console's default (the system +// IGraphicsScaleController::GetCurrentGraphicsScale); the other two words are zero. +static void ov_oe_scaling_parameters(Cpu *c) { + ret_x(c, 10); + c->x[1] = 0; +} +HOST_OVERRIDE("_ZN2nn2oe28GetGraphicsScalingParametersEv", ov_oe_scaling_parameters); // void SetPerformanceConfiguration(PerformanceMode mode, int config) static void ov_performance_config(Cpu *c) { @@ -266,6 +288,24 @@ static void ov_report_save(Cpu *c) { // Result PlayReport::Save() and Save(con ret_x(c, 0); } static void ov_result_ok(Cpu *c) { ret_x(c, 0); } // RequestImmediateTransmission, AddInternetConnectionStatus +// Switch 2: Result PlayReport::Send(const account::Uid &) replaces Save (kept local like it); +// Result AsyncContext::GetCompletionEvent(SystemEvent *out): out becomes a signalled event; AsyncContext::GetResult(); +// SystemEvent *GetOverSaveFrequencyLimitEvent(): an event that never fires (no save limit here) +static void ov_report_event(Cpu *c) { + void *out = arg_p(c, 1); + if (out) { + kern_bind_system_event(c, (uint64_t)(uintptr_t)out, 0); + kern_signal_system_event((uint64_t)(uintptr_t)out); + } + ret_x(c, 0); +} +static void ov_report_limit_event(Cpu *c) { + static uint8_t ev[0x40]; // an nn::os::SystemEventType + static volatile int32_t bound; + if (!plat_xchg32(&bound, 1)) kern_bind_system_event(c, (uint64_t)(uintptr_t)ev, 0); + ret_x(c, (uint64_t)(uintptr_t)ev); +} +static void ov_nothing(Cpu *c) { (void)c; } // nn::nfp: NFC reader & amiibo support static volatile int32_t nfp_state; @@ -404,6 +444,17 @@ static void ov_aoc_list(Cpu *c) { if (offset < 0 || count < 0) rt_fatal("aoc: ListAddOnContent range %d+%d", offset, count); ret_x(c, 0); } +// ==== nn::nifm: this build is offline. Initialize succeeds without the nifm:u session, the network is never +// available, a request completes at once (unaccepted), and the calls that report its error return +// NetworkCommunicationDisabled (nifm module 110, description 1111: the system service/nifm/nifm.cpp). +#define NIFM_R_COMMUNICATION_DISABLED 0x8ae6eu +static void ov_nifm_initialize(Cpu *c) { rt_log("nifm: offline, no network"); ret_x(c, 0); } +static void ov_nifm_available(Cpu *c) { ret_x(c, 0); } +static void ov_nifm_request(Cpu *c) { (void)c; } +static void ov_nifm_error(Cpu *c) { ret_x(c, NIFM_R_COMMUNICATION_DISABLED); } + +// int CountAddOnContent(): the SDK's own reads the aoc service object the overridden Initialize never opens. +static void ov_aoc_count(Cpu *c) { ret_x(c, 0); } static void ov_err_context(Cpu *c) { (void)c; rt_log("err: error contexts stay local"); } static void ov_pctl_initialize(Cpu *c) { (void)c; rt_log("pctl: no parental controls"); } static void ov_abort_observer(Cpu *c) { (void)c; rt_log("diag: the host crash reporter observes aborts"); } @@ -693,6 +744,30 @@ static void ov_current_time(Cpu *c) { // returning the 0x18-byte struct through x8 would write through a stale pointer (time_host.c). static void ov_steady_now(Cpu *c) { ret_x(c, (uint64_t)steady_now().value); } +// Result GetStandardSteadyClockCurrentTimespec(timespec *) / GetStandardUserSystemClockCurrentTimespec(timespec *): +// the SDK libc's clock_gettime(CLOCK_MONOTONIC / CLOCK_REALTIME) once a title selects the nn::time clocks +// (sdk+0xaf0f8 mode 3; Switch 2 titles do, via std::chrono::steady_clock). The SDK's own versions need the time +// service state the overridden Initialize never builds, fail, and libc++ then throws std::system_error. +typedef struct { int64_t sec, nsec; } GuestTimespec; +static void ov_steady_timespec(Cpu *c) { + GuestTimespec *out = arg_p(c, 0); + if (!out) { ret_x(c, TIME_R_INVALID); return; } + uint64_t ns = plat_now_ns(); + out->sec = (int64_t)(ns / 1000000000u); + out->nsec = (int64_t)(ns % 1000000000u); + ret_x(c, 0); +} +static void ov_user_timespec(Cpu *c) { + GuestTimespec *out = arg_p(c, 0); + if (!out) { ret_x(c, TIME_R_INVALID); return; } + uint64_t ns = plat_unix_time_ns(); + out->sec = (int64_t)(ns / 1000000000u); + out->nsec = (int64_t)(ns % 1000000000u); + ret_x(c, 0); +} +HOST_OVERRIDE("_ZN2nn4time37GetStandardSteadyClockCurrentTimespecEP8timespec", ov_steady_timespec); +HOST_OVERRIDE("_ZN2nn4time41GetStandardUserSystemClockCurrentTimespecEP8timespec", ov_user_timespec); + // Result ToCalendarTime(CalendarTime *out, CalendarAdditionalInfo *info, const PosixTime &t): local time static void ov_to_calendar(Cpu *c) { CalendarTime *out = arg_p(c, 0); @@ -860,7 +935,7 @@ static void ov_font_size(Cpu *c) { } // ==== registration -HOST_OVERRIDE("_ZN2nn2oe10InitializeEv", ov_oe_initialize); +/* installed as HOST_OBSERVER above */ HOST_OVERRIDE("_ZN2nn2oe18GetDesiredLanguageEv", ov_desired_language); HOST_OVERRIDE("_ZN2nn2oe27SetPerformanceConfigurationENS0_15PerformanceModeEi", ov_performance_config); HOST_OVERRIDE("_ZN2nn2oe15SetCpuBoostModeENS0_12CpuBoostModeE", ov_cpu_boost); @@ -887,6 +962,11 @@ HOST_OVERRIDE("_ZN2nn2oe22PopNotificationMessageEv", ov_pop); HOST_OVERRIDE("_ZN2nn5prepo10PlayReport4SaveEv", ov_report_save); HOST_OVERRIDE("_ZN2nn5prepo10PlayReport4SaveERKNS_7account3UidE", ov_report_save); HOST_OVERRIDE("_ZN2nn5prepo28RequestImmediateTransmissionEv", ov_result_ok); +HOST_OVERRIDE("_ZN2nn5prepo10PlayReport4SendERKNS_7account3UidE", ov_report_save); +HOST_OVERRIDE("_ZN2nn5prepo12AsyncContext18GetCompletionEventEPNS_2os11SystemEventE", ov_report_event); +HOST_OVERRIDE("_ZN2nn5prepo12AsyncContext9GetResultEv", ov_result_ok); +HOST_OVERRIDE("_ZN2nn5prepo12AsyncContextD1Ev", ov_nothing); +HOST_OVERRIDE("_ZN2nn5prepo30GetOverSaveFrequencyLimitEventEv", ov_report_limit_event); HOST_OVERRIDE("_ZN2nn5prepo27AddInternetConnectionStatusEPNS0_10PlayReportEPKcS4_S4_", ov_result_ok); HOST_OVERRIDE("_ZN2nn3nfp10InitializeEv", ov_nfp_initialize); HOST_OVERRIDE("_ZN2nn3nfp8FinalizeEv", ov_nfp_finalize); @@ -906,7 +986,14 @@ HOST_OVERRIDE("_ZN2nn3err9ShowErrorENS_6ResultE", ov_show_error); HOST_OVERRIDE("nndiagStartup", ov_diag_startup); HOST_OVERRIDE("_ZN2nn3aoc10InitializeEv", ov_aoc_initialize); +HOST_OVERRIDE("_ZN2nn4nifm10InitializeEv", ov_nifm_initialize); +HOST_OVERRIDE("_ZN2nn4nifm18IsNetworkAvailableEv", ov_nifm_available); +HOST_OVERRIDE("_ZN2nn4nifm27SubmitNetworkRequestAndWaitEv", ov_nifm_request); +HOST_OVERRIDE("_ZN2nn4nifm20CancelNetworkRequestEv", ov_nifm_request); +HOST_OVERRIDE("_ZN2nn4nifm31HandleNetworkRequestErrorResultEv", ov_nifm_error); +HOST_OVERRIDE("_ZN2nn4nifm22GetCurrentProxySettingEPNS0_12ProxySettingE", ov_nifm_error); HOST_OVERRIDE("_ZN2nn3aoc16ListAddOnContentEPiii", ov_aoc_list); +HOST_OVERRIDE("_ZN2nn3aoc17CountAddOnContentEv", ov_aoc_count); HOST_OVERRIDE("_ZN2nn3err26InitializeContextWriterApiEv", ov_err_context); HOST_OVERRIDE("_ZN2nn4pctl10InitializeEv", ov_pctl_initialize); HOST_OVERRIDE("_ZN2nn4diag34InitializeApplicationAbortObserverEv", ov_abort_observer); diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index aa6c449c..14633350 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -966,6 +966,13 @@ void plat_mem_fault_hook(int (*fn)(uint64_t addr)) { fault_install(); } +void plat_crash_context(const char *s) { (void)s; } // the Windows crash notes only + +int plat_frame_capture(PlatJmp *out, int (*want)(uintptr_t rip, void *arg), void *arg) { // not on Linux yet + (void)out, (void)want, (void)arg; + return 0; +} + void plat_crash_hook(void (*fn)(const PlatCrash *c)) { crash_fn = fn; fault_install(); diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 02d59ff4..0d3cea87 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -78,6 +78,11 @@ typedef struct { jmp_buf b; } PlatJmp; #define plat_jmp_save(j) setjmp((j)->b) #define plat_jmp_go(j) longjmp((j)->b, 1) #endif +// The host state of a caller further up this thread's stack, as it will be when the call it is making returns: the +// stack is unwound frame by frame (from the caller of this function) until want(return address, arg) accepts one, and +// that frame's callee-saved registers, stack pointer and return address go to *out for plat_jmp_go, which then makes +// that call return a second time (eax 1). 1 on success, 0 when no frame is accepted (or on a platform without it). +int plat_frame_capture(PlatJmp *out, int (*want)(uintptr_t rip, void *arg), void *arg); // Scheduling priority, Windows' seven levels: of the calling thread, or of t. On Linux a thread lowered without the // right to raise its priority again (CAP_SYS_NICE, or an RLIMIT_NICE of 20) could never return to normal, and the // runtime lowers threads that later take shared locks at normal priority; without that right every change is skipped @@ -239,6 +244,9 @@ typedef struct { const char *module; // the module holding rip: NULL for this executable, "" for none (a jump to address 0) } PlatCrash; void plat_crash_hook(void (*fn)(const PlatCrash *c)); +// What this thread is doing, named by the crash notes (a stack overflow leaves no room for the full report); NULL +// clears it. s must stay valid until it is cleared. +void plat_crash_context(const char *s); int plat_mem_read(uint64_t addr, void *out, size_t n); // copies n bytes, 0 instead of faulting on unmapped memory int plat_mem_readable(uint64_t addr, uint64_t n); // every page of [addr, addr + n) can be read; no fault void *plat_aligned_alloc(size_t size, size_t align); // free with plat_aligned_free; NULL on failure diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index 21bc6211..c643ebcb 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -262,6 +262,34 @@ __asm__(".text\n" " movl $1, %eax\n" " jmpq *72(%rcx)\n"); +int plat_frame_capture(PlatJmp *out, int (*want)(uintptr_t rip, void *arg), void *arg) { + CONTEXT c; + RtlCaptureContext(&c); + ULONG_PTR low, high; + GetCurrentThreadStackLimits(&low, &high); + for (int i = 0; i < 256 && c.Rsp >= low && c.Rsp + 8 <= high; i++) { + DWORD64 img; + RUNTIME_FUNCTION *fe = RtlLookupFunctionEntry(c.Rip, &img, NULL); + if (fe) { + void *data; + DWORD64 frame; + RtlVirtualUnwind(UNW_FLAG_NHANDLER, img, c.Rip, fe, &c, &data, &frame, NULL); + } else { // a leaf: the return address is on top of the stack + c.Rip = *(const DWORD64 *)c.Rsp; + c.Rsp += 8; + } + if (!c.Rip) return 0; + if (!want((uintptr_t)c.Rip, arg)) continue; + out->gpr[0] = c.Rbx, out->gpr[1] = c.Rbp, out->gpr[2] = c.Rdi, out->gpr[3] = c.Rsi; + out->gpr[4] = c.R12, out->gpr[5] = c.R13, out->gpr[6] = c.R14, out->gpr[7] = c.R15; + out->gpr[8] = c.Rsp, out->gpr[9] = c.Rip; + __asm__ volatile("stmxcsr %0\n\tfnstcw %1" : "=m"(out->mxcsr), "=m"(out->fpcw)); + memcpy(out->xmm, &c.Xmm6, 160); // xmm6-xmm15, consecutive in CONTEXT + return 1; + } + return 0; +} + int plat_thread_spawn(PlatThreadFn fn, void *arg, const char *name, size_t stack) { PlatThread *t = plat_thread_new_ex(fn, arg, name, stack); if (t) plat_thread_detach(t); @@ -626,8 +654,33 @@ void plat_mem_fault_hook(int (*fn)(uint64_t addr)) { static void (*crash_fn)(const PlatCrash *c); +// A line written straight to the standard error handle, for the crash path: no C runtime buffering or locks. +static const char *image_name(const char *base); +static __thread const char *crash_ctx; +void plat_crash_context(const char *s) { crash_ctx = s; } +static void raw_note(const char *tag, DWORD code, uintptr_t rip, uintptr_t addr) { + char b[256]; + void *base = NULL; + RtlPcToFileHeader((void *)rip, &base); + ULONG_PTR lo = 0, hi = 0; + GetCurrentThreadStackLimits(&lo, &hi); + int n = snprintf(b, sizeof b, "[host] %s: exception %#lx at %p (%s+%#llx), address %p, thread %lu, stack %llu KiB reserved\n", tag, + (unsigned long)code, (void *)rip, base ? image_name(base) : "no module", + (unsigned long long)(rip - (uintptr_t)base), (void *)addr, (unsigned long)GetCurrentThreadId(), + (unsigned long long)(hi - lo) >> 10); + DWORD w; + if (n > 0) WriteFile(GetStdHandle(STD_ERROR_HANDLE), b, (DWORD)n, &w, NULL); + if (crash_ctx) { + n = snprintf(b, sizeof b, "[host] %s: the thread was %s\n", tag, crash_ctx); + if (n > 0) WriteFile(GetStdHandle(STD_ERROR_HANDLE), b, (DWORD)n, &w, NULL); + } +} + static LONG CALLBACK crash_veh(EXCEPTION_POINTERS *ep) { DWORD code = ep->ExceptionRecord->ExceptionCode; + if (code == EXCEPTION_ACCESS_VIOLATION || code == EXCEPTION_STACK_OVERFLOW || code == EXCEPTION_ILLEGAL_INSTRUCTION) + raw_note("crash_veh", code, (uintptr_t)ep->ContextRecord->Rip, + ep->ExceptionRecord->NumberParameters >= 2 ? (uintptr_t)ep->ExceptionRecord->ExceptionInformation[1] : 0); const char *what = code == EXCEPTION_ACCESS_VIOLATION ? "access violation" : code == EXCEPTION_ILLEGAL_INSTRUCTION ? "illegal instruction" : code == EXCEPTION_INT_DIVIDE_BY_ZERO ? "integer division by zero" @@ -643,6 +696,13 @@ static LONG CALLBACK crash_veh(EXCEPTION_POINTERS *ep) { PlatCrash c = {what, (uint32_t)code, rip, rip - (uintptr_t)mi.lpBaseOfDll, -1, 0}; if (code == EXCEPTION_ACCESS_VIOLATION && ep->ExceptionRecord->NumberParameters >= 2) c.access = (int)ep->ExceptionRecord->ExceptionInformation[0], c.addr = ep->ExceptionRecord->ExceptionInformation[1]; + { // the faulting host stack, so the report can name the runtime frame that took it + void *f[32]; + USHORT n = RtlCaptureStackBackTrace(0, 32, f, NULL); + for (USHORT i = 0; i < n && c.nframes < (int)(sizeof c.frames / sizeof *c.frames); i++) + if ((uintptr_t)f[i] - (uintptr_t)mi.lpBaseOfDll < mi.SizeOfImage) + c.frames[c.nframes++] = (uintptr_t)f[i]; + } crash_fn(&c); return EXCEPTION_CONTINUE_SEARCH; } @@ -708,6 +768,7 @@ static LONG WINAPI crash_filter(EXCEPTION_POINTERS *ep) { const EXCEPTION_RECORD *er = ep->ExceptionRecord; DWORD code = er->ExceptionCode; uintptr_t rip = (uintptr_t)ep->ContextRecord->Rip; + raw_note("crash_filter", code, rip, er->NumberParameters >= 2 ? (uintptr_t)er->ExceptionInformation[1] : 0); PlatCrash c = {code == EXCEPTION_ACCESS_VIOLATION ? "access violation" : code == EXCEPTION_ILLEGAL_INSTRUCTION ? "illegal instruction" : code == EXCEPTION_INT_DIVIDE_BY_ZERO ? "integer division by zero" @@ -754,6 +815,7 @@ void plat_crash_hook(void (*fn)(const PlatCrash *c)) { // A fresh process gets a fresh layout: Windows places DLLs, heaps and thread stacks at random addresses. The parent // waits and exits with the child's code. void plat_reexec(void) { + fprintf(stderr, "[host] memory: a fixed region is taken; re-running the process\n"); fflush(stdout); fflush(stderr); STARTUPINFOW si = {.cb = sizeof si}; diff --git a/runtime/romfs.c b/runtime/romfs.c index b8980293..b8f43d7e 100644 --- a/runtime/romfs.c +++ b/runtime/romfs.c @@ -64,7 +64,23 @@ static PlatFile *image; static const uint8_t *image_map; static char image_file[2048]; static char index_dir[2048]; // the directory of romfs-index.bin: the proof file's default place -static char mount_name[32]; +// The names the RomFS is mounted under: one for NX2NAVI; Switch 2 titles mount it again under other names +// (nn::fs::MountRom(const char *)), each an alias of the same image. +#define NROM_NAMES 4 +static char mount_names[NROM_NAMES][32]; + +// The path relative to the RomFS root when path is ":/...", else NULL. +// Repeated '/' inside the path read as one (romfs_path_collapse); the result lives in a per-thread buffer until the +// thread's next call. +static const char *rom_relative(const char *path) { + static __thread char folded[2048]; + if (path && strstr(path, "//") && romfs_path_collapse(path, folded, sizeof folded)) path = folded; + for (int i = 0; i < NROM_NAMES && mount_names[i][0]; i++) { + const char *p = romfs_path_relative(mount_names[i], path); + if (p) return p; + } + return NULL; +} static PlatMutex lock = PLAT_MUTEX_INIT; static uint64_t handles[NH]; // file index + 1; 0 is free static uint32_t handle_ovl[NH]; // overlay slot + 1 of an open handle, 0 = original @@ -1023,30 +1039,54 @@ const char *romfs_overlay(size_t k, const char **file, uint64_t *size, int *is_a } int romfs_owns(const char *path) { - size_t n = strlen(mount_name); - return n && path && !strncmp(path, mount_name, n) && path[n] == ':'; + if (!path) return 0; + for (int i = 0; i < NROM_NAMES && mount_names[i][0]; i++) { + size_t n = strlen(mount_names[i]); + if (!strncmp(path, mount_names[i], n) && path[n] == ':') return 1; + } + return 0; } int romfs_handle(uint64_t h) { return h > ROMFS_HANDLE_BASE && h < ROMFS_HANDLE_BASE + NH; } uint32_t romfs_mount(const char *name) { fs_init(); - if (!name || !romfs_path_valid_relative(name) || strlen(name) >= sizeof mount_name || strchr(name, '/') || + if (!name || !romfs_path_valid_relative(name) || strlen(name) >= sizeof mount_names[0] || strchr(name, '/') || strchr(name, ':')) return FS_R_INVALID; plat_mutex_lock(&lock); - if (*mount_name && strcmp(name, mount_name)) rt_fatal("second RomFS mount %s (%s is mounted)", name, mount_name); - strcpy(mount_name, name); + int i = 0; + while (i < NROM_NAMES && mount_names[i][0] && strcmp(mount_names[i], name)) i++; + if (i == NROM_NAMES) rt_fatal("RomFS mounted under more than %d names (%s)", NROM_NAMES, name); + strcpy(mount_names[i], name); plat_mutex_unlock(&lock); rt_log("fs mount %s: -> RomFS", name); return 0; } +// NX2NAVI_MOVIES=off skips movies: the RomFS reports .webm files as absent, and the game's movie player +// (main+0x3b5eb40), which opens the file with nn::fs::OpenFile before it creates a decoder, abandons the movie +// cleanly. Movies play otherwise (ov_movie2.c decodes them). +static int movie_skipped(const char *p) { + size_t n = strlen(p); + if (n < 5 || strcmp(p + n - 5, ".webm")) return 0; + const char *v = cfg_get("NX2NAVI_MOVIES"); + if (!v || strcmp(v, "off")) return 0; // movies play (ov_movie2.c) unless NX2NAVI_MOVIES=off + rt_log("fs: movie %s skipped (NX2NAVI_MOVIES=off)", p); + return 1; +} + uint32_t romfs_open(uint64_t *out, const char *path, int mode) { fs_trace("rom", path); if (!out || mode != FS_OPEN_READ) return FS_R_UNSUPPORTED_OPEN; - const char *p = romfs_path_relative(mount_name, path); + const char *p = rom_relative(path); if (!p) return FS_R_PATH_NOT_FOUND; + if (movie_skipped(p)) return FS_R_PATH_NOT_FOUND; + // NX2NAVI_NPLN=off: the online client's configuration is absent (diagnosis of the crashes that follow its start) + if (!strcmp(p, "npln_config.json") && cfg_get("NX2NAVI_NPLN") && !strcmp(cfg_get("NX2NAVI_NPLN"), "off")) { + rt_log("fs: %s hidden (NX2NAVI_NPLN=off)", p); + return FS_R_PATH_NOT_FOUND; + } uint64_t i = file_index(p); if (i == UINT64_MAX) return FS_R_PATH_NOT_FOUND; uint32_t ov = overlay_slot(i); @@ -1142,7 +1182,7 @@ void romfs_close(uint64_t h) { uint32_t romfs_entry_type(int *out, const char *path) { if (!out) return FS_R_INVALID; - const char *p = romfs_path_relative(mount_name, path); + const char *p = rom_relative(path); if (!p) return FS_R_PATH_NOT_FOUND; if (file_index(p) != UINT64_MAX) { *out = FS_ENTRY_FILE; @@ -1157,9 +1197,40 @@ uint32_t romfs_entry_type(int *out, const char *path) { return FS_R_PATH_NOT_FOUND; } +// Directory listing over the sorted index: every path under "

/" is contiguous, and so is every path under one +// subdirectory, so one pass emits each file and each distinct subdirectory once. Files the RomFS-mods manifest adds +// (added[]) are not listed yet. +uint32_t romfs_list(const char *path, int (*fn)(const char *name, int is_dir, uint64_t size, void *ctx), void *ctx) { + const char *p = rom_relative(path); + if (!p) return FS_R_PATH_NOT_FOUND; + char dir[4096]; + if (snprintf(dir, sizeof dir, *p ? "%s/" : "%s", p) >= (int)sizeof dir) return FS_R_INVALID; + size_t n = strlen(dir); + if (n && !folder_exists(dir)) return FS_R_PATH_NOT_FOUND; + char last[4096] = ""; + for (uint64_t i = lower(dir); i < file_count; i++) { + Entry e = entry(i); + const char *name = entry_name(e); + if (strncmp(name, dir, n)) break; + const char *rest = name + n, *slash = strchr(rest, '/'); + if (!slash) { + if (!fn(rest, 0, e.size, ctx)) break; + continue; + } + size_t k = (size_t)(slash - rest); + if (k >= sizeof last) continue; + if (!strncmp(last, rest, k) && last[k] == 0) continue; + memcpy(last, rest, k); + last[k] = 0; + if (!fn(last, 1, 0, ctx)) break; + } + return 0; +} + // ---- overrides // nn::Result nn::fs::MountRom(const char *name, void *cache, size_t cache_size) static void ov_mount_rom(Cpu *c) { ret_x(c, romfs_mount(arg_p(c, 0))); } +// nn::Result nn::fs::MountRom(const char *name): the Switch 2 SDK's form, without a cache // nn::Result nn::fs::QueryMountRomCacheSize(size_t *out): no cache needed, the host reads the image directly static void ov_query_cache(Cpu *c) { uint64_t *out = arg_p(c, 0); @@ -1173,5 +1244,6 @@ static void ov_initialize(Cpu *c) { } HOST_OVERRIDE("_ZN2nn2fs8MountRomEPKcPvm", ov_mount_rom); +HOST_OVERRIDE("_ZN2nn2fs8MountRomEPKc", ov_mount_rom); HOST_OVERRIDE("_ZN2nn2fs22QueryMountRomCacheSizeEPm", ov_query_cache); HOST_OVERRIDE("_ZN2nn2fs24InitializeForApplicationEv", ov_initialize); diff --git a/runtime/romfs_path.h b/runtime/romfs_path.h index a6a7c789..5f3edb0d 100644 --- a/runtime/romfs_path.h +++ b/runtime/romfs_path.h @@ -18,6 +18,22 @@ static inline int romfs_path_valid_relative(const char *path) { return 1; } +// Copies path into out (cap bytes) with each run of '/' after the first ':' folded into one, as a file system +// reads "a//b". Switch 2 voice requests join "Voice/Resource/" and "/Stream/..." (Ocarina's +// "content://Voice/Resource//Stream/USen_...bwav"). 0 when it does not fit. +static inline int romfs_path_collapse(const char *path, char *out, size_t cap) { + const char *colon = path ? strchr(path, ':') : NULL; + size_t n = 0; + for (const char *p = path; p && *p; p++) { + if (colon && p > colon + 1 && *p == '/' && p[-1] == '/') continue; // keep ":/" and one more + if (n + 1 >= cap) return 0; + out[n++] = *p; + } + if (!cap) return 0; + out[n] = 0; + return 1; +} + // ":/" -> "", or NULL. Audio stream requests join a mount root and a leading-slash resource, // producing "content://Sound/...": repeated root separators are skipped; traversal, backslashes and empty interior // components are still rejected. diff --git a/runtime/shaders/fallback_tonemap.spvasm b/runtime/shaders/fallback_tonemap.spvasm new file mode 100644 index 00000000..85153635 --- /dev/null +++ b/runtime/shaders/fallback_tonemap.spvasm @@ -0,0 +1,115 @@ +; Tonemap and scale for the untranslatable upscaler (runtime/vk_exec_draw.c untranslated_dispatch_fallback). +; Written in SPIR-V assembly (spirv-as; no GLSL compiler in the build). As GLSL: +; layout(local_size_x = 8, local_size_y = 8) in; +; layout(binding = 0) uniform sampler2D src; // the HDR scene (linear filtering) +; layout(binding = 1, rgba16f) uniform writeonly image2D dst; // the pass output (vk_tonemap.c), blitted into the target +; layout(push_constant) uniform P { vec2 inv; float exposure; float pad; ivec2 size; } pc; +; void main() { +; ivec2 p = ivec2(gl_GlobalInvocationID.xy); +; if (p.x >= pc.size.x || p.y >= pc.size.y) return; +; vec3 c = textureLod(src, (vec2(p) + 0.5) * pc.inv, 0.0).rgb * pc.exposure; +; c = clamp(c * (2.51 * c + 0.03) / (c * (2.43 * c + 0.59) + 0.14), 0.0, 1.0); // Narkowicz's ACES fit +; imageStore(dst, p, vec4(c, 1.0)); +; } + OpCapability Shader + %1 = OpExtInstImport "GLSL.std.450" + OpMemoryModel Logical GLSL450 + OpEntryPoint GLCompute %main "main" %gid %src %dst %pc + OpExecutionMode %main LocalSize 8 8 1 + OpDecorate %gid BuiltIn GlobalInvocationId + OpDecorate %src DescriptorSet 0 + OpDecorate %src Binding 0 + OpDecorate %dst DescriptorSet 0 + OpDecorate %dst Binding 1 + OpDecorate %dst NonReadable + OpDecorate %P Block + OpMemberDecorate %P 0 Offset 0 + OpMemberDecorate %P 1 Offset 8 + OpMemberDecorate %P 2 Offset 12 + OpMemberDecorate %P 3 Offset 16 + %void = OpTypeVoid + %fnvoid = OpTypeFunction %void + %float = OpTypeFloat 32 + %int = OpTypeInt 32 1 + %uint = OpTypeInt 32 0 + %bool = OpTypeBool + %v2float = OpTypeVector %float 2 + %v3float = OpTypeVector %float 3 + %v4float = OpTypeVector %float 4 + %v2int = OpTypeVector %int 2 + %v3uint = OpTypeVector %uint 3 + %v2uint = OpTypeVector %uint 2 + %ptr_in_v3uint = OpTypePointer Input %v3uint + %gid = OpVariable %ptr_in_v3uint Input + %img2d = OpTypeImage %float 2D 0 0 0 1 Unknown + %simg2d = OpTypeSampledImage %img2d + %ptr_uc_simg = OpTypePointer UniformConstant %simg2d + %src = OpVariable %ptr_uc_simg UniformConstant + %stimg = OpTypeImage %float 2D 0 0 0 2 Rgba16f + %ptr_uc_stimg = OpTypePointer UniformConstant %stimg + %dst = OpVariable %ptr_uc_stimg UniformConstant + %P = OpTypeStruct %v2float %float %float %v2int + %ptr_pc_P = OpTypePointer PushConstant %P + %pc = OpVariable %ptr_pc_P PushConstant + %ptr_pc_v2float = OpTypePointer PushConstant %v2float + %ptr_pc_float = OpTypePointer PushConstant %float + %ptr_pc_v2int = OpTypePointer PushConstant %v2int + %int_0 = OpConstant %int 0 + %int_1 = OpConstant %int 1 + %int_3 = OpConstant %int 3 + %float_0 = OpConstant %float 0 + %float_1 = OpConstant %float 1 + %float_half = OpConstant %float 0.5 + %float_2_51 = OpConstant %float 2.51 + %float_0_03 = OpConstant %float 0.03 + %float_2_43 = OpConstant %float 2.43 + %float_0_59 = OpConstant %float 0.59 + %float_0_14 = OpConstant %float 0.14 + %v2half = OpConstantComposite %v2float %float_half %float_half + %v3_0 = OpConstantComposite %v3float %float_0 %float_0 %float_0 + %v3_1 = OpConstantComposite %v3float %float_1 %float_1 %float_1 + %v3_2_51 = OpConstantComposite %v3float %float_2_51 %float_2_51 %float_2_51 + %v3_0_03 = OpConstantComposite %v3float %float_0_03 %float_0_03 %float_0_03 + %v3_2_43 = OpConstantComposite %v3float %float_2_43 %float_2_43 %float_2_43 + %v3_0_59 = OpConstantComposite %v3float %float_0_59 %float_0_59 %float_0_59 + %v3_0_14 = OpConstantComposite %v3float %float_0_14 %float_0_14 %float_0_14 + %v2bool = OpTypeVector %bool 2 + %main = OpFunction %void None %fnvoid + %entry = OpLabel + %g3 = OpLoad %v3uint %gid + %gu = OpVectorShuffle %v2uint %g3 %g3 0 1 + %p = OpBitcast %v2int %gu + %psize = OpAccessChain %ptr_pc_v2int %pc %int_3 + %size = OpLoad %v2int %psize + %ge = OpSGreaterThanEqual %v2bool %p %size + %out = OpAny %bool %ge + OpSelectionMerge %body None + OpBranchConditional %out %done %body + %done = OpLabel + OpReturn + %body = OpLabel + %pf = OpConvertSToF %v2float %p + %pfh = OpFAdd %v2float %pf %v2half + %pinv = OpAccessChain %ptr_pc_v2float %pc %int_0 + %inv = OpLoad %v2float %pinv + %uv = OpFMul %v2float %pfh %inv + %si = OpLoad %simg2d %src + %s4 = OpImageSampleExplicitLod %v4float %si %uv Lod %float_0 + %s3 = OpVectorShuffle %v3float %s4 %s4 0 1 2 + %pexp = OpAccessChain %ptr_pc_float %pc %int_1 + %exp = OpLoad %float %pexp + %c = OpVectorTimesScalar %v3float %s3 %exp + %a1 = OpFMul %v3float %c %v3_2_51 + %a2 = OpFAdd %v3float %a1 %v3_0_03 + %num = OpFMul %v3float %c %a2 + %b1 = OpFMul %v3float %c %v3_2_43 + %b2 = OpFAdd %v3float %b1 %v3_0_59 + %b3 = OpFMul %v3float %c %b2 + %den = OpFAdd %v3float %b3 %v3_0_14 + %t = OpFDiv %v3float %num %den + %tc = OpExtInst %v3float %1 FClamp %t %v3_0 %v3_1 + %o4 = OpCompositeConstruct %v4float %tc %float_1 + %di = OpLoad %stimg %dst + OpImageWrite %di %p %o4 + OpReturn + OpFunctionEnd diff --git a/runtime/svc.c b/runtime/svc.c index a62976a3..3940bd62 100644 --- a/runtime/svc.c +++ b/runtime/svc.c @@ -111,7 +111,7 @@ static void get_info(Cpu *c, uint64_t pc) { return; } switch (id) { - case 0: v = 7; break; // core mask + case 0: v = 0x3f; break; // core mask: cores 0-5 (Switch 2 application cores; see kernel.c NCORE) case 1: v = 0x0ffffffff0000000ull; break; // priority mask case 8: v = 0; break; // debugger attached case 10: v = 0; break; // idle tick count @@ -121,6 +121,9 @@ static void get_info(Cpu *c, uint64_t pc) { case 18: v = PROGRAM_ID; break; case 20: v = user_exception_context(); break; // the SDK keeps e.g. the dying message at +0x1C0 case 23: v = 1; break; + case 29: // region the SDK records at nnosInitialize (address, then + case 30: v = 0; break; + case 33: v = 0x1000; break; // VAMM control region size: mapped at 0x300000000 and made into a unit heap default: { char why[64]; snprintf(why, sizeof why, "GetInfo id %u (handle %#x, sub id %#llx)", id, (uint32_t)c->x[2], @@ -197,6 +200,33 @@ static void svc_body(Cpu *c, uint32_t imm, uint64_t pc) { x[0] = R_OK; return; } + { // the SDK keeps the abort text in the user exception context: show any readable run + const char *ctx = (const char *)(uintptr_t)user_exception_context(); + char run[161]; + int n = 0; + for (int i = 0; i < 0x400; i++) { + unsigned char ch = (unsigned char)ctx[i]; + if (ch >= 0x20 && ch < 0x7f) { + if (n < 160) run[n++] = (char)ch; + } else if (n >= 4) { + run[n] = 0; + rt_log("guest abort text at ctx+%#x: %s", i - n, run); + n = 0; + } else { + n = 0; + } + } + } + { // the object the aborting code was working on: its first bytes carry the state machine's byte + uint64_t obj = c->x[19]; + if (obj && readable(obj, 32)) { + const unsigned char *p = (const unsigned char *)(uintptr_t)obj; + rt_log("guest abort object %#llx: %02x %02x %02x %02x %02x %02x %02x %02x " + "%02x %02x %02x %02x %02x %02x %02x %02x", (unsigned long long)obj, + p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], + p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); + } + } svc_fatal(c, imm, pc, "Break (the SDK aborted)"); } case 0x27: { // OutputDebugString(x0 text, x1 size) diff --git a/runtime/tex_deswizzle.c b/runtime/tex_deswizzle.c index 66945a59..3ba700d9 100644 --- a/runtime/tex_deswizzle.c +++ b/runtime/tex_deswizzle.c @@ -8,10 +8,24 @@ // in x -> y -> z row-major order. // Level and layer start offsets and the block height and depth of a level come from nvn_tex.c; this file only does the // addressing inside a level and the copy. +// +// Switch 2 (NVN2 titles, NVN API major version 200 and up) stores the 16-byte pieces of a GOB in another order: +// (x%64/32)*256 + (y%8/4)*128 + (x%32/16)*64 + (y%4)*16 + x%16 +// (four rows of one 16-byte column before the next column). Found on Ocarina of Time's packaged textures: with it the +// ASTC 4x4 title logo and BC4 masks decode without seams (an edge-continuity search over all 120 orders of the five +// address bits picks it for both; OCARINA-BRINGUP.md round 18). Block and level arithmetic are unchanged. +#include "host.h" #include "tex.h" #include +static int gob_nvn2(void) { + static int v = -1; + if (v < 0) v = rt_nvn_api_version[0] >= 200; + return v; +} + static uint32_t gob_off(uint32_t x, uint32_t y) { + if (gob_nvn2()) return (x % 64 / 32) * 256 + (y % 8 / 4) * 128 + (x % 32 / 16) * 64 + (y % 4) * 16 + x % 16; return (x % 64 / 32) * 256 + (y % 8 / 2) * 64 + (x % 32 / 16) * 32 + (y % 2) * 16 + x % 16; } @@ -47,10 +61,11 @@ int nvn_deswizzle_geom(const nvn_texdesc *d, unsigned level, unsigned layer, } // nvn_bl_offset per row instead of per 16 bytes: within a row, GOB column k starts k blocks further, and the four -// 16-byte segments of a GOB row (x % 64 = 0, 16, 32, 48) sit at 0, 32, 256 and 288 of it. +// 16-byte segments of a GOB row (x % 64 = 0, 16, 32, 48) sit at 0, 32, 256 and 288 of it (Switch 2: 0, 64, 256, 320). void tex_bl_copy(uint8_t *bl, uint8_t *lin, size_t pitch, uint32_t row_bytes, uint32_t rows, uint32_t slices, unsigned th, unsigned td, int to_linear) { - static const uint32_t seg[4] = {0, 32, 256, 288}; + static const uint32_t seg_x1[4] = {0, 32, 256, 288}, seg_nvn2[4] = {0, 64, 256, 320}; + const uint32_t *seg = gob_nvn2() ? seg_nvn2 : seg_x1; uint64_t block = 512ull << th << td; for (uint32_t z = 0; z < slices; z++) for (uint32_t y = 0; y < rows; y++) { diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 4553fa8f..a56970e0 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -20,7 +20,11 @@ static uint8_t rnd8(void) { } // Byte offset inside a 64 x 8 GOB from the bit layout: x[3:0] -> 0-3, y[0] -> 4, x[4] -> 5, y[2:1] -> 6-7, x[5] -> 8. +// Bit by bit, independent of tex_deswizzle.c's arithmetic: Tegra X1, or Switch 2 for NVN2 titles (the order +// tex_deswizzle.c describes). static unsigned gob_bits(unsigned x, unsigned y) { + if (rt_nvn_api_version[0] >= 200) + return (x & 15) | (y & 3) << 4 | (x >> 4 & 1) << 6 | (y >> 2 & 1) << 7 | (x >> 5 & 1) << 8; return (x & 15) | (y & 1) << 4 | (x >> 4 & 1) << 5 | (y >> 1 & 3) << 6 | (x >> 5 & 1) << 8; } diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index de90fd9d..e49cc03e 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -197,27 +197,50 @@ static float menu_scale(uint32_t h) { return ((float)h >= least ? (float)h : least) / 1080.0f; } -// SavageWindRecomp's colours: dark green panels, gold accents. +// Ocarina of Time colours: a night-sky navy panel, Kokiri green for what is selected, Triforce gold for accents and +// the cursor, the Ocarina's blue for the active control. +static const ImVec4 kGold(0.95f, 0.78f, 0.30f, 1), kGreen(0.16f, 0.42f, 0.26f, 1), kNavy(0.05f, 0.07f, 0.13f, 0.96f), + kOcarina(0.26f, 0.48f, 0.82f, 1); static void style_init(ImGuiStyle &s) { ImGui::StyleColorsDark(&s); - s.WindowRounding = 14; s.FrameRounding = 8; s.GrabRounding = 8; s.TabRounding = 8; s.ChildRounding = 10; - s.WindowPadding = ImVec2(28, 22); s.FramePadding = ImVec2(14, 8); s.ItemSpacing = ImVec2(14, 12); - s.WindowBorderSize = 0; s.ScrollbarSize = 18; + s.WindowRounding = 10; s.FrameRounding = 6; s.GrabRounding = 6; s.TabRounding = 6; s.ChildRounding = 8; + s.WindowPadding = ImVec2(30, 22); s.FramePadding = ImVec2(14, 8); s.ItemSpacing = ImVec2(14, 12); + s.WindowBorderSize = 2; s.ScrollbarSize = 16; ImVec4 *c = s.Colors; - ImVec4 gold(0.93f, 0.76f, 0.33f, 1), green(0.15f, 0.36f, 0.25f, 1); - c[ImGuiCol_WindowBg] = ImVec4(0.06f, 0.09f, 0.08f, 0.94f); + c[ImGuiCol_WindowBg] = kNavy; + c[ImGuiCol_Border] = ImVec4(kGold.x, kGold.y, kGold.z, 0.55f); c[ImGuiCol_ChildBg] = ImVec4(0, 0, 0, 0); - c[ImGuiCol_Text] = ImVec4(0.96f, 0.94f, 0.88f, 1); - c[ImGuiCol_TextDisabled] = ImVec4(0.62f, 0.64f, 0.58f, 1); - c[ImGuiCol_Button] = ImVec4(0.14f, 0.22f, 0.18f, 1); - c[ImGuiCol_ButtonHovered] = green; - c[ImGuiCol_ButtonActive] = ImVec4(0.21f, 0.50f, 0.34f, 1); - c[ImGuiCol_Tab] = ImVec4(0.12f, 0.18f, 0.15f, 1); - c[ImGuiCol_TabHovered] = green; - c[ImGuiCol_TabSelected] = ImVec4(0.21f, 0.44f, 0.31f, 1); - c[ImGuiCol_TabSelectedOverline] = gold; - c[ImGuiCol_NavCursor] = gold; - c[ImGuiCol_Separator] = ImVec4(0.30f, 0.36f, 0.30f, 1); + c[ImGuiCol_PopupBg] = kNavy; + c[ImGuiCol_Text] = ImVec4(0.97f, 0.95f, 0.88f, 1); + c[ImGuiCol_TextDisabled] = ImVec4(0.60f, 0.64f, 0.70f, 1); + c[ImGuiCol_FrameBg] = ImVec4(0.10f, 0.14f, 0.22f, 1); + c[ImGuiCol_FrameBgHovered] = ImVec4(0.13f, 0.24f, 0.22f, 1); + c[ImGuiCol_FrameBgActive] = kGreen; + c[ImGuiCol_SliderGrab] = kOcarina; + c[ImGuiCol_SliderGrabActive] = kGold; + c[ImGuiCol_CheckMark] = kGold; + c[ImGuiCol_Button] = ImVec4(0.12f, 0.18f, 0.28f, 1); + c[ImGuiCol_ButtonHovered] = kGreen; + c[ImGuiCol_ButtonActive] = kOcarina; + c[ImGuiCol_Header] = ImVec4(0.13f, 0.24f, 0.22f, 1); + c[ImGuiCol_HeaderHovered] = kGreen; + c[ImGuiCol_HeaderActive] = kOcarina; + c[ImGuiCol_Tab] = ImVec4(0.09f, 0.13f, 0.21f, 1); + c[ImGuiCol_TabHovered] = kGreen; + c[ImGuiCol_TabSelected] = ImVec4(0.14f, 0.36f, 0.24f, 1); + c[ImGuiCol_TabSelectedOverline] = kGold; + c[ImGuiCol_NavCursor] = kGold; + c[ImGuiCol_Separator] = ImVec4(kGold.x, kGold.y, kGold.z, 0.30f); + c[ImGuiCol_ScrollbarGrab] = ImVec4(0.20f, 0.28f, 0.40f, 1); +} + +// The Triforce, its top at (x, y), h high, drawn with the title. +static void triforce(ImDrawList *dl, float x, float y, float h, ImU32 col) { + float w = h * 1.1547f; // equilateral: width = height * 2 / sqrt(3) + ImVec2 t(x, y), l(x - w / 2, y + h), r(x + w / 2, y + h), ml(x - w / 4, y + h / 2), mr(x + w / 4, y + h / 2), mb(x, y + h); + dl->AddTriangleFilled(t, mr, ml, col); + dl->AddTriangleFilled(ml, mb, l, col); + dl->AddTriangleFilled(mr, r, mb, col); } static void make_context(void) { @@ -380,7 +403,7 @@ extern "C" void menu_vk_release(void) { // NX2NAVI_MENU_TEST_SET=:=[,=...][;:...]: cfg_set of each key (NX2NAVI_ prefix // optional) at that presented frame; changes live settings with no input, menu open or not. nx2navi.ini is written only by // the pseudo-key SAVE=1 (cfg_save, as a row change does; a failed write shows the menu's save note). -enum { TAB_SCREEN, TAB_GAME, TAB_SOUND, TAB_CONTROLS, TAB_AMIIBO, TAB_MENU, TAB_ADVANCED, TAB_MODS, TAB_CHEATS, TAB_COUNT }; +enum { TAB_SCREEN, TAB_GAME, TAB_SOUND, TAB_CONTROLS, TAB_COUNT }; // Ocarina of Time: the options this port has enum { TEST_PAGE_FRAMES = 30, TEST_SETS = 16 }; struct TestSet { uint64_t frame; char key[64], value[64]; }; static struct { @@ -661,38 +684,12 @@ static void tab_screen(float scale) { opt_choice("NX2NAVI_PERF_OVERLAY", UI_MENU_PERF_OVERLAY, perf, 3, UI_MENU_PERF_OVERLAY_DESC); opt_group(UI_MENU_GRAPHICS); row_output_res(); - row_render_scale(); - row_shadow_res(); static const UiTextId up[] = {UI_MENU_UPSCALER_FSR, UI_MENU_UPSCALER_BILINEAR}; opt_choice("NX2NAVI_UPSCALER", UI_MENU_UPSCALER, up, 2, UI_MENU_UPSCALER_DESC); opt_slider("NX2NAVI_SHARPNESS", UI_MENU_SHARPNESS, 0, 100, UI_MENU_SHARPNESS_DESC, 5, "%.0f %%"); static const UiTextId aspect[] = {UI_MENU_ASPECT_FIT, UI_MENU_ASPECT_STRETCH}; opt_choice("NX2NAVI_ASPECT", UI_MENU_ASPECT, aspect, 2, UI_MENU_ASPECT_DESC); - opt_slider("NX2NAVI_FOV", UI_MENU_FOV, 70, 140, UI_MENU_FOV_DESC, 5, "%.0f %%"); - static const char *const aniso[] = {"game", "2", "4", "8", "16"}; - const char *aniso_names[] = {menu_text(UI_MENU_ANISO_GAME), "2x", "4x", "8x", "16x"}; - opt_choice_list("NX2NAVI_ANISO", UI_MENU_ANISO, aniso, aniso_names, 5, UI_MENU_ANISO_DESC); - row_shadow_scale(); - row_shadow_range(); - opt_slider("NX2NAVI_PCSS", UI_MENU_PCSS, 0, 200, UI_MENU_PCSS_DESC, 10, "%.0f %%"); - opt_slider("NX2NAVI_CONTACT_SHADOWS", UI_MENU_CONTACT_SHADOWS, 0, 100, UI_MENU_CONTACT_SHADOWS_DESC, 10, "%.0f %%"); - opt_toggle("NX2NAVI_DLAA", UI_MENU_DLAA, UI_MENU_DLAA_DESC); - static const UiTextId dlss[] = {UI_MENU_DLSS_DLAA, UI_MENU_DLSS_QUALITY, UI_MENU_DLSS_BALANCED, - UI_MENU_DLSS_PERFORMANCE, UI_MENU_DLSS_ULTRA}; - opt_choice("NX2NAVI_DLSS", UI_MENU_DLSS, dlss, 5, UI_MENU_DLSS_DESC); - static const UiTextId preset[] = {UI_MENU_DLSS_PRESET_DEFAULT, UI_MENU_DLSS_PRESET_K, UI_MENU_DLSS_PRESET_J}; - opt_choice("NX2NAVI_DLSS_PRESET", UI_MENU_DLSS_PRESET, preset, 3, UI_MENU_DLSS_PRESET_DESC); - static const UiTextId chars[] = {UI_MENU_DLSS_CHARS_OFF, UI_MENU_DLSS_CHARS_LIGHT, UI_MENU_DLSS_CHARS_MEDIUM, - UI_MENU_DLSS_CHARS_FULL}; - opt_choice("NX2NAVI_DLSS_CHAR_MASK", UI_MENU_DLSS_CHARS, chars, 4, UI_MENU_DLSS_CHARS_DESC); - opt_toggle("NX2NAVI_FXAA", UI_MENU_FXAA, UI_MENU_FXAA_DESC); rows_color(); - opt_group(UI_MENU_HDR_GROUP); - opt_toggle("NX2NAVI_HDR", UI_MENU_HDR, UI_MENU_HDR_DESC); - opt_slider("NX2NAVI_HDR_PEAK_NITS", UI_MENU_HDR_PEAK, 400, 10000, UI_MENU_HDR_PEAK_DESC, 50, "%.0f"); - opt_slider("NX2NAVI_HDR_GAME_NITS", UI_MENU_HDR_GAME, 80, 500, UI_MENU_HDR_GAME_DESC, 5, "%.0f"); - opt_slider("NX2NAVI_HDR_UI_NITS", UI_MENU_HDR_UI, 80, 500, UI_MENU_HDR_UI_DESC, 5, "%.0f"); - opt_slider("NX2NAVI_HDR_HUE", UI_MENU_HDR_HUE, 0, 100, UI_MENU_HDR_HUE_DESC, 5, "%.0f %%"); opt_end(); } @@ -830,24 +827,10 @@ static void tab_controls(float scale) { opt_begin("##controls", scale); static const UiTextId layout[] = {UI_MENU_PAD_LAYOUT_AUTO, UI_MENU_PAD_LAYOUT_NINTENDO, UI_MENU_PAD_LAYOUT_XBOX}; opt_choice("NX2NAVI_PAD_LAYOUT", UI_MENU_PAD_LAYOUT, layout, 3, UI_MENU_PAD_LAYOUT_DESC, pad_layout_note()); - static const UiTextId prompts[] = {UI_MENU_PROMPTS_NINTENDO, UI_MENU_PROMPTS_XBOX, UI_MENU_PROMPTS_PC}; - bool restart; - const char *note = prompts_note(&restart); - opt_choice("NX2NAVI_PROMPTS", UI_MENU_PROMPTS, prompts, 3, UI_MENU_PROMPTS_DESC, note); - bool ask = restart && opt_button("##restart", UI_MENU_RESTART, UI_MENU_RESTART_DESC); opt_toggle("NX2NAVI_MOTION", UI_MENU_MOTION, UI_MENU_MOTION_DESC); opt_slider("NX2NAVI_RUMBLE_STRENGTH", UI_MENU_RUMBLE_STRENGTH, 0, 300, UI_MENU_RUMBLE_STRENGTH_DESC, 10, "%.0f %%"); opt_slider("NX2NAVI_RUMBLE_CURVE", UI_MENU_RUMBLE_CURVE, 0.2f, 1.5f, UI_MENU_RUMBLE_CURVE_DESC, 0.05f, "%.2f"); opt_toggle("NX2NAVI_RUMBLE_TRIGGERS", UI_MENU_RUMBLE_TRIGGERS, UI_MENU_RUMBLE_TRIGGERS_DESC); - static const UiTextId mouse_modes[] = {UI_MENU_MOUSE_STICK, UI_MENU_MOUSE_DIRECT}; - opt_choice("NX2NAVI_MOUSE_MODE", UI_MENU_MOUSE_MODE, mouse_modes, 2, UI_MENU_MOUSE_MODE_DESC); - opt_slider("NX2NAVI_MOUSE_SENS", UI_MENU_MOUSE_SENS, 0, 10, UI_MENU_MOUSE_SENS_DESC, 0.1f, "%.1f"); - opt_toggle("NX2NAVI_MENU_CURSOR", UI_MENU_MENU_CURSOR, UI_MENU_MENU_CURSOR_DESC); - row_walk_key(); - opt_slider("NX2NAVI_WALK_SCALE", UI_MENU_WALK_SCALE, 0.2f, 0.9f, UI_MENU_WALK_SCALE_DESC, 0.01f, "%.2f"); - // Analog sticks: one dead zone row per stick, falling back to NX2NAVI_STICK_DEADZONE (both sticks) while unset. With - // a gamepad connected each row shows its stick's deflection now, before the dead zone: a drifting stick's resting - // value, to set it above. opt_group(UI_MENU_GROUP_STICKS); double now[2]; bool pad = input_pad_stick_percent(now) != 0; @@ -862,181 +845,9 @@ static void tab_controls(float scale) { binds_rows(scale); opt_end(); binds_after_page(); - if (ask) ImGui::OpenPopup("##restart_confirm"); // at the level BeginPopupModal looks it up - restart_confirm(); } -static void tab_menu(float scale) { - opt_begin("##menu", scale); - static const UiTextId langs[] = {UI_MENU_MENU_LANG_AUTO, UI_MENU_LANGNAME_EN, UI_MENU_LANGNAME_RU, - UI_MENU_LANGNAME_PT_BR, UI_MENU_LANGNAME_ZH_HANS}; - opt_choice("NX2NAVI_MENU_LANG", UI_MENU_MENU_LANG, langs, sizeof langs / sizeof *langs, UI_MENU_MENU_LANG_DESC); - opt_toggle("NX2NAVI_DISCORD_RPC", UI_MENU_DISCORD_RPC, UI_MENU_DISCORD_RPC_DESC); - opt_end(); -} - -// ---- Advanced: the testing and troubleshooting keys, the bug report helpers and the reset of every setting - -// A thread count with no fixed default: Automatic (unset), then the common counts. -static void row_threads(const char *key, UiTextId label, UiTextId desc) { - static const char *const vals[] = {"", "1", "2", "4", "6", "8", "12", "16", "24", "32", "48", "64"}; - enum { N = sizeof vals / sizeof *vals }; - const char *names[N]; - names[0] = menu_text(UI_MENU_AUTO); - for (int i = 1; i < N; i++) names[i] = vals[i]; - opt_choice_list(key, label, vals, names, N, desc); -} - -// A free-text key shown as Off (unset) / On (value set): On writes on_value, any set value reads as On. -static void row_set_or_not(const char *key, UiTextId label, UiTextId desc, const char *on_value) { - const char *vals[] = {"", on_value}, *names[] = {menu_text(UI_MENU_OFF), menu_text(UI_MENU_ON)}; - opt_choice_list(key, label, vals, names, 2, desc, *cfg_value(key) ? 1 : 0); -} - -static bool validation_layer_present(void) { - static int have = -1; - if (have < 0) { - uint32_t n = 0; - vkEnumerateInstanceLayerProperties(&n, nullptr); - VkLayerProperties *l = (VkLayerProperties *)calloc(n ? n : 1, sizeof *l); - if (l && vkEnumerateInstanceLayerProperties(&n, l) >= 0) - for (uint32_t i = 0; i < n && have <= 0; i++) have = !strcmp(l[i].layerName, "VK_LAYER_KHRONOS_validation"); - free(l); - if (have < 0) have = 0; - } - return have != 0; -} - -// Every setting the menu shows back to its default (the environment's keys stay), written to nx2navi.ini. -static void reset_all(void) { - const char *keys[128]; - int n = 0; - for (const CfgMeta *m; n < 128 && (m = cfg_meta_at(n)); n++) keys[n] = m->key; - int done = cfg_reset(keys, n); - opt_save_soon(); - rt_log("menu: reset %d settings to their defaults", done); -} - -// A button's result shown in its value for a few seconds: 1 done (done_text), -1 failed. -struct ActionNote { uint64_t until; int result; }; - -static const char *action_value(const ActionNote &a, UiTextId done_text) { - if (!a.result || plat_now_ns() > a.until) return ""; - return menu_text(a.result > 0 ? done_text : UI_MENU_ACTION_FAILED); -} - -static void action_done(ActionNote &a, bool ok) { a = {plat_now_ns() + 3000000000ull, ok ? 1 : -1}; } - -static void tab_advanced(float scale) { - static ActionNote copy_note, log_note; - opt_begin("##advanced", scale); - opt_text(UI_MENU_ADV_WARNING, true); - - opt_group(UI_MENU_GROUP_SCHED); - static const UiTextId preempt[] = {UI_MENU_PREEMPT_OFF, UI_MENU_PREEMPT_ON, UI_MENU_PREEMPT_INHERIT}; // 0|1|2 - opt_choice("NX2NAVI_ASYNC_PREEMPT", UI_MENU_ASYNC_PREEMPT, preempt, 3, UI_MENU_ASYNC_PREEMPT_DESC); - opt_toggle("NX2NAVI_REPLAY_ASYNC", UI_MENU_REPLAY_ASYNC, UI_MENU_REPLAY_ASYNC_DESC); - opt_toggle("NX2NAVI_TIMER_1MS", UI_MENU_TIMER_1MS, UI_MENU_TIMER_1MS_DESC); - - opt_group(UI_MENU_GROUP_SHADERS); - static const UiTextId wait[] = {UI_MENU_PIPE_WAIT_DEFAULT, UI_MENU_PIPE_WAIT_DRAW}; - opt_choice("NX2NAVI_PIPE_WAIT", UI_MENU_PIPE_WAIT, wait, 2, UI_MENU_PIPE_WAIT_DESC); - static const UiTextId skip[] = {UI_MENU_PIPE_SKIP_STRICT, UI_MENU_PIPE_SKIP_ANY, UI_MENU_PIPE_SKIP_FULLSCREEN}; - opt_choice("NX2NAVI_PIPE_SKIP", UI_MENU_PIPE_SKIP, skip, 3, UI_MENU_PIPE_SKIP_DESC); - opt_toggle("NX2NAVI_LIB_PREBUILD", UI_MENU_LIB_PREBUILD, UI_MENU_LIB_PREBUILD_DESC); - row_threads("NX2NAVI_LIB_PREBUILD_THREADS", UI_MENU_LIB_PREBUILD_THREADS, UI_MENU_LIB_PREBUILD_THREADS_DESC); - opt_toggle("NX2NAVI_PIPE_PREBUILD", UI_MENU_PIPE_PREBUILD, UI_MENU_PIPE_PREBUILD_DESC); - row_threads("NX2NAVI_PREWARM_XLATE", UI_MENU_PREWARM_XLATE, UI_MENU_PREWARM_XLATE_DESC); - opt_toggle("NX2NAVI_GPL", UI_MENU_GPL, UI_MENU_GPL_DESC); - - opt_group(UI_MENU_GROUP_TEXMEM); - static const UiTextId astc[] = {UI_MENU_ASTC_AUTO, UI_MENU_ASTC_GPU, UI_MENU_ASTC_NATIVE, UI_MENU_ASTC_CPU}; - opt_choice("NX2NAVI_ASTC", UI_MENU_ASTC, astc, 4, UI_MENU_ASTC_DESC); - static const UiTextId mem[] = {UI_MENU_MEMORY_AUTO, UI_MENU_MEMORY_IMPORT, UI_MENU_MEMORY_MIRROR, - UI_MENU_MEMORY_VERIFY}; - opt_choice("NX2NAVI_MEMORY", UI_MENU_MEMORY, mem, 4, UI_MENU_MEMORY_DESC); - opt_toggle("NX2NAVI_TEX_ALIAS", UI_MENU_TEX_ALIAS, UI_MENU_TEX_ALIAS_DESC); - opt_slider("NX2NAVI_TEX_VRAM_PRESSURE", UI_MENU_TEX_VRAM_PRESSURE, 0, 100, UI_MENU_TEX_VRAM_PRESSURE_DESC, 1, "%.0f%%"); - static const UiTextId astc_bc[] = {UI_MENU_AUTO, UI_MENU_OFF, UI_MENU_ON}; // unset, 0, 1 - opt_choice("NX2NAVI_ASTC_BC", UI_MENU_ASTC_BC, astc_bc, 3, UI_MENU_ASTC_BC_DESC); - - opt_group(UI_MENU_GROUP_FIXES); - opt_toggle("NX2NAVI_LOD_GUARD", UI_MENU_LOD_GUARD, UI_MENU_LOD_GUARD_DESC); - opt_toggle("NX2NAVI_FIX_C0", UI_MENU_FIX_C0, UI_MENU_FIX_C0_DESC); - opt_toggle("NX2NAVI_FIX_ZERO", UI_MENU_FIX_ZERO, UI_MENU_FIX_ZERO_DESC); - opt_toggle("NX2NAVI_BIND_SKIP", UI_MENU_BIND_SKIP, UI_MENU_BIND_SKIP_DESC); - opt_toggle("NX2NAVI_SHADER_MEMO", UI_MENU_SHADER_MEMO, UI_MENU_SHADER_MEMO_DESC); - opt_toggle("NX2NAVI_PIPE_MEMO", UI_MENU_PIPE_MEMO, UI_MENU_PIPE_MEMO_DESC); - opt_toggle("NX2NAVI_TEX_MEMO", UI_MENU_TEX_MEMO, UI_MENU_TEX_MEMO_DESC); - opt_toggle("NX2NAVI_DESC_TEMPLATE", UI_MENU_DESC_TEMPLATE, UI_MENU_DESC_TEMPLATE_DESC); - opt_toggle("NX2NAVI_PUSH_SKIP", UI_MENU_PUSH_SKIP, UI_MENU_PUSH_SKIP_DESC); - opt_slider("NX2NAVI_SPLIT_DRAWS", UI_MENU_SPLIT_DRAWS, 0, 1000, UI_MENU_SPLIT_DRAWS_DESC, 8, "%.0f"); - - opt_group(UI_MENU_GROUP_VULKAN); - if (!validation_layer_present() && !cfg_bool("NX2NAVI_VK_VALIDATION")) opt_disable_next(UI_MENU_VK_VALIDATION_MISSING); - opt_toggle("NX2NAVI_VK_VALIDATION", UI_MENU_VK_VALIDATION, UI_MENU_VK_VALIDATION_DESC); - static const UiTextId dyn[] = {UI_MENU_AUTO, UI_MENU_OFF, UI_MENU_ON}; // unset, 0, 1 - opt_choice("NX2NAVI_VK_DYN_VERTEX", UI_MENU_VK_DYN_VERTEX, dyn, 3, UI_MENU_VK_DYN_VERTEX_DESC); - opt_choice("NX2NAVI_VK_DYN_A2C", UI_MENU_VK_DYN_A2C, dyn, 3, UI_MENU_VK_DYN_A2C_DESC); - opt_toggle("NX2NAVI_VK_SUBGROUP_PIN", UI_MENU_VK_SUBGROUP_PIN, UI_MENU_VK_SUBGROUP_PIN_DESC); - - opt_group(UI_MENU_GROUP_DIAG); - row_set_or_not("NX2NAVI_TRACE", UI_MENU_TRACE, UI_MENU_TRACE_DESC, "perf"); - opt_toggle("NX2NAVI_FLIGHT", UI_MENU_FLIGHT, UI_MENU_FLIGHT_DESC); - opt_slider("NX2NAVI_FLIGHT_SPIKE_MS", UI_MENU_FLIGHT_SPIKE_MS, 20, 1000, UI_MENU_FLIGHT_SPIKE_MS_DESC, 10, "%.0f"); - opt_choice("NX2NAVI_PASS_CRUMBS", UI_MENU_PASS_CRUMBS, dyn, 3, UI_MENU_PASS_CRUMBS_DESC); // vk_crumbs.c pass marks - static char rec_path[600]; // the recording goes beside the log - if (!rec_path[0]) { - char dir[512]; - snprintf(rec_path, sizeof rec_path, "%s/input-record.txt", sysinfo_log_folder(dir, sizeof dir) ? dir : "."); - } - row_set_or_not("NX2NAVI_INPUT_RECORD", UI_MENU_INPUT_RECORD, UI_MENU_INPUT_RECORD_DESC, rec_path); - if (opt_button("##open_log", UI_MENU_OPEN_LOG, UI_MENU_OPEN_LOG_DESC, action_value(log_note, UI_MENU_OPEN_LOG))) { - char dir[512]; - bool ok = sysinfo_log_folder(dir, sizeof dir) && plat_open_folder(dir); - if (!ok) action_done(log_note, false); // success shows as the opened folder - } - if (opt_button("##copy_info", UI_MENU_COPY_INFO, UI_MENU_COPY_INFO_DESC, - action_value(copy_note, UI_MENU_COPY_INFO_DONE))) { - static char text[16384]; - action_done(copy_note, sysinfo_text(text, sizeof text) > 0 && plat_clipboard_set(text)); - } - bool ask = opt_button("##reset", UI_MENU_RESET, UI_MENU_RESET_DESC); - opt_end(); - if (ask) ImGui::OpenPopup("##reset_confirm"); // at the level BeginPopupModal looks it up - - // the confirmation, over the menu; Cancel has the cursor first - g_modal = false; - ImGui::SetNextWindowPos(ImGui::GetMainViewport()->GetCenter(), ImGuiCond_Always, ImVec2(0.5f, 0.5f)); - if (ImGui::BeginPopupModal("##reset_confirm", nullptr, - ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_AlwaysAutoResize)) { - g_modal = true; - ImGui::TextUnformatted(menu_text(UI_MENU_RESET_CONFIRM)); - ImGui::Spacing(); - const char *yes = menu_text(UI_MENU_RESET_YES), *no = menu_text(UI_MENU_RESET_NO); - const ImGuiStyle &st = ImGui::GetStyle(); - float bw = fmaxf(ImGui::CalcTextSize(yes).x, ImGui::CalcTextSize(no).x) + st.FramePadding.x * 4; - bool close = false; - if (ImGui::Button(no, ImVec2(bw, 0))) close = true; - if (ImGui::IsWindowAppearing()) ImGui::SetItemDefaultFocus(); - ImGui::SameLine(); - if (ImGui::Button(yes, ImVec2(bw, 0))) { - reset_all(); - close = true; - } - if (ImGui::IsKeyPressed(ImGuiKey_Escape, false) || ImGui::IsKeyPressed(ImGuiKey_GamepadFaceRight, false)) - close = true; - if (close) { - ImGui::CloseCurrentPopup(); - opt_focus_key("##reset"); // back on the row it came from - } - ImGui::EndPopup(); - } -} - -static const UiTextId kTabNames[TAB_COUNT] = {UI_MENU_TAB_SCREEN, UI_MENU_TAB_GAME, UI_MENU_TAB_SOUND, - UI_MENU_TAB_CONTROLS, UI_MENU_TAB_AMIIBO, UI_MENU_TAB_MENU, - UI_MENU_TAB_ADVANCED, UI_MENU_TAB_MODS, UI_MENU_TAB_CHEATS}; +static const UiTextId kTabNames[TAB_COUNT] = {UI_MENU_TAB_SCREEN, UI_MENU_TAB_GAME, UI_MENU_TAB_SOUND, UI_MENU_TAB_CONTROLS}; static int g_tab; // the tab shown static int g_tab_request = -1; // select this tab at the next layout; -1 none static int g_focus_page = -1; // this tab's first row takes the cursor once it is shown; -1 none @@ -1047,112 +858,28 @@ static void go_to(int tab, const char *key) { opt_focus_key(key); } -// ---- Mods: the mods folder (mods.c) with switches and the load order, then the port's own patches. The folder was -// read at start; the rows change the order and switches for the next start. -static void mods_builtin_rows(void) { - opt_group(UI_MENU_MODS_BUILTIN); - char fps[64], res[64]; - const char *lim = cfg_value("NX2NAVI_FPS_LIMIT"); - if (!strcmp(lim, "display")) snprintf(fps, sizeof fps, "%s", menu_text(UI_MENU_FPS_DISPLAY)); - else snprintf(fps, sizeof fps, "%s FPS", lim); - if (opt_button("##mods_fps60", UI_MENU_MODS_FPS60, UI_MENU_MODS_FPS60_DESC, fps)) go_to(TAB_GAME, "NX2NAVI_FPS_LIMIT"); - int w = 0, h = 0; - sscanf(cfg_value("NX2NAVI_OUTPUT_RES"), "%dx%d", &w, &h); - snprintf(res, sizeof res, "%d × %d", w, h); // U+00D7, as row_output_res shows it - if (opt_button("##mods_res", UI_MENU_MODS_RES, UI_MENU_MODS_RES_DESC, res)) go_to(TAB_SCREEN, "NX2NAVI_OUTPUT_RES"); - opt_toggle("NX2NAVI_REALTIME", UI_MENU_REALTIME, UI_MENU_REALTIME_DESC); - opt_toggle("NX2NAVI_CLOTH_REALTIME", UI_MENU_CLOTH_REALTIME, UI_MENU_CLOTH_REALTIME_DESC); - opt_toggle("NX2NAVI_HIGHFPS_FIX", UI_MENU_HIGHFPS_FIX, UI_MENU_HIGHFPS_FIX_DESC); - opt_button("##mods_quickmenu", UI_MENU_MODS_QUICKMENU, UI_MENU_MODS_QUICKMENU_DESC, - menu_text(romfs_patches_loaded() ? UI_MENU_ON : UI_MENU_OFF)); -} - -static void tab_mods(float scale) { - opt_begin("##mods", scale); - opt_group(UI_MENU_MODS_INSTALLED); - int n = mods_count(); - if (!n) opt_text(UI_MENU_MODS_NONE); - const char *after = mods_changed() ? menu_text(UI_MENU_AFTER_RESTART) : nullptr; - for (int i = 0; i < n; i++) { - const ModInfo *m = mods_at(i); - char id[32], desc[1536]; - snprintf(id, sizeof id, "##mod%d", i); - char over[512] = "", other[512] = ""; - if (m->overridden) snprintf(over, sizeof over, menu_text(UI_MENU_MODS_OVERRIDDEN), m->overridden, m->overridden_by); - if (m->other_version) snprintf(other, sizeof other, menu_text(UI_MENU_MODS_OTHER_VERSION), m->other_version); - char cheat[256] = ""; - if (m->kind & MOD_CHEATS) { // its cheats as read at start, or the file they come from - int count = 0; - for (int j = 0; j < cheats_count(); j++) - count += !strcmp(cheats_info(j)->mod, m->name) && cheats_info(j)->kind != CHEAT_HEADING; - if (count) snprintf(cheat, sizeof cheat, menu_text(UI_MENU_MODS_CHEATS), count); - else snprintf(cheat, sizeof cheat, menu_text(UI_MENU_MODS_CHEATS_FILE), cheats_file()); - } - char code[512] = "", problem[512] = ""; - if (m->nsets && m->overrides[0]) - snprintf(code, sizeof code, menu_text(UI_MENU_MODS_REPLACES_BUILTIN), m->nsets, m->overrides); - else if (m->nsets) - snprintf(code, sizeof code, menu_text(UI_MENU_MODS_EXEFS_ON), m->nsets); - switch (m->problem) { - case MOD_EMPTY: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EMPTY)); break; - case MOD_PLUGIN: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_SUBSDK)); break; - case MOD_EXEFS_LATER: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EXEFS_LATER)); break; - case MOD_EXEFS_UNSUPPORTED: - snprintf(problem, sizeof problem, menu_text(UI_MENU_MODS_EXEFS_UNSUPPORTED), m->note); - break; - case MOD_EXEFS_OTHER: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_EXEFS_OTHER)); break; - case MOD_BAD_NAME: snprintf(problem, sizeof problem, "%s", menu_text(UI_MENU_MODS_BAD_NAME)); break; - default: break; - } - int k = snprintf(desc, sizeof desc, menu_text(UI_MENU_MODS_ROW_DESC), m->replaced, m->added); - const char *parts[] = {over, other, cheat, code, problem, menu_text(UI_MENU_MODS_MOVE_HINT)}; - for (const char *part : parts) - if (*part && k >= 0 && k < (int)sizeof desc) k += snprintf(desc + k, sizeof desc - k, "\n\n%s", part); - const char *note = m->enabled && m->problem != MOD_OK ? menu_text(UI_MENU_MODS_UNUSED) : after; - int s = opt_text_row(id, m->name, menu_text(m->enabled ? UI_MENU_ON : UI_MENU_OFF), note, desc, true); - if (s == 2 || s == -1 || s == 1) { - mods_set_enabled(i, !m->enabled); - opt_save_soon(); - } else if ((s == 3 && i > 0) || (s == 4 && i + 1 < n)) { - int to = s == 3 ? i - 1 : i + 1; - mods_move(i, to - i); - snprintf(id, sizeof id, "##mod%d", to); - opt_focus_key(id); // the cursor follows the mod - opt_save_soon(); - } - } - bool rebuild = - cheats_rebuild_pending() && opt_button("##restart", UI_MENU_CHEATS_REBUILD, UI_MENU_CHEATS_REBUILD_DESC); - static uint64_t open_failed; // plat_now_ns until which the button says it failed - if (opt_button("##mods_open", UI_MENU_MODS_OPEN, UI_MENU_MODS_OPEN_DESC, - plat_now_ns() < open_failed ? menu_text(UI_MENU_ACTION_FAILED) : "")) { - bool ok = mods_dir()[0] != 0; - if (ok) plat_file_mkdir_p(mods_dir()); - if (!ok || !plat_open_folder(mods_dir())) open_failed = plat_now_ns() + 3000000000ull; - } - mods_builtin_rows(); - opt_end(); - if (rebuild) { - restart_rebuild = true; - ImGui::OpenPopup("##restart_confirm"); - } - restart_confirm(); -} - -// ---- the window: title, tabs (L / R or Q / E switch), the page, and the bottom row with "Continue" and "Quit game". -// B on the gamepad and Esc close it. static void layout_menu(uint32_t w, uint32_t h, float scale) { - ImGui::GetBackgroundDrawList()->AddRectFilled(ImVec2(0, 0), ImVec2((float)w, (float)h), IM_COL32(0, 0, 0, 120)); + ImGui::GetBackgroundDrawList()->AddRectFilled(ImVec2(0, 0), ImVec2((float)w, (float)h), IM_COL32(4, 8, 20, 140)); float ww = fminf(w * 0.88f, 1400 * scale), wh = fminf(h * 0.86f, 820 * scale); ImGui::SetNextWindowPos(ImVec2(w * 0.5f, h * 0.5f), ImGuiCond_Always, ImVec2(0.5f, 0.5f)); ImGui::SetNextWindowSize(ImVec2(ww, wh), ImGuiCond_Always); ImGui::Begin("##menu", nullptr, ImGuiWindowFlags_NoDecoration | ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoSavedSettings); - ImGui::TextColored(ImVec4(0.93f, 0.76f, 0.33f, 1), "%s", menu_text(UI_MENU_TITLE)); + { // the Triforce, then "Ocarina of Time" and the page title + ImVec2 at = ImGui::GetCursorScreenPos(); + float th = ImGui::GetTextLineHeight(); + triforce(ImGui::GetWindowDrawList(), at.x + th * 0.6f, at.y + th * 0.02f, th * 0.96f, ImGui::GetColorU32(kGold)); + ImGui::SetCursorScreenPos(ImVec2(at.x + th * 1.5f, at.y)); + ImGui::TextColored(kGold, "Ocarina of Time"); + ImGui::SameLine(); + ImGui::TextDisabled("|"); + ImGui::SameLine(); + ImGui::TextUnformatted(menu_text(UI_MENU_TITLE)); + } // the port version and the game update, right-aligned on the title line: what a bug report needs first char game[16], version[96]; snprintf(game, sizeof game, "%d.%d.%d", rt_game_version / 100, rt_game_version / 10 % 10, rt_game_version % 10); - snprintf(version, sizeof version, menu_text(UI_MENU_VERSION), rt_port_version, game); + snprintf(version, sizeof version, "Ocarina Recomp %s - game %s", rt_port_version, game); ImGui::SameLine(ImGui::GetWindowWidth() - ImGui::GetStyle().WindowPadding.x - ImGui::CalcTextSize(version).x); ImGui::TextDisabled("%s", version); @@ -1183,19 +910,8 @@ static void layout_menu(uint32_t w, uint32_t h, float scale) { switch (k) { case TAB_SCREEN: tab_screen(scale); break; case TAB_GAME: tab_game(scale); break; - case TAB_MODS: tab_mods(scale); break; case TAB_SOUND: tab_sound(scale); break; case TAB_CONTROLS: tab_controls(scale); break; - case TAB_AMIIBO: tab_amiibo(scale); break; - case TAB_MENU: tab_menu(scale); break; - case TAB_CHEATS: - if (tab_cheats(scale)) { - restart_rebuild = true; - ImGui::OpenPopup("##restart_confirm"); - } - restart_confirm(); - break; - case TAB_ADVANCED: tab_advanced(scale); break; } ImGui::EndTabItem(); } @@ -1217,7 +933,7 @@ static void layout_menu(uint32_t w, uint32_t h, float scale) { ImGui::SameLine(); if (ImGui::Button(quit, ImVec2(bw, 0))) gfx_window_request_close(); bool was_modal = g_modal || binds_modal(); - if (g_tab != TAB_ADVANCED && g_tab != TAB_CONTROLS) g_modal = false; + if (g_tab != TAB_CONTROLS) g_modal = false; if (!was_modal && (ImGui::IsKeyPressed(ImGuiKey_Escape, false) || ImGui::IsKeyPressed(ImGuiKey_GamepadFaceRight, false))) set_open(0); ImGui::End(); diff --git a/runtime/vk.h b/runtime/vk.h index 5c022655..a870a732 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -36,7 +36,8 @@ typedef struct VkCtx { has_nv_checkpoints, // VK_NV_device_diagnostic_checkpoints (NX2NAVI_PASS_CRUMBS=1, vk_crumbs.c) has_astc_ldr, // textureCompressionASTC_LDR (enabled when reported) has_calibrated, // VK_KHR/EXT_calibrated_timestamps (GPU idle split in the perf line) - has_hdr_metadata; // VK_EXT_hdr_metadata (vk_hdr.c vk_hdr_metadata) + has_hdr_metadata, // VK_EXT_hdr_metadata (vk_hdr.c vk_hdr_metadata) + has_mesh_shader; // VK_EXT_mesh_shader with taskShader and meshShader (NVN2 task/mesh stages) // Feedback loops (maxwell.md 1.9, vk_exec_draw.c feedback_prepare): a draw samples a target of its open pass that // an earlier draw of the pass wrote. NX2NAVI_FEEDBACK=0: VK_FEEDBACK_OFF (sampled in GENERAL without a barrier, the // old behaviour); VK_FEEDBACK_LAYOUT with VK_EXT_attachment_feedback_loop_layout and dynamicRenderingLocalRead @@ -403,6 +404,23 @@ VkSampler vk_sampler_by_id_locked(int id); // for an empty slot. VkSampler vk_sampler_point_by_id_locked(int id); uint32_t vk_texture_slots_gen(void); +int vk_texture_slot_count_locked(void); +void vk_texture_create_budget_locked(int n); // slot lookups create at most n entries (mapped storage only); -1 any +int vk_texture_create_deferred_locked(void); // a lookup since the last call left a creation to the budget // one past the highest texture slot registered so far (or more) +int vk_sampler_slot_count_locked(void); +// Bindless pools (vk_exec_draw.c, "texture and sampler pools"): set 2 of every pipeline layout. Shaders that index +// the texture or sampler pool with handles from memory have their pool arrays moved there (vk_pipeline.c +// spirv_pool_rebind); VK_POOL_KINDS image bindings, one per view kind, then the sampler pool. +#define VK_POOL_TEX 32768 // as the translator's Accessor.TexturePoolLength +#define VK_POOL_SAMP 4096 // Accessor.SamplerPoolLength +#define VK_POOL_KINDS 16 // kind = depth * 8 + dim * 2 + arrayed, dim 1D 2D 3D CUBE +extern volatile uint32_t vk_pool_kinds_used; // bit k: a pipeline was made with a kind-k pool array +VkDescriptorSetLayout vk_pool_set_layout(void); // the pools' layout (made on first use) +VkDescriptorSetLayout vk_empty_set_layout(void); // set 1 of layouts without one +// The upscaler stand-in (vk_tonemap.c): src (sampled) scaled to dw x dh and tonemapped into its own RGBA16F image, which +// it returns for the caller to blit into the target; VK_NULL_HANDLE when it cannot. The caller binds its own compute +// pipeline again afterwards. +VkImage vk_tonemap_record(VkCommandBuffer cb, VkImage src, VkFormat sf, uint32_t dw, uint32_t dh); // What vk_texture_by_id(id) returns can change only when this does: the slot is registered again, or a texture entry at // its storage address is created, dropped, released or taken back into use. Unlocked; read it before the lookup it // guards (a lookup may create an entry). 0 when the slot was never registered: nothing is kept under 0. @@ -478,7 +496,9 @@ void vk_backend_shutdown(void); // presents, vk_sync_shutdown, vk_mem // Everything a graphics pipeline depends on, as the op stream carries it. Fields covered by dynamic state are read by // vk_pipeline_set_dynamic only and left out of the pipeline key. typedef struct VkPipeState { - const GfxShader *vs, *fs; // gfx_shader_get results; fs NULL: no fragment stage (depth only) + const GfxShader *vs, *fs; // gfx_shader_get results; fs NULL: no fragment stage (depth only). vs is + // the mesh stage of a mesh draw (gfx_shader_stage GFX_STAGE_MS) + const GfxShader *ts; // mesh draws: the task stage, or NULL int32_t color_fmt[8]; // NVNformat of colour target i (OP_SET_RT), 0 none int32_t depth_fmt; // NVNformat of the depth/stencil target, 0 none uint8_t samples; // sample count of the targets; 0 or 1 single-sampled @@ -531,6 +551,7 @@ typedef struct { // Descriptor update template per set (vk_pipeline.c make_tpl): reads one VkDescSlot per descriptor, in binding order, // from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs. VkDescriptorUpdateTemplate tpl[2]; + int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools) } VkPipeInfo; // One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the // bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte. diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 152e5e9d..3a83f72c 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -217,6 +217,7 @@ typedef struct { VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT gpl; VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbl; VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr; + VkPhysicalDeviceMeshShaderFeaturesEXT mesh; } FeatureChain; static int create_device(void) { @@ -251,6 +252,8 @@ static int create_device(void) { // Ryujinx's SPIR-V declares fragment shader interlock whenever the host reports it (vk_pipeline.c spirv_fixup). OPTIONAL_EXT(has_fragment_shader_interlock, VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME, fsi, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT); + // NVN2 task and mesh programs (gameplay geometry) draw through VK_EXT_mesh_shader (vk_exec_draw.c op_draw_mesh) + OPTIONAL_EXT(has_mesh_shader, VK_EXT_MESH_SHADER_EXTENSION_NAME, mesh, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_FEATURES_EXT); // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -313,7 +316,14 @@ static int create_device(void) { else if (!h->geometryShader) miss = "geometryShader"; else if (!h->textureCompressionBC) miss = "textureCompressionBC"; if (miss) { rt_log("vk: required feature missing on %s: %s", g_vk.props.deviceName, miss); free(c); return 0; } - if (!c->fault.deviceFault) g_vk.has_device_fault = 0; // vkGetDeviceFaultInfoEXT needs the feature, not just the extension + if (!c->fault.deviceFault) g_vk.has_device_fault = 0; + if (g_vk.has_mesh_shader) { + // these two need features that are not enabled (multiview, primitive fragment shading rate) + c->mesh.multiviewMeshShader = c->mesh.primitiveFragmentShadingRateMeshShader = VK_FALSE; + c->mesh.meshShaderQueries = VK_FALSE; + if (!c->mesh.meshShader || !c->mesh.taskShader) g_vk.has_mesh_shader = 0; + rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "not supported (task/mesh draws are skipped)"); + } // vkGetDeviceFaultInfoEXT needs the feature, not just the extension g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index efe1bafb..19c85e13 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -33,7 +33,10 @@ #include "host.h" #include "nvn_api.h" #include "vk.h" + #include "vk_exec.h" +uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r); // vk_exec_draw.c +int vk_exec_raw_9e5(VkImage img, VkFormat f); // vk_exec_draw.c #include "platform/platform.h" #define MAX_PASSES 400 @@ -562,10 +565,28 @@ static void describe_bindings(Str *b, const VkExec *ex, const VkPipeInfo *info) if (r && bd->kind == GFX_RES_SSBO) { // where the address lives (c0: the stage's NVN slot table) and the stage's slots int st = nvn_stage(bd->stage); sp(b, " %s ssbo at c%d+%#x%s, bound:", stn[st], r->cbuf_slot, r->handle * 4, r->flags & 1 ? " written" : ""); + if (st == 5 && r->cbuf_slot == 0 && r->handle * 4 >= 0x10 && r->handle * 4 + 12 <= 0x10 + (int)sizeof ex->cs_push) { + uint64_t pa; uint32_t ps; // an NVN2 compute push-constant pointer {address, size} + memcpy(&pa, ex->cs_push + r->handle * 4 - 0x10, 8), memcpy(&ps, ex->cs_push + r->handle * 4 - 0x10 + 8, 4); + sp(b, " (push pointer %#llx+%#x)", (unsigned long long)pa, ps); + } + if (st == 5) sp(b, " -> %#llx", (unsigned long long)vk_exec_cs_storage_addr(ex, r)); for (int k = 0; k < 16; k++) if (ex->ssbo[st][k].addr) sp(b, " %d=%#llx+%#llx", k, (unsigned long long)ex->ssbo[st][k].addr, (unsigned long long)ex->ssbo[st][k].size); sp(b, "\n"); + // NX2NAVI_DUMP_UBO: also the small storage buffers' guest bytes (GPU results as of the last write-back); per-view + // constants (exposure and such) live in such buffers. + for (int k = 0; dump_ubo && k < 16; k++) { + const p_bind_buf *u = &ex->ssbo[st][k]; + VkMemSpan ms; + if (!u->addr || !u->size || u->size > 0x100 || !vk_mem_resolve(u->addr, 1, &ms)) continue; + const float *f = (const float *)(uintptr_t)u->addr; + sp(b, " %s ssbo %d %#llx:", stn[st], k, (unsigned long long)u->addr); + for (uint64_t q = 0; q * 16 < u->size && q < (uint64_t)dump_ubo; q++) + sp(b, " [%llu] %g %g %g %g", (unsigned long long)q, f[4 * q], f[4 * q + 1], f[4 * q + 2], f[4 * q + 3]); + sp(b, "\n"); + } continue; } if (r && bd->kind == GFX_RES_UBO && dump_ubo && r->slot >= 3 && r->slot - 3 < 18) { @@ -775,6 +796,15 @@ void vk_dump_dispatch(VkExec *ex, const GfxShader *cs, VkPipeline p, const VkPip d->z, (unsigned long long)shader_hash(cs), (unsigned long long)(ex->prog ? ex->prog->serial : 0), (unsigned long long)d->caller, (unsigned long long)ex->fence, b.s ? b.s : ""); free(b.s); + if (dump_ubo) { // NX2NAVI_DUMP_UBO: the NVN2 push constants as 64-bit words + fprintf(passes_f, " push:"); + for (int q = 0; q < 0x80; q += 8) { + uint64_t v; + memcpy(&v, ex->cs_push + q, 8); + fprintf(passes_f, " %02x:%llx", q, (unsigned long long)v); + } + fprintf(passes_f, "\n"); + } for (uint32_t i = 0; i < info->n_bindings && !pass_skip; i++) { // storage images the dispatch writes const VkPipeBinding *bd = &info->bindings[i]; if (!bd->res || bd->kind != GFX_RES_IMG || !(bd->res->flags & 4) || bd->type != VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) @@ -796,7 +826,8 @@ void vk_dump_dispatch(VkExec *ex, const GfxShader *cs, VkPipeline p, const VkPip if (luts_done[k] == t->image) luts_done[k] = VK_NULL_HANDLE; } const char *why = t->samples > 1 ? "multisampled" : - copy_out(ex, t->image, t->format, t->aspect, lv, layer, w, hh, dd, name); + copy_out(ex, t->image, vk_exec_raw_9e5(t->image, t->format) ? VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 : t->format, + t->aspect, lv, layer, w, hh, dd, name); fprintf(passes_f, " writes img slot %d level %u layer %u depth %u -> %s.png%s%s\n", bd->res->slot, lv, layer, dd, name, why ? " skipped: " : "", why ? why : ""); } diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 0c4a24cb..e2af956b 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -739,8 +739,21 @@ static void clear_attachment(VkExec *ex, const VkClearAttachment *a) { ex->fb_dirty |= a->aspectMask & VK_IMAGE_ASPECT_COLOR_BIT ? 1u << a->colorAttachment : 0x100u; } +// NX2NAVI_LOG_RCOND=1: clears issued under a render condition (the first 400), run or skipped, with their target. +static void clear_cond_log(const VkExec *ex, const char *what, int index) { + static int on = -1, n; + if (on < 0) on = rt_knob("LOG_RCOND", "0"); + if (!on || !ex->has_render_cond || n >= 400) return; + n++; + const VkExecTarget *t = index < 0 ? &ex->depth : index < (int)ex->pass_ncolor ? &ex->color[index] : NULL; + rt_log("vk: %s clear %d under the condition at %#llx (mode %d): %s, target %ux%u", what, index, + (unsigned long long)ex->render_cond.addr, ex->render_cond.mode, ex->render_disabled ? "skipped" : "run", + t ? t->width : 0, t ? t->height : 0); +} + static void clear_color(VkExec *ex, const p_clear_color *c) { vk_exec_frame.clears++; + clear_cond_log(ex, "colour", c->index); if (ex->render_disabled) return; resolve_rt(ex); int i = c->index; @@ -771,6 +784,7 @@ static void clear_color(VkExec *ex, const p_clear_color *c) { } static void clear_ds(VkExec *ex, const p_clear_ds *c) { + clear_cond_log(ex, "depth", -1); if (ex->render_disabled) return; resolve_rt(ex); const VkExecTarget *t = &ex->depth; @@ -908,7 +922,8 @@ static void walk(VkExec *ex, const op_hdr *h, int depth) { if (h->op == OP_DRAW || h->op == OP_DRAW_IDX || h->op == OP_DRAW_IDX_IND) vk_exec_frame.draws_seen++, split_draws++; if (h->op == OP_DISPATCH || h->op == OP_DISPATCH_IND) vk_exec_frame.dispatches++, split_draws++; case OP_BIND_PROGRAM: case OP_BIND_STATE: case OP_BIND_VATTR: case OP_BIND_VSTREAM: case OP_BIND_VBUF: - case OP_BIND_UBO: case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: + case OP_BIND_UBO: case OP_BIND_SSBO: case OP_BIND_TEX: case OP_BIND_SAMP: case OP_BIND_IMG: case OP_PUSH_CC: + case OP_DRAW_MESH: case OP_DRAW_MESH_IND: vk_exec_op_draw(ex, h->op, a); break; case OP_COPY_B2B: case OP_COPY_B2T: case OP_COPY_T2B: case OP_COPY_T2T: case OP_CLEAR_TEX: diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 4a94fc46..ad437edd 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -87,8 +87,9 @@ typedef struct VkExec { const prog_t *prog; // OP_BIND_PROGRAM int32_t prog_stages; NVNbufferRange vbuf[16]; // OP_BIND_VBUF - p_bind_buf ubo[6][18], ssbo[6][16]; // OP_BIND_UBO / OP_BIND_SSBO per stage and index - uint64_t tex[6][128], samp[6][128], img[6][32]; // OP_BIND_TEX / OP_BIND_SAMP / OP_BIND_IMG handles per stage + p_bind_buf ubo[8][18], ssbo[8][16]; // OP_BIND_UBO / OP_BIND_SSBO per NVN stage (6 task, 7 mesh) and index + uint8_t cs_push[1024]; // OP_PUSH_CC: NVN2 compute push constants (c0 from 0x10) + uint64_t tex[8][128], samp[8][128], img[8][32]; // OP_BIND_TEX / OP_BIND_SAMP / OP_BIND_IMG handles per stage // ---- vk_exec_xfer.c slots (render enable, conditional rendering): left empty by vk_exec.c int render_disabled; // OP_RENDER_ENABLE false or a failed OP_RENDER_COND: clears and draws skip diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 25872554..a8f3f2fa 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -26,6 +26,10 @@ #include "nvn_api.h" #include "nvn_hmap.h" #include "vk.h" + +#ifndef NX2NAVI_TONEMAP_STANDIN +#define NX2NAVI_TONEMAP_STANDIN 0 // vk_tonemap.c, see untranslated_dispatch_fallback +#endif #include "vk_exec.h" #include "platform/platform.h" #include "vk_dlaa.h" @@ -39,9 +43,11 @@ #define NVN_STAGE_VS 0 #define NVN_STAGE_FS 1 #define NVN_STAGE_CS 5 +#define NVN_STAGE_TASK 6 // NVN2 +#define NVN_STAGE_MESH 7 #define ONESHOT_WAIT_MS 30000u // prepare_draw: how long a one-shot draw waits for its pipeline before it is skipped -enum { W_PROGRAM, W_SHADER, W_PIPELINE, W_NO_VS, W_INDEX, W_UB, W_BIND, W_SSBO_ALIGN, W_SSBO_SLOT, W_RESTART, W_INDIRECT, W_COUNT }; +enum { W_PROGRAM, W_SHADER, W_PIPELINE, W_NO_VS, W_INDEX, W_UB, W_BIND, W_SSBO_ALIGN, W_SSBO_SLOT, W_RESTART, W_INDIRECT, W_MESH, W_COUNT }; static void warn_once(int key, const char *fmt, ...) { static volatile int32_t seen[W_COUNT]; @@ -115,7 +121,7 @@ static struct { VkBuffer c0_buf[3]; VkDeviceSize c0_off[3]; int c0_valid[3]; - struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; } c0_in[3]; // what vk_c0_fill read for c0[k] + struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; uint8_t push[1024]; } c0_in[3]; // what vk_c0_fill read for c0[k] VkPipelineLayout ds_layout; // graphics layout the sets below were last pushed or bound with VkDescriptorSet ds_bound[2]; // allocated set bound per set number (0: none or pushed since) VkDescriptorSet ds_last[2]; // last allocated and written set per set number, contents in DS @@ -287,6 +293,8 @@ static struct { VkBuffer zbuf; VmaAllocation zalloc; VkBufferView zview; + VkBuffer sink; // ssbo_info: an unresolved storage buffer a shader writes (zbuf stays zero) + VmaAllocation sink_alloc; VkSampler samp, samp_shadow; struct { VkImage image; VmaAllocation alloc; VkImageView view, sview; } img[DV_COUNT]; } Z; @@ -379,6 +387,10 @@ static void dummies_init(void) { VkBufferViewCreateInfo bv = {.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, .buffer = Z.zbuf, .format = VK_FORMAT_R32G32B32A32_SFLOAT, .range = ZERO_BYTES}; if (vkCreateBufferView(c->device, &bv, NULL, &Z.zview) != VK_SUCCESS) rt_fatal("vk: zero buffer view failed"); + VkBufferCreateInfo sbi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ZERO_BYTES, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo sai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateBuffer(c->vma, &sbi, &sai, &Z.sink, &Z.sink_alloc, NULL) != VK_SUCCESS) rt_fatal("vk: sink buffer failed"); samp_par sp; memset(&sp, 0, sizeof sp); sp.lod_max = 1000.0f; @@ -418,6 +430,7 @@ void vk_exec_draw_shutdown(void) { vk_usnap_shutdown(); vkDestroyBufferView(c->device, Z.zview, NULL); vmaDestroyBuffer(c->vma, Z.zbuf, Z.zalloc); + if (Z.sink) vmaDestroyBuffer(c->vma, Z.sink, Z.sink_alloc); memset(&Z, 0, sizeof Z); } @@ -489,6 +502,87 @@ typedef struct ViewNode { struct ViewNode *next; ViewKey k; VkImageView view; } static nvn_hmap view_of; // ViewKey -> VkImageView static nvn_hmap views_by_image; // VkImage -> its first ViewNode +// Integer formats a texture view may give an image (nvnTextureViewSetFormat): the Vulkan format and the Ryujinx +// TextureFormat code (gfx_shader.c fmt_names) the translator declares a storage image with. A shader that packs texels +// itself writes raw words through such a view (Ocarina's last post-process pass stores R11G11B10 bits through an R32UI +// view of an R11G11B10F target); through a view of the texture's own format the store converted them (a black picture +// with red blobs). 0: not an integer format. +static int int_view_format(uint32_t nvn, VkFormat *vk) { + static const struct { uint16_t nvn; uint8_t code; VkFormat vk; } T[] = { + {NVN_FORMAT_R8UI, 3, VK_FORMAT_R8_UINT}, {NVN_FORMAT_R8I, 4, VK_FORMAT_R8_SINT}, + {NVN_FORMAT_R16UI, 8, VK_FORMAT_R16_UINT}, {NVN_FORMAT_R16I, 9, VK_FORMAT_R16_SINT}, + {NVN_FORMAT_R32UI, 11, VK_FORMAT_R32_UINT}, {NVN_FORMAT_R32I, 12, VK_FORMAT_R32_SINT}, + {NVN_FORMAT_RG8UI, 15, VK_FORMAT_R8G8_UINT}, {NVN_FORMAT_RG8I, 16, VK_FORMAT_R8G8_SINT}, + {NVN_FORMAT_RG16UI, 20, VK_FORMAT_R16G16_UINT}, {NVN_FORMAT_RG16I, 21, VK_FORMAT_R16G16_SINT}, + {NVN_FORMAT_RG32UI, 23, VK_FORMAT_R32G32_UINT}, {NVN_FORMAT_RG32I, 24, VK_FORMAT_R32G32_SINT}, + {NVN_FORMAT_RGBA8UI, 27, VK_FORMAT_R8G8B8A8_UINT}, {NVN_FORMAT_RGBA8I, 28, VK_FORMAT_R8G8B8A8_SINT}, + {NVN_FORMAT_RGBA16UI, 32, VK_FORMAT_R16G16B16A16_UINT}, {NVN_FORMAT_RGBA16I, 33, VK_FORMAT_R16G16B16A16_SINT}, + {NVN_FORMAT_RGBA32UI, 35, VK_FORMAT_R32G32B32A32_UINT}, {NVN_FORMAT_RGBA32I, 36, VK_FORMAT_R32G32B32A32_SINT}, + {NVN_FORMAT_RGB10A2UI, 38, VK_FORMAT_A2B10G10R10_UINT_PACK32}}; + for (size_t i = 0; i < sizeof T / sizeof *T; i++) + if (T[i].nvn == nvn) { + if (vk) *vk = T[i].vk; + return T[i].code; + } + return 0; +} + +// Bytes per texel of the formats a storage view may reinterpret (int_view_format's and the 32/64/128-bit float and +// unorm ones they alias); 0 unknown. +static int texel_bytes(VkFormat f) { + switch (f) { + case VK_FORMAT_R8_UINT: case VK_FORMAT_R8_SINT: case VK_FORMAT_R8_UNORM: return 1; + case VK_FORMAT_R16_UINT: case VK_FORMAT_R16_SINT: case VK_FORMAT_R16_SFLOAT: case VK_FORMAT_R16_UNORM: + case VK_FORMAT_R8G8_UINT: case VK_FORMAT_R8G8_SINT: case VK_FORMAT_R8G8_UNORM: return 2; + case VK_FORMAT_R32_UINT: case VK_FORMAT_R32_SINT: case VK_FORMAT_R32_SFLOAT: case VK_FORMAT_R16G16_UINT: + case VK_FORMAT_R16G16_SINT: case VK_FORMAT_R16G16_SFLOAT: case VK_FORMAT_R16G16_UNORM: case VK_FORMAT_R8G8B8A8_UINT: + case VK_FORMAT_R8G8B8A8_SINT: case VK_FORMAT_R8G8B8A8_UNORM: case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_A2B10G10R10_UINT_PACK32: case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_B10G11R11_UFLOAT_PACK32: + return 4; + case VK_FORMAT_R32G32_UINT: case VK_FORMAT_R32G32_SINT: case VK_FORMAT_R32G32_SFLOAT: case VK_FORMAT_R16G16B16A16_UINT: + case VK_FORMAT_R16G16B16A16_SINT: case VK_FORMAT_R16G16B16A16_SFLOAT: case VK_FORMAT_R16G16B16A16_UNORM: return 8; + case VK_FORMAT_R32G32B32A32_UINT: case VK_FORMAT_R32G32B32A32_SINT: case VK_FORMAT_R32G32B32A32_SFLOAT: return 16; + default: return 0; + } +} + +// R11G11B10F images written this frame as raw words (storage_view_format chose an integer view): Ocarina's last +// post-process pass stores shared-exponent RGB9E5 texels into its R11G11B10F scene target that way, and the upscaler +// that reads them (tensor-core, untranslatable) decodes them itself; the fallback blit must too (raw_9e5). +static struct { VkImage img; uint64_t frame; } RAW[8]; + +static void raw_note(VkImage img) { + uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + int k = 0; + for (int i = 0; i < 8; i++) { + if (RAW[i].img == img) { RAW[i].frame = f; return; } + if (RAW[i].frame < RAW[k].frame) k = i; + } + RAW[k].img = img, RAW[k].frame = f; +} + +static int raw_9e5(VkImage img, VkFormat f) { + if (f != VK_FORMAT_B10G11R11_UFLOAT_PACK32) return 0; + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + for (int i = 0; i < 8; i++) + if (RAW[i].img == img && now - RAW[i].frame <= 1) return 1; + return 0; +} + +// vk_dump.c: 1 when img was written as raw RGB9E5 words this frame (dump it decoded as such). +int vk_exec_raw_9e5(VkImage img, VkFormat f) { return raw_9e5(img, f); } + +// The format a storage view of img takes: the view's integer format when it sets one of the image's texel size (the +// image is MUTABLE_FORMAT, vk_texture.c), else the image's own. +static VkFormat storage_view_format(const VkTexImage *img, const view_t *v, int hv) { + VkFormat f; + if (hv && v->set_format && int_view_format(v->format, &f) && texel_bytes(f) && texel_bytes(f) == texel_bytes(img->format)) { + if (img->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32) raw_note(img->image); + return f; + } + return img->format; +} + // An image view of a registered texture for a shader resource of type `rtype` (GfxShaderRes.type); VK_NULL_HANDLE when // the image cannot serve it (the caller binds a stand-in). Texture lock held. static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, int hv, uint16_t rtype, int storage) { @@ -539,12 +633,13 @@ static VkImageView get_view(VkExec *ex, const VkTexImage *img, const view_t *v, } } uint32_t swz = (uint32_t)sw[0] | (uint32_t)sw[1] << 8 | (uint32_t)sw[2] << 16 | (uint32_t)sw[3] << 24; - ViewKey k = {img->image, img->format, vt, aspect, l0, nl, y0, ny, swz}; + VkFormat vf = storage ? storage_view_format(img, v, hv) : img->format; + ViewKey k = {img->image, vf, vt, aspect, l0, nl, y0, ny, swz}; if (!view_of.klen) nvn_hm_init(&view_of, sizeof(ViewKey)), nvn_hm_init(&views_by_image, sizeof(VkImage)); VkImageView found = (VkImageView)nvn_hm_get(&view_of, &k); if (found) return found; VkImageViewCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img->image, .viewType = vt, - .format = img->format, + .format = vf, .components = {swz_of(sw[0]), swz_of(sw[1]), swz_of(sw[2]), swz_of(sw[3])}, .subresourceRange = {aspect, l0, nl, y0, ny}}; VkImageViewUsageCreateInfo use = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO, @@ -610,14 +705,43 @@ void vk_exec_image_gone(VkImage image) { typedef struct { uint64_t serial; int stage; int used; GfxShaderAnswers a; const GfxShader *s; } ShaderEnt; static ShaderEnt shader_tab[SHADER_SETS][SHADER_WAYS]; +// Stage and code length of a control section. NVN1 (0x880 bytes): stage at +0x714, length at +0x704, c1 offset at +// +0x700, workgroup size at +0x748. NVN2 (Switch 2, 0xe0 bytes, magic 0x98761234 at +0): stage in bits 16+ of +0x54, +// length at +0x38, workgroup size at +0x58, no c1 part (gfx_shader.c, "NVN2"). +static int ctl_nvn2(const uint8_t *c) { + uint32_t magic; + memcpy(&magic, c, 4); + return magic == 0x98761234u; +} +static void ctl_stage_len(const uint8_t *c, uint32_t *st, uint32_t *len) { + if (ctl_nvn2(c)) { + uint32_t c1s, c1e; + memcpy(st, c + 0x54, 4), *st >>= 16; + memcpy(len, c + 0x38, 4); + // NVN2: the c1 constants (BRX jump tables) follow the code from +0x40 to +0x44, in the same storage; the + // length covers them so gfx_shader.c req_init sends them (with the code length alone it never could) + memcpy(&c1s, c + 0x40, 4), memcpy(&c1e, c + 0x44, 4); + if (c1s >= *len && c1e > c1s && c1e - *len < (1u << 20)) *len = c1e; + } else { + memcpy(st, c + 0x714, 4); + memcpy(len, c + 0x704, 4); + } +} +static uint32_t ctl_c1(const uint8_t *c, uint32_t len) { // offset of the c1 constants (len: none) + uint32_t c1 = len; + if (!ctl_nvn2(c)) memcpy(&c1, c + 0x700, 4); + else memcpy(&c1, c + 0x40, 4); // NVN2: after the code and its padding, up to +0x44 + return c1; +} +static const uint8_t *ctl_local(const uint8_t *c) { return c + (ctl_nvn2(c) ? 0x58 : 0x748); } + static int find_stage_scan(const prog_t *p, int nvn_stage, const uint8_t **ctl, const uint8_t **code, uint32_t *len) { for (uint32_t i = 0; i < p->nstage && i < 6; i++) { const uint8_t *c = p->sd[i].control; if (!c || !p->sd[i].data) continue; uint32_t st; - memcpy(&st, c + 0x714, 4); + ctl_stage_len(c, &st, len); if ((int)st != nvn_stage) continue; - memcpy(len, c + 0x704, 4); *ctl = c, *code = (const uint8_t *)(uintptr_t)p->sd[i].data; return 1; } @@ -639,9 +763,9 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const for (uint32_t i = 0; i < p->nstage && i < 6; i++) { const uint8_t *c = p->sd[i].control; if (!c || !p->sd[i].data) continue; - uint32_t st; - memcpy(&st, c + 0x714, 4); - if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, memcpy(&m->len[st], c + 0x704, 4); + uint32_t st, len; + ctl_stage_len(c, &st, &len); + if (st < 6 && m->idx[st] < 0) m->idx[st] = (int8_t)i, m->len[st] = len; } } int i = m->idx[nvn_stage]; @@ -653,7 +777,7 @@ static int find_stage(const prog_t *p, int nvn_stage, const uint8_t **ctl, const // Texture and image types of a stage's first 32 slots, each kept with the handle and the generation and slot token it // was looked up under: a slot is looked up again, a hash probe, only when the handle changed or its slot may resolve // otherwise (slot_still). -static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32]; } AC[6]; +static struct { uint64_t tex[32], img[32], ttok[32], itok[32]; uint32_t tgen[32], igen[32]; uint8_t tt[32], it[32], ifmt[32]; } AC[8]; // vk_texture_slot_token of the slot a texture or image handle names; an empty handle always resolves to nothing. static uint64_t token_of(uint64_t h) { return h ? vk_texture_slot_token((int)(h & 0xfffff)) : 1; } @@ -695,6 +819,22 @@ static uint8_t type_of_handle(uint64_t h) { // texture lock held return img ? (uint8_t)gfx_type_of(img, &v, hv) : 0; } +// The img_format answer of an image handle: the integer format its view gives it (int_view_format, as get_view binds +// it), else 0 (the translator's default, a float format). Texture lock held. +static uint8_t format_of_handle(uint64_t h) { + view_t v; + int hv; + const VkTexImage *img = h ? vk_texture_by_id_locked((int)(h & 0xfffff), &v, &hv) : NULL; + VkFormat f = VK_FORMAT_UNDEFINED; + static int logged; + if (img && hv && v.set_format && int_view_format(v.format, NULL) && logged < 8) + logged++, rt_log("vk: image view format: handle %#llx view nvn format %#x on vk format %d -> %d", + (unsigned long long)h, v.format, img->format, int_view_format(v.format, NULL)); + if (!img || !hv || !v.set_format || !int_view_format(v.format, &f) || texel_bytes(f) != texel_bytes(img->format) || !texel_bytes(f)) + return 0; + return (uint8_t)int_view_format(v.format, NULL); +} + // The answers of stage st for the slots in mask; the types of other slots stay zero. static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, GfxShaderAnswers *a) { memset(a, 0, sizeof *a); @@ -703,7 +843,7 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G else for (int k = 0; k < 3; k++) { uint32_t l; - memcpy(&l, ctl + 0x748 + 4 * k, 4); + memcpy(&l, ctl_local(ctl) + 4 * k, 4); a->local[k] = (uint16_t)(l ? l : 1); } typeof(AC[0]) *c = &AC[st]; @@ -720,10 +860,12 @@ static void answers_for(VkExec *ex, int st, const uint8_t *ctl, uint32_t mask, G uint64_t tt = tmiss ? token_of(h) : 0, it = imiss ? token_of(hi) : 0; if ((tmiss || imiss || check) && !locked) vk_texture_lock(), locked = 1, BS.ac_miss++; if (tmiss) c->tt[slot] = type_of_handle(h), c->tex[slot] = h, c->ttok[slot] = tt, c->tgen[slot] = gen; - if (imiss) c->it[slot] = type_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it, c->igen[slot] = gen; + if (imiss) + c->it[slot] = type_of_handle(hi), c->ifmt[slot] = format_of_handle(hi), c->img[slot] = hi, c->itok[slot] = it, + c->igen[slot] = gen; if (check && (type_of_handle(h) != c->tt[slot] || type_of_handle(hi) != c->it[slot])) // NX2NAVI_SHADER_MEMO=check rt_fatal("vk_exec: kept texture types of stage %d slot %d differ from a lookup", st, slot); - a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot]; + a->tex_type[slot] = c->tt[slot], a->img_type[slot] = c->it[slot], a->img_format[slot] = c->ifmt[slot]; } if (locked) vk_texture_unlock(); } @@ -759,7 +901,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin return set[0].s; } } - int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : GFX_STAGE_CS; + int gs = st == NVN_STAGE_VS ? GFX_STAGE_VS : st == NVN_STAGE_FS ? GFX_STAGE_FS : st == NVN_STAGE_TASK ? GFX_STAGE_TS + : st == NVN_STAGE_MESH ? GFX_STAGE_MS : GFX_STAGE_CS; const GfxShader *s = await ? gfx_shader_await(gs, ctl, code, len, &a) : pending && (vk_pipeline_async_on() || a.mvprev) ? gfx_shader_get_async(gs, ctl, code, len, &a, pending) : gfx_shader_get(gs, ctl, code, len, NULL, &a); @@ -777,7 +920,8 @@ static const GfxShader *shader_for(VkExec *ex, int st, int *present, int *pendin // ---- descriptors static int nvn_stage_of(int gfx_stage) { - return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS : NVN_STAGE_CS; + return gfx_stage == GFX_STAGE_VS ? NVN_STAGE_VS : gfx_stage == GFX_STAGE_FS ? NVN_STAGE_FS + : gfx_stage == GFX_STAGE_TS ? NVN_STAGE_TASK : gfx_stage == GFX_STAGE_MS ? NVN_STAGE_MESH : NVN_STAGE_CS; } static VkDescriptorBufferInfo zero_info(uint64_t max) { @@ -839,14 +983,23 @@ static void c0_table(uint8_t *at, const p_bind_buf b[16]) { // 16 bytes at 0x110 (vertex), 0x510 (fragment), 0x310 (compute): Ryujinx's GlobalToStorage fallback loads each // candidate base from here and compares the global pointer against it. Compute: dispatch size at 0x200, the uniform // buffer table at 0x210 + 16 r. Graphics: the largest point size (Maxwell's limit, GLSLC clamps gl_PointSize to it) at -// 0x610, the base instance at 0x644. Other stages: zeros. Unbound slots read zero, since the replay keeps them zero. +// 0x610, the base instance at 0x644. NVN2 task and mesh stages: storage tables at 0x670 and 0x770 (the game binds +// slots 0-2 there, and their programs read 0x670.. and 0x770..). Other stages: zeros. Unbound slots read zero, since the +// replay keeps them zero. // Layout: nxnative dm/gfx/nvn/nvn_mtl.m (MIT). +// Byte offset of a stage's storage buffer table in c0, -1 for none. +static int c0_ssbo_base(int stage) { + return stage == NVN_STAGE_VS ? 0x110 : stage == NVN_STAGE_FS ? 0x510 : stage == NVN_STAGE_CS ? 0x310 + : stage == NVN_STAGE_TASK ? 0x670 : stage == NVN_STAGE_MESH ? 0x770 : -1; +} + void vk_c0_fill(uint8_t bank[VK_C0_BYTES], int stage, const p_bind_buf ssbo[16], const p_bind_buf ubo[16], const uint32_t dispatch[3], uint32_t base_instance) { memset(bank, 0, VK_C0_BYTES); - int base = stage == NVN_STAGE_VS ? 0x110 : stage == NVN_STAGE_FS ? 0x510 : stage == NVN_STAGE_CS ? 0x310 : -1; + int base = c0_ssbo_base(stage); if (base < 0) return; c0_table(bank + base, ssbo); + if (stage == NVN_STAGE_TASK || stage == NVN_STAGE_MESH) return; if (stage == NVN_STAGE_CS) { memcpy(bank + 0x200, dispatch, 12); c0_table(bank + 0x210, ubo); @@ -866,6 +1019,10 @@ static int c0_on(void) { // NX2NAVI_FIX_C0=0: c0 reads zeros instead of the dr } // c0 from replay scratch; a bank equal to the stage's previous one reuses its range, so it costs no copy. +static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos); +static uint32_t nvn2_cs_tables_at(const uint8_t *ctl); +static uint32_t cs_push_bytes; // the size of the last PushComputeConstants (OP_PUSH_CC), replay thread + static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; // vk_c0_fill reads the storage slots, and the uniform slots and dispatch size for compute or the base instance for @@ -875,11 +1032,30 @@ static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { memcpy(in.ssbo, ex->ssbo[st], sizeof in.ssbo); if (st == NVN_STAGE_CS) memcpy(in.ubo, ex->ubo[st], sizeof in.ubo), memcpy(in.dim, c0_dim, sizeof in.dim); else in.binst = c0_binst; + // NVN2 compute: the bank holds the push constants from byte 0x10 (then the program's handle table, which the + // host does not read: textures and images are bound by slot). + const uint8_t *pctl, *pcode; + uint32_t plen; + int nvn2_push = st == NVN_STAGE_CS && ex->prog && find_stage(ex->prog, st, &pctl, &pcode, &plen) && ctl_nvn2(pctl); + if (nvn2_push) memcpy(in.push, ex->cs_push, sizeof in.push); if (B.c0_valid[k] && !memcmp(&in, &B.c0_in[k], sizeof in)) return (VkDescriptorBufferInfo){B.c0_buf[k], B.c0_off[k], VK_C0_BYTES}; B.c0_in[k] = in; uint8_t tmp[VK_C0_BYTES]; vk_c0_fill(tmp, st, ex->ssbo[st], ex->ubo[st], c0_dim, c0_binst); + if (nvn2_push) { + // the pushed constants from 0x10, the driver's tables after them (push_ptr, nvn2_cs_tables_at) + uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl); + nvn2_cs_tables(pctl, &nu, &ns); + uint32_t pn = ta - 0x10 < sizeof ex->cs_push ? ta - 0x10 : sizeof ex->cs_push; + memcpy(tmp + 0x10, ex->cs_push, pn < VK_C0_BYTES - 0x10 ? pn : VK_C0_BYTES - 0x10); + for (uint32_t k = 0; k < nu + ns && ta + 16 * k + 12 <= VK_C0_BYTES; k++) { + const p_bind_buf *sb = k < nu ? (k < 18 ? &ex->ubo[st][k] : NULL) : (k - nu < 16 ? &ex->ssbo[st][k - nu] : NULL); + if (!sb) continue; + uint32_t sz = (uint32_t)sb->size; + memcpy(tmp + ta + 16 * k, &sb->addr, 8), memcpy(tmp + ta + 16 * k + 8, &sz, 4); + } + } if (!B.c0_valid[k] || memcmp(tmp, B.c0[k], VK_C0_BYTES)) { memcpy(scratch(ex, VK_C0_BYTES, &B.c0_buf[k], &B.c0_off[k]), tmp, VK_C0_BYTES); memcpy(B.c0[k], tmp, VK_C0_BYTES); @@ -914,7 +1090,8 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { const uint8_t *ctl, *code; uint32_t len, c1; if (find_stage(ex->prog, st, &ctl, &code, &len)) { - memcpy(&c1, ctl + 0x700, 4); + c1 = ctl_c1(ctl, len); + if (ctl_nvn2(ctl)) memcpy(&len, ctl + 0x44, 4); // NVN2: c1 is [+0x40, +0x44) after the code if (c1 < len && bind_range(ex, (uint64_t)(uintptr_t)code + c1, len - c1, align, max, VK_MUSE_UBO, 0, &bi)) return bi; } @@ -938,7 +1115,7 @@ static VkDescriptorBufferInfo ubo_info(VkExec *ex, int st, int slot) { static int ssbo_slot(const GfxShaderRes *r, int st, int *ubo) { *ubo = 0; if (r->cbuf_slot != 0) return -1; - int off = r->handle * 4, base = st == NVN_STAGE_VS ? 0x110 : st == NVN_STAGE_FS ? 0x510 : st == NVN_STAGE_CS ? 0x310 : -1; + int off = r->handle * 4, base = c0_ssbo_base(st); if (st == NVN_STAGE_CS && off >= 0x210 && off < 0x310 && (off - 0x210) % 16 == 0) { *ubo = 1; return (off - 0x210) / 16; @@ -994,6 +1171,72 @@ static uint64_t ubo_ptr(const VkExec *ex, int st, const GfxShaderRes *r) { return a; } +// NVN2 compute c0 (fitted to the storage offsets of 2549 compute programs against their control sections, round 24): +// from byte 0x10 the constants the game pushed (PushComputeConstants), then a table of uniform buffers and one of +// storage buffers, {u64 address, u32 size} every 16 bytes (counts: bytes 1 and 2 of control +0xb0), then 8-byte entries +// (byte 3 of +0xb0, then byte 2 of +0xb4); +0xb8 (high half) is the end of it all, which gives the pushed size. Entry k +// of the storage part is storage slot k (BindStorageBuffer), entry k of the uniform part uniform slot k. Progs 98 and +// 99 (the compute after the composite: auto exposure) push 0x74 bytes, rounded to 0x80, and read storage entries 9 and 8 +// at c0+0x120 / +0x1c0; most programs push nothing and have their tables at 0x10. +static void nvn2_cs_tables(const uint8_t *ctl, uint32_t *ubos, uint32_t *ssbos) { + uint32_t w; + memcpy(&w, ctl + 0xb0, 4); + *ubos = w >> 8 & 0xff, *ssbos = w >> 16 & 0xff; +} + +// Where the tables start: after the pushed constants. +static uint32_t nvn2_cs_tables_at(const uint8_t *ctl) { + uint32_t b0, b4, b8; + memcpy(&b0, ctl + 0xb0, 4), memcpy(&b4, ctl + 0xb4, 4), memcpy(&b8, ctl + 0xb8, 4); + uint32_t rest = 16 * ((b0 >> 8 & 0xff) + (b0 >> 16 & 0xff)) + 8 * ((b0 >> 24) + (b4 >> 16 & 0xff)), end = b8 >> 16; + if (!(b0 >> 8 & 0xffff)) return 0x10 + 1024; // no tables: all of c0 from 0x10 is the pushed constants + return end >= 0x10 + rest ? end - rest : 0x10; +} + +// An NVN2 compute program that takes a storage buffer's address from c0 (c0_info): an entry of the driver's tables +// (the bound uniform or storage slot), or the {u64 address, u32 size} at that place of the constants the game pushed. +// 1 with *out filled; 0 when the program is not NVN2 or the offset is in neither. +static int push_ptr(const VkExec *ex, int st, const GfxShaderRes *r, p_bind_buf *out) { + const uint8_t *pctl, *pcode; + uint32_t plen; + uint32_t o = (uint32_t)r->handle * 4; // c0 byte offset + if (st != NVN_STAGE_CS || r->cbuf_slot != 0 || o < 0x10 || !ex->prog || !find_stage(ex->prog, st, &pctl, &pcode, &plen) || + !ctl_nvn2(pctl)) + return 0; + uint32_t nu, ns, ta = nvn2_cs_tables_at(pctl); + nvn2_cs_tables(pctl, &nu, &ns); + if (o < ta) { // in the pushed constants: a pointer the game passes itself + uint64_t a; + uint32_t n = 0; + if (o - 0x10 + 8 > sizeof ex->cs_push) return 0; + memcpy(&a, ex->cs_push + (o - 0x10), 8); + if (o - 0x10 + 12 <= sizeof ex->cs_push) memcpy(&n, ex->cs_push + (o - 0x10) + 8, 4); + *out = (p_bind_buf){0, 0, a, n < (1u << 30) ? n : 0}; // size 0: up to the end of the backing + return 1; + } + uint32_t t = (o - ta) / 16; + if ((o - ta) % 16 == 0 && t < nu + ns) { + *out = t < nu ? (t < 18 ? ex->ubo[st][t] : (p_bind_buf){0}) : (t - nu < 16 ? ex->ssbo[st][t - nu] : (p_bind_buf){0}); + return 1; + } + static int logged; // diagnosis: offsets past the tables + if (logged++ < 16) { + uint32_t b0, b4, b8; + memcpy(&b0, pctl + 0xb0, 4), memcpy(&b4, pctl + 0xb4, 4), memcpy(&b8, pctl + 0xb8, 4); + rt_log("vk: NVN2 compute c0+%#x is past the tables (prog %llu, +0xb0 %08x +0xb4 %08x +0xb8 %08x, tables at %#x)", o, + (unsigned long long)ex->prog->serial, b0, b4, b8, ta); + } + return 0; +} + +// vk_dump.c: the guest address a compute storage resource resolves to through c0 (push_ptr), 0 when none. +uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r) { + p_bind_buf up = {0}; + int ubo, slot = ssbo_slot(r, NVN_STAGE_CS, &ubo); + if (slot >= 0 && slot < 16) return ubo ? ex->ubo[NVN_STAGE_CS][slot].addr : ex->ssbo[NVN_STAGE_CS][slot].addr; + return push_ptr(ex, NVN_STAGE_CS, r, &up) ? up.addr : 0; +} + static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes *r, int written) { if (!__atomic_load_n(&LIM.ubo_max, __ATOMIC_ACQUIRE)) limits_init(); uint64_t max = LIM.sbo_max, align = LIM.sbo_align; @@ -1003,6 +1246,7 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * const p_bind_buf *b = NULL; if (slot >= 0 && slot < 16) b = ubo ? &ex->ubo[st][slot] : &ex->ssbo[st][slot]; else if (ubo_ptr_res(r)) up.addr = ubo_ptr(ex, st, r), b = &up; // size 0: up to the end of the backing + else if (push_ptr(ex, st, r, &up)) b = &up; if (b) { // vk_caps.c: shaders translated for sbuf_align address the buffer from its address rounded down to it. uint64_t lead = vk_caps.sbuf_align ? b->addr & (vk_caps.sbuf_align - 1) : 0; @@ -1016,7 +1260,31 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * } else { warn_once(W_SSBO_SLOT, "a storage buffer whose address is at c%d+%#x (stage %d) is not replayed: a zero buffer is bound", r->cbuf_slot, r->handle * 4, st); + static int logged; // diagnosis: what c1 of an NVN2 compute program is (its control section's c1 range, the + const uint8_t *ctl, *code; // uniform slots the game bound) + uint32_t len; + if (logged < 4 && r->cbuf_slot == 1 && ex->prog && find_stage(ex->prog, st, &ctl, &code, &len)) { + uint32_t c1s, c1e, size; + memcpy(&size, ctl + 0x38, 4), memcpy(&c1s, ctl + 0x40, 4), memcpy(&c1e, ctl + 0x44, 4); + char line[512]; + int o = snprintf(line, sizeof line, "vk: c1 storage address, prog %llu: code %#x, c1 [%#x, %#x), len %#x; c1 words:", + (unsigned long long)ex->prog->serial, size, c1s, c1e, len); + for (uint32_t k = 0; k < 4 && c1s + 4 * k + 4 <= len; k++) { + uint32_t wv; + memcpy(&wv, code + c1s + 4 * k, 4); + o += snprintf(line + o, sizeof line - o, " %08x", wv); + } + for (int k = 0; k < 6 && o < (int)sizeof line - 40; k++) + if (ex->ubo[st][k].addr) + o += snprintf(line + o, sizeof line - o, " ubo%d=%#llx+%#x", k + 3, (unsigned long long)ex->ubo[st][k].addr, + (unsigned)ex->ubo[st][k].size); + rt_log("%s", line); + logged++; + } } + // A shader that writes it gets the sink: the zero buffer also stands in for missing vertex buffers and read-only + // storage, and words written into it came back as garbage loop counts (a GPU hang at the first gameplay frame). + if (written && Z.sink) return (VkDescriptorBufferInfo){Z.sink, 0, max < ZERO_BYTES ? max : ZERO_BYTES}; return zero_info(max); } @@ -1249,6 +1517,217 @@ static void tex_info_full(VkExec *ex, const GfxShaderRes *r, uint64_t h, int sto *out = (VkDescriptorImageInfo){separate || storage ? VK_NULL_HANDLE : samp, view, layout}; } +// ---- texture and sampler pools (bindless) +// Shaders that take texture handles from memory (TEX.B and kin) index the whole texture pool: the translator declares +// arrays of VK_POOL_TEX images and VK_POOL_SAMP samplers (Ryujinx BindlessElimination GenerateBindlessAccess, index = +// handle & 0xfffff and handle >> 20), which vk_pipeline.c spirv_pool_rebind moves to set 2, one image binding per view +// kind (vk.h). One set serves every such pipeline: entry id of binding k holds texture slot id viewed as kind k (a +// stand-in when the slot is empty or cannot be viewed so), entry id of binding VK_POOL_KINDS sampler slot id. pool_sync +// rewrites the entries whose slot changed (vk_texture_slot_token) and fills the kinds a new pipeline asked for; with +// update-after-bind (descriptorIndexing) a write never invalidates a command buffer being recorded. Each submit's first +// sync also looks every registered slot up once, the freshness check a bound texture gets (vk_texture_by_id_locked). +// The entries are not double-buffered: a slot registered again while a frame using it is in flight is read new by that +// frame, as on the Switch, where the pool is memory the GPU reads as it goes. +volatile uint32_t vk_pool_kinds_used; +static struct { + VkDescriptorSetLayout layout, empty; + VkDescriptorPool dpool; + VkDescriptorSet set; + int ready, uab; + uint32_t kinds, gen, epoch, sgen; + int retry; // the last sync left slots to the creation budget + uint64_t *tok; // per texture slot: vk_texture_slot_token it was filled under (0: stand-ins) + VkImageView *view; // [kind * VK_POOL_TEX + id]: what the entry holds + VkSampler *samp; // [id] + uint64_t writes; +} PL; + +static VkDescriptorSetLayout pool_layout_make(void) { + VkCtx *c = vk_ctx(); + VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES}; + VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12}; + vkGetPhysicalDeviceFeatures2(c->phys, &f2); // create_device enables every supported feature + PL.uab = f12.descriptorBindingSampledImageUpdateAfterBind && f12.descriptorBindingPartiallyBound && + f12.descriptorBindingUpdateUnusedWhilePending; + VkDescriptorSetLayoutBinding b[VK_POOL_KINDS + 1]; + VkDescriptorBindingFlags fl[VK_POOL_KINDS + 1]; + VkDescriptorBindingFlags f = (f12.descriptorBindingPartiallyBound ? VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT : 0) | + (PL.uab ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT | VK_DESCRIPTOR_BINDING_UPDATE_UNUSED_WHILE_PENDING_BIT : 0); + for (uint32_t k = 0; k <= VK_POOL_KINDS; k++) { + b[k] = (VkDescriptorSetLayoutBinding){k, k < VK_POOL_KINDS ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, + k < VK_POOL_KINDS ? VK_POOL_TEX : VK_POOL_SAMP, VK_SHADER_STAGE_ALL, NULL}; + fl[k] = f; + } + VkDescriptorSetLayoutBindingFlagsCreateInfo bf = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO, + .bindingCount = VK_POOL_KINDS + 1, .pBindingFlags = fl}; + VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, .pNext = &bf, + .flags = PL.uab ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0, + .bindingCount = VK_POOL_KINDS + 1, .pBindings = b}; + VkDescriptorSetLayout l; + if (vkCreateDescriptorSetLayout(c->device, &li, NULL, &l) != VK_SUCCESS) rt_fatal("vk: the texture pool layout failed"); + VkDescriptorPoolSize ps[2] = {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, VK_POOL_KINDS * VK_POOL_TEX}, + {VK_DESCRIPTOR_TYPE_SAMPLER, VK_POOL_SAMP}}; + VkDescriptorPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, + .flags = PL.uab ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0, + .maxSets = 1, .poolSizeCount = 2, .pPoolSizes = ps}; + if (vkCreateDescriptorPool(c->device, &pi, NULL, &PL.dpool) != VK_SUCCESS) rt_fatal("vk: the texture pool descriptor pool failed"); + VkDescriptorSetAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, .descriptorPool = PL.dpool, + .descriptorSetCount = 1, .pSetLayouts = &l}; + if (vkAllocateDescriptorSets(c->device, &ai, &PL.set) != VK_SUCCESS) rt_fatal("vk: the texture pool set failed"); + PL.tok = calloc(VK_POOL_TEX, sizeof *PL.tok); + PL.view = calloc((size_t)VK_POOL_KINDS * VK_POOL_TEX, sizeof *PL.view); + PL.samp = calloc(VK_POOL_SAMP, sizeof *PL.samp); + if (!PL.tok || !PL.view || !PL.samp) rt_fatal("vk: out of memory"); + rt_log("vk: bindless texture pools: %d kinds x %d textures, %d samplers, %s", VK_POOL_KINDS, VK_POOL_TEX, VK_POOL_SAMP, + PL.uab ? "update after bind" : "no update-after-bind on this GPU (entries written while bound are undefined)"); + return l; +} + +static PlatMutex pool_lock_ = PLAT_MUTEX_INIT; + +VkDescriptorSetLayout vk_pool_set_layout(void) { + plat_mutex_lock(&pool_lock_); + if (!PL.layout) PL.layout = pool_layout_make(); + plat_mutex_unlock(&pool_lock_); + return PL.layout; +} + +VkDescriptorSetLayout vk_empty_set_layout(void) { + plat_mutex_lock(&pool_lock_); + if (!PL.empty) { + VkDescriptorSetLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + if (vkCreateDescriptorSetLayout(vk_ctx()->device, &li, NULL, &PL.empty) != VK_SUCCESS) rt_fatal("vk: empty set layout failed"); + } + plat_mutex_unlock(&pool_lock_); + return PL.empty; +} + +// Ryujinx sampler type (GfxShaderRes.type) of pool kind k. +static uint16_t pool_kind_type(int k) { + static const uint16_t dim[4] = {1, 3, 4, 5}; // 1D, 2D, 3D, CUBE + return (uint16_t)(dim[k >> 1 & 3] | (k & 1 ? 0x100 : 0) | (k & 8 ? 0x400 : 0)); +} + +#define POOL_WBATCH 512 +#define POOL_CREATE_BUDGET 64 // texture entries a pool sync may create (images and uploads), the rest next time +static VkWriteDescriptorSet pool_w[POOL_WBATCH]; +static VkDescriptorImageInfo pool_ii[POOL_WBATCH]; +static uint32_t pool_nw; + +static void pool_flush(void) { + if (pool_nw) vkUpdateDescriptorSets(vk_ctx()->device, pool_nw, pool_w, 0, NULL); + PL.writes += pool_nw; + pool_nw = 0; +} + +static void pool_write(uint32_t binding, uint32_t id, VkImageView view, VkSampler samp) { + if (pool_nw == POOL_WBATCH) pool_flush(); + pool_ii[pool_nw] = (VkDescriptorImageInfo){samp, view, VK_IMAGE_LAYOUT_GENERAL}; + pool_w[pool_nw] = (VkWriteDescriptorSet){.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstSet = PL.set, + .dstBinding = binding, .dstArrayElement = id, .descriptorCount = 1, + .descriptorType = view ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, + .pImageInfo = &pool_ii[pool_nw]}; + pool_nw++; +} + +// Entry id of every kind in mask: the slot's view of that kind or the kind's stand-in. Texture lock held. 0: the +// slot's texture was left to a later sync by the creation budget (pool_sync). +// NX2NAVI_SKIP_SHADERS=,...: draws and dispatches whose shader's SPIR-V has one of these FNV-1a hashes (the "vs"/"fs"/ +// "cs" hashes of the frame dump's passes.txt) are skipped (diagnosis: which pass makes an effect). +static int shader_skipped(const GfxShader *sh) { + static uint64_t list[16]; + static int n = -1; + static struct { const GfxShader *s; uint8_t skip; } memo[256]; + if (n < 0) { + n = 0; + const char *e = cfg_get("NX2NAVI_SKIP_SHADERS"); + for (const char *p = e; p && *p && n < 16;) { + char *end; + uint64_t v = strtoull(p, &end, 16); + if (end == p) break; + list[n++] = v; + p = *end ? end + 1 : end; + } + if (n) rt_log("vk: NX2NAVI_SKIP_SHADERS: %d shader hashes skipped", n); + } + if (!n || !sh) return 0; + uint32_t i = (uint32_t)(((uintptr_t)sh * 0x9E3779B97F4A7C15ull) >> 56); + if (memo[i].s == sh) return memo[i].skip; + uint64_t words = 0, h = 0xcbf29ce484222325ull; // as vk_dump.c shader_hash + const uint8_t *b = (const uint8_t *)gfx_shader_spirv(sh, &words); + for (uint64_t k = 0; b && k < words * 4; k++) h = (h ^ b[k]) * 0x100000001b3ull; + uint8_t skip = 0; + for (int k = 0; k < n; k++) skip |= list[k] == h; + memo[i].s = sh, memo[i].skip = skip; + return skip; +} + +// NX2NAVI_POOL=0: draws and dispatches of pipelines that index the pools are skipped (A/B against bindless shaders). +static int pool_on(void) { + static int on = -1; + if (on < 0) on = rt_knob("POOL", "1"); + return on; +} + +static int pool_fill(VkExec *ex, uint32_t id, uint32_t mask, uint64_t tok) { + vk_texture_create_deferred_locked(); + for (int k = 0; k < VK_POOL_KINDS; k++) { + if (!(mask >> k & 1)) continue; + GfxShaderRes r = {0}; + r.type = pool_kind_type(k); + VkDescriptorImageInfo ii; + VkImage img; + tex_info_full(ex, &r, tok ? id | 1ull << 63 : 0, 0, 1, &ii, &img); + VkImageView *e = &PL.view[(size_t)k * VK_POOL_TEX + id]; + if (*e != ii.imageView) *e = ii.imageView, pool_write((uint32_t)k, id, ii.imageView, VK_NULL_HANDLE); + } + return !vk_texture_create_deferred_locked(); +} + +static void pool_sync(VkExec *ex) { + if (!PL.layout) vk_pool_set_layout(); + uint32_t used = __atomic_load_n(&vk_pool_kinds_used, __ATOMIC_RELAXED); + uint32_t gen = vk_texture_slots_gen(), ep = vk_texture_epoch(), sg = vk_sampler_gen(); + if (PL.ready && !PL.retry && used == PL.kinds && gen == PL.gen && ep == PL.epoch && sg == PL.sgen) return; + vk_texture_lock(); + uint32_t fresh = used & ~PL.kinds; + int n = vk_texture_slot_count_locked(); + if (n > VK_POOL_TEX) n = VK_POOL_TEX; + if (!PL.ready) { // every entry a stand-in first: partially bound arrays are still read out of range by guests + for (uint32_t id = 0; id < VK_POOL_TEX; id++) pool_fill(ex, id, 0xffffu, 0); + for (uint32_t id = 0; id < VK_POOL_SAMP; id++) PL.samp[id] = Z.samp, pool_write(VK_POOL_KINDS, id, VK_NULL_HANDLE, Z.samp); + PL.ready = 1, PL.sgen = 0; + } + int retry = 0; // slots whose texture the creation budget left: filled again by the next sync + if (fresh || gen != PL.gen || ep != PL.epoch || PL.retry) { + vk_texture_create_budget_locked(POOL_CREATE_BUDGET); + for (int id = 0; id < n; id++) { // slots past n were never registered: stand-ins since the first sync + uint64_t t = vk_texture_slot_token(id); + if (t != PL.tok[id]) { + if (pool_fill(ex, (uint32_t)id, used, t)) PL.tok[id] = t; + else retry = 1; + } else if (t && fresh) { + pool_fill(ex, (uint32_t)id, fresh, t); + } else if (t && ep != PL.epoch) { + vk_texture_by_id_locked(id, NULL, NULL); // the submit's freshness check + } + } + vk_texture_create_budget_locked(-1); + } + PL.retry = retry; + if (sg != PL.sgen) { + int ns = vk_sampler_slot_count_locked(); + for (int id = 0; id < VK_POOL_SAMP; id++) { + VkSampler s = id < ns ? vk_sampler_by_id_locked(id) : VK_NULL_HANDLE; + if (!s) s = Z.samp; + if (s != PL.samp[id]) PL.samp[id] = s, pool_write(VK_POOL_KINDS, (uint32_t)id, VK_NULL_HANDLE, s); + } + } + pool_flush(); + vk_texture_unlock(); + PL.kinds = used, PL.gen = gen, PL.epoch = ep, PL.sgen = sg; +} + // ---- depth twins: colour images under a compare sampler in a compute shader // NX2navi's compute passes sample an R32_SFLOAT texture through a shadow sampler. Maxwell compares any single-channel // float or unorm texel; Vulkan only formats with SAMPLED_IMAGE_DEPTH_COMPARISON (VUID-vkCmdDispatch-None-06479), which @@ -1891,6 +2370,11 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { } } } + if (info->pool) { // set 2: the texture and sampler pools + pool_sync(ex); + vkCmdBindDescriptorSets(ex->cb, info->bind_point, info->layout, 2, 1, &PL.set, 0, NULL); + RS.volatile_ = 1; // the pools may change under an equal draw: build the next one again + } if (skip) { RS.rec = 0; // An equal next draw records nothing when every push set holds its push (NX2NAVI_PUSH_SKIP on) and nothing made @@ -2087,8 +2571,56 @@ static int pipe_memo_on(void) { } // Pass, shaders, pipeline, fixed state and resources for a draw. 0: the draw is skipped. -static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { - if (ex->render_disabled) return 0; +// ---- draw accounting: why draws were not drawn +// Every draw prepare_draw_ex turns away is counted by reason, vertex and mesh draws apart. The first draw of each kind +// skipped for each reason is logged with its program, and every DRAW_STATS_FRAMES presented frames one line sums up +// the draws issued and skipped since the last line (only when something was skipped). A black screen with draws +// skipped for "shader failed" or "pipeline failed" points at the translator or the pipeline; draws issued and still +// nothing on screen point at the shaders' results (NX2NAVI_DUMP). +enum { SK_DISABLED, SK_SCISSOR, SK_NO_PROGRAM, SK_NO_VERTEX_STAGE, SK_SHADER_PENDING, SK_SHADER_FAILED, SK_DLAA, + SK_PIPE_PENDING, SK_PIPE_FAILED, SK_POOL_OFF, SK_COUNT }; +static const char *const sk_names[SK_COUNT] = { + "rendering disabled (render condition)", "empty scissor", "no program bound", + "no vertex (or mesh) stage in the program", "shader still translating", + "shader failed to translate (see gfx_shader: translation failed)", "replaced by the upscaler", + "pipeline still building", "pipeline could not be created", "uses the texture pools (NX2NAVI_POOL=0)"}; +#define DRAW_STATS_FRAMES 600 +static struct { uint64_t issued[2], skipped[2][SK_COUNT], frame; uint32_t first[2]; } DRS; + +static void draw_stats_tick(void) { + uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (!DRS.frame) DRS.frame = f; + if (f - DRS.frame < DRAW_STATS_FRAMES) return; + for (int m = 0; m < 2; m++) { + uint64_t any = 0; + for (int k = 0; k < SK_COUNT; k++) any += DRS.skipped[m][k]; + if (!any) continue; + char line[512]; + int o = snprintf(line, sizeof line, "vk: %s draws over %llu frames: %llu issued, skipped:", m ? "mesh" : "vertex", + (unsigned long long)(f - DRS.frame), (unsigned long long)DRS.issued[m]); + for (int k = 0; k < SK_COUNT && o < (int)sizeof line - 60; k++) + if (DRS.skipped[m][k]) o += snprintf(line + o, sizeof line - o, " %s %llu;", sk_names[k], (unsigned long long)DRS.skipped[m][k]); + rt_log("%s", line); + } + memset(&DRS, 0, sizeof DRS.issued + sizeof DRS.skipped); + DRS.frame = f; +} + +static int draw_skipped(const VkExec *ex, int mesh, int why) { + DRS.skipped[mesh][why]++; + if (!(DRS.first[mesh] >> why & 1)) { + DRS.first[mesh] |= 1u << why; + rt_log("vk: first %s draw skipped because: %s (program %llu)", mesh ? "mesh" : "vertex", sk_names[why], + ex->prog ? (unsigned long long)ex->prog->serial : 0ull); + } + draw_stats_tick(); + return 0; +} + +// mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing +// vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. +static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int mesh) { + if (ex->render_disabled) return draw_skipped(ex, mesh, SK_DISABLED); cb_res(ex); dummies_init(); if (ex->pass_noatt) { // no attachment: the pass covers this draw's viewport or scissor, reopened when it needs more @@ -2100,10 +2632,11 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { if (!ex->in_pass) ex->dirty |= VK_EXEC_DIRTY_VIEWPORT | VK_EXEC_DIRTY_SCISSOR; vk_exec_pass_begin(ex); VkRect2D sc = vk_exec_scissor(ex); - if (!sc.extent.width || !sc.extent.height) return 0; - if (!ex->prog) return warn_once(W_PROGRAM, "a draw without a bound program is skipped"), 0; - int has_vs, has_fs, vs_wait = 0, fs_wait = 0, memo = shader_memo_on(); - const GfxShader *vs, *fs; + if (!sc.extent.width || !sc.extent.height) return draw_skipped(ex, mesh, SK_SCISSOR); + if (!ex->prog) return draw_skipped(ex, mesh, SK_NO_PROGRAM); + int has_vs, has_fs, vs_wait = 0, fs_wait = 0, memo = mesh ? 0 : shader_memo_on(); + int has_ts = 0, ts_wait = 0, vst = mesh ? NVN_STAGE_MESH : NVN_STAGE_VS; + const GfxShader *vs, *fs, *ts = NULL; uint32_t gen = memo ? vk_texture_slots_gen() : 0; // before shader_for, whose lookups may bump it if (memo && SM.valid && memo_match(ex, gen, &SM.k)) { vs = SM.vs, fs = SM.fs, has_vs = 1, has_fs = SM.has_fs; @@ -2115,27 +2648,31 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { } } else { SM.valid = 0; - vs = shader_for(ex, NVN_STAGE_VS, &has_vs, &vs_wait, 0); + vs = shader_for(ex, vst, &has_vs, &vs_wait, 0); + if (mesh) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, &ts_wait, 0); fs = shader_for(ex, NVN_STAGE_FS, &has_fs, &fs_wait, 0); if (memo && vs && !vs_wait && !fs_wait && (!has_fs || fs)) memo_key(ex, gen, &SM.k), SM.vs = vs, SM.fs = fs, SM.has_fs = has_fs, SM.valid = 1; } - if (!has_vs) return warn_once(W_NO_VS, "a draw with a program that has no vertex stage is skipped"), 0; + if (!has_vs) return draw_skipped(ex, mesh, SK_NO_VERTEX_STAGE); // A draw into a target the previous frame did not render (or into none) waits for a pending shader or pipeline: // one-shot draws (volumes and LUTs generated once, later only sampled) would leave their target empty for the // session. A draw into per-frame targets is skipped until they are ready and comes back next frame. int oneshot = -1, skip = pipe_skip_mode(); // -1: not looked up - if (vs_wait || fs_wait) { - if (!(oneshot = !vk_exec_targets_recent(ex, skip >= 1))) return perf_count(PERF_CNT_SHADER_SKIP), 0; // skipped until ready + if (vs_wait || fs_wait || ts_wait) { + if (!(oneshot = !vk_exec_targets_recent(ex, skip >= 1))) // skipped until ready + return perf_count(PERF_CNT_SHADER_SKIP), draw_skipped(ex, mesh, SK_SHADER_PENDING); perf_count(PERF_CNT_ONESHOT_WAIT); - if (vs_wait) vs = shader_for(ex, NVN_STAGE_VS, &has_vs, NULL, 1); + if (vs_wait) vs = shader_for(ex, vst, &has_vs, NULL, 1); + if (ts_wait) ts = shader_for(ex, NVN_STAGE_TASK, &has_ts, NULL, 1); if (fs_wait) fs = shader_for(ex, NVN_STAGE_FS, &has_fs, NULL, 1); } - if (!vs || (has_fs && !fs)) return 0; - if (vk_dlaa_skip_draw(ex, fs)) return 0; // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c) + if (!vs || (has_fs && !fs) || (has_ts && !ts)) return draw_skipped(ex, mesh, SK_SHADER_FAILED); + if (shader_skipped(vs) || shader_skipped(fs)) return draw_skipped(ex, mesh, SK_POOL_OFF); // NX2NAVI_SKIP_SHADERS + if (vk_dlaa_skip_draw(ex, fs)) return draw_skipped(ex, mesh, SK_DLAA); // the game's FXAA off, or its scaling replaced by DLSS (vk_dlaa.c) uint32_t fbl = feedback_prepare(ex, fs); VkPipeState s = ex->pipe; - s.vs = vs, s.fs = fs; + s.vs = vs, s.fs = fs, s.ts = ts; s.feedback = (uint8_t)((fbl & 0xff ? 1 : 0) | (fbl & 0x100 ? 2 : 0)); s.prim = (uint8_t)prim; s.depth_minus_w = (uint8_t)(nvn_depth_minus_w() != 0); @@ -2195,8 +2732,9 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { (unsigned long long)__atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + 1, waited / 1e6, why, pending ? ", skipped after all" : "", count, ex->pass_w, ex->pass_h, tg); } - if (pending) return 0; // a background worker is building it: the draw is skipped until it is ready - if (!p || !info) return warn_once(W_PIPELINE, "a graphics pipeline could not be created: draws with it are skipped"), 0; + if (pending) return draw_skipped(ex, mesh, SK_PIPE_PENDING); // a background worker is building it: skipped until it is ready + if (!p || !info) return draw_skipped(ex, mesh, SK_PIPE_FAILED); + if (info->pool && !pool_on()) return draw_skipped(ex, mesh, SK_POOL_OFF); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_GRAPHICS, p); if (!B.st_valid || memcmp(&B.st, &s, sizeof s)) { vk_pipeline_set_dynamic(ex->cb, &s); @@ -2204,21 +2742,29 @@ static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { B.st = s, B.st_valid = 1; if (ex->mark && vk_exec_frame.draws == ex->mark_draws0) ex->mark->first[0] = vs, ex->mark->first[1] = fs; } - vk_dlaa_draw(ex); - vk_mvsurvey_draw(ex); - vk_mvprev_draw(ex); + if (!mesh) { + vk_dlaa_draw(ex); + vk_mvsurvey_draw(ex); + vk_mvprev_draw(ex); + } record_fixed(ex); repeat_stats(ex, info); bind_resources(ex, info); if (fbl & ex->fb_dirty) feedback_barrier(ex); feedback_written(ex, &s); - bind_vertex_buffers(ex); - if (vk_dump_armed) vk_dump_draw(ex, &s, p, info); + if (!mesh) bind_vertex_buffers(ex); + if (vk_dump_armed && !mesh) vk_dump_draw(ex, &s, p, info); if (vk_hdr_track) vk_hdr_draw(ex, info); vk_exec_oq_draw(ex); + DRS.issued[mesh]++; + draw_stats_tick(); return 1; } +static int prepare_draw(VkExec *ex, int prim, int32_t count, int restart) { + return prepare_draw_ex(ex, prim, count, restart, 0); +} + // ---- per-object motion // The camera pass (vk_dlaa.c mv_pass) gives everything the camera's motion, so a character that moves with the camera, // or on its own, gets the wrong one and DLSS throws its history away there. Each solid scene draw whose position reads @@ -2364,7 +2910,7 @@ static VkWriteDescriptorSet mo_write(const VkPipeBinding *b, const VkDescSlot *d // would hold the same); a pushed set is pushed by mo_emit. 0 for a pipeline with storage buffers, images or texel // buffers (not drawn again). static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q) { - if (info->n_bindings > MAX_BINDINGS) return 0; + if (info->n_bindings > MAX_BINDINGS || info->pool) return 0; uint32_t total = 0; for (uint32_t i = 0; i < info->n_bindings; i++) total += info->bindings[i].count ? info->bindings[i].count : 1; if (total > MAX_DESC) return 0; @@ -2975,6 +3521,73 @@ int vk_mvobj_execute(VkExec *ex, uint64_t cam, VkImageView mv, VkImageView depth return 1; } +// NVN2 mesh draws (DrawMeshTasks): count task workgroups, or mesh workgroups for a program without a task stage, through +// VK_EXT_mesh_shader. The stages are the translator's TaskEXT / MeshEXT modules (tools/xlate/MeshSpirv.cs). +static PFN_vkCmdDrawMeshTasksEXT draw_mesh_fn(void) { + static PFN_vkCmdDrawMeshTasksEXT fn; + static int looked; + if (!looked) { + looked = 1; + if (vk_ctx()->has_mesh_shader) fn = (PFN_vkCmdDrawMeshTasksEXT)vkGetDeviceProcAddr(vk_ctx()->device, "vkCmdDrawMeshTasksEXT"); + } + return fn; +} + +static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { + PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); + if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + static int on = -1; + if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) + if (!d->count || !on) return; + c0_binst = 0; + if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; + if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); + vk_exec_frame.draws++; + static uint32_t n_logged; + if (n_logged < 8 && rt_knob("LOG_MESH", "0")) { + n_logged++; + rt_log("vk: mesh draw: %u workgroups, program %llu", d->count, (unsigned long long)ex->prog->serial); + } + rec_draw_mesh(fn, ex->cb, d->count, 1, 1); + ex->recorded++; +} + +// DrawMeshTasksIndirect: the arguments are written by the GPU earlier in the frame (they never come back to guest +// memory), so the draw reads them from the device copy, taken as VkDrawMeshTasksIndirectCommandEXT {x, y, z}. If the +// game's layout were NV's {count, first} instead, y would be 0 and the draw would draw nothing. One draw; x2-x4 of the +// call vary like leftovers, so they are not read as a count or stride. +static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { + static PFN_vkCmdDrawMeshTasksIndirectEXT fn; + static int looked; + if (!looked) { + looked = 1; + if (vk_ctx()->has_mesh_shader) + fn = (PFN_vkCmdDrawMeshTasksIndirectEXT)vkGetDeviceProcAddr(vk_ctx()->device, "vkCmdDrawMeshTasksIndirectEXT"); + } + if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + static int on = -1; + if (on < 0) on = rt_knob("MESH", "1"); + if (!on) return; + VkMemSpan ip; + if (!vk_mem_resolve(d->ind, 12, &ip) || ip.offset % 4) + return warn_once(W_INDIRECT, "indirect mesh draw arguments at %#llx are not backed or not aligned: skipped", + (unsigned long long)d->ind); + c0_binst = 0; + if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; + vk_exec_frame.draws++; + static uint32_t n_logged; + if (n_logged < 8 && rt_knob("LOG_MESH", "0")) { + n_logged++; + rt_log("vk: indirect mesh draw: arguments at %#llx, program %llu", (unsigned long long)d->ind, + (unsigned long long)ex->prog->serial); + } + VK_MEM_USE(ip); + rec_draw_mesh_ind(fn, ex->cb, ip.buffer, ip.offset, 1, 12); + ex->recorded++; + vk_mem_count(ip.cls, VK_MUSE_IND, 12); + VK_MEM_BOUND(ip); +} + static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) { VkBuffer b; VkDeviceSize o; @@ -3095,6 +3708,240 @@ static void op_draw_idx_ind(VkExec *ex, const p_draw_idx_ind *d) { // ---- dispatch and uniform buffer updates +// A compute dispatch whose shader cannot be translated: Switch 2 titles run tensor-core networks (HMMA/LDSM, an +// AI upscaler: Ocarina's title movie goes 1920x1080 -> 2560x1440 through ten such passes) that the Maxwell translator +// has no model for. Instead of leaving the outputs untouched (black), the largest bound storage image receives a +// linear blit of the largest bound sampled RGBA8 texture that is a different image and no larger (the networks' +// 10-bit intermediates are only partly meaningful as pictures, so they are not a source): what an upscaler +// approximates. In-place passes (the output also sampled) are left alone. NX2NAVI_CS_FALLBACK=0 turns it off. +// sRGB-ness of the RGBA8 formats the fallback blits: a blit converts between encodings, the game's network writes the +// bits through a UNORM view, so a UNORM source and an sRGB target go through a scratch image in the source's format +// (blit, scaling only) and then a raw copy. +static int fmt_srgb(VkFormat f) { return f == VK_FORMAT_R8G8B8A8_SRGB || f == VK_FORMAT_B8G8R8A8_SRGB || f == VK_FORMAT_A8B8G8R8_SRGB_PACK32; } +static int fmt_rgba8(VkFormat f) { + return f == VK_FORMAT_R8G8B8A8_UNORM || f == VK_FORMAT_R8G8B8A8_SRGB || f == VK_FORMAT_B8G8R8A8_UNORM || + f == VK_FORMAT_B8G8R8A8_SRGB || f == VK_FORMAT_A8B8G8R8_UNORM_PACK32 || f == VK_FORMAT_A8B8G8R8_SRGB_PACK32; +} + +// Scratch images for those blits, one per (size, format), kept for the life of the process (a handful). +static VkImage fallback_scratch(VkExec *ex, uint32_t w, uint32_t h, VkFormat f) { + static struct { uint32_t w, h; VkFormat f; VkImage img; VmaAllocation a; } cache[8]; + static int n; + for (int i = 0; i < n; i++) if (cache[i].w == w && cache[i].h == h && cache[i].f == f) return cache[i].img; + if (n == 8) return VK_NULL_HANDLE; + VkCtx *c = vk_ctx(); + VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, .format = f, + .extent = {w, h, 1}, .mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, + .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | + (f == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 && rt_knob("TM_DISPATCH", "1") != 2 ? VK_IMAGE_USAGE_SAMPLED_BIT : 0), + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; + VmaAllocationCreateInfo ia = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + VkImage img; VmaAllocation a; + if (vmaCreateImage(c->vma, &ii, &ia, &img, &a, NULL) != VK_SUCCESS) return VK_NULL_HANDLE; + VkImageMemoryBarrier2 bar = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = img, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &bar}; + vkCmdPipelineBarrier2(ex->cb, &di); + cache[n].w = w, cache[n].h = h, cache[n].f = f, cache[n].img = img, cache[n].a = a, n++; + return img; +} + +// 1 when the bound compute program uses tensor-core instructions (HMMA 0x23c, IMMA 0x237, LDSM 0x83b): the game's +// upscalers (the movie's and the 3D scene's), which the translator cannot run and the fallback blit stands in for. +// Other untranslated programs (bindless texturing, indirect branches) bind the output image too, and a blit for them +// pasted G-buffer normals and intermediates over the picture. Kept per program serial. +static int uses_tensor_ops(const VkExec *ex) { + static struct { uint64_t serial; int8_t v; } memo[256]; + const uint8_t *ctl, *code; + uint32_t len; + if (!ex->prog || !find_stage(ex->prog, NVN_STAGE_CS, &ctl, &code, &len)) return 0; + uint64_t sr = ex->prog->serial; + typeof(memo[0]) *m = &memo[(sr * 0x9E3779B97F4A7C15ull) >> 56]; + if (sr && m->serial == sr) return m->v; + int v = 0; + if (ctl_nvn2(ctl)) + for (uint32_t o = 0; o + 16 <= len && !v; o += 16) { + uint32_t op = (uint32_t)(code[o] | code[o + 1] << 8) & 0xfff; + v = op == 0x23c || op == 0x237 || op == 0x83b; + } + m->serial = sr, m->v = (int8_t)v; + return v; +} + +// NX2NAVI_NATIVE_TENSOR=1: tensor-core programs run their translation (HMMA, LDSM and MOVM lowered to warp shuffles in +// tools/xlate/AmpereLowering.cs Hmma, Ldsm, Movm). Off by default: the lowered networks do not match the hardware yet +// (Ocarina's scene upscaler wrote infinities and noise), so they take the fallback below as untranslated ones do. +static int native_tensor(const VkExec *ex) { + static int on = -1; + if (on < 0) on = rt_knob("NATIVE_TENSOR", "0"); + return on || !uses_tensor_ops(ex); +} + +static void untranslated_dispatch_fallback(VkExec *ex) { + static int on = -1; + if (on < 0) on = rt_knob("CS_FALLBACK", "1"); + if (!on || !uses_tensor_ops(ex)) return; + const VkTexImage *dst = NULL, *src = NULL; + vk_texture_lock(); + for (int i = 0; i < 32; i++) { + uint64_t h = ex->img[NVN_STAGE_CS][i]; + view_t v; int hv = 0; + const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL; + if (t && t->aspect == VK_IMAGE_ASPECT_COLOR_BIT && t->type == VK_IMAGE_TYPE_2D && t->samples == 1 && + (!dst || (uint64_t)t->width * t->height > (uint64_t)dst->width * dst->height)) + dst = t; + } + for (int i = 0; dst && i < 128; i++) { // an in-place pass (its output also sampled) is left alone + uint64_t h = ex->tex[NVN_STAGE_CS][i]; + view_t v; int hv = 0; + const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL; + if (t && t->image == dst->image) { dst = NULL; break; } + } + for (int i = 0; dst && i < 128; i++) { + uint64_t h = ex->tex[NVN_STAGE_CS][i]; + view_t v; int hv = 0; + const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL; + // RGBA8 sources (the movie's frames; its 10-bit intermediates gave black bands), or one in the output's own + // format: the 3D scene (r11g11b10f) the game's upscaler enlarges to the output resolution + int rgba8 = t && (t->format == VK_FORMAT_R8G8B8A8_UNORM || t->format == VK_FORMAT_R8G8B8A8_SRGB); + int same = t && t->format == dst->format; + if (t && (rgba8 || same) && t->image != dst->image && t->aspect == VK_IMAGE_ASPECT_COLOR_BIT && t->type == VK_IMAGE_TYPE_2D && + t->samples == 1 && t->width <= dst->width && t->height <= dst->height && t->width >= 16 && t->height >= 16 && + (!src || (uint64_t)t->width * t->height > (uint64_t)src->width * src->height)) + src = t; + } + if (dst && src) vk_texture_storage_bound_locked(dst); + if (dst && src && cfg_get("NX2NAVI_LOG_CS_BINDS")) { // each distinct (program, source, target) once, 256 at most + static struct { uint64_t prog, s, d; } seen[256]; + static int n; + uint64_t pr = ex->prog ? ex->prog->serial : 0; + int k = 0; + while (k < n && (seen[k].prog != pr || seen[k].s != src->addr || seen[k].d != dst->addr)) k++; + if (k == n && n < 256) seen[n++] = (typeof(seen[0])){pr, src->addr, dst->addr}, k = -1; + if (k < 0) { + rt_log("vk_exec: fallback blit prog %llu: %ux%u fmt %d at %#llx -> %ux%u fmt %d at %#llx", + (unsigned long long)(ex->prog ? ex->prog->serial : 0), src->width, src->height, src->format, + (unsigned long long)src->addr, dst->width, dst->height, dst->format, (unsigned long long)dst->addr); + char line[1600]; // everything the untranslated program binds: what a stand-in could use instead + int o = 0; + for (int i = 0; i < 160 && o < (int)sizeof line - 80; i++) { + int im = i >= 128; + uint64_t h = im ? ex->img[NVN_STAGE_CS][i - 128 < 32 ? i - 128 : 0] : ex->tex[NVN_STAGE_CS][i]; + if (im && i - 128 >= 32) break; + view_t v; int hv = 0; + const VkTexImage *t = h ? vk_texture_by_id_locked(NVN_HANDLE_TEX(h), &v, &hv) : NULL; + if (t && (i == 0 || h != ex->tex[NVN_STAGE_CS][0] || im)) + o += snprintf(line + o, sizeof line - o, " %s%d=%ux%u/f%d@%#llx", im ? "img" : "tex", im ? i - 128 : i, + t->width, t->height, t->format, (unsigned long long)t->addr); + } + for (int i = 0; i < 18 && o < (int)sizeof line - 60; i++) + if (ex->ubo[NVN_STAGE_CS][i].size) + o += snprintf(line + o, sizeof line - o, " ubo%d=%#llx+%#x", i + 3, + (unsigned long long)ex->ubo[NVN_STAGE_CS][i].addr, (unsigned)ex->ubo[NVN_STAGE_CS][i].size); + rt_log("vk_exec: its bindings:%s", line); + } + } + VkImage si = src ? src->image : VK_NULL_HANDLE, di = dst ? dst->image : VK_NULL_HANDLE; + VkImageView dst_view = VK_NULL_HANDLE; // for the tonemap pass, while the lock keeps the image + if (src && dst && (dst->usage & VK_IMAGE_USAGE_STORAGE_BIT) && raw_9e5(src->image, src->format) && + dst->format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && rt_knob("TM_DISPATCH", "1") != 2) { + view_t nv; + memset(&nv, 0, sizeof nv); + dst_view = get_view(ex, dst, &nv, 0, 3, 1); // 3: Ryujinx Texture2D + } + uint32_t sw = src ? src->width : 0, sh = src ? src->height : 0, dw = dst ? dst->width : 0, dh = dst ? dst->height : 0; + VkFormat sf = src ? src->format : VK_FORMAT_UNDEFINED, df = dst ? dst->format : VK_FORMAT_UNDEFINED; + vk_texture_unlock(); + if (!si || !di) return; + static int logged; + if (!logged++) rt_log("vk_exec: untranslated compute dispatch: its output image (%ux%u) gets a linear blit of its " + "largest input (%ux%u) instead (NX2NAVI_CS_FALLBACK=0 turns this off)", dw, dh, sw, sh); + vk_exec_pass_end(ex); + vk_exec_barrier(ex); + VkImageBlit bl = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}}; + bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1}; + bl.dstOffsets[1] = (VkOffset3D){(int32_t)dw, (int32_t)dh, 1}; + VkImage scratch = fmt_rgba8(df) && fmt_srgb(sf) != fmt_srgb(df) ? fallback_scratch(ex, dw, dh, sf) : VK_NULL_HANDLE; + static int e5_blit = -1; // E5B9G9R9 can be a blit source (with linear filtering) + if (e5_blit < 0) { + VkFormatProperties fp; + vkGetPhysicalDeviceFormatProperties(vk_ctx()->phys, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, &fp); + VkFormatFeatureFlags need = VK_FORMAT_FEATURE_BLIT_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT; + e5_blit = (fp.optimalTilingFeatures & need) == need ? 1 + !!(fp.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT) : 0; + rt_log("vk_exec: RGB9E5 fallback decode %s", e5_blit ? "on" : "unavailable (E5B9G9R9 is no blit source here)"); + } + VkImage e5 = !scratch && e5_blit && raw_9e5(si, sf) ? fallback_scratch(ex, sw, sh, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32) : VK_NULL_HANDLE; + if (e5) { // the raw words into a shared-exponent image (same texel size: a copy keeps the bits), then the blit + VkImageCopy cp = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .extent = {sw, sh, 1}}; + vkCmdCopyImage(ex->cb, si, VK_IMAGE_LAYOUT_GENERAL, e5, VK_IMAGE_LAYOUT_GENERAL, 1, &cp); + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT}; + VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(ex->cb, &dep); + si = e5; + } +#if NX2NAVI_TONEMAP_STANDIN + // Off: every build with this call lost the device right after the first gameplay load, even with the call never + // reached at run time (NX2NAVI_TM_DISPATCH=2/3), while the build without it ran clean; not understood yet (round 24). + VkImage tm = e5 && dst_view ? vk_tonemap_record(ex->cb, e5, VK_FORMAT_E5B9G9R9_UFLOAT_PACK32, dw, dh) : VK_NULL_HANDLE; + if (tm) { // the tonemapped picture, already at the target's size, then the same blit as below + B.pipe[1] = VK_NULL_HANDLE; // the replay must bind its compute pipeline again + si = tm, sw = dw, sh = dh; + bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1}; + } +#else + VkImage tm = VK_NULL_HANDLE; + (void)dst_view; +#endif + // The scene upscaler's input is the post pass's picture already sRGB-encoded (prog 102 packs encode(composite) + // into RGB9E5), and its output is linear: the HUD is drawn over it and the frame's last pass encodes to sRGB. A + // plain blit carried the encoded values into the linear target, so the picture was encoded twice (washed out). + // The stand-in decodes: scale into an RGBA8 UNORM scratch (the encoded values), copy those bits into an RGBA8 sRGB + // scratch, and blit that into the target, which reads the sRGB image as linear. + int filter_nearest = tm || (e5 && e5_blit == 1); + VkImage g8 = VK_NULL_HANDLE, g8s = VK_NULL_HANDLE; + if (e5 && !tm && !scratch && !fmt_rgba8(df)) { + g8 = fallback_scratch(ex, dw, dh, VK_FORMAT_R8G8B8A8_UNORM); + g8s = g8 ? fallback_scratch(ex, dw, dh, VK_FORMAT_R8G8B8A8_SRGB) : VK_NULL_HANDLE; + } + if (g8 && g8s) { + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT}; + VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdBlitImage(ex->cb, si, VK_IMAGE_LAYOUT_GENERAL, g8, VK_IMAGE_LAYOUT_GENERAL, 1, &bl, + filter_nearest ? VK_FILTER_NEAREST : VK_FILTER_LINEAR); + vkCmdPipelineBarrier2(ex->cb, &dep); + VkImageCopy cp = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .extent = {dw, dh, 1}}; + vkCmdCopyImage(ex->cb, g8, VK_IMAGE_LAYOUT_GENERAL, g8s, VK_IMAGE_LAYOUT_GENERAL, 1, &cp); + vkCmdPipelineBarrier2(ex->cb, &dep); + si = g8s, sw = dw, sh = dh, filter_nearest = 1; + bl.srcOffsets[1] = (VkOffset3D){(int32_t)sw, (int32_t)sh, 1}; + } + vkCmdBlitImage(ex->cb, si, VK_IMAGE_LAYOUT_GENERAL, scratch ? scratch : di, VK_IMAGE_LAYOUT_GENERAL, 1, &bl, + filter_nearest ? VK_FILTER_NEAREST : VK_FILTER_LINEAR); + if (scratch) { // the bits as they are: a UNORM picture stays the same picture in an sRGB image's memory + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT}; + VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(ex->cb, &dep); + VkImageCopy cp = {.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, .dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, + .extent = {dw, dh, 1}}; + vkCmdCopyImage(ex->cb, scratch, VK_IMAGE_LAYOUT_GENERAL, di, VK_IMAGE_LAYOUT_GENERAL, 1, &cp); + } + ex->recorded++; + vk_exec_barrier(ex); +} + static void op_dispatch(VkExec *ex, const p_dispatch *d) { if (d->x <= 0 || d->y <= 0 || d->z <= 0) return; if (!ex->prog) return warn_once(W_PROGRAM, "a dispatch without a bound program is skipped"); @@ -3103,10 +3950,28 @@ static void op_dispatch(VkExec *ex, const p_dispatch *d) { dummies_init(); int present; const GfxShader *cs = shader_for(ex, NVN_STAGE_CS, &present, NULL, 0); // synchronous: a skipped dispatch changes results - if (!cs) return; + if (cfg_get("NX2NAVI_LOG_CS_BINDS")) { // diagnosis: the compute bindings of the first dispatch of each program + static uint64_t seen[64]; static int nseen; + int known = 0; + for (int i = 0; i < nseen; i++) known |= seen[i] == ex->prog->serial; + if (!known && nseen < 64) { + seen[nseen++] = ex->prog->serial; + char line[2048]; int o = snprintf(line, sizeof line, "cs binds prog %llu %dx%dx%d:", (unsigned long long)ex->prog->serial, d->x, d->y, d->z); + for (int i = 0; i < 128 && o < 1900; i++) if (ex->tex[NVN_STAGE_CS][i]) o += snprintf(line + o, sizeof line - o, " tex%d=%llx", i, (unsigned long long)ex->tex[NVN_STAGE_CS][i]); + for (int i = 0; i < 32 && o < 1900; i++) if (ex->img[NVN_STAGE_CS][i]) o += snprintf(line + o, sizeof line - o, " img%d=%llx", i, (unsigned long long)ex->img[NVN_STAGE_CS][i]); + const GfxShaderRes *rs; + int nt = cs ? gfx_shader_resources(cs, GFX_RES_TEX, &rs) : 0; + for (int i = 0; i < nt && o < 1900; i++) o += snprintf(line + o, sizeof line - o, " want-tex(h%u,slot%d)", rs[i].handle, rs[i].slot); + int ni = cs ? gfx_shader_resources(cs, GFX_RES_IMG, &rs) : 0; + for (int i = 0; i < ni && o < 1900; i++) o += snprintf(line + o, sizeof line - o, " want-img(h%u,slot%d)", rs[i].handle, rs[i].slot); + rt_log("%s%s", line, cs ? "" : " (not translated)"); + } + } + if (!cs || !native_tensor(ex)) { untranslated_dispatch_fallback(ex); return; } const VkPipeInfo *info = NULL; VkPipeline p = vk_pipeline_compute(cs, &info); if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped"); + if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS vk_exec_barrier(ex); // ends the pass; earlier writes are visible to the shader bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); c0_dim[0] = (uint32_t)d->x, c0_dim[1] = (uint32_t)d->y, c0_dim[2] = (uint32_t)d->z; @@ -3131,10 +3996,11 @@ static void op_dispatch_ind(VkExec *ex, const p_dispatch_ind *d) { dummies_init(); int present; const GfxShader *cs = shader_for(ex, NVN_STAGE_CS, &present, NULL, 0); - if (!cs) return; + if (!cs || !native_tensor(ex)) { untranslated_dispatch_fallback(ex); return; } const VkPipeInfo *info = NULL; VkPipeline p = vk_pipeline_compute(cs, &info); if (!p || !info) return warn_once(W_PIPELINE, "a compute pipeline could not be created: dispatches with it are skipped"); + if ((info->pool && !pool_on()) || shader_skipped(cs)) return; // NX2NAVI_POOL=0, NX2NAVI_SKIP_SHADERS vk_exec_barrier(ex); VK_MEM_USE(ip); bind_pipeline(ex, VK_PIPELINE_BIND_POINT_COMPUTE, p); @@ -3182,7 +4048,7 @@ static void op_update_ub(VkExec *ex, const p_update_ub *u) { // ---- bind ops static void bind_handles(uint64_t *dst, int cap, const p_bind_handles *b) { - if (b->stage < 0 || b->stage >= 6 || b->first < 0 || b->n < 0 || b->n > 32 || b->first + b->n > cap) { + if (b->stage < 0 || b->stage >= 8 || b->first < 0 || b->n < 0 || b->n > 32 || b->first + b->n > cap) { warn_once(W_BIND, "a bind of %d handles at %d of stage %d is out of range: ignored", b->n, b->first, b->stage); return; } @@ -3236,37 +4102,59 @@ void vk_exec_op_draw(VkExec *ex, uint32_t op, const void *payload) { } case OP_BIND_UBO: { const p_bind_buf *b = payload; - if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 18) ex->ubo[b->stage][b->index] = *b; + if (b->stage >= 0 && b->stage < 8 && b->index >= 0 && b->index < 18) ex->ubo[b->stage][b->index] = *b; + else if (b->index >= 18 && b->index < 64) { // NVN2 has more banks than Maxwell's 18: say which, once each + static uint64_t seen; + if (!(__atomic_fetch_or(&seen, 1ull << b->index, __ATOMIC_RELAXED) >> b->index & 1)) + rt_log("vk: uniform buffer bind to index %d (bank %d), stage %d, %llu bytes: beyond Maxwell's banks, dropped", + b->index, b->index + 3, b->stage, (unsigned long long)b->size); + } bind_gen++; break; } case OP_BIND_SSBO: { const p_bind_buf *b = payload; - if (b->stage >= 0 && b->stage < 6 && b->index >= 0 && b->index < 16) ex->ssbo[b->stage][b->index] = *b; + if (b->stage >= 0 && b->stage < 8 && b->index >= 0 && b->index < 16) ex->ssbo[b->stage][b->index] = *b; + if (b->stage >= NVN_STAGE_TASK && b->stage <= NVN_STAGE_MESH && b->index >= 0 && b->index < 16) { // once per slot + static uint32_t seen; + uint32_t bit = 1u << (16 * (b->stage - NVN_STAGE_TASK) + b->index); + if (!(__atomic_fetch_or(&seen, bit, __ATOMIC_RELAXED) & bit)) + rt_log("vk: storage buffer bind, stage %d slot %d: %#llx, %llu bytes", b->stage, b->index, + (unsigned long long)b->addr, (unsigned long long)b->size); + } bind_gen++; break; } case OP_BIND_TEX: { const p_bind_handles *b = payload; - bind_handles(b->stage >= 0 && b->stage < 6 ? ex->tex[b->stage] : NULL, 128, b); + bind_handles(b->stage >= 0 && b->stage < 8 ? ex->tex[b->stage] : NULL, 128, b); ex->tex_serial++; bind_gen++; break; } case OP_BIND_SAMP: { const p_bind_handles *b = payload; - bind_handles(b->stage >= 0 && b->stage < 6 ? ex->samp[b->stage] : NULL, 128, b); + bind_handles(b->stage >= 0 && b->stage < 8 ? ex->samp[b->stage] : NULL, 128, b); break; } case OP_BIND_IMG: { const p_bind_handles *b = payload; - bind_handles(b->stage >= 0 && b->stage < 6 ? ex->img[b->stage] : NULL, 32, b); + bind_handles(b->stage >= 0 && b->stage < 8 ? ex->img[b->stage] : NULL, 32, b); ex->tex_serial++; // the shader memo's handle check (feedback_scan reads only textures: a rescan at most) bind_gen++; break; } case OP_UPDATE_UB: op_update_ub(ex, payload); break; + case OP_PUSH_CC: { + const p_push_cc *p = payload; + if (p->offset + (uint64_t)p->size <= sizeof ex->cs_push) memcpy(ex->cs_push + p->offset, p->data, p->size); + cs_push_bytes = p->offset + p->size; + bind_gen++; // storage buffers may come from these words (push_ptr) + break; + } case OP_DRAW: op_draw(ex, payload); break; + case OP_DRAW_MESH: vk_exec_frame.draws_seen++; op_draw_mesh(ex, payload); break; + case OP_DRAW_MESH_IND: vk_exec_frame.draws_seen++; op_draw_mesh_ind(ex, payload); break; case OP_DRAW_IDX: op_draw_idx(ex, payload); break; case OP_DISPATCH: op_dispatch(ex, payload); break; case OP_DRAW_IDX_IND: op_draw_idx_ind(ex, payload); break; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index b8ee1dad..c218288f 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -96,17 +96,68 @@ static void write_counter(uint64_t addr, uint64_t counter) { memcpy((void *)(uintptr_t)addr, rec, sizeof rec); } +// Reports other than samples passed (timestamps; the statistics counters are not modelled) write a counter of 0. The +// counter word matters: Ocarina writes {1, 0} into a record from the CPU, reports TIMESTAMP_TOP into it and gates +// clears on it differing from a zero record (mode 1), clears that must not run. +static void pending_written(uint64_t addr, uint64_t seq); // below, with the pending table + +typedef struct { uint64_t addr, seq; } CounterWrite; + static void counter_cb(void *arg) { - uint64_t *addr = arg; - write_counter(*addr, 0); - free(addr); + CounterWrite *w = arg; + write_counter(w->addr, 0); + pending_written(w->addr, w->seq); + free(w); } // Reports queued by the replay that the GPU has not finished. The condition of RENDER_COND reads a timestamp one as 0 // and a samples-passed one as unknown. -#define PENDING 256 -static struct { uint64_t addr, fence; int unknown; } pending[PENDING]; -static unsigned pending_head; +// The newest report per address, in a table probed over PENDING_PROBE slots from the address's hash: Ocarina issues +// hundreds of occlusion reports a frame and conditions draws on them in the same frame, so a ring of the last 256 +// reports had dropped the report a condition referred to, which then read an older query's count (often 0) and culled +// visible ground and trees. It also rotates through tens of thousands of record addresses, so an entry whose value is +// in memory (written >= seq: a condition reads memory anyway) gives its slot to a new address; a table that only grew +// filled up after a minute and conditions on the newer records read the CPU's 1 again. +#define PENDING (1u << 16) +#define PENDING_PROBE 32 +// seq: the newest report to addr; written: the newest whose value the completion thread has put in guest memory. A +// report counts as pending until then, not until its fence passes: the callback runs after the fence, and in between +// the record still held what the CPU wrote (Ocarina re-arms its condition records with 1 for the next frame), so a +// condition read 1 where the GPU had written 0, a depth clear ran after the depth copy and frames showed only sky. +typedef struct { uint64_t addr, seq; volatile uint64_t written; int unknown, type; } Pending; +static Pending pending[PENDING]; +static uint64_t report_seq; + +static Pending *pending_slot(uint64_t addr, int insert); + +// Completion thread: the value of report seq to addr is in guest memory. +static void pending_written(uint64_t addr, uint64_t seq) { + Pending *e = pending_slot(addr, 0); + if (!e) return; + uint64_t w = __atomic_load_n(&e->written, __ATOMIC_RELAXED); + while (w < seq && !__atomic_compare_exchange_n(&e->written, &w, seq, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {} +} + +// Lookups run on the replay thread and the completion thread; only the replay thread inserts. +static Pending *pending_slot(uint64_t addr, int insert) { + if (!addr) return NULL; + uint32_t i = (uint32_t)((addr * 0x9E3779B97F4A7C15ull) >> 48) & (PENDING - 1); + Pending *free_ = NULL, *victim = NULL; + for (unsigned n = 0; n < PENDING_PROBE; n++, i = (i + 1) & (PENDING - 1)) { + Pending *e = &pending[i]; + uint64_t a = __atomic_load_n(&e->addr, __ATOMIC_ACQUIRE); + if (a == addr) return e; + if (!insert) continue; + if (!a) { if (!free_) free_ = e; continue; } + if (__atomic_load_n(&e->written, __ATOMIC_ACQUIRE) >= e->seq && (!victim || e->seq < victim->seq)) victim = e; + } + Pending *e = free_ ? free_ : victim; + if (!e) return NULL; // every probed entry still has a report in flight + e->seq = 0, e->unknown = 0, e->type = -1; + __atomic_store_n(&e->written, 0, __ATOMIC_RELEASE); + __atomic_store_n(&e->addr, addr, __ATOMIC_RELEASE); + return e; +} // ---- occlusion counts (samples passed) // @@ -137,7 +188,7 @@ static uint32_t oq_nseg, oq_capseg; static int oq_lost; // replay: the segment missed a draw (ring full) static uint64_t oq_count; // completion thread: samples since the last reset -typedef struct { uint64_t addr; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment; +typedef struct { uint64_t addr, seq; int reset, lost; uint32_t n; uint32_t q[]; } OqSegment; static int oq_ready(void) { if (oq_state) return oq_state > 0; @@ -231,7 +282,7 @@ static void oq_cb(void *arg) { else { oq_count += sum; if (s->lost && !oq_count) oq_count = 1; - if (s->addr) write_counter(s->addr, oq_count); + if (s->addr) write_counter(s->addr, oq_count), pending_written(s->addr, s->seq); } free(s); } @@ -241,7 +292,7 @@ static void oq_segment(VkExec *ex, uint64_t addr, int reset) { oq_close(ex); OqSegment *s = malloc(sizeof *s + sizeof *s->q * oq_nseg); if (!s) rt_fatal("vk: out of memory"); - s->addr = addr, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; + s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); oq_nseg = 0, oq_lost = 0; vk_completion_add(ex->fence, oq_cb, s); @@ -274,6 +325,7 @@ static void signal_event(VkExec *ex, const p_signal_event *p) { static void report_counter(VkExec *ex, const p_report_counter *p) { int samples = p->type == NVN_COUNTER_TYPE_SAMPLES_PASSED && oq_ready(); + report_seq++; // before oq_report / counter_cb take it if (nvn_gpu_only(p->addr)) { // CPU-invisible pool: no host memory behind it if (samples) oq_report(ex, 0); return; @@ -284,23 +336,23 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData)); if (samples) oq_report(ex, p->addr); else { - uint64_t *a = malloc(sizeof *a); + CounterWrite *a = malloc(sizeof *a); if (!a) rt_fatal("vk: out of memory"); - *a = p->addr; + *a = (CounterWrite){p->addr, report_seq}; vk_completion_add(ex->fence, counter_cb, a); } - pending[pending_head++ % PENDING] = (typeof(pending[0])){p->addr, ex->fence, samples}; + Pending *e = pending_slot(p->addr, 1); + if (e) e->seq = report_seq, e->unknown = samples, e->type = (int)p->type; + else LOG_ONCE("vk: a counter report's table neighbourhood is all in flight: conditions on %#llx read memory", + (unsigned long long)p->addr); } // ---- conditional rendering -// 1: *v read, 0: not readable, 2: a samples-passed report not done yet (unknown) +// 1: *v read, 0: not readable, 2: a samples-passed report not in memory yet (unknown) static int counter_word(uint64_t addr, uint64_t *v) { - uint64_t done = vk_sync_fence_done(); - for (unsigned k = 1; k <= PENDING; k++) { // newest first: the last report to addr decides - unsigned i = (pending_head - k) % PENDING; - if (pending[i].addr == addr && pending[i].fence > done) return *v = 0, pending[i].unknown ? 2 : 1; - } + const Pending *e = pending_slot(addr, 0); // the last report to addr decides + if (e && __atomic_load_n(&e->written, __ATOMIC_ACQUIRE) < e->seq) return *v = 0, e->unknown ? 2 : 1; VkMemSpan m; if (nvn_gpu_only(addr) || !vk_mem_resolve(addr, sizeof *v, &m)) return 0; memcpy(v, (const void *)(uintptr_t)addr, sizeof *v); @@ -308,6 +360,12 @@ static int counter_word(uint64_t addr, uint64_t *v) { return 1; } +// The counter type of the newest report to addr the ring still holds, -1 when there is none (NX2NAVI_LOG_RCOND). +static int last_type(uint64_t addr) { + const Pending *e = pending_slot(addr, 0); + return e ? e->type : -1; +} + static int cond_enabled(const p_render_cond *c) { uint64_t x, y; int rx = counter_word(c->addr, &x), ry = counter_word(c->addr + 16, &y); @@ -316,7 +374,24 @@ static int cond_enabled(const p_render_cond *c) { return 1; } if (rx == 2 || ry == 2) return 1; // a samples-passed count not done yet - return c->mode == 0 ? x == y : x != y; + int on = c->mode == 0 ? x == y : x != y; + // NX2NAVI_LOG_RCOND=1: one condition in 499 (up to 300 lines), on or off, with the counter types last reported there + static int log_on = -1, nlog; + static unsigned ncond; + if (log_on < 0) log_on = rt_knob("LOG_RCOND", "0"); + if (log_on && ncond++ % 499 == 0 && nlog < 300) { + nlog++; + // (pending: the report value has not reached guest memory yet) + int px = 0, py = 0; // the word came from a report not done yet (forced to 0) rather than memory + const Pending *ex_ = pending_slot(c->addr, 0), *ey_ = pending_slot(c->addr + 16, 0); + px = ex_ && ex_->written < ex_->seq, py = ey_ && ey_->written < ey_->seq; + const uint64_t *m = (const uint64_t *)(uintptr_t)c->addr; + rt_log("vk: render condition %s at %#llx mode %d: %llu (type %d%s) vs %llu (type %d%s); memory %llx %llx %llx %llx", + on ? "on" : "off", (unsigned long long)c->addr, c->mode, (unsigned long long)x, last_type(c->addr), px ? ", pending" : "", + (unsigned long long)y, last_type(c->addr + 16), py ? ", pending" : "", (unsigned long long)m[0], + (unsigned long long)m[1], (unsigned long long)m[2], (unsigned long long)m[3]); + } + return on; } // ---- textures diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index eccf9d19..4c2f3359 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -877,10 +877,70 @@ static void pin_subgroup32(VkPipelineShaderStageCreateInfo *st, VkPipelineShader st->pNext = rq; } +// Pool arrays (the translator's bindless texture and sampler arrays of VK_POOL_TEX / VK_POOL_SAMP elements, found by +// their types) move to set 2, at the binding of their view kind (vk.h VK_POOL_KINDS) or VK_POOL_KINDS for samplers; +// the kinds used are noted in vk_pool_kinds_used so the replay fills them. NULL: nothing to move (or no memory). +static uint32_t *spirv_pool_rebind(const uint32_t *w, uint64_t n) { + enum { OP_TYPE_IMAGE = 25, OP_TYPE_SAMPLER = 26, OP_TYPE_ARRAY_ = 28 }; + uint32_t bound = n >= 5 && w[3] < (1u << 22) ? w[3] : 0; + if (!bound) return NULL; + int32_t *kind = malloc(bound * sizeof *kind); // per id: image kind, 100 sampler, -1 other (types, then variables) + uint32_t *cval = calloc(bound, sizeof *cval); + if (!kind || !cval) return free(kind), free(cval), NULL; + memset(kind, 0xff, bound * sizeof *kind); + int any = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) break; + if (op == OP_TYPE_IMAGE && len >= 9 && w[i + 1] < bound && w[i + 7] == 1) { // sampled image + uint32_t dim = w[i + 3], depth = w[i + 4], arrayed = w[i + 5], ms = w[i + 6]; + if (dim <= 3 && !ms) kind[w[i + 1]] = (int32_t)((depth == 1 ? 8 : 0) + dim * 2 + (arrayed ? 1 : 0)); + } else if (op == OP_TYPE_SAMPLER && len >= 2 && w[i + 1] < bound) { + kind[w[i + 1]] = 100; + } else if (op == SPV_OP_CONSTANT && len >= 4 && w[i + 2] < bound) { + cval[w[i + 2]] = w[i + 3]; + } else if (op == OP_TYPE_ARRAY_ && len >= 4 && w[i + 1] < bound && w[i + 2] < bound && w[i + 3] < bound) { + int32_t k = kind[w[i + 2]]; + uint32_t want = k == 100 ? VK_POOL_SAMP : VK_POOL_TEX; + kind[w[i + 1]] = k >= 0 && cval[w[i + 3]] == want ? k + 1000 : -1; // + 1000: a pool array + } else if (op == SPV_OP_TYPE_POINTER && len >= 4 && w[i + 1] < bound && w[i + 3] < bound) { + kind[w[i + 1]] = kind[w[i + 3]] >= 1000 ? kind[w[i + 3]] : -1; + } else if (op == SPV_OP_VARIABLE && len >= 4 && w[i + 1] < bound && w[i + 2] < bound) { + kind[w[i + 2]] = kind[w[i + 1]] >= 1000 ? kind[w[i + 1]] + 1000 : -1; // + 2000: a pool variable + if (kind[w[i + 2]] >= 2000) any = 1; + } + i += len; + } + uint32_t *o = NULL; + if (any && (o = malloc(n * 4))) { + memcpy(o, w, n * 4); + uint32_t used = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = o[i] & 0xffff, len = o[i] >> 16; + if (!len || i + len > n) break; + if (op == SPV_OP_DECORATE && len >= 4 && o[i + 1] < bound && kind[o[i + 1]] >= 2000) { + int32_t k = kind[o[i + 1]] - 2000; + if (o[i + 2] == SPV_DEC_DESCRIPTOR_SET) o[i + 3] = 2; + else if (o[i + 2] == SPV_DEC_BINDING) o[i + 3] = k == 100 ? VK_POOL_KINDS : (uint32_t)k; + if (k != 100) used |= 1u << k; + } + i += len; + } + __atomic_fetch_or(&vk_pool_kinds_used, used, __ATOMIC_RELAXED); + } + free(kind), free(cval); + return o; +} + static VkShaderModule make_module(const uint32_t *w, uint64_t words) { uint64_t fixed_n = 0; uint32_t *fixed = spirv_fixup(w, words, &fixed_n); if (fixed) w = fixed, words = fixed_n; + uint32_t *pooled = spirv_pool_rebind(w, words); + if (pooled) { + free(fixed); + w = fixed = pooled; + } VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w}; VkShaderModule m = VK_NULL_HANDLE; VkResult r = vkCreateShaderModule(P.device, &ci, NULL, &m); @@ -894,6 +954,7 @@ typedef struct { uint32_t n; VkDescriptorSetLayoutBinding b[MAX_BINDINGS]; VkPipeBinding pb[MAX_BINDINGS]; + int pool; // a stage indexes the texture or sampler pool (set 2, not in b / pb) } Bindings; static VkDescriptorType descriptor_type(int kind, const GfxShaderRes *r) { @@ -911,7 +972,26 @@ static VkDescriptorType descriptor_type(int kind, const GfxShaderRes *r) { static VkShaderStageFlags stage_bit(int stage) { return stage == GFX_STAGE_VS ? VK_SHADER_STAGE_VERTEX_BIT : stage == GFX_STAGE_FS ? VK_SHADER_STAGE_FRAGMENT_BIT - : VK_SHADER_STAGE_COMPUTE_BIT; + : stage == GFX_STAGE_TS ? VK_SHADER_STAGE_TASK_BIT_EXT : stage == GFX_STAGE_MS ? VK_SHADER_STAGE_MESH_BIT_EXT + : VK_SHADER_STAGE_COMPUTE_BIT; +} + +static int is_mesh(const VkPipeState *s) { return s->vs && gfx_shader_stage(s->vs) == GFX_STAGE_MS; } + +// The shaders of the pipeline this thread builds, for the crash notes (platform.h plat_crash_context): the first 8 +// bytes of each cache key, as gfx_shader.c names its cache files. +static void crash_note(const char *what, const GfxShader *a, const GfxShader *b, const GfxShader *c) { + static __thread char buf[160]; + char *o = buf; + o += snprintf(o, 40, "building %s", what); + const GfxShader *sh[3] = {a, b, c}; + for (int i = 0; i < 3; i++) + if (sh[i]) { + const uint8_t *k = gfx_shader_key(sh[i]); + o += snprintf(o, 40, " stage %d %02x%02x%02x%02x%02x%02x%02x%02x", gfx_shader_stage(sh[i]), k[0], k[1], k[2], k[3], + k[4], k[5], k[6], k[7]); + } + plat_crash_context(buf); } // One past the highest binding a shader uses (1 with no resources: binding 0 is the support buffer). @@ -941,6 +1021,10 @@ static int add_stage(Bindings *B, const GfxShader *s, uint32_t off) { const GfxShaderRes *r; int n = gfx_shader_resources(s, k, &r); for (int i = 0; i < n; i++) { + if (r[i].pool) { // set 2 (spirv_pool_rebind), bound by the replay from the pools + B->pool = 1; + continue; + } if (B->n == MAX_BINDINGS || !r[i].binding) return 0; VkDescriptorType t = descriptor_type(k, &r[i]); uint32_t binding = r[i].binding + off, count = r[i].count ? r[i].count : 1; @@ -957,10 +1041,15 @@ static int add_stage(Bindings *B, const GfxShader *s, uint32_t off) { static int module_binding_missing(const GfxShader *s, const VkDescriptorSetLayoutBinding *b, uint32_t n, uint32_t off) { uint64_t words = 0; const uint32_t *w = gfx_shader_spirv(s, &words); + const GfxShaderRes *tr; + int nt = gfx_shader_resources(s, GFX_RES_TEX, &tr); for (uint64_t i = 5; w && i < words;) { uint32_t op = w[i] & 0xffff, len = w[i] >> 16; if (!len || i + len > words) break; - if (op == SPV_OP_DECORATE && len >= 4 && w[i + 2] == SPV_DEC_BINDING) { + int pooled = 0; // a pool array: set 2, not in this layout (spirv_pool_rebind) + for (int t = 0; t < nt && op == SPV_OP_DECORATE && len >= 4; t++) + if (tr[t].pool && tr[t].binding == (w[i + 3] & 0xffff)) pooled = 1; // GfxShaderRes.binding is 16 bits + if (op == SPV_OP_DECORATE && len >= 4 && w[i + 2] == SPV_DEC_BINDING && !pooled) { uint32_t want = w[i + 3] + (w[i + 3] >= 1 ? off : 0), j = 0; while (j < n && b[j].binding != want) j++; if (j == n) return (int)w[i + 3]; @@ -1051,8 +1140,9 @@ static Layout *layout_get(Bindings *B) { free(l); return NULL; } - VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, - .pSetLayouts = &l->set}; + VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()}; // set 2: the pools + VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3, + .pSetLayouts = sets3}; if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) { vkDestroyDescriptorSetLayout(P.device, l->set, NULL); free(l); @@ -1106,9 +1196,9 @@ static Layout *layout_pair(Bindings *B, Bindings *F) { free(l); return NULL; } - VkDescriptorSetLayout sets[2] = {l->set, l->set1}; + VkDescriptorSetLayout sets[3] = {l->set, l->set1, vk_pool_set_layout()}; VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, - .flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 2, + .flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3, .pSetLayouts = sets}; if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) { vkDestroyDescriptorSetLayout(P.device, l->set, NULL); @@ -1190,6 +1280,7 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel p->info.set_layout1 = l ? l->set1 : VK_NULL_HANDLE; p->info.push1 = l ? l->push1 : 0; p->info.n_sets = l && l->set1 ? 2 : 1; + p->info.pool = B ? B->pool : 0; if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1]; p->lay = l; Pipe **head = &pipes[h % PIPE_BUCKETS]; @@ -1201,7 +1292,7 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel // ---- graphics pipelines typedef struct { // normalized: zero where the state is dynamic or unused - const GfxShader *vs, *fs; + const GfxShader *vs, *fs, *ts; // ts: the task stage of a mesh draw uint32_t color[8], depth, stencil; // VkFormat uint8_t n_color, samples, topo_class, minus_w; uint8_t clamp, poly_mode, a2c, logic_op; // logic_op: VkLogicOp + 1 when enabled @@ -1259,7 +1350,7 @@ uint32_t vk_pipeline_target_wmask(const VkPipeState *s, int i) { return target_w static void gkey_fill(const VkPipeState *s, GKey *k) { memset(k, 0, sizeof *k); - k->vs = s->vs, k->fs = s->fs; + k->vs = s->vs, k->fs = s->fs, k->ts = s->ts; k->indep = (uint8_t)P.indep; k->n_color = (uint8_t)target_count(s); for (int i = 0; i < k->n_color; i++) k->color[i] = target_format(s->color_fmt[i], NULL, NULL); @@ -1287,7 +1378,8 @@ static void gkey_fill(const VkPipeState *s, GKey *k) { o[3] = (uint8_t)blend_factor(b->src_a), o[4] = (uint8_t)blend_factor(b->dst_a), o[5] = (uint8_t)blend_op(b->eq_a); } } - if (!P.dyn_vi) { + if (is_mesh(s)) k->topo_class = 2; // mesh draws: no vertex input, the mesh stage declares its primitives + if (!P.dyn_vi && !is_mesh(s)) { VtxInput v; vertex_input(s, &v); for (uint32_t i = 0; i < v.n_attr; i++) { @@ -1328,7 +1420,17 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay uint64_t vw, fw; const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw); uint32_t *vpatched = NULL, *fpatched = NULL; - if (l->set1) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader + int mesh = is_mesh(s); + VkShaderModule tm = VK_NULL_HANDLE; + if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss) + uint64_t tw; + const uint32_t *tcode = gfx_shader_spirv(s->ts, &tw); + uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0); + uint32_t *t = tcode && toff ? spirv_rebind(tcode, tw, toff) : NULL; + tm = make_module(t ? t : tcode, tw); + free(t); + } + if (l->set1 && !mesh) { // independent layouts: outputs from the vertex shader's union, not from this fragment shader GfxShaderIo fin[32]; uint32_t missing = vs_union_missing(s, fin); if (missing) { @@ -1340,11 +1442,11 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay } if (s->fs) { GfxShaderIo fin[32], vout[32]; - uint32_t need = gfx_shader_inputs(s->fs, fin), have = gfx_shader_outputs(s->vs, vout), missing = 0; + uint32_t need = gfx_shader_inputs(s->fs, fin), have = mesh ? ~0u : gfx_shader_outputs(s->vs, vout), missing = 0; for (int i = 0; i < 32; i++) { if (!(need >> i & 1) || fin[i].packed) continue; if (!(have >> i & 1)) missing |= 1u << i; - else if (vout[i].type != fin[i].type) + else if (!mesh && vout[i].type != fin[i].type) LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type); } if (missing && !l->set1) { @@ -1371,11 +1473,19 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay vm = make_module(vcode, vw); free(vpatched), free(fpatched); VkPipeline pipe = VK_NULL_HANDLE; - if (!vm || (s->fs && !fm)) goto out; + if (!vm || (s->fs && !fm) || (mesh && s->ts && !tm)) goto out; - VkPipelineShaderStageCreateInfo stages[2] = { - {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_VERTEX_BIT, .module = vm, .pName = "main"}, - {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_FRAGMENT_BIT, .module = fm, .pName = "main"}}; + VkPipelineShaderStageCreateInfo stages[3] = { + {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = mesh ? VK_SHADER_STAGE_MESH_BIT_EXT : VK_SHADER_STAGE_VERTEX_BIT, + .module = vm, .pName = "main"}, + {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_FRAGMENT_BIT, .module = fm, .pName = "main"}, + {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .stage = VK_SHADER_STAGE_TASK_BIT_EXT, .module = tm, .pName = "main"}}; + uint32_t n_stages = 1; + if (s->fs) n_stages = 2; + if (tm) { + if (!s->fs) stages[1] = stages[2]; + n_stages++; + } VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf; uint64_t sw; const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now @@ -1429,18 +1539,29 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay .attachmentCount = k->n_color, .pAttachments = cba}; VkPipelineDynamicStateCreateInfo dy = {.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, .dynamicStateCount = P.n_dyn, .pDynamicStates = P.dyn}; + VkDynamicState mdyn[sizeof P.dyn / sizeof P.dyn[0]]; + if (mesh) { // a mesh pipeline must not list vertex input, topology or primitive restart as dynamic + uint32_t n = 0; + for (uint32_t i = 0; i < P.n_dyn; i++) + if (P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_EXT && P.dyn[i] != VK_DYNAMIC_STATE_VERTEX_INPUT_BINDING_STRIDE && + P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_TOPOLOGY && P.dyn[i] != VK_DYNAMIC_STATE_PRIMITIVE_RESTART_ENABLE) + mdyn[n++] = P.dyn[i]; + dy.dynamicStateCount = n, dy.pDynamicStates = mdyn; + } VkPipelineRenderingCreateInfo ri = {.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, .colorAttachmentCount = k->n_color, .pColorAttachmentFormats = (const VkFormat *)k->color, .depthAttachmentFormat = (VkFormat)k->depth, .stencilAttachmentFormat = (VkFormat)k->stencil}; VkGraphicsPipelineCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, .pNext = &ri, - .stageCount = s->fs ? 2 : 1, .pStages = stages, .pVertexInputState = &vi, - .pInputAssemblyState = &ia, .pViewportState = &vp, .pRasterizationState = &rs, + .stageCount = n_stages, .pStages = stages, .pVertexInputState = mesh ? NULL : &vi, + .pInputAssemblyState = mesh ? NULL : &ia, .pViewportState = &vp, .pRasterizationState = &rs, .pMultisampleState = &ms, .pDepthStencilState = &ds, .pColorBlendState = &cb, .pDynamicState = &dy, .layout = l->layout}; if (k->fbl & 1) pi.flags |= VK_PIPELINE_CREATE_COLOR_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; if (k->fbl & 2) pi.flags |= VK_PIPELINE_CREATE_DEPTH_STENCIL_ATTACHMENT_FEEDBACK_LOOP_BIT_EXT; uint64_t p0 = perf_begin(); + crash_note("a graphics pipeline", s->vs, s->fs, s->ts); VkResult r = vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe); + plat_crash_context(NULL); perf_end(PERF_PIPE, p0); // counts on the render thread only, so background builds stay out of it if (r != VK_SUCCESS) { rt_log("vk_pipeline: vkCreateGraphicsPipelines failed (%d)", (int)r); @@ -1451,6 +1572,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay out: if (vm) vkDestroyShaderModule(P.device, vm, NULL); if (fm) vkDestroyShaderModule(P.device, fm, NULL); + if (tm) vkDestroyShaderModule(P.device, tm, NULL); return pipe; } @@ -1485,9 +1607,9 @@ static double ms_since(uint64_t t0) { return (double)(qpc() - t0) / 1e6; } static VkPipelineLayout half_layout(Layout *l, int h) { if (l->half[h]) return l->half[h]; - VkDescriptorSetLayout sets[2] = {h ? VK_NULL_HANDLE : l->set, h ? l->set1 : VK_NULL_HANDLE}; + VkDescriptorSetLayout sets[3] = {h ? VK_NULL_HANDLE : l->set, h ? l->set1 : VK_NULL_HANDLE, vk_pool_set_layout()}; VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, - .flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 2, + .flags = VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, .setLayoutCount = 3, .pSetLayouts = sets}; if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->half[h]) != VK_SUCCESS) l->half[h] = VK_NULL_HANDLE; return l->half[h]; @@ -1591,7 +1713,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi ri.depthAttachmentFormat = (VkFormat)k->depth, ri.stencilAttachmentFormat = (VkFormat)k->stencil; pi.pColorBlendState = &cb, pi.pMultisampleState = &ms; } + crash_note("a pipeline library", s->vs, s->fs, NULL); if (vkCreateGraphicsPipelines(P.device, P.cache, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_crash_context(NULL); out: if (m) vkDestroyShaderModule(P.device, m, NULL); free(patched); @@ -1783,7 +1907,9 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc .layout = l->layout}; if (build) plat_mutex_unlock(&P.lock); uint64_t t1 = qpc(); + crash_note("a library link", s->vs, s->fs, NULL); if (vkCreateGraphicsPipelines(P.device, VK_NULL_HANDLE, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_crash_context(NULL); double ms = ms_since(t1); // the link alone: library builds are counted apart if (build) plat_mutex_lock(&P.lock); P.g.link_ms += ms; @@ -2156,12 +2282,24 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode Bindings B = {0}; Layout *l = NULL; uint32_t span = binding_span(s->vs); - if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch"; + int mesh = is_mesh(s); + if (mesh) { + // Mesh draws (VK_EXT_mesh_shader): one set holding the mesh stage's bindings, the fragment stage's after them and + // the task stage's after those (create_graphics renumbers both modules); built here, without libraries or workers. + uint32_t fspan = s->fs ? binding_span(s->fs) : 1; + mode = 0; + if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders"; + else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch"; + else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) || + (s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings"; + else if (!(l = layout_get(&B))) why = "descriptor layout creation failed"; + } else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch"; else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated"; else if (P.indep) { Bindings F = {0}; if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&F, s->fs, 0))) why = "too many bindings"; else if (!(l = layout_pair(&B, &F))) why = "descriptor layout creation failed"; + B.pool |= F.pool; span = 1; } else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1))) why = "too many bindings"; else if (!(l = layout_get(&B))) why = "descriptor layout creation failed"; @@ -2179,7 +2317,7 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode rt_log("vk_pipeline: graphics pipeline not built: %s", why); } else if (!j) { P.stats.graphics++; - if (mode != 2) keys_note(s); + if (mode != 2 && !mesh) keys_note(s); // the key file holds vertex pipelines only } Pipe *p = pipe_add(k, sizeof *k, h, pipe, l, l ? &B : NULL, VK_PIPELINE_BIND_POINT_GRAPHICS, gfx_shader_inputs(s->vs, NULL)); if (!j || !p) { @@ -2525,7 +2663,9 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l) { .layout = l->layout}; pin_subgroup32(&ci.stage, &rqc, code, words); VkPipeline pipe = VK_NULL_HANDLE; + crash_note("a compute pipeline", cs, NULL, NULL); if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + plat_crash_context(NULL); vkDestroyShaderModule(P.device, m, NULL); return pipe; } diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 287171e1..096eaafd 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -44,7 +44,7 @@ enum { R_SetRasterizerDiscardEnable, R_SetScissorWithCount, R_SetStencilCompareMask, R_SetStencilOp, R_SetStencilReference, R_SetStencilTestEnable, R_SetStencilWriteMask, R_SetViewportWithCount, R_WriteTimestamp, R_WriteTimestamp2, R_PushTpl, R_Push, R_VertexInput, R_Bool, R_PolygonMode, R_BlendEnable, R_BlendEquation, R_WriteMask, R_EndCb, - R_PassMark, + R_PassMark, R_DrawMesh, R_DrawMeshInd, }; static uint64_t now_ns(void) { return plat_now_ns(); } @@ -617,6 +617,23 @@ void rec_polygon_mode(PFN_vkCmdSetPolygonModeEXT fn, VkCommandBuffer cb, VkPolyg pub(); } +typedef struct { PFN_vkCmdDrawMeshTasksEXT fn; uint32_t x, y, z; } A_DrawMesh; +void rec_draw_mesh(PFN_vkCmdDrawMeshTasksEXT fn, VkCommandBuffer cb, uint32_t x, uint32_t y, uint32_t z) { + if (!OWNED(cb)) { fn(cb, x, y, z); return; } + A_DrawMesh *a = put(R_DrawMesh, sizeof *a); + *a = (A_DrawMesh){fn, x, y, z}; + pub(); +} + +typedef struct { PFN_vkCmdDrawMeshTasksIndirectEXT fn; VkBuffer b; VkDeviceSize o; uint32_t n, stride; } A_DrawMeshInd; +void rec_draw_mesh_ind(PFN_vkCmdDrawMeshTasksIndirectEXT fn, VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, + uint32_t stride) { + if (!OWNED(cb)) { fn(cb, b, o, n, stride); return; } + A_DrawMeshInd *a = put(R_DrawMeshInd, sizeof *a); + *a = (A_DrawMeshInd){fn, b, o, n, stride}; + pub(); +} + typedef struct { uint32_t id; int end; } A_PassMark; void rec_pass_mark(VkCommandBuffer cb, uint32_t id, int end) { if (!OWNED(cb)) { vk_pass_mark_cmd(cb, id, end); return; } @@ -714,6 +731,8 @@ static void exec_packet(uint32_t op, void *p) { case R_BlendEquation: { A_Targets *a = p; ((PFN_vkCmdSetColorBlendEquationEXT)a->fn)(cb, a->first, a->n, a->v); break; } case R_WriteMask: { A_Targets *a = p; ((PFN_vkCmdSetColorWriteMaskEXT)a->fn)(cb, a->first, a->n, a->v); break; } case R_PassMark: { A_PassMark *a = p; vk_pass_mark_cmd(cb, a->id, a->end); break; } + case R_DrawMesh: { A_DrawMesh *a = p; a->fn(cb, a->x, a->y, a->z); break; } + case R_DrawMeshInd: { A_DrawMeshInd *a = p; a->fn(cb, a->b, a->o, a->n, a->stride); break; } case R_EndCb: { A_EndCb *a = p; if (a->kind) vk_crumb(cb, a->kind, a->n); diff --git a/runtime/vk_rec.h b/runtime/vk_rec.h index afc5d515..f8758c5e 100644 --- a/runtime/vk_rec.h +++ b/runtime/vk_rec.h @@ -44,6 +44,9 @@ void rec_blend_equation(PFN_vkCmdSetColorBlendEquationEXT fn, VkCommandBuffer cb const VkColorBlendEquationEXT *v); void rec_write_mask(PFN_vkCmdSetColorWriteMaskEXT fn, VkCommandBuffer cb, uint32_t first, uint32_t n, const VkColorComponentFlags *v); +void rec_draw_mesh(PFN_vkCmdDrawMeshTasksEXT fn, VkCommandBuffer cb, uint32_t x, uint32_t y, uint32_t z); +void rec_draw_mesh_ind(PFN_vkCmdDrawMeshTasksIndirectEXT fn, VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, + uint32_t stride); // A pass mark (vk_crumbs.c vk_pass_mark_cmd: a buffer marker and a checkpoint), in order with the commands around it. void rec_pass_mark(VkCommandBuffer cb, uint32_t id, int end); diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 0ddd9393..4dc2d9cd 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -362,6 +362,11 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { if (!ds && (ff & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT)) usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; if ((d->flags & NVN_TEXTURE_FLAGS_IMAGE) && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) usage |= VK_IMAGE_USAGE_STORAGE_BIT; + // NVN2 titles (API 200+) number the texture flags differently: Ocarina's compute targets (the title movie's RGBA8 + // frames) carry 0x1, NVN1's DISPLAY. Every single-sample colour texture whose format allows it can be a storage + // image there, so a compute write into one is never dropped. + if (rt_nvn_api_version[0] >= 200 && !ds && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && samples == 1) + usage |= VK_IMAGE_USAGE_STORAGE_BIT; // contact shadows (vk_contact.c) darken the game's B10G11R11 scene colour in place from a compute shader if (f.vk == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && (usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) && samples == 1 && (ff & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) && w >= 640 && vk_contact_strength()) @@ -670,6 +675,13 @@ static void upload(Tex *e) { const nvn_texdesc *d = &t->d; int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); if (!bpb || !t->mem) return; + if (!mem_readable(e->pub.addr, e->t.L.size)) { // a stale slot (the bindless pools look every slot up): no upload; + static volatile int32_t n_unmapped; // a later write to mapped storage uploads it (make_fresh_snap) + log_limited(&n_unmapped, "texture %#llx (%llu bytes): storage not mapped; not uploaded", + (unsigned long long)e->pub.addr, (unsigned long long)e->t.L.size); + e->uploaded = 1; + return; + } uint64_t p0 = perf_begin(), u0 = kern_now_ns(); int linear = nvn_tex_is_linear(d), is3d = d->target == NVN_TEXTURE_TARGET_3D, one_d = e->pub.type == VK_IMAGE_TYPE_1D; uint32_t levels = linear ? 1 : e->pub.levels, layers = linear ? 1 : e->pub.layers; @@ -1115,6 +1127,14 @@ void vk_texture_register(int id, const tex_t *t, const view_t *v) { // The entry slot s resolves to, and the view and has_view a caller gets with it (the slot's, with the descriptor's // swizzle when the entry is another descriptor at the same storage). NULL when the slot names no texture. +// Entry creation budget of slot lookups (vk_texture_create_budget_locked): -1 none. The bindless pools look up every +// registered slot, and most of a pool is stale descriptors (storage freed or reused); with a budget a lookup creates at +// most that many entries, only for mapped storage, and notes the slots it left for later (create_deferred). +static int create_budget = -1, create_deferred; + +void vk_texture_create_budget_locked(int n) { create_budget = n, create_deferred = 0; } +int vk_texture_create_deferred_locked(void) { int d = create_deferred; create_deferred = 0; return d; } + static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { *view = s->v; *has_view = s->has_view; @@ -1134,7 +1154,11 @@ static Tex *slot_resolve(TexSlot *s, view_t *view, int *has_view) { } if (!e) e = any ? any : own; } - if (!e) e = s->t.mem ? entry_of(&s->t) : NULL; + if (!e && s->t.mem) { + if (create_budget < 0) e = entry_of(&s->t); + else if (!create_budget) create_deferred = 1; + else if (mem_readable((uint64_t)(uintptr_t)s->t.mem, s->t.L.size)) e = entry_of(&s->t), create_budget--; + } // A followed texture is another descriptor at the same storage: its image holds the content, but sampling takes the // slot descriptor's swizzle. The UI renders text into RGBA8 and samples it through a (A,A,A,A) descriptor; with the // render target's RGBA swizzle the text blur saw R = 1 everywhere and drew solid boxes behind tips and labels. @@ -1165,6 +1189,8 @@ uint64_t vk_texture_slot_token(int id) { return v ? (v >> 16) << 32 | __atomic_load_n(&agen[v & 0xffff], __ATOMIC_ACQUIRE) : 0; } uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAXED); } +int vk_texture_slot_count_locked(void) { return ntslots; } +int vk_sampler_slot_count_locked(void) { return nsslots; } const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { lock(); diff --git a/runtime/vk_tonemap.c b/runtime/vk_tonemap.c new file mode 100644 index 00000000..a6dd4624 --- /dev/null +++ b/runtime/vk_tonemap.c @@ -0,0 +1,152 @@ +// Tonemap pass standing in for the scene upscaler (vk.h vk_tonemap_record; used by vk_exec_draw.c +// untranslated_dispatch_fallback). +#include "vk.h" + +#include +#include + +#include "cfg.h" +#include "fallback_tonemap_spv.h" +#include "host.h" + +// The game's upscaler (tensor-core, untranslatable) also tonemaps: its input is the HDR scene in RGB9E5 (values up to a +// few hundred), its output the display picture the HUD is drawn over, and the present after it only converts to sRGB. +// A plain blit left the picture white. This pass (shaders/fallback_tonemap.spvasm) scales with linear filtering and +// maps the colour, times NX2NAVI_FALLBACK_EXPOSURE (default 0.04), through Narkowicz's ACES fit. Not the game's own curve +// or auto exposure: those live in the shader we cannot run. +static struct { + int tried, ok; + PFN_vkCmdPushDescriptorSetKHR push; + VkDescriptorSetLayout dsl; + VkPipelineLayout layout; + VkPipeline pipe; + VkSampler sampler; + struct { VkImage img; VkFormat f; VkImageView view; } views[16]; + int nviews; + VkImage out; // the pass's own output (RGBA16F, the target's size); the caller blits it in + VmaAllocation out_alloc; + VkImageView out_view; + uint32_t ow, oh; +} TM; + +// The output image at w x h, made again when the size changes (the old one is left to the deferred release). +static int tm_out(VkCommandBuffer cb, uint32_t w, uint32_t h) { + if (TM.out && TM.ow == w && TM.oh == h) return 1; + VkCtx *c = vk_ctx(); + if (TM.out) vk_defer_view(TM.out_view), vk_defer_image(TM.out, TM.out_alloc), TM.out = VK_NULL_HANDLE; + VkImageCreateInfo ii = {.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .imageType = VK_IMAGE_TYPE_2D, + .format = VK_FORMAT_R16G16B16A16_SFLOAT, .extent = {w, h, 1}, .mipLevels = 1, .arrayLayers = 1, + .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, + .usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT, + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (vmaCreateImage(c->vma, &ii, &ai, &TM.out, &TM.out_alloc, NULL) != VK_SUCCESS) return TM.out = VK_NULL_HANDLE, 0; + VkImageViewCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = TM.out, + .viewType = VK_IMAGE_VIEW_TYPE_2D, .format = VK_FORMAT_R16G16B16A16_SFLOAT, + .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}}; + if (vkCreateImageView(c->device, &vi, NULL, &TM.out_view) != VK_SUCCESS) { + vmaDestroyImage(c->vma, TM.out, TM.out_alloc); + return TM.out = VK_NULL_HANDLE, 0; + } + VkImageMemoryBarrier2 b = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = TM.out, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}}; + VkDependencyInfo d = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .imageMemoryBarrierCount = 1, .pImageMemoryBarriers = &b}; + vkCmdPipelineBarrier2(cb, &d); + TM.ow = w, TM.oh = h; + return 1; +} + +static int tm_init(void) { + if (TM.tried) return TM.ok; + TM.tried = 1; + VkCtx *c = vk_ctx(); + TM.push = (PFN_vkCmdPushDescriptorSetKHR)vkGetDeviceProcAddr(c->device, "vkCmdPushDescriptorSetKHR"); + VkDescriptorSetLayoutBinding b[2] = { + {.binding = 0, .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}, + {.binding = 1, .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT}}; + VkDescriptorSetLayoutCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, + .bindingCount = 2, .pBindings = b}; + VkPushConstantRange pr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 24}; + VkSamplerCreateInfo si = {.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, .magFilter = VK_FILTER_LINEAR, + .minFilter = VK_FILTER_LINEAR, .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST, + .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, + .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, .maxLod = 0}; + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, + .codeSize = sizeof fallback_tonemap_spv, .pCode = fallback_tonemap_spv}; + VkShaderModule m = VK_NULL_HANDLE; + int ok = TM.push && vkCreateDescriptorSetLayout(c->device, &dci, NULL, &TM.dsl) == VK_SUCCESS; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 1, + .pSetLayouts = &TM.dsl, .pushConstantRangeCount = 1, .pPushConstantRanges = &pr}; + ok = ok && vkCreatePipelineLayout(c->device, &li, NULL, &TM.layout) == VK_SUCCESS && + vkCreateSampler(c->device, &si, NULL, &TM.sampler) == VK_SUCCESS && + vkCreateShaderModule(c->device, &mi, NULL, &m) == VK_SUCCESS; + if (ok) { + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = TM.layout}; + ok = vkCreateComputePipelines(c->device, VK_NULL_HANDLE, 1, &ci, NULL, &TM.pipe) == VK_SUCCESS; + } + if (m) vkDestroyShaderModule(c->device, m, NULL); + rt_log("vk_exec: upscaler stand-in: %s", ok ? "tonemap pass (NX2NAVI_FALLBACK_EXPOSURE)" : "blit (the tonemap pass could not be created)"); + return TM.ok = ok; +} + +// Views of the scratch images the pass samples, kept for the process (the scratch images live as long). +static VkImageView tm_view(VkImage img, VkFormat f) { + for (int i = 0; i < TM.nviews; i++) + if (TM.views[i].img == img && TM.views[i].f == f) return TM.views[i].view; + if (TM.nviews == 16) return VK_NULL_HANDLE; + VkImageViewCreateInfo vi = {.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .image = img, .viewType = VK_IMAGE_VIEW_TYPE_2D, + .format = f, .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}}; + VkImageView v; + if (vkCreateImageView(vk_ctx()->device, &vi, NULL, &v) != VK_SUCCESS) return VK_NULL_HANDLE; + TM.views[TM.nviews].img = img, TM.views[TM.nviews].f = f, TM.views[TM.nviews].view = v, TM.nviews++; + return v; +} + +// Records the pass from src (sampled) into the pass's own RGBA16F image of dw x dh and returns it (the caller blits it +// into the target); VK_NULL_HANDLE when it cannot (the caller blits the source instead). +VkImage vk_tonemap_record(VkCommandBuffer cb, VkImage src, VkFormat sf, uint32_t dw, uint32_t dh) { + static int disp = -1; // NX2NAVI_TM_DISPATCH=0: everything but the dispatch, =2: not at all (diagnosis) + if (disp < 0) disp = rt_knob("TM_DISPATCH", "1"); + if (disp == 2 || !tm_init() || !tm_out(cb, dw, dh)) return VK_NULL_HANDLE; + VkImageView sv = tm_view(src, sf), dv = TM.out_view; + if (!sv) return VK_NULL_HANDLE; + static float exposure = -1; + if (exposure < 0) { + const char *e = cfg_get("NX2NAVI_FALLBACK_EXPOSURE"); + exposure = e && *e ? (float)atof(e) : 0.04f; + } + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT}; + VkDependencyInfo dep = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &dep); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, TM.pipe); + VkDescriptorImageInfo ii[2] = {{TM.sampler, sv, VK_IMAGE_LAYOUT_GENERAL}, {VK_NULL_HANDLE, dv, VK_IMAGE_LAYOUT_GENERAL}}; + VkWriteDescriptorSet w[2] = { + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 0, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, .pImageInfo = &ii[0]}, + {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, .dstBinding = 1, .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .pImageInfo = &ii[1]}}; + rec_push(TM.push, cb, VK_PIPELINE_BIND_POINT_COMPUTE, TM.layout, 0, 2, w); // through the recorder (vk_rec.h), in order + struct { float inv[2], exposure, pad; int32_t size[2]; } pc = {{1.0f / (float)dw, 1.0f / (float)dh}, exposure, 0, + {(int32_t)dw, (int32_t)dh}}; + vkCmdPushConstants(cb, TM.layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc); + if (disp) vkCmdDispatch(cb, (dw + 7) / 8, (dh + 7) / 8, 1); + VkMemoryBarrier2 after = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT}; + VkDependencyInfo dep2 = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &after}; + vkCmdPipelineBarrier2(cb, &dep2); + return TM.out; +} + diff --git a/tools/build_game.py b/tools/build_game.py index ee05869b..f7687426 100644 --- a/tools/build_game.py +++ b/tools/build_game.py @@ -354,7 +354,9 @@ def module_glue(layout, linked, hashes, paths, version): for name in ('exefs', 'romfs', 'profile'): out.append(f'const char rt_default_{name}[] = {c_string(paths[name].as_posix())};') number = int(DISPLAY_VERSIONS[version].replace('.', '')) if version in DISPLAY_VERSIONS else 0 # 1.4.3 -> 143 - out += [f'const int rt_game_version = {number};', f'const char rt_port_version[] = {c_string(port_version())};', ''] + out += ['const unsigned long long rt_app_pool_bytes = 0xCD500000ull;', # Switch 1 retail application pool + 'const int rt_nvn_api_version[2] = {55, 15};', # NVN1: NX2NAVI main requires major 55, minor >= 15 + f'const int rt_game_version = {number};', f'const char rt_port_version[] = {c_string(port_version())};', ''] return '\n'.join(out) diff --git a/tools/build_xlate.py b/tools/build_xlate.py index fee41897..a6bcacf4 100644 --- a/tools/build_xlate.py +++ b/tools/build_xlate.py @@ -1,7 +1,7 @@ """Build the shader translator: build/xlate/nx2navi_xlate.exe (Ryujinx 1.3.2 + patches, NativeAOT, SPIR-V for Vulkan). Copies tools/vendor/ryujinx to build/xlate/ryujinx, applies tools/xlate/patches/*.patch there (tools/vendor stays -byte-identical; 0001-0004 are MNG's, 0005-0013, 0015 and 0016 are ours), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build +byte-identical; 0001-0004 are MNG's, 0005-0013 and 0015-0020 are ours), then runs `dotnet publish`. Needs the .NET 10 SDK, the MSVC build tools and vswhere.exe (docs/building.md). Run with .venv. On Linux it builds build/xlate/nx2navi_xlate for linux-x64, which NativeAOT links with the system's clang (no MSVC); NativeAOT builds for the OS it runs on only. @@ -99,8 +99,8 @@ def prepare_sources() -> Path: for junk in list(src.glob("src/*/obj")) + list(src.glob("src/*/bin")): shutil.rmtree(junk) patches = sorted((XLATE / "patches").glob("*.patch")) - if len(patches) != 15: - sys.exit(f"expected 15 patches, found {len(patches)}") + if len(patches) != 19: + sys.exit(f"expected 19 patches, found {len(patches)}") for patch in patches: # git apply works outside a repository when given a directory; --check first so a failure names the patch. for extra in (["--check"], []): @@ -117,7 +117,7 @@ def prepare_sources() -> Path: def identity() -> str: """sha256 over the translator's inputs, in a fixed order: path and content hash of each file.""" h = hashlib.sha256() - files = [Path(__file__).resolve(), XLATE / "Program.cs", XLATE / "nx2navi_xlate.csproj", + files = [Path(__file__).resolve(), XLATE / "Program.cs", XLATE / "AmpereLowering.cs", XLATE / "MeshSpirv.cs", XLATE / "nx2navi_xlate.csproj", *sorted((XLATE / "patches").glob("*.patch"))] for f in files: h.update(f"{f.name}\0{hashlib.sha256(f.read_bytes()).hexdigest()}\n".encode()) @@ -161,7 +161,7 @@ def main() -> None: work = BUILD / "xlate" shutil.rmtree(work, ignore_errors=True) work.mkdir(parents=True) - for name in ("Program.cs", "nx2navi_xlate.csproj"): + for name in ("Program.cs", "AmpereLowering.cs", "MeshSpirv.cs", "nx2navi_xlate.csproj"): shutil.copy2(XLATE / name, work / name) out = BUILD / "publish" shutil.rmtree(out, ignore_errors=True) diff --git a/tools/gen_nvn_procs.py b/tools/gen_nvn_procs.py index e199b21f..9625e5e7 100644 --- a/tools/gen_nvn_procs.py +++ b/tools/gen_nvn_procs.py @@ -24,6 +24,17 @@ SUBSDK0 = [ # section 0.1: asked by the movie library through nvnDeviceGetProcA "nvnTextureGetNvRmSurfaceNVX", ] MAIN_COUNT = 534 +# NVN2 (Switch 2) procs this host implements; the rest of the NVN2 names get traps (runtime/nvn_procs.c nvn_trap). +NVN2 = [ + "nvnCommandBufferDrawMeshTasks", + "nvnCommandBufferDrawMeshTasksIndirect", + "nvnCommandBufferPushComputeConstants", + "nvnCommandBufferSetGeometryProcessingMode", + "nvnProgramBuilderSetDefaults", + "nvnProgramBuilderSetDevice", + "nvnProgramBuilderSetShaders", + "nvnProgramInitializeFromBuilder", +] def main() -> int: @@ -39,7 +50,7 @@ def main() -> int: return 1 # nvnDeviceGetProcAddress is in both lists; keep one row per name. extra_sub = [n for n in SUBSDK0 if n not in main_names] - rows = sorted(main_names) + sorted(extra_sub) + rows = sorted(main_names) + sorted(extra_sub) + sorted(n for n in NVN2 if n not in main_names) text = ("// Generated by tools/gen_nvn_procs.py: the NVN proc names of NX2navi 1.2.1.\n" + "".join(f"NVN_PROC({n})\n" for n in rows)) OUT.write_text(text, encoding="utf-8", newline="\n") diff --git a/tools/xlate/AmpereLowering.cs b/tools/xlate/AmpereLowering.cs new file mode 100644 index 00000000..0a4be334 --- /dev/null +++ b/tools/xlate/AmpereLowering.cs @@ -0,0 +1,2387 @@ +// Switch 2 (NVN2) shader programs are SM86 (Ampere) code; the Ryujinx translator decodes Maxwell (SM5x) only. This file +// rewrites each SM86 instruction as one or more Maxwell instructions with the same meaning, so Ryujinx's Maxwell decoder, +// IR and SPIR-V backend run unchanged. Instructions this file does not know make the whole program fail with the opcode +// and the reason, so coverage grows by census (python nvn2_shaders.py over a NX2NAVI_SHADER_DUMP directory). +// +// Encoding facts: SM70+ instruction fields as in Mesa NAK's sm70_encode.rs (MIT); the Maxwell side is Ryujinx's +// Decoders/InstTable.cs and InstDecoders.cs. Behaviour where SM86 and the translator's Maxwell model differ (MUFU.SIN/COS +// take turns, BSSY/BSYNC are reconvergence hints, bank 24 is never bound) was cross-checked against the system's +// shader_recompiler/frontend/ampere/lower_to_maxwell.h (GPL-3.0, read for behaviour only; no code taken). +// +// Layout: graphics programs keep the first 0x50 bytes of the 0x80-byte Turing+ header (the same fields as Maxwell's +// header), then the instructions; Maxwell puts a scheduling word first in every 32 bytes counted from the first +// instruction (Ryujinx's Decoder skips them), written as 0. Compute programs start at address 0. +// +// Registers: guest registers keep their numbers. Uniform registers live just above the highest register the program +// uses (UrBase, even so pairs stay aligned; URZ is RZ) and must end below R239. R239-R254 are this file's scratch +// registers; R237 and R238 hold the uniform predicate state (see "uniform predicates" below). A program that leaves +// no room for them (task and mesh programs use up to R252) is lowered again in spill mode: guest registers from +// SpillFrom up live in local memory after the program's own, and an instruction that names one works on a block of +// four temporaries (R188-R203, one block per aligned group of four guest registers, so pairs and quads stay +// contiguous) loaded before it and stored after it; uniform registers then take blocks of four from R204-R235 (eight +// groups: an Ocarina mesh program used more than the four R220-R235 held). +using System; +using System.Collections.Generic; + +static class AmpereLowering +{ + const int RZ = 255, PT = 7, ScratchBase = 239; + // Uniform predicates: UPk is bit k of RUP. An instruction that reads or writes uniform predicates runs with UPk + // swapped into Pk: the guest's Pk saved to USAVE (P2R), UPk loaded (R2P), the instruction lowered as if its + // predicate fields named P registers, written UPs merged back into RUP (P2R), the guest's P restored (R2P). + const int RUP = 238, USAVE = 237, UrLimit = 237; + // Scratch: operand A/B/C materialisation, a spare, texture coordinates, texture meta values, texture results. + const int SA = 252, SB = 253, SC = 254, SX = 239, TexCoord = 240, TexMeta = 244, TexOut = 248; + + public class LoweringException : Exception + { + public LoweringException(string message) : base(message) { } + } + + // The registers do not fit the plain layout: Lower retries in spill mode. + sealed class RegisterPressureException : LoweringException + { + public RegisterPressureException(string message) : base(message) { } + } + + const int SpillFrom = 188, TempBase = 188, UrPool = 204, UrPoolEnd = 236, SpillBytes = (256 - SpillFrom) * 4; + + // Task (NVN2 stage 6) and mesh (stage 7) programs. Both are lowered as compute programs: the stage buffer they + // write with ISBEWR and read with ISBERD becomes shared memory, laid out by this record, and MeshSpirv.cs turns + // the compute SPIR-V into a TaskEXT or MeshEXT entry point that hands the buffer to Vulkan. + // Mesh: vertex outputs at Attr: 32-vertex groups, a 32-word line per output component, word + // ((v >> 5) * Lines + line) * 32 + (v & 31); the primitive count at Map + 0 (ISBEWR region 1) and the + // primitive's vertex indices as bytes from Map + 4 (region 0); the task payload at Pay (ISBERD region 7). + // Task: the payload at Pay: the mesh workgroup count at Pay + 4, the data from Pay + 0x10 (region 7). + // ISBERD of address 0 in region 15 (no task) or 7 reads the workgroup's index. Control section fields + // (runtime/gfx_shader.c build_request): +0x58 low 16 bits the workgroup size, bits 24-31 the payload size in 256 + // bytes; +0x5c byte 3 the primitive type (1 points, 2 lines, 3 triangles), byte 2 set when a task stage feeds the + // mesh stage; +0x60 low 16 bits the vertex limit, high 16 bits the primitive limit. + public sealed class MeshInfo + { + public int Stage, Threads, MaxVertices, MaxPrimitives, Topology, PayloadBytes; + public bool HasTask; + // Output lines in order: Maxwell attribute address of each component (0x60.. system values, 0x80 + 16g + 4c + // generic g component c). + public readonly List Lines = new(); + public int Attr, Map, Pay, SharedBytes; + public int Groups => (MaxVertices + 31) / 32; + public int IndicesPerPrimitive => Topology switch { 1 => 1, 2 => 2, _ => 3 }; + + // Lines from the output map of the program header (Turing+ VTG header, the same bytes as Maxwell's: system + // values B at byte 0x35, attributes 0x60-0x7c one bit each; generic vectors from byte 0x36, a nibble each, + // component x in bit 0). System values A (0x34) and C (0x48) are not expected in mesh programs. + public void Layout(byte[] sph) + { + if (Stage == 7) + { + if (sph[0x34] != 0 || sph[0x48] != 0 || sph[0x49] != 0) throw new LoweringException("mesh output map: system values A or C"); + for (int b = 0; b < 8; b++) if ((sph[0x35] >> b & 1) != 0) Lines.Add(0x60 + 4 * b); + for (int g = 0; g < 32; g++) + for (int c = 0; c < 4; c++) + if ((sph[0x36 + g / 2] >> (4 * (g & 1) + c) & 1) != 0) Lines.Add(0x80 + 16 * g + 4 * c); + Attr = 0; + Map = (Attr + Groups * Lines.Count * 128 + 15) & ~15; + Pay = (Map + 4 + IndicesPerPrimitive * MaxPrimitives + 15) & ~15; + SharedBytes = Pay + (HasTask ? PayloadBytes : 0); + } + else { Pay = 0; SharedBytes = PayloadBytes; } + if (SharedBytes == 0) SharedBytes = 16; + } + } + + struct Insn + { + public ulong Lo, Hi; + public uint W0 => (uint)Lo; + public uint W1 => (uint)(Lo >> 32); + public uint W2 => (uint)Hi; + public uint Bits(int offset, int count) + { + ulong v = offset >= 64 ? Hi >> (offset - 64) : offset + count <= 64 ? Lo >> offset : (Lo >> offset) | (Hi << (64 - offset)); + return (uint)(v & ((1UL << count) - 1)); + } + public bool Bit(int offset) => Bits(offset, 1) != 0; + } + + sealed class Lowerer + { + readonly Insn[] _code; + readonly List _out = new(); + readonly List<(int output, int targetSource)> _branches = new(); + readonly int[] _firstOutput; + Insn _i; + int _index; + ulong _guard; // the SM86 instruction's predicate, in Maxwell position (bits 16-19) + + readonly byte[] _header; // the program header, null for compute + readonly int UrBase; // R(UrBase + n) holds URn; guest registers are below it + readonly bool _compute; + readonly int[] _table; // NVN2 compute binding table: start byte, texture entries, image entries (or null) + readonly MeshInfo _mesh; // task or mesh program (lowered as compute), else null + public byte[] C1; // the program's c1 constants (NVN2: after the code; BRX jump tables), or null + readonly int _spillAt = -1; // spill mode: byte offset of guest register SpillFrom in local memory; -1 otherwise + readonly Dictionary _temps = new(); // spill mode: guest register group (r & ~3) -> its temporaries, this instruction + readonly Dictionary _urBlocks = new(); // spill mode: uniform register group (n & ~3) -> its registers + public Lowerer(Insn[] code, byte[] header, bool compute, int[] table, MeshInfo mesh, int spillAt = -1) + { + _code = code; _header = header; _compute = compute; _table = table; _mesh = mesh; _firstOutput = new int[code.Length + 1]; + _spillAt = spillAt; + int highest = HighestGpr(code); + if (spillAt < 0 && highest >= UrLimit) throw new RegisterPressureException($"guest registers up to R{highest}"); + UrBase = spillAt >= 0 ? SpillFrom : (highest + 2) & ~1; + } + + // Spill mode: the temporary that holds guest register r for this instruction, loading its group on first use. + int Spilled(int r) + { + int g = r & ~3; + if (!_temps.TryGetValue(g, out int t)) + { + if (_temps.Count == 4) throw Fail("more than four spilled register groups in one instruction"); + t = TempBase + 4 * _temps.Count; + _temps[g] = t; + for (int k = 0; k < 4; k++) // LDL.32 t+k, [RZ + slot] + Emit(Unguarded(0xef40000000000000UL | (uint)(t + k) | ((ulong)RZ << 8) | ((ulong)(uint)(_spillAt + 4 * (g - SpillFrom + k)) << 20) | (4UL << 48))); + } + return t + (r & 3); + } + + // Spill mode: the temporaries of this instruction back to local memory. + void SpillStore() + { + foreach (var (g, t) in _temps) + for (int k = 0; k < 4; k++) // STL.32 [RZ + slot], t+k + Emit(Unguarded(0xef50000000000000UL | (uint)(t + k) | ((ulong)RZ << 8) | ((ulong)(uint)(_spillAt + 4 * (g - SpillFrom + k)) << 20) | (4UL << 48))); + _temps.Clear(); + } + + // The highest general register the program names (destination, A, B in the register form, C), counting the + // register fields of uniform-datapath instructions too: an overestimate only moves the uniform registers up. + static int HighestGpr(Insn[] code) + { + int max = -1; + foreach (var i in code) + { + void see(uint r) { if (r < RZ && r > max) max = (int)r; } + uint op = i.Bits(0, 12), b = op & 0x1ff; + if (b is >= 0x140 and <= 0x15f) continue; // branches, calls, exits, barriers: no register fields + see(i.Bits(16, 8)); see(i.Bits(24, 8)); + // Register ranges the base fields do not show: LDSM writes Rd..Rd+count-1; TXD reads Ra..Ra+2 and Rb..Rb+3 + // and writes the second destination pair; HMMA reads A, B and C fragments and writes the D pair. + if (op == 0x83b && i.Bits(16, 8) != RZ) see(i.Bits(16, 8) + (1u << (int)i.Bits(72, 2)) - 1); + if (op == 0xb6c) { see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 2); see(i.Bits(32, 8) + 3); see(i.Bits(64, 8) + 1); } + if (op == 0x23c) + { + uint k16 = i.Bits(75, 1); + see(i.Bits(16, 8) + 1); see(i.Bits(24, 8) + 1 + 2 * k16); see(i.Bits(32, 8) + k16); see(i.Bits(64, 8) + 1); + } + if (b < 0x080 || b is >= 0x160 and <= 0x16f) see(i.Bits(64, 8)); // C of ALU forms, TEX second destinations + if (i.Bits(9, 3) == 1) see(i.Bits(32, 8)); + } + return max; + } + + LoweringException Fail(string reason) => + new($"sm86 instruction {_index} (0x{_index * 16:x}) op {_i.Bits(0, 12):x3} [{_i.W0:x8} {_i.W1:x8} {_i.W2:x8} {(uint)(_i.Hi >> 32):x8}]: {reason}"); + + void Emit(ulong word) => _out.Add(word); + // Guarded by the instruction's own predicate. + void EmitG(ulong word) => _out.Add(word | _guard); + static ulong Unguarded(ulong word) => word | ((ulong)PT << 16); + + // ---- Maxwell building blocks ------------------------------------------------------------------------------- + // guarded: under the instruction's predicate (a guest-visible write); otherwise unconditional (scratch). + void Put(ulong word, bool guarded) => _out.Add(guarded ? word | _guard : Unguarded(word)); + void Mov(int dst, int src, bool guarded = true) => Put(0x5c98000000000000UL | (15UL << 39) | ((ulong)src << 20) | (uint)dst, guarded); + void Mov32i(int dst, uint imm, bool guarded = false) => Put(0x0100000000000000UL | (15UL << 12) | ((ulong)imm << 20) | (uint)dst, guarded); + void MovC(int dst, int bank, int byteOffset, bool guarded = false) => + Put(0x4c98000000000000UL | (15UL << 39) | ((ulong)(byteOffset / 4) << 20) | ((ulong)bank << 34) | (uint)dst, guarded); + void Lop32i(int dst, int a, uint imm, int op) => Emit(Unguarded(0x0400000000000000UL | ((ulong)op << 53) | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)dst)); + void Fmul32i(int dst, int a, uint imm) => Emit(Unguarded(0x1e00000000000000UL | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)dst)); + void Iadd3(int dst, int a, int b, int c, bool guarded, bool negB = false) => + Put(0x5cc0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst | (negB ? 1UL << 50 : 0), guarded); + // Rd = p ? a : b (p: index | negate << 3) + void Sel(int dst, int a, int b, uint p, bool guarded) => + Put(0x5ca0000000000000UL | ((ulong)(p & 7) << 39) | ((ulong)(p >> 3) << 42) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst, guarded); + // Pd = a < b, unsigned (the carry out of a + b is sum < a) + void IsetpLtU(int pd, int a, int b) => + EmitG(0x5b60000000000000UL | (1UL << 49) | ((ulong)PT << 39) | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)pd << 3) | PT); + + // ---- SM86 operand decoding --------------------------------------------------------------------------------- + uint Form => _i.Bits(9, 3); + int Dst => (int)_i.Bits(16, 8); + int RegA => (int)_i.Bits(24, 8); + + int Ur(uint reg) + { + if (reg == 63) return RZ; + if (reg > 62) throw Fail($"UR{reg}"); + if (_spillAt >= 0) + { + int q = (int)reg & ~3; + if (!_urBlocks.TryGetValue(q, out int b)) + { + b = UrPool + 4 * _urBlocks.Count; + if (b + 4 > UrPoolEnd) throw Fail("more uniform register groups than spill mode has room for"); + _urBlocks[q] = b; + } + return b + ((int)reg & 3); + } + if (UrBase + (int)reg >= UrLimit) throw new RegisterPressureException($"UR{reg} has no free register (guest registers up to R{UrBase - 1})"); + return UrBase + (int)reg; + } + + // While a uniform-datapath instruction is lowered through its vector twin (UniformAlu), register fields name + // uniform registers. + bool _uniform; + + int Gpr(uint reg) + { + if (_uniform) return reg == RZ ? RZ : Ur(reg); + if (reg != RZ && _spillAt >= 0 && reg >= SpillFrom) return Spilled((int)reg); + if (reg != RZ && reg >= UrBase) throw Fail($"R{reg} above the registers the program was scanned for"); + return (int)reg; + } + + // Constant bank 24 holds NVN2 driver data the shaders read (fullscreen passes: texel size, viewport rectangle); + // Maxwell has 18 banks, so it is bank 17 here and the runtime binds it there. + static int Bank(uint bank) => bank == 24 ? 17 : (int)bank; + + // Constant-buffer operand: byte offset 38-53, bank 54-58. + (int bank, int offset) Cbuf() + { + int bank = Bank(_i.Bits(54, 5)), offset = (int)_i.Bits(38, 16); + if (bank >= 18 || (offset & 3) != 0 || (_i.Bit(91) && !_uniform)) throw Fail($"constant buffer operand c[{_i.Bits(54, 5)}][0x{offset:x}]"); + return (bank, offset); + } + + // Source 0 (A), 1 (B) or 2 (C) with its float modifiers. Forms (bits 9-11): 1 B and C registers, 2 C immediate, + // 3 C constant, 4 B immediate, 5 B constant, 6 B uniform, 7 C uniform. A wide (immediate/constant/uniform) operand + // uses bits 32-63; the register operand that would sit there then moves to bits 64-71. + (int reg, bool abs, bool neg) Raw(int which, int scratch) + { + if (which == 0) return (Gpr(_i.Bits(24, 8)), _i.Bit(73), _i.Bit(72)); + uint form = Form; + if (form == 0) throw Fail("source form 0"); + bool wide = which == 1 ? form is 4 or 5 or 6 : form is 2 or 3 or 7; + if (wide) + { + if (form is 4 or 2) { Mov32i(scratch, _i.W1); return (scratch, false, false); } + if (form is 5 or 3) { var (bank, off) = Cbuf(); MovC(scratch, bank, off); return (scratch, _i.Bit(62), _i.Bit(63)); } + return (Ur(_i.Bits(32, 8)), _i.Bit(62), _i.Bit(63)); + } + bool upper = which == 2 || form is 2 or 3 or 7; + return upper ? (Gpr(_i.Bits(64, 8)), _i.Bit(74), _i.Bit(75)) : (Gpr(_i.Bits(32, 8)), _i.Bit(62), _i.Bit(63)); + } + + // The second operand of a two-source instruction sits in the B slot, or in the C slot for forms 2, 3 and 7. + int SecondSlot => Form is 2 or 3 or 7 ? 2 : 1; + + // Applies float abs/neg in a scratch register (bit operations, so -0.0 and NaN payloads stay exact). + int Mat(int reg, bool abs, bool neg, int scratch) + { + if (!abs && !neg) return reg; + if (abs) { Lop32i(scratch, reg, 0x7fffffff, 0); reg = scratch; } + if (neg) { Lop32i(scratch, reg, 0x80000000, 2); reg = scratch; } + return reg; + } + + int Plain(int which, int scratch) + { + var (r, _, _) = Raw(which, scratch); + return r; + } + + int FloatMat(int which, int scratch) + { + var (r, a, n) = Raw(which, scratch); + return Mat(r, a, n, scratch); + } + + // A uniform guard is swapped in by Run like any other uniform predicate operand. + void RequireUniformGuardTrue() { } + + // Uniform-datapath instructions: their guard and predicate fields name uniform predicates. + static bool IsUniformOp(uint op) => ((op & 0x1ff) is >= 0x080 and <= 0x0bf && op != 0x886) || op == 0x9c3; // VOTEU: Vote + + // Uniform predicates the instruction reads and writes, as masks of UP indices (UPT excluded). + (uint inUp, uint outUp) UniformPredicates() + { + uint op = _i.Bits(0, 12); + if (!IsUniformOp(op)) return (0, 0); + uint inUp = 0, outUp = 0; + void use(ref uint mask, uint p) { if ((p & 7) != 7) mask |= 1u << (int)(p & 7); } + use(ref inUp, _i.Bits(12, 3)); + switch (op & 0x1ff) + { + case 0x08c: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); use(ref inUp, _i.Bits(87, 3)); break; // UISETP + case 0x09c: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); // UPLOP3 + use(ref inUp, _i.Bits(87, 3)); use(ref inUp, _i.Bits(77, 3)); use(ref inUp, _i.Bits(68, 3)); break; + case 0x090: use(ref outUp, _i.Bits(81, 3)); use(ref outUp, _i.Bits(84, 3)); // UIADD3 + if (_i.Bit(74)) { use(ref inUp, _i.Bits(87, 3)); use(ref inUp, _i.Bits(77, 3)); } break; + case 0x092: use(ref outUp, _i.Bits(81, 3)); use(ref inUp, _i.Bits(87, 3)); break; // ULOP3 + case 0x087: use(ref inUp, _i.Bits(87, 3)); break; // USEL + case 0x091: case 0x0a4: case 0x0a5: use(ref outUp, _i.Bits(81, 3)); if (_i.Bit(74)) use(ref inUp, _i.Bits(87, 3)); break; + } + return (inUp, outUp); + } + + // ---- driver ---------------------------------------------------------------------------------------------- + public List Run(out List<(int output, int targetSource)> branches, out int[] firstOutput) + { + Mov32i(RUP, 0); // uniform predicates start false + for (_index = 0; _index < _code.Length; _index++) + { + _i = _code[_index]; + _firstOutput[_index] = _out.Count; + if (_i.Bits(12, 4) == 15) continue; // @!PT: never runs (compiler padding) + _guard = (ulong)_i.Bits(12, 4) << 16; + var (inUp, outUp) = UniformPredicates(); + uint m = inUp | outUp; + if (m != 0) + { + Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RZ << 8) | USAVE)); // P2R: save guest P + Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)RUP << 8))); // R2P: load UPs + } + LowerOne(); + if (_temps.Count != 0) SpillStore(); + if (m != 0) + { + if (outUp != 0) Emit(Unguarded(0x38e8000000000000UL | ((ulong)outUp << 20) | ((ulong)RUP << 8) | RUP)); // P2R merge + Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)USAVE << 8))); // R2P: restore + } + } + _firstOutput[_code.Length] = _out.Count; + branches = _branches; + firstOutput = _firstOutput; + return _out; + } + + void LowerOne() + { + uint op = _i.Bits(0, 12); + switch (op) + { + // Scheduling, dependency and reconvergence hints: the translator runs each thread on its own. + case 0x918: // NOP + case 0x91a: // DEPBAR + case 0x945: // BSSY + case 0x941: // BSYNC + case 0x942: // BREAK + case 0x948: // WARPSYNC + return; + + case 0x202: case 0x802: case 0xa02: case 0xc02: Mov_(op); return; + case 0x882: case 0xc82: Umov(op); return; + case 0xab9: Uldc(); return; + case 0x290: case 0x890: case 0xa90: // UIADD3 + case 0x2a4: case 0x4a4: case 0x8a4: case 0xaa4: case 0x2a5: case 0x8a5: // UIMAD, UIMAD.WIDE + case 0x292: case 0x892: case 0xa92: // ULOP3 + case 0x299: case 0x499: case 0x899: // USHF + case 0x291: case 0x891: // ULEA + case 0x287: case 0x887: // USEL + case 0x28c: case 0x88c: case 0xa8c: // UISETP + case 0x89c: // UPLOP3 + case 0x296: case 0x896: // UPRMT + UniformAlu(op); return; + case 0x81c: Plop3(); return; + case 0xb63: Tld4(); return; + case 0x388: case 0x984: case 0x988: SharedAccess(op); return; + case 0x83b: Ldsm(); return; + case 0xb1d: Bar(); return; + case 0x305: case 0xb05: case 0xd05: F2i(op); return; + case 0xb82: Ldc(); return; + case 0x381: case 0x981: Ldg(op); return; + case 0x386: case 0x986: Stg(op); return; + case 0xb66: Tld(); return; + case 0xb6c: Txd(); return; + + case 0x221: case 0x421: case 0x621: case 0x821: case 0xa21: case 0xc21: case 0xe21: Fadd(); return; + case 0x220: case 0x420: case 0x620: case 0x820: case 0xa20: case 0xc20: Fmul(); return; + case 0x223: case 0x423: case 0x623: case 0x823: case 0xa23: case 0xc23: case 0xe23: Ffma(); return; + case 0x209: case 0x409: case 0x609: case 0x809: case 0xa09: case 0xc09: Fmnmx(); return; + case 0x308: case 0x908: case 0xb08: case 0xd08: Mufu(op); return; + case 0x307: case 0xb07: Frnd(op); return; + case 0x20b: case 0x40b: case 0x60b: case 0x80b: case 0xa0b: case 0xc0b: Setp(true); return; + case 0x20c: case 0x40c: case 0x60c: case 0x80c: case 0xa0c: case 0xc0c: Setp(false); return; + case 0x20a: case 0x40a: case 0x60a: case 0x80a: case 0xa0a: case 0xc0a: Fset(); return; + case 0x207: case 0x807: case 0xa07: case 0xc07: Sel_(false); return; + case 0x208: case 0x808: case 0xa08: case 0xc08: Sel_(true); return; + case 0x245: case 0x845: case 0xa45: case 0xc45: I2fp(); return; + + case 0x210: case 0x410: case 0x610: case 0x810: case 0xa10: case 0xc10: Iadd3_(); return; + case 0x224: case 0x424: case 0x624: case 0x824: case 0xa24: case 0xc24: case 0xe24: Imad(); return; + case 0x212: case 0x812: case 0xa12: case 0xc12: Lop3(); return; + case 0x217: case 0x817: case 0xa17: case 0xc17: Imnmx(); return; + case 0x219: case 0x419: case 0x619: case 0x819: case 0xa19: case 0xc19: Shf(); return; + case 0x23e: case 0x43e: case 0x83e: case 0xa3e: case 0xc3e: F2fp(); return; + case 0x216: case 0x416: case 0x616: case 0x816: case 0xa16: case 0xc16: Prmt(); return; + case 0x309: case 0xd09: Popc(); return; + case 0x300: case 0xd00: Flo(); return; + case 0x301: Brev(); return; + case 0x240: case 0x840: Hmnmx2(); return; + case 0x234: case 0x434: Hsetp2(); return; + case 0x803: case 0x804: PredicateRegister(op == 0x803); return; + case 0x59f: Sured(); return; + case 0x3c4: Redux(); return; + case 0x81a: Sgxt(); return; + case 0x806: case 0x886: Vote(op == 0x886); return; + case 0x3a8: case 0x9a8: Atomg(op == 0x9a8); return; + case 0x38c: Atoms(); return; + case 0x98e: Red(); return; + case 0x389: case 0x589: case 0x989: case 0xf89: Shfl(op); return; + case 0x822: Fswzadd(); return; + case 0x230: case 0x430: case 0x630: Half(0x5d10000000000000UL); return; // HADD2 + case 0x232: case 0x832: case 0xa32: Half(0x5d08000000000000UL); return; // HMUL2 + case 0x231: case 0x431: case 0x631: case 0x831: case 0xa31: Half(0x5d00000000000000UL); return; // HFMA2 + case 0xb9b: SurfaceAccess(true); return; + case 0xb97: SurfaceAccess(false); return; + case 0x211: case 0x811: case 0xa11: case 0xc11: Lea(); return; + + case 0x321: case 0x322: AttributeAccess(op == 0x322); return; + case 0x326: Ipa(); return; + case 0x805: Cs2r(); return; + case 0x919: case 0x9c3: S2r(op == 0x9c3); return; + case 0x927: Isbewr(); return; + case 0x949: Brx(false); return; + case 0x958: Brx(true); return; + case 0x923: Isberd(); return; + case 0x3c2: R2ur(); return; + case 0x225: case 0x227: case 0x625: case 0x825: case 0x827: case 0xa27: ImadWide(op); return; + case 0x213: case 0xa13: case 0xc13: Iabs(); return; + case 0xb6f: Txq(); return; + case 0xb60: case 0xf60: case 0x361: Tex(op); return; + case 0xb69: case 0xf69: case 0x36a: Tmml(op); return; + case 0xf66: TldBindless(); return; + case 0x370: TxqBindless(); return; + case 0x95b: Kill(); return; + case 0x94d: Exit(); return; + case 0x947: Bra(); return; + case 0x944: Call(); return; + case 0x950: Ret(); return; + case 0x2bd: Uflo(); return; + case 0x2be: Ubrev(); return; + case 0x2bf: Upopc(); return; + case 0x883: Up2ur(); return; + case 0x243: F2ipByte(); return; + case 0x23c: Hmma(); return; + case 0x23a: Movm(); return; + case 0x306: case 0xb06: case 0xd06: I2f(op); return; + case 0x387: case 0x983: LocalAccess(op == 0x387); return; + case 0x526: IpaIndexed(); return; + case 0xabb: UldcIndexed(); return; + } + throw Fail("opcode not lowered yet"); + } + + // ---- moves and uniform datapath ---------------------------------------------------------------------------- + void Mov_(uint op) + { + if (_i.Bits(72, 4) != 15) throw Fail("MOV lane mask"); + int d = Gpr((uint)Dst); + switch (op) + { + case 0x802: Mov32i(d, _i.W1, true); return; + case 0x202: Mov(d, Gpr(_i.Bits(32, 8))); return; + case 0xc02: Mov(d, Ur(_i.Bits(32, 8))); return; + default: { var (bank, off) = Cbuf(); MovC(d, bank, off, true); return; } + } + } + + void Umov(uint op) + { + RequireUniformGuardTrue(); + int d = Ur((uint)Dst); + if (d == RZ) return; + if (op == 0x882) Mov32i(d, _i.W1, true); + else EmitG(0x5c98000000000000UL | (15UL << 39) | ((ulong)Ur(_i.Bits(32, 8)) << 20) | (uint)d); + } + + // ULDC: width (bits 73-75) 4 = 32 bit, 5 = 64 bit into a uniform pair. + void Uldc() + { + RequireUniformGuardTrue(); + uint width = _i.Bits(73, 3); + int d = Ur((uint)Dst); + if (width < 4) // U8, S8, U16, S16: Maxwell LDC with the same width code, immediate address (any alignment) + { + int cb = Bank(_i.Bits(54, 5)), at = (int)_i.Bits(38, 16); + if (cb >= 18 || at >= 0x8000 || _i.Bits(24, 6) != 63) throw Fail($"ULDC width {width} c[{_i.Bits(54, 5)}][0x{at:x}]"); + if (d != RZ) + EmitG(0xef90000000000000UL | ((ulong)RZ << 8) | ((ulong)at << 20) | ((ulong)cb << 36) | ((ulong)width << 48) | (uint)d); + return; + } + var (bank, off) = Cbuf(); + if (width == 4) { if (d != RZ) MovC(d, bank, off, true); return; } + if (width == 5 && (Dst & 1) == 0) { if (d != RZ) { MovC(d, bank, off, true); MovC(d + 1, bank, off + 4, true); } return; } + throw Fail($"ULDC width {width}"); + } + + // Uniform-datapath arithmetic (UIADD3, UIMAD, UIMAD.WIDE, ULOP3, USHF, ULEA, USEL) has the encoding of its vector + // twin (base opcode - 0x80) with uniform registers in the register fields and bit 91 set; it is lowered as the + // twin with the fields read as uniform registers; its predicate fields name uniform predicates, which Run has + // swapped into the P registers of the same numbers. + void UniformAlu(uint op) + { + uint twin = (op & 0x1ff) - 0x80; + Insn saved = _i; + _i.Lo = (_i.Lo & ~0x1ffUL) | twin; + _i.Hi &= ~(1UL << 27); // bit 91 + _uniform = true; + try { LowerOne(); } + finally { _uniform = false; _i = saved; } + } + + // ---- floating point ---------------------------------------------------------------------------------------- + void Fadd() + { + if (_i.Bits(81, 10) != 0) throw Fail("FADD mode"); + var (a, aAbs, aNeg) = Raw(0, SA); + var (b, bAbs, bNeg) = Raw(SecondSlot, SB); + EmitG(0x5c58000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | (aNeg ? 1UL << 48 : 0) | (aAbs ? 1UL << 46 : 0) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0) + | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | ((ulong)_i.Bits(77, 1) << 50)); + } + + // FMUL: scale (bits 84-86) 1-3 divide by 8/4/2, 4 none, 5-7 multiply by 2/4/8; Maxwell's scale field is 0 none, + // 1-3 divide by 2/4/8, 4-6 multiply by 8/4/2. + static readonly uint[] FmulScale = { 0, 3, 2, 1, 0, 6, 5, 4 }; + void Fmul() + { + if (_i.Bit(76) || _i.Bits(81, 3) != 0 || _i.Bits(87, 4) != 0 || _i.Bits(84, 3) == 0) throw Fail("FMUL mode"); + var (a, aAbs, aNeg) = Raw(0, SA); + var (b, bAbs, bNeg) = Raw(1, SB); + a = Mat(a, aAbs, false, SA); + b = Mat(b, bAbs, false, SB); + EmitG(0x5c68000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | (aNeg != bNeg ? 1UL << 48 : 0) | ((ulong)FmulScale[_i.Bits(84, 3)] << 41) + | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) | ((ulong)_i.Bits(77, 1) << 50)); + } + + void Ffma() + { + if (_i.Bit(76) || _i.Bits(81, 10) != 0) throw Fail("FFMA mode"); + var (a, aAbs, aNeg) = Raw(0, SA); + var (b, bAbs, bNeg) = Raw(1, SB); + var (c, cAbs, cNeg) = Raw(2, SC); + a = Mat(a, aAbs, false, SA); + b = Mat(b, bAbs, false, SB); + c = Mat(c, cAbs, false, SC); + EmitG(0x5980000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | (aNeg != bNeg ? 1UL << 48 : 0) | (cNeg ? 1UL << 49 : 0) + | ((ulong)_i.Bits(77, 1) << 50) | ((ulong)_i.Bits(78, 2) << 51) | ((ulong)_i.Bits(80, 1) << 53)); + } + + // FMNMX: Rd = p ? min : max (p in bits 87-90). .NAN (bit 81) is not lowered. + void Fmnmx() + { + if (_i.Bits(74, 6) != 0 || _i.Bits(81, 6) != 0) throw Fail("FMNMX mode"); + var (a, aAbs, aNeg) = Raw(0, SA); + var (b, bAbs, bNeg) = Raw(SecondSlot, SB); + EmitG(0x5c60000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | (aNeg ? 1UL << 48 : 0) | (aAbs ? 1UL << 46 : 0) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0) + | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(80, 1) << 44)); + } + + // MUFU: operation in bits 74-79, numbered as Maxwell's (0 COS, 1 SIN, 2 EX2, 3 LG2, 4 RCP, 5 RSQ, 8 SQRT). + // SM70+ COS/SIN take the angle in turns (the compiler multiplies by 1/2pi first); the translator's take radians. + void Mufu(uint op) + { + uint operation = _i.Bits(74, 6); + if (operation > 8 || operation is 6 or 7 || _i.Bit(73)) throw Fail($"MUFU operation {operation}"); + int src; bool abs, neg; + if (op == 0x908) { Mov32i(SB, _i.W1); (src, abs, neg) = (SB, false, false); } + else (src, abs, neg) = Raw(1, SB); + if (operation <= 1) + { + src = Mat(src, abs, neg, SB); + Fmul32i(SB, src, 0x40c90fdb); // 2pi + src = SB; abs = neg = false; + } + EmitG(0x5080000000000000UL | ((ulong)operation << 20) | ((ulong)src << 8) | (uint)Gpr((uint)Dst) + | (abs ? 1UL << 46 : 0) | (neg ? 1UL << 48 : 0)); + } + + // FRND.F32: rounding (bits 78-79: nearest/floor/ceil/trunc) as Maxwell F2F.F32.F32 with integer rounding. + void Frnd(uint op) + { + if (_i.Bits(75, 2) != 2 || _i.Bits(84, 2) != 2 || _i.Bits(81, 3) != 0 || _i.Bits(86, 6) != 0) throw Fail("FRND format"); + var (b, bAbs, bNeg) = Raw(1, SB); + EmitG(0x5ca8000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)b << 20) | (uint)Gpr((uint)Dst) + | ((ulong)(8 + _i.Bits(78, 2)) << 39) | ((ulong)_i.Bits(80, 1) << 44) | (bNeg ? 1UL << 45 : 0) | (bAbs ? 1UL << 49 : 0)); + } + + // FSETP / ISETP: P (bits 81-83), Q (84-86), combined with Pc (87-90) by the operation in 74-75. + void Setp(bool floating) + { + if (!floating && _i.Bit(72)) { SetpExtended(); return; } + uint combine = _i.Bits(74, 2); + if (combine > 2) throw Fail("predicate combination"); + var (a, aAbs, aNeg) = Raw(0, SA); + var (b, bAbs, bNeg) = Raw(1, SB); + ulong w = (floating ? 0x5bb0000000000000UL : 0x5b60000000000000UL) | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) + | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)combine << 45); + if (floating) + w |= ((ulong)_i.Bits(80, 1) << 47) | ((ulong)_i.Bits(76, 4) << 48) + | (aNeg ? 1UL << 43 : 0) | (aAbs ? 1UL << 7 : 0) | (bNeg ? 1UL << 6 : 0) | (bAbs ? 1UL << 44 : 0); + else + w |= ((ulong)_i.Bits(73, 1) << 48) | ((ulong)_i.Bits(76, 3) << 49); + EmitG(w); + } + + // FSET.BF: Rd = 1.0 or 0.0. + void Fset() + { + if (_i.Bits(64, 8) != 0 || _i.Bits(74, 2) == 3 || _i.Bits(81, 6) != 0) throw Fail("FSET output mode"); + var (a, aAbs, aNeg) = Raw(0, SA); + var (b, bAbs, bNeg) = Raw(1, SB); + EmitG(0x5800000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(74, 2) << 45) + | ((ulong)_i.Bits(76, 4) << 48) | (1UL << 52) | ((ulong)_i.Bits(80, 1) << 55) + | (aNeg ? 1UL << 43 : 0) | (bNeg ? 1UL << 53 : 0) | (aAbs ? 1UL << 54 : 0) | (bAbs ? 1UL << 44 : 0)); + } + + // SEL / FSEL: Rd = p ? a : b (p in bits 87-90). FSEL applies the float modifiers first. + void Sel_(bool floating) + { + int a = floating ? FloatMat(0, SA) : Plain(0, SA); + int b = floating ? FloatMat(1, SB) : Plain(1, SB); + Sel(Gpr((uint)Dst), a, b, _i.Bits(87, 4), true); + } + + // I2FP.F32.{U32,S32}: bit 74 signed, rounding in 78-79. + void I2fp() + { + uint w2 = _i.W2 & ~(0x400u | 0xc000u | 0x08000000u); + if (w2 != 0x00201000) throw Fail("I2FP format"); + int b = Plain(1, SB); + EmitG(0x5cb8000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)_i.Bits(74, 1) << 13) | ((ulong)b << 20) + | ((ulong)_i.Bits(78, 2) << 39) | (uint)Gpr((uint)Dst)); + } + + // ---- integer ----------------------------------------------------------------------------------------------- + // IADD3 Rd, [Pu,] a, b, c. Negations are arithmetic (Maxwell IADD3's too). One carry out (bits 81-83) with c = RZ, + // or .X (bit 74) adding the carry-in predicate of bits 87-90. + void Iadd3_() + { + uint op = _i.Bits(0, 12); + bool negA = _i.Bit(72), negB = op != 0x810 && _i.Bit(63) && Form is not 2 and not 3, negC = _i.Bit(75), extended = _i.Bit(74); + uint carry1 = _i.Bits(81, 3), carry2 = _i.Bits(84, 3); + int a = Plain(0, SA), b = Plain(1, SB), c = Plain(2, SC); + int d = Gpr((uint)Dst); + if (Form is 2 or 3 && (_i.Bit(63) || negC)) throw Fail("IADD3 negation with a wide third source"); + if (!extended && carry1 == 7 && carry2 == 7) + { + EmitG(0x5cc0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)d + | (negA ? 1UL << 51 : 0) | (negB ? 1UL << 50 : 0) | (negC ? 1UL << 49 : 0)); + return; + } + // The 34-bit sum: low word in R244, carry count in R245 (IADD.CC / IADD.X). A negated source is ~x, plus 1 + // unless .X, whose carry-ins (bits 87-90 and 77-80) are added instead. Pu = bit 32, Pv = bit 33. + const int lo = 244, hi = 245, one = 246; + int inv(int r, bool neg, int t) { if (!neg) return r; Lop32i(t, r, 0xffffffff, 2); return t; } + a = inv(a, negA, 241); b = inv(b, negB, 242); c = inv(c, negC, 243); + void addc(int r) + { + Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)r << 20) | ((ulong)lo << 8) | lo)); // IADD.CC + Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)RZ << 20) | ((ulong)hi << 8) | hi)); // IADD.X + } + Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)b << 20) | ((ulong)a << 8) | lo)); + Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)RZ << 20) | ((ulong)RZ << 8) | hi)); + addc(c); + if (extended) + foreach (uint p in new[] { _i.Bits(87, 4), _i.Bits(77, 4) }) + { + if (p == 15) continue; + Mov32i(one, 1); + if (p != 7) Sel(one, one, RZ, p, false); + addc(one); + } + else if (negA || negB || negC) + { + Mov32i(one, (uint)((negA ? 1 : 0) + (negB ? 1 : 0) + (negC ? 1 : 0))); + addc(one); + } + if (d != RZ) Mov(d, lo); + uint guard = _i.Bits(12, 3); + foreach (int k in guard == carry1 ? new[] { 1, 0 } : new[] { 0, 1 }) // an output that is the guard goes last + { + uint p = k == 0 ? carry1 : carry2; + if (p == 7) continue; + Lop32i(one, hi, k == 0 ? 1u : 2u, 0); + EmitG(0x5b60000000000000UL | (5UL << 49) | ((ulong)PT << 39) | ((ulong)RZ << 20) | ((ulong)one << 8) | ((ulong)p << 3) | PT); + } + } + + // IMAD Rd, a, b, c (low word); bit 73 signed, bit 75 negates c (~c with .X), .X (bit 74) adds the carry-in + // predicate of bits 87-90. A carry out (bits 81-83) is not lowered; .HI/.WIDE are ImadWide. + void Imad() + { + uint known = 0xffu | 1u << 9 | 1u << 10 | 1u << 11 | 7u << 17 | 0xfu << 23 | (Form is 6 or 7 ? 1u << 27 : 0u); + bool x = _i.Bit(74); + if ((_i.W2 & ~known) != 0 || _i.Bits(81, 3) != 7 || (!x && _i.Bits(87, 4) != 15)) throw Fail("IMAD mode"); + int a = Plain(0, SA), b = Plain(1, SB), c = Plain(2, SC); + if (_i.Bit(75) && c != RZ) + { + if (x) Lop32i(SC, c, 0xffffffff, 2); // .X: ~c, the +1 comes as the carry + else Iadd3(SC, RZ, c, RZ, false, negB: true); + c = SC; + } + ulong signed = _i.Bit(73) ? (1UL << 48) | (1UL << 53) : 0; + int d = Gpr((uint)Dst); + ulong imad(int dst) => 0x5a00000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)dst | signed; + if (!x) { EmitG(imad(d)); return; } + Mov32i(SX, 1); + Sel(SX, SX, RZ, _i.Bits(87, 4), false); + Emit(Unguarded(imad(SA))); + Iadd3(d, SA, SX, RZ, true); + } + + // IMAD.HI Rd / IMAD.WIDE Rd:Rd+1 = a * b + (c+1:c), bit 73 signed. b: a register (bits 32-39), an immediate + // (0x8xx) or a constant (0xa27); 0x625 takes b from bits 64-71 and adds a constant pair. The 64-bit sum uses + // Maxwell IMUL.LO/.HI and IADD.CC/.X into R247:R248. A carry out (bits 81-83) is not lowered. + void ImadWide(uint op) + { + bool wide = op is 0x225 or 0x625 or 0x825, signedMul = _i.Bit(73); + if ((_i.W2 & ~0x000e02ffu) != 0x07800000 || _i.Bits(81, 3) != 7) throw Fail("IMAD.HI/WIDE mode"); + int a = Gpr(_i.Bits(24, 8)), b, cLo, cHi; + if (op is 0x825 or 0x827) { Mov32i(SB, _i.W1); b = SB; } + else if (op == 0xa27) { var (bank, off) = Cbuf(); MovC(SB, bank, off); b = SB; } + else b = Gpr(_i.Bits(op == 0x625 ? 64 : 32, 8)); + if (op == 0x625) { var (bank, off) = Cbuf(); MovC(SC, bank, off); MovC(SX, bank, off + 4); cLo = SC; cHi = SX; } + else + { + int c = Gpr(_i.Bits(64, 8)); + if (c != RZ && (c & 1) != 0) throw Fail("IMAD.WIDE odd addend pair"); + cLo = c; cHi = c == RZ ? RZ : c + 1; + } + int d = Gpr((uint)Dst); + if (wide && (d == RZ || (d & 1) != 0)) throw Fail("IMAD.WIDE destination pair"); + ulong sign = signedMul ? (1UL << 40) | (1UL << 41) : 0; + Emit(Unguarded(0x5c38000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | 247 | sign)); // IMUL lo + Emit(Unguarded(0x5c38000000000000UL | (1UL << 39) | ((ulong)b << 20) | ((ulong)a << 8) | 248 | sign)); // IMUL.HI + Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)cLo << 20) | (247UL << 8) | 247)); // IADD.CC + Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)cHi << 20) | (248UL << 8) | 248)); // IADD.X + if (wide) { Mov(d, 247); Mov(d + 1, 248); } + else Mov(d, 248); + } + + // IABS Rd, b: max(b, -b), signed. + void Iabs() + { + uint op = _i.Bits(0, 12); + if ((_i.W2 & ~(op == 0xc13 ? 0x08000000u : 0u)) != 0) throw Fail("IABS mode"); + int b = Plain(1, SB); + Iadd3(SX, RZ, b, RZ, false, negB: true); + EmitG(0x5c20000000000000UL | ((ulong)SX << 20) | ((ulong)b << 8) | (uint)Gpr((uint)Dst) | (15UL << 39) | (1UL << 48)); + } + + // LOP3.LUT: table in bits 72-79, optional P = result != 0 (bits 81-83). + void Lop3() + { + if (_i.Bit(80) || _i.Bits(87, 4) != 15) throw Fail("LOP3 predicate mode"); + int a = Plain(0, SA), b = Plain(1, SB), c = Gpr(_i.Bits(64, 8)); + uint p = _i.Bits(81, 3); + EmitG(0x5be0000000000000UL | ((ulong)c << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | ((ulong)_i.Bits(72, 8) << 28) | ((ulong)(p == 7 ? 0 : 3) << 36) | ((ulong)p << 48)); + } + + // IMNMX: Rd = p ? min : max (p in bits 87-90), bit 73 signed. + void Imnmx() + { + int a = Plain(0, SA), b = Plain(1, SB); + EmitG(0x5c20000000000000UL | ((ulong)b << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst) + | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(73, 1) << 48)); + } + + // SHF Rd, low (A), shift (B), high (C): bits 73-74 type (0 S64, 1 U64, 2 S32, 3 U32), 75 .W (wrap), 76 right, + // 80 .HI (the high word of the 64-bit result). + void Shf() + { + if (_i.Bits(81, 10) != 0) throw Fail("SHF mode"); + uint type = _i.Bits(73, 2); + bool wrap = _i.Bit(75), right = _i.Bit(76), hi = _i.Bit(80), wide = type <= 1, signedType = type == 0 || type == 2; + int low = Plain(0, SA), s = Plain(1, SB), high = Plain(2, SC); + int d = Gpr((uint)Dst); + ulong maxShift = type == 0 ? 3UL : type == 1 ? 2UL : 0UL; + if (!right && hi) // high word of (high:low) << s + EmitG(0x5bf8000000000000UL | ((ulong)high << 39) | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 50 : 0) | (maxShift << 37)); + else if (!right) // low word: low << s + { + if (wrap && wide) throw Fail("SHF.L.W 64-bit low word"); + EmitG(0x5c48000000000000UL | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 39 : 0)); + } + else if (!hi) // low word of (high:low) >> s + EmitG(0x5cf8000000000000UL | ((ulong)high << 39) | ((ulong)s << 20) | ((ulong)low << 8) | (uint)d | (wrap ? 1UL << 50 : 0) | (maxShift << 37)); + else // high word: high >> s + { + if (wrap && wide) throw Fail("SHF.R.W 64-bit high word"); + EmitG(0x5c28000000000000UL | ((ulong)s << 20) | ((ulong)high << 8) | (uint)d | (wrap ? 1UL << 39 : 0) | (signedType ? 1UL << 48 : 0)); + } + } + + // LEA Rd, [Pu,] a, b, s: (a << s) + b, shift in bits 75-79, bit 63 negates b. LEA.HI (bit 80): b + high word of + // (c:a) << s (bit 73 sign-extends a when c is RZ), .X (bit 74) adds the carry predicate of bits 87-90. + void Lea() + { + uint op = _i.Bits(0, 12); + int a = Plain(0, SA), b = Plain(1, SB), d = Gpr((uint)Dst); + uint shift = _i.Bits(75, 5), carry = _i.Bits(81, 3); + bool negB = op != 0x811 && _i.Bit(63); + if (_i.Bit(72)) throw Fail("LEA negated a"); + if (!_i.Bit(80)) + { + if (_i.Bit(74)) throw Fail("LEA.X without .HI"); + ulong iscadd(int dst, int bb) => 0x5c18000000000000UL | ((ulong)shift << 39) | ((ulong)bb << 20) | ((ulong)a << 8) | (uint)dst | (negB ? 1UL << 48 : 0); + if (carry == 7) { EmitG(iscadd(d, b)); return; } + if (negB) throw Fail("LEA carry out with negated b"); + Mov(SX, b); + EmitG(iscadd(SC, SX)); + if (d != RZ) Mov(d, SC); + IsetpLtU((int)carry, SC, SX); // carry = sum < b + return; + } + if (carry != 7) throw Fail("LEA.HI carry out"); + int high = Gpr(_i.Bits(64, 8)); + // With c = RZ the high word is a right shift of a by 32 - s (arithmetic when bit 73 sign-extends a); + // otherwise SHF.L.HI of (c:a) by s. + Mov32i(SX, high == RZ ? 32 - shift : shift); + if (high == RZ) + EmitG(0x5c28000000000000UL | ((ulong)SX << 20) | ((ulong)a << 8) | SC | (_i.Bit(73) ? 1UL << 48 : 0)); + else + EmitG(0x5bf8000000000000UL | ((ulong)high << 39) | ((ulong)SX << 20) | ((ulong)a << 8) | SC); + if (_i.Bit(74)) + { + Mov32i(SX, 1); + Sel(SX, SX, RZ, _i.Bits(87, 4), false); + Iadd3(SC, SC, SX, RZ, true); + } + Iadd3(d, SC, b, RZ, true, negB); + } + + // ---- attributes, special registers ------------------------------------------------------------------------ + // ALD / AST: address bits 40-49, components 74-75 (+1), patch 76, output 79; vertex register 32 (ALD) / 64 (AST). + void AttributeAccess(bool store) + { + bool patch = _i.Bit(76), output = _i.Bit(79); + uint address = _i.Bits(40, 10), components = _i.Bits(74, 2) + 1; + if (_i.Bit(77) || patch || (output && store) || (address & 3) != 0 || address + components * 4 > 0x400 || _i.Bits(24, 8) != RZ) + throw Fail("indexed/patch attribute access"); + // a[0x5c] is an Ampere per-primitive hint (shading rate); nothing reads it back. + if (store && address == 0x5c && components == 1) return; + int value = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); + if (value == RZ) + { + if (!store) return; + for (int k = 0; k < components; k++) Mov32i(TexOut + k, 0); + value = TexOut; + } + int vertex = Gpr(store ? _i.Bits(64, 8) : _i.Bits(32, 8)); + EmitG((store ? 0xeff0000000000000UL : 0xefd8000000000000UL) | (uint)value | ((ulong)RZ << 8) | ((ulong)address << 20) + | (output ? 1UL << 32 : 0) | ((ulong)vertex << 39) | ((ulong)(components - 1) << 47)); + } + + // IPA: attribute (bits 64-71, in words), sample mode 76-77 (default/centroid), bit 78 flat. Maxwell IPA.PASS: + // like Maxwell's, SM86 IPA of a perspective input returns the value times 1/w, and the NVN2 compiler multiplies + // by w itself (IPA a[0x7c], MUFU.RCP, FMUL). Treating it as perspective-correct and adding IPA.MULTIPLY by w + // scaled every texture coordinate by w a second time: Ocarina's floors and walls smeared along the view. + void Ipa() + { + if (_i.Bits(32, 8) != RZ || _i.Bits(76, 2) > 1 || _i.Bit(79) || _i.Bits(81, 3) != 7) throw Fail("IPA mode"); + EmitG(0xe000000000000000UL | ((ulong)RZ << 8) | ((ulong)RZ << 20) | ((ulong)_i.Bits(64, 8) << 30) + | ((ulong)_i.Bits(76, 2) << 52) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)Gpr((uint)Dst)); + } + + // CS2R Rd, SRZ: a zeroed register pair. + void Cs2r() + { + if (_i.Bits(72, 8) != RZ || !_i.Bit(80)) throw Fail("CS2R special register"); + int d = Gpr((uint)Dst); + if (d == RZ) return; + Mov32i(d, 0, true); + Mov32i(d + 1, 0, true); + } + + // ISBEWR [Ra + imm], Rb: a stage buffer write of a task or mesh program (MeshInfo): Ra 24-31, Rb 32-39, the byte + // offset 40-63, the region 75-78, bit 79 set (a write). Region 15 the vertex outputs, 0 the primitive indices + // (bytes), 1 the primitive count, 7 the task payload. A shared memory store here. + void Isbewr() + { + if (_mesh == null) throw Fail("ISBEWR outside a task or mesh program"); + uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24); + if (!_i.Bit(79) || (_i.W2 & ~0xf800u) != 0 || _i.Bits(64, 8) != 0 || offset >= 0x10000) throw Fail("ISBEWR mode"); + int a = Gpr(_i.Bits(24, 8)), data = Gpr(_i.Bits(32, 8)); + int at; uint width = 4; + if (_mesh.Stage == 7 && region == 15) at = _mesh.Attr + (int)offset; + else if (_mesh.Stage == 7 && region == 0) { at = _mesh.Map + (int)offset; width = 0; } + else if (_mesh.Stage == 7 && region == 1) + { + if (a != RZ) throw Fail("ISBEWR primitive count at a register address"); + at = _mesh.Map; + } + else if (_mesh.Stage == 6 && region == 7) at = _mesh.Pay + (int)offset; + else throw Fail($"ISBEWR region {region} in a stage {_mesh.Stage} program"); + EmitG(0xef58000000000000UL | (uint)data | ((ulong)a << 8) | ((ulong)(uint)at << 20) | ((ulong)width << 48)); + } + + // ISBERD Rd, [Ra + imm]: a stage buffer read (fields as ISBEWR's, bit 79 clear). Address 0 of region 15 (a mesh + // program without a task stage, a task program) or of region 7 (a mesh program after a task stage) is the + // workgroup's index; region 7 elsewhere is the task payload. + void Isberd() + { + if (_mesh == null) throw Fail("ISBERD outside a task or mesh program"); + uint region = _i.Bits(75, 4), offset = _i.Bits(40, 24); + if (_i.Bit(79) || (_i.W2 & ~0x7800u) != 0 || _i.Bits(32, 8) != 0 || _i.Bits(64, 8) != 0 || offset >= 0x10000) throw Fail("ISBERD mode"); + int a = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst); + if (d == RZ) return; + if ((region == 15 || region == 7) && a == RZ && offset == 0) + { + Put(0xf0c8000000000000UL | (0x25UL << 20) | (uint)d, true); // S2R SR_CTAID.X + return; + } + if (region != 7 || _mesh.Stage != 7 || !_mesh.HasTask) throw Fail($"ISBERD region {region} at a nonzero address"); + EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | ((ulong)(uint)(_mesh.Pay + (int)offset) << 20) | (4UL << 48)); + } + + // S2R / S2UR: special register in bits 72-79, numbered as Maxwell's (lane id 0, thread and workgroup ids + // 33-39, lane masks 56-60, ...). SR 132 (variable-rate shading: fragment width and height in bytes 1 and 2) + // reads as 1x1. + void S2r(bool uniform) + { + uint special = _i.Bits(72, 8); + if (_i.W1 != 0 || _i.W2 != special << 8) throw Fail("special register mode"); + if (uniform) RequireUniformGuardTrue(); + int d = uniform ? Ur((uint)Dst) : Gpr((uint)Dst); + if (d == RZ) return; + if (special == 132) { Mov32i(d, 0x00010100, true); return; } + Put(0xf0c8000000000000UL | ((ulong)special << 20) | (uint)d, true); + } + + // R2UR: a uniform register takes the value of the first active lane. Uniform registers are per-thread here, so + // this is a move (the compiler emits R2UR for values that are uniform, or loops until they are). + void R2ur() + { + if (_i.W1 != 0 || _i.W2 != 0x000e0000) throw Fail("R2UR mode"); + int d = Ur((uint)Dst); + if (d != RZ) Mov(d, Gpr(_i.Bits(24, 8))); + } + + // ---- texture --------------------------------------------------------------------------------------------- + // TEX (bound): Rd 16-23, Ra 24-31, Rb 32-39, descriptor word offset 40-53, its bank 54-58, dimension 61-63 + // (1D, 2D, 3D, CUBE, ARRAY_1D, ARRAY_2D, -, ARRAY_CUBE), second destination 64-71, mask 72-75, offsets 76, + // depth compare 78, FP16 results 79, LOD mode 87-89 (numbered as Maxwell's), bit 90 no-dependency. + // Operands: array index, coordinates, then LOD/bias, offsets, depth reference; up to four values are split as + // halves over Ra and Rb (1+1, 2+1, 2+2), five or more as four in Ra and the rest in Rb. Maxwell TEX takes the + // coordinates in Ra.. and the rest in Rb.., and writes the masked components to consecutive registers. + static readonly int[] TexCoords = { 1, 2, 3, 3, 2, 3, 0, 4 }; + static readonly int[] TexMaxwellDim = { 0, 2, 4, 6, 1, 3, 0, 7 }; + // TEX. 0xb60 a bound descriptor; the bindless forms take the 32-bit handle (texture index in bits 0-19, sampler + // index above) from a uniform register in bits 40-45 (0xf60) or, with bit 59 (.B) set, as one more operand + // right after the coordinates (0x361; with bit 60 clear Ra holds all coordinates and Rb the handle and the + // rest). Bindless becomes Maxwell TEX.B, the handle first among the meta registers (offsets bit 36, LOD mode 37): + // the translator then indexes its texture-pool arrays with it (Ryujinx BindlessElimination GenerateBindlessAccess, + // the handle being loaded from memory). Operand packing as the system's lower_to_maxwell.h describes (behaviour only). + void Tex(uint op) + { + bool bindless = op != 0xb60; + uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4); + bool offsets = _i.Bit(76), dc = _i.Bit(78); + if (dim == 6 || _i.Bit(77) || _i.Bits(81, 3) != 7 || _i.Bit(59) != (op == 0x361) || (lod > 3 && lod != 6 && lod != 7) || mask == 0) + throw Fail("TEX mode"); + uint handle = 0; + int handleReg = RZ; + if (op == 0xb60) + { + handle = Handle(false); + uint bank = _i.Bits(54, 5); + if (handle > 0x1fff) throw Fail("TEX descriptor offset"); + if (!TexBank(bank)) throw Fail($"TEX descriptor bank {bank}"); + } + else if (op == 0xf60) + { + if (_i.Bits(46, 13) != 0) throw Fail("bindless TEX mode"); + handleReg = Ur(_i.Bits(40, 6)); + } + else if (_i.Bits(40, 19) != 0) throw Fail("TEX.B mode"); + bool hasLod = lod is 2 or 3 or 6 or 7; + int coords = TexCoords[dim], handleSlot = op == 0x361 ? 1 : 0; + int count = coords + handleSlot + (hasLod ? 1 : 0) + (offsets ? 1 : 0) + (dc ? 1 : 0); + int inA = op == 0x361 && !_i.Bit(60) ? coords : count <= 4 ? (count + 1) / 2 : op == 0x361 ? coords : 4; + int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); + int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); + for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false); + int next = coords + handleSlot, meta = TexMeta; + if (bindless) Mov(meta++, op == 0x361 ? Value(coords) : handleReg, false); + if (hasLod) Mov(meta++, Value(next++), false); + if (offsets) Mov(meta++, Value(next++), false); + if (dc) Mov(meta++, Value(next++), false); + if (bindless) + EmitG(0xde80000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)TexMaxwellDim[dim] << 28) + | ((ulong)mask << 31) | (offsets ? 1UL << 36 : 0) | ((ulong)lod << 37) | ((ulong)_i.Bits(90, 1) << 49) + | (dc ? 1UL << 50 : 0) | (7UL << 51)); + else + EmitG(0xc000000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) + | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) + | (dc ? 1UL << 50 : 0) | (7UL << 51) | (offsets ? 1UL << 54 : 0) | ((ulong)lod << 55)); + TexResults(mask, _i.Bit(79)); + } + + // TMML (LOD query; results 8.8 fixed point as Maxwell's), 1D/2D/3D/CUBE. 0xb69 a bound descriptor; 0xf69 the + // handle in a uniform register (bits 40-45, bit 91 set); 0x36a (TMML.B, bit 59) the handle in Rb and every + // coordinate in Ra. Coordinates come from Ra, or from Ra and Rb as TEX's when Rb is a register. Maxwell TMML + // (0xdf58, descriptor in 36-48) / TMML.B (0xdf60, handle in Rb), dimension in Maxwell's TexDim numbering. + void Tmml(uint op) + { + bool bindless = op != 0xb69; + uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4); + int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); + int coords = dim < 4 ? new[] { 1, 2, 3, 3 }[dim] : 0; + if (coords == 0 || _i.Bits(76, 4) != 0 || _i.Bits(80, 10) != 0 || mask == 0 || _i.Bit(59) != (op == 0x36a)) + throw Fail("TMML mode"); + uint handle = 0; + int handleReg = RZ; + if (op == 0xb69) + { + handle = Handle(false); + if (handle > 0x1fff || !TexBank(_i.Bits(54, 5))) throw Fail("TMML descriptor"); + } + else if (op == 0xf69) + { + if (_i.Bits(46, 13) != 0 || !_i.Bit(91)) throw Fail("bindless TMML mode"); + handleReg = Ur(_i.Bits(40, 6)); + } + else + { + if (_i.Bits(40, 19) != 0 || _i.Bit(91)) throw Fail("TMML.B mode"); + handleReg = Gpr((uint)rb); + } + int inA = rb == RZ || op == 0x36a ? coords : (coords + 1) / 2; + for (int k = 0; k < coords; k++) + Mov(TexCoord + k, k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))), false); + if (bindless) Mov(TexMeta, handleReg, false); + EmitG((bindless ? 0xdf60000000000000UL : 0xdf58000000000000UL) | TexOut | ((ulong)TexCoord << 8) + | ((ulong)(bindless ? TexMeta : RZ) << 20) | ((ulong)(dim * 2) << 28) | ((ulong)mask << 31) + | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49)); + TexResults(mask, false); + } + + // TLD.LZ with the handle in a uniform register (0xf66: bits 40-45, bit 91 and bit 60 set): Maxwell TLD.B, the + // coordinates x | y, layer, x | y or x, y | z over Ra and Rb (1D, 2D, ARRAY_2D, 3D) and the handle in Rb. + void TldBindless() + { + uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4); + bool array = dim == 5, volume = dim == 2; + if (_i.Bits(46, 13) != 0 || !_i.Bit(91) || !_i.Bit(60) || _i.Bit(59) || _i.Bits(76, 5) != 0 || _i.Bits(81, 3) != 7 + || _i.Bits(87, 3) != 1 || (dim != 0 && dim != 1 && !array && !volume) || mask == 0) + throw Fail("bindless TLD mode"); + int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); + int handleReg = Ur(_i.Bits(40, 6)); + Mov(TexCoord, ra == RZ ? RZ : Gpr((uint)ra), false); + if (array || volume) Mov(TexCoord + 1, ra == RZ ? RZ : Gpr((uint)(ra + 1)), false); + if (dim != 0) Mov(array || volume ? TexCoord + 2 : TexCoord + 1, rb == RZ ? RZ : Gpr((uint)rb), false); + Mov(TexMeta, handleReg, false); + EmitG(0xdd00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) + | ((ulong)(array ? 3u : volume ? 4u : dim == 0 ? 0u : 2u) << 28) | ((ulong)mask << 31) + | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51)); + TexResults(mask, false); + } + + // TXQ.B (0x370): the handle in Ra and the level in Ra+1, as Maxwell TXQ.B (0xdf50) takes them; query 1 + // (dimensions), results as TXQ's. + void TxqBindless() + { + uint mask = _i.Bits(72, 4); + if (_i.Bits(59, 5) != 1 || mask == 0 || _i.Bits(76, 14) != 0 || _i.Bit(91) || _i.Bits(32, 27) != 0) throw Fail("TXQ.B mode"); + int ra = (int)_i.Bits(24, 8); + Mov(TexCoord, ra == RZ ? RZ : Gpr((uint)ra), false); + Mov(TexCoord + 1, ra == RZ ? RZ : Gpr((uint)(ra + 1)), false); + EmitG(0xdf50000000000000UL | TexOut | ((ulong)TexCoord << 8) | (1UL << 22) | ((ulong)mask << 31) | ((ulong)_i.Bits(90, 1) << 49)); + TexResults(mask, false); + } + + // F2I: destination signed (bit 72) and size (75-76: 8/16/32 bits), rounding 78-79 (nearest, floor, ceil, + // trunc), FTZ 80, source size 84-85 (F16, F32). Maxwell F2I has the same rounding order. + void F2i(uint op) + { + const uint known = 1u << 8 | 3u << 11 | 1u << 13 | 3u << 14 | 1u << 16 | 3u << 20; + uint w2 = _i.W2 & ~(op == 0xd05 ? 0x08000000u : 0u); + uint dstSize = _i.Bits(75, 2), srcSize = _i.Bits(84, 2); + if ((w2 & ~known) != 0 || dstSize > 2 || srcSize is not 1 and not 2 || (srcSize == 1 && _i.Bits(60, 2) != 0)) + throw Fail("F2I format"); + var (b, abs, neg) = Raw(1, SB); + uint fmt = dstSize | (_i.Bit(72) ? 4u : 0u); // ISrcDstFmt: U8 0, U16 1, U32 2, +4 signed + EmitG(0x5cb0000000000000UL | ((ulong)b << 20) | (uint)Gpr((uint)Dst) | ((ulong)(fmt & 3) << 8) | ((ulong)(fmt >> 2) << 12) + | ((ulong)srcSize << 10) | ((ulong)_i.Bits(78, 2) << 39) | ((ulong)_i.Bits(80, 1) << 44) + | (neg ? 1UL << 45 : 0) | (abs ? 1UL << 49 : 0)); + } + + // LDC Rd, c[bank][Ra + offset]: offset 38-53, bank 54-58, size 73-75 (as Maxwell's: U8 S8 U16 S16 32 64), + // mode 78-79 (0, or 2 .IS: Ra addresses all banks). + void Ldc() + { + uint width = _i.Bits(73, 3), mode = _i.Bits(78, 2); + int bank = Bank(_i.Bits(54, 5)); + if (width > 5 || mode is not 0 and not 2 || (_i.W2 & ~0xce00u) != 0 || (_i.W1 & 0xf800003fu) != 0 || bank >= 18) + throw Fail("LDC mode"); + int d = Gpr((uint)Dst); + if (width == 5 && (d & 1) != 0) throw Fail("LDC.64 odd destination"); + EmitG(0xef90000000000000UL | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.Bits(38, 16) << 20) | ((ulong)bank << 36) + | ((ulong)mode << 44) | ((ulong)width << 48) | (uint)d); + } + + // Global memory address of LDG/STG: a 64-bit register pair (Ra, .E), or a uniform pair (bits 32-37 for loads, + // 64-69 for stores) plus Ra (64-bit when bit 90 is set, else zero-extended), or with the "desc[UR][Ra.64]" + // forms Ra alone (the uniform pair is a cache-policy descriptor). Sums go to R244:R245. + int GlobalAddress(bool uniformForm, bool store) + { + int ra = (int)_i.Bits(24, 8); + if (!uniformForm) + { + if (store ? (_i.W2 & ~0xe00u) != 0x0010a100 : (_i.W2 & ~0x2e00u) != 0x001e8100) throw Fail("global access mode"); + if (ra != RZ && ((ra & 1) != 0)) throw Fail("global address pair"); + return ra == RZ ? RZ : Gpr((uint)ra); + } + uint ur = store ? _i.Bits(64, 6) : _i.Bits(32, 6); + bool descriptor = store ? (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c101100 + : (_i.W2 & ~(0x3fu | 0xe00u)) == 0x0c1e1100 || _i.W2 == 0x0c1ebd00; + uint plain = store ? 0x0810a100u : 0x081e8100u; + if (!descriptor && (_i.W2 & ~(0x04000000u | 0x3fu | 0xe00u | (store ? 0u : 0x2000u))) != plain) throw Fail("global access uniform mode"); + if (!store && _i.Bits(38, 2) != 0) throw Fail("global access uniform mode"); + if (descriptor) + { + if ((ra & 1) != 0 || ra == RZ) throw Fail("global address pair"); + return Gpr((uint)ra); + } + int u = Ur(ur); + if (u != RZ && (ur & 1) != 0) throw Fail("global uniform address pair"); + if (ra == RZ) return u; + bool wide = _i.Bit(90); + if (wide && (ra & 1) != 0) throw Fail("global address pair"); + int r = Gpr((uint)ra); + Emit(Unguarded(0x5c10000000000000UL | (1UL << 47) | ((ulong)r << 20) | ((ulong)u << 8) | 244)); // IADD.CC + Emit(Unguarded(0x5c10000000000000UL | (1UL << 43) | ((ulong)(wide ? r + 1 : RZ) << 20) | ((ulong)(u == RZ ? RZ : u + 1) << 8) | 245)); // IADD.X + return 244; + } + + // LDG Rd, [address + offset]: offset 40-63 (signed), size 73-75 (U8 S8 U16 S16 32 64 128, as Maxwell's). + void Ldg(uint op) + { + uint size = _i.Bits(73, 3); + int d = Gpr((uint)Dst); + if (size > 6 || (size == 5 && (d & 1) != 0) || (size == 6 && (d & 3) != 0)) throw Fail("LDG size"); + if (op == 0x381 && _i.Bits(32, 8) != 0) throw Fail("LDG mode"); + int addr = GlobalAddress(op == 0x981, false); + EmitG(0xeed0000000000000UL | (uint)d | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | (1UL << 45) | ((ulong)size << 48)); + } + + // STG [address + offset], Rdata: data 32-39, offset 40-63, size 73-75; .E (bit 72) required. + void Stg(uint op) + { + uint size = _i.Bits(73, 3); + int data = Gpr(_i.Bits(32, 8)); + if (size > 6 || !_i.Bit(72) || (data != RZ && ((size == 5 && (data & 1) != 0) || (size == 6 && (data & 3) != 0)))) throw Fail("STG size"); + if (data == RZ && size > 4) + { + for (int k = 0; k < (1 << (int)(size - 4)); k++) Mov32i(TexOut + k, 0); + data = TexOut; + } + int addr = GlobalAddress(op == 0x986, true); + EmitG(0xeed8000000000000UL | (uint)data | ((ulong)addr << 8) | ((ulong)_i.Bits(40, 24) << 20) | (1UL << 45) | ((ulong)size << 48)); + } + + // TLD (texel fetch, integer coordinates): fields as TEX; LOD mode 87-89 is 1 (.LZ) or 3 (.LL, the level after + // the coordinates), offsets bit 76. Maxwell TLD takes its operands as Maxwell TEX does. + void Tld() + { + uint dim = _i.Bits(61, 3), lod = _i.Bits(87, 3), mask = _i.Bits(72, 4); + bool offsets = _i.Bit(76); + if (dim == 6 || _i.Bits(77, 3) != 0 || _i.Bits(81, 3) != 7 || _i.Bit(59) || lod is not 1 and not 3 || mask == 0) + throw Fail("TLD mode"); + uint handle = Handle(false), bank = _i.Bits(54, 5); + if (handle > 0x1fff) throw Fail("TLD descriptor offset"); + if (!TexBank(bank)) throw Fail($"TLD descriptor bank {bank}"); + int coords = TexCoords[dim]; + int count = coords + (lod == 3 ? 1 : 0) + (offsets ? 1 : 0); + int inA = count <= 4 ? (count + 1) / 2 : 4; + int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); + int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); + for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false); + int next = coords, meta = TexMeta; + if (lod == 3) Mov(meta++, Value(next++), false); + if (offsets) Mov(meta++, Value(next++), false); + EmitG(0xdc00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) + | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | (offsets ? 1UL << 35 : 0) | ((ulong)handle << 36) + | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51) | (lod == 3 ? 1UL << 55 : 0)); + TexResults(mask, false); + } + + // TXQ (dimensions): level in Ra, descriptor offset 40-53 in bank 54-58 (2), mask 72-75, second destination + // 64-71 for the third and fourth values. Maxwell TXQ with query 1 (dimension) writes them to consecutive + // registers. + void Txq() + { + uint mask = _i.Bits(72, 4), handle = Handle(false), bank = _i.Bits(54, 5); + if (_i.Bits(59, 5) != 0 || mask == 0 || _i.Bits(76, 14) != 0 || _i.Bit(91) || handle > 0x1fff) throw Fail("TXQ mode"); + if (!TexBank(bank)) throw Fail($"TXQ descriptor bank {bank}"); + int ra = Gpr(_i.Bits(24, 8)); + EmitG(0xdf48000000000000UL | TexOut | ((ulong)ra << 8) | (1UL << 22) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49)); + TexResults(mask, false); + } + + // Texture results: the components of the mask, from TexOut on, to Rd, Rd+1 and then the second destination + // (bits 64-71) when there is one. FP16 results pack (c0, c1) into the first register and (c2, c3) into the + // second, the first of each pair in the low half (F2F.F16 writes its half there; ISCADD merges). + void TexResults(uint mask, bool f16) + { + int d = Dst, d2 = (int)_i.Bits(64, 8); + int n = System.Numerics.BitOperations.PopCount(mask); + if (!f16) + { + for (int k = 0; k < n; k++) + { + int target = d2 == RZ || k < 2 ? (d == RZ ? RZ : d + k) : d2 + (k - 2); + if (target != RZ) Mov(Gpr((uint)target), TexOut + k); + } + return; + } + void pack(int target, int lo, int hi) + { + if (target == RZ) return; + Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)lo << 20) | 246)); + if (hi == RZ) Mov32i(247, 0); + else Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)hi << 20) | 247)); + EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)target)); + } + pack(d, TexOut, n > 1 ? TexOut + 1 : RZ); + if (n > 2) pack(d2 != RZ ? d2 : d == RZ ? RZ : d + 1, TexOut + 2, n > 3 ? TexOut + 3 : RZ); + } + + // PRMT Rd, a, selector (B slot), c: byte i of the result is byte (s & 7) of c:a, its sign repeated when s & 8; + // the mode (bits 72-75) other than the default index mode is not lowered. Maxwell PRMT has the same meaning. + void Prmt() + { + if (_i.Bits(72, 4) != 0 || _i.Bits(76, 15) != 0) throw Fail("PRMT mode"); + int a = Plain(0, SA), sel = Plain(1, SB), c = Plain(2, SC); + EmitG(0x5bc0000000000000UL | ((ulong)c << 39) | ((ulong)sel << 20) | ((ulong)a << 8) | (uint)Gpr((uint)Dst)); + } + + // FLO Rd, b: the index of the highest set bit (bit 73 signed, bit 74 .SH: the shift amount, bit 63 of ~b). + void Flo() + { + if ((_i.W2 & ~(0x600u | 0x08000000u)) != 0x000e0000) throw Fail("FLO mode"); + var (b, _, inv) = Raw(1, SB); + EmitG(0x5c30000000000000UL | ((ulong)b << 20) | (inv ? 1UL << 40 : 0) | ((ulong)_i.Bits(74, 1) << 41) + | ((ulong)_i.Bits(73, 1) << 48) | (uint)Gpr((uint)Dst)); + } + + // HSETP2 P, Q, a, b: compare 76-79, FTZ 80, P/Q 81-86, combined with Pc (87-90) by bits 69-70, H_AND 71; + // A: reg 24, neg 72, abs 73, swizzle 74-75; b: a register (swizzle 60-61, abs 62, neg 63) or an immediate half + // pair. Maxwell HSETP2 has the same fields at its own positions. + void Hsetp2() + { + uint op = _i.Bits(0, 12); + ulong source; + if (op == 0x234) + { + if ((_i.W1 & 0x0fffff00u) != 0 || _i.Bits(64, 5) != 0) throw Fail("HSETP2 mode"); + source = ((ulong)Gpr(_i.Bits(32, 8)) << 20) | ((ulong)_i.Bits(60, 2) << 28) | ((ulong)_i.Bits(62, 1) << 30) | ((ulong)_i.Bits(63, 1) << 31); + } + else { Mov32i(SB, _i.W1); source = (ulong)SB << 20; } + EmitG(0x5d20000000000000UL | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)_i.Bits(80, 1) << 6) + | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | source | ((ulong)_i.Bits(76, 4) << 35) | ((ulong)_i.Bits(87, 3) << 39) + | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)_i.Bits(72, 1) << 43) | ((ulong)_i.Bits(73, 1) << 44) + | ((ulong)_i.Bits(69, 2) << 45) | ((ulong)_i.Bits(74, 2) << 47) | ((ulong)_i.Bits(71, 1) << 49)); + } + + // P2R Rd, PR, Ra, mask / R2P PR, Ra, mask: immediate mask (bits 32-38), byte select 76-77, Ra the merge source + // (P2R) or the value (R2P), as Maxwell's P2R/R2P with an immediate. + void PredicateRegister(bool toRegister) + { + if (_i.W1 > 0x7f || (_i.W2 & ~0x3000u) != 0 || (!toRegister && _i.Bits(16, 8) != 0)) throw Fail("P2R/R2P mode"); + ulong w = (toRegister ? 0x38e8000000000000UL | (uint)Gpr((uint)Dst) : 0x38f0000000000000UL) + | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.W1 << 20) | ((ulong)_i.Bits(76, 2) << 41); + EmitG(w); + } + + // SURED.D.op U32 (bound descriptor): data 32-39, coordinates 24-31, operation 87-89 (ADD MIN MAX INC DEC AND OR + // XOR, as Maxwell's), dimension 61-63 (1D buffer or 2D), descriptor offset 40-53 in bank 54-58. + void Sured() + { + uint dim = _i.Bits(61, 3), handle = Handle(true), bank = _i.Bits(54, 5); + if ((_i.W2 & ~(7u << 23)) != 0x0010e000 || dim is not 1 and not 3 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank)) + throw Fail("SURED mode"); + EmitG(0xeb48000000000000UL | (uint)Gpr(_i.Bits(32, 8)) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)_i.Bits(87, 3) << 24) + | ((ulong)dim << 33) | ((ulong)handle << 36)); + } + + // REDUX URd, Ra: Ra reduced over the warp's active lanes into a uniform register. Mode (bits 78-80): SUM 3 + // (modulo 2^32), MIN 4, MAX 5 (unsigned, signed with bit 73), OR 1, XOR 2. Lowered as VOTE.ANY (the active-lane + // mask) and one SHFL.IDX per lane; an inactive lane contributes the operation's identity. + void Redux() + { + uint mode = (_i.W2 & ~0x200u) >> 14; + bool signedOrder = _i.Bit(73); + if (_i.W1 != 0 || (_i.W2 & ~0x200u & ~(7u << 14)) != 0 || mode is not (1 or 2 or 3 or 4 or 5) + || (signedOrder && mode is not (4 or 5)) || _i.Bits(12, 4) != 7) + throw Fail("REDUX mode"); + int t = Ur((uint)Dst), src = Gpr(_i.Bits(24, 8)); + if (t == RZ) return; + const int ballot = 240, lanev = 241, keep = 242; + bool min = mode == 4; + Emit(Unguarded(0x50d8000000000000UL | (1UL << 48) | (7UL << 45) | ((ulong)PT << 39) | ballot)); // VOTE.ANY ballot + Mov32i(t, min ? 0xffffffffu : 0u); + ulong lop3(int d, int a, int b, uint lut) => Unguarded(0x5be0000000000000UL | ((ulong)RZ << 39) | ((ulong)b << 20) | ((ulong)a << 8) | (uint)d | ((ulong)lut << 28) | (7UL << 48)); + for (uint lane = 0; lane < 32; lane++) + { + Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | (1UL << 28) | ((ulong)lane << 20) | ((ulong)src << 8) | lanev)); + if (signedOrder) Lop32i(lanev, lanev, 0x80000000, 2); // signed order as unsigned + Emit(Unguarded(0x3828000000000000UL | ((ulong)lane << 20) | ((ulong)ballot << 8) | keep)); // SHR imm + Lop32i(keep, keep, 1, 0); + Iadd3(keep, RZ, keep, RZ, false, negB: true); // 0 or ~0 + Emit(lop3(lanev, lanev, keep, min ? 0xF3u : 0xC0u)); // x | ~m, x & m + switch (mode) + { + case 4: Emit(Unguarded(0x5c20000000000000UL | ((ulong)lanev << 20) | ((ulong)t << 8) | (uint)t | ((ulong)PT << 39))); break; // min + case 5: Emit(Unguarded(0x5c20000000000000UL | ((ulong)lanev << 20) | ((ulong)t << 8) | (uint)t | (15UL << 39))); break; // max + case 1: Emit(lop3(t, t, lanev, 0xFC)); break; + case 2: Emit(lop3(t, t, lanev, 0x3C)); break; + default: Iadd3(t, t, lanev, RZ, false); break; + } + } + if (signedOrder) Lop32i(t, t, 0x80000000, 2); + } + + // VOTE Rd, Pu, Pin: ballot into Rd (URd for VOTEU), mode 72-73 (ALL, ANY, EQ) over the vector predicate of bits + // 87-90, result predicate 81-83 (a uniform predicate for VOTEU). Maxwell VOTE has the same fields. VOTEU's + // output goes through Pk (k its UP index) with the guest's Pk saved around it; no uniform predicate is read. + void Vote(bool uniform) + { + if (_i.W1 != 0 || _i.Bits(72, 2) > 2 || (_i.W2 & ~0x078e0300u) != 0 || (uniform && (_i.Bits(24, 8) != 0 || _i.Bits(12, 4) != 7))) + throw Fail("VOTE mode"); + uint pout = _i.Bits(81, 3); + int d = uniform ? Ur((uint)Dst) : Gpr((uint)Dst); + ulong vote = 0x50d8000000000000UL | (uint)d | ((ulong)_i.Bits(87, 4) << 39) | ((ulong)pout << 45) | ((ulong)_i.Bits(72, 2) << 48); + if (!uniform || pout == 7) { EmitG(vote); return; } + uint m = 1u << (int)pout; + Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RZ << 8) | USAVE)); + EmitG(vote); + Emit(Unguarded(0x38e8000000000000UL | ((ulong)m << 20) | ((ulong)RUP << 8) | RUP)); + Emit(Unguarded(0x38f0000000000000UL | ((ulong)m << 20) | ((ulong)USAVE << 8))); + } + + // ATOMG.E U32: Rd = atomic op (87-90, numbered as Maxwell ATOM's: ADD MIN MAX INC DEC AND OR XOR EXCH) of + // [address + offset 40-63] with Rb (32-39); the address a register pair (24-31) or a uniform pair (64-69). + void Atomg(bool uniformAddress) + { + uint operation = _i.Bits(87, 4); + int offset = (int)(_i.Bits(40, 24) << 8) >> 8; + uint w2 = _i.W2 & ~(0xfu << 23); + int addr; + if (uniformAddress) + { + if (_i.Bits(24, 8) != RZ || (w2 & ~0x3fu) != 0x081ee140 || (_i.Bits(64, 6) & 1) != 0) throw Fail("ATOMG mode"); + addr = Ur(_i.Bits(64, 6)); + } + else + { + if (w2 != 0x001ee100 || (_i.Bits(24, 8) & 1) != 0) throw Fail("ATOMG mode"); + addr = Gpr(_i.Bits(24, 8)); + } + if (operation > 8 || offset < -0x80000 || offset > 0x7ffff) throw Fail("ATOMG operation/offset"); + EmitG(0xed00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)addr << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) + | ((ulong)((uint)offset & 0xfffff) << 28) | (1UL << 48) | ((ulong)operation << 52)); + } + + // RED.E (global reduction, no result): op 87-89 (ADD MIN MAX INC DEC AND OR XOR), type 73-75 (U32, S32, F32 add), + // data 32-39, address a register pair (24-31) or, with Ra = RZ, a uniform pair (64-69), offset 40-63. + void Red() + { + uint type = _i.Bits(73, 3), operation = _i.Bits(87, 3); + bool ur = _i.Bits(24, 8) == RZ; + uint variable = 0x03800e00u | (ur ? 0x3fu : 0u), mode = ur ? 0x0c10e100u : 0x0010e100u; + int offset = (int)(_i.Bits(40, 24) << 8) >> 8; + if ((_i.W2 & ~variable) != mode || type is not (0 or 1 or 3) || (type == 3 && operation != 0)) throw Fail("RED mode"); + uint a = ur ? _i.Bits(64, 6) : _i.Bits(24, 8); + if ((a & 1) != 0 || offset < -0x80000 || offset > 0x7ffff) throw Fail("RED address"); + int addr = ur ? Ur(a) : Gpr(a); + EmitG(0xebf8000000000000UL | (uint)Gpr(_i.Bits(32, 8)) | ((ulong)addr << 8) | ((ulong)type << 20) | ((ulong)operation << 23) + | ((ulong)((uint)offset & 0xfffff) << 28) | (1UL << 48)); + } + + // SHFL Rd, [Pout,] a, lane, mask: lane an immediate (53-57; 0x989, 0xf89) or a register (32-39), the + // clamp/segment mask an immediate (40-52; 0x589, 0xf89) or a register (64-71), mode 58-59 (IDX UP DOWN BFLY), + // lane-valid predicate 81-83. Maxwell SHFL has the same choices. + void Shfl(uint op) + { + bool laneImm = op is 0x989 or 0xf89, maskImm = op is 0x589 or 0xf89; + if ((_i.W2 & ~(0x000e0000u | (maskImm ? 0u : 0xffu))) != 0 || _i.Bits(60, 4) != 0 || (laneImm && _i.Bits(32, 8) != 0) || (!laneImm && _i.Bits(53, 5) != 0)) + throw Fail("SHFL mode"); + ulong lane = laneImm ? ((ulong)_i.Bits(53, 5) << 20) | (1UL << 28) : (ulong)Gpr(_i.Bits(32, 8)) << 20; + ulong mask = maskImm ? ((ulong)_i.Bits(40, 13) << 34) | (1UL << 29) : (ulong)Gpr(_i.Bits(64, 8)) << 39; + EmitG(0xef10000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | lane | mask + | ((ulong)_i.Bits(58, 2) << 30) | ((ulong)_i.Bits(81, 3) << 48)); + } + + // FSWZADD Rd, a, b (64-71), per-lane operation codes (bits 32-39): 0 a+b, 1 a-b, 2 b-a, 3 b. Maxwell numbers 1 + // and 2 the other way round. + void Fswzadd() + { + uint swz = _i.W1; + if (swz > 0xff || _i.W2 > 255) throw Fail("FSWZADD mode"); + uint maxwell = ((swz & 0x55) << 1) | ((swz & 0xaa) >> 1); + EmitG(0x50f8000000000000UL | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(64, 8)) << 20) | ((ulong)maxwell << 28) | (uint)Gpr((uint)Dst)); + } + + // ATOMS (shared) U32/S32 (bit 73): Rd = op (87-90, as Maxwell ATOMS's) of [Ra + offset 40-63] with Rb. + void Atoms() + { + uint offset = _i.Bits(40, 24), operation = _i.Bits(87, 4); + if ((_i.W2 & ~0x07800200u) != 0 || operation > 8 || (offset & 3) != 0) throw Fail("ATOMS mode"); + EmitG(0xec00000000000000UL | (uint)Gpr((uint)Dst) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | ((ulong)Gpr(_i.Bits(32, 8)) << 20) + | ((ulong)_i.Bits(73, 1) << 28) | ((ulong)(offset >> 2) << 30) | ((ulong)operation << 52)); + } + + // SGXT Rd, a, n (immediate): the low n bits of a, sign-extended with bit 73 (else zero-extended): Maxwell BFE + // from bit 0, n bits. + void Sgxt() + { + if ((_i.W2 & ~0x200u) != 0 || _i.W1 > 32) throw Fail("SGXT mode"); + EmitG(0x3800000000000000UL | ((ulong)(_i.W1 << 8) << 20) | ((ulong)_i.Bits(73, 1) << 48) | ((ulong)Gpr(_i.Bits(24, 8)) << 8) | (uint)Gpr((uint)Dst)); + } + + // BREV Rd, b (register): Maxwell SHR.BREV by 0. + void Brev() + { + if (_i.W2 != 0 || _i.W1 > 255) throw Fail("BREV mode"); + EmitG(0x3828000000000000UL | ((ulong)Gpr(_i.Bits(32, 8)) << 8) | (1UL << 39) | (1UL << 40) | (uint)Gpr((uint)Dst)); + } + + // HMNMX2: per half lane, both operands widened to F32 (F2F, exact), FMNMX with the predicate of bits 87-90 + // (true: min), narrowed back and packed (lane 1 in the high half). + void Hmnmx2() + { + uint op = _i.Bits(0, 12); + if (_i.Bits(76, 11) != 0 || _i.Bit(91)) throw Fail("HMNMX2 mode"); + var a = HalfSrc(0, SA); + if (a.swz == 1) throw Fail("HMNMX2 F32 operand"); + (int reg, uint swz, bool abs, bool neg) b = op == 0x840 ? (SB, 0u, false, false) : HalfSrc(1, SB); + if (op == 0x840) Mov32i(SB, _i.W1); + if (b.swz == 1) throw Fail("HMNMX2 F32 operand"); + void widen(int target, (int reg, uint swz, bool abs, bool neg) h, int lane) + { + bool high = h.swz == 3 || (h.swz == 0 && lane == 1); + Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)h.reg << 20) | (high ? 1UL << 41 : 0) + | (h.neg ? 1UL << 45 : 0) | (h.abs ? 1UL << 49 : 0) | (uint)target)); + } + for (int lane = 0; lane < 2; lane++) + { + int t = lane == 0 ? 246 : 247; + widen(244, a, lane); + widen(245, b, lane); + Emit(Unguarded(0x5c60000000000000UL | (245UL << 20) | (244UL << 8) | (uint)t | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42))); + Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)t << 20) | (uint)t)); + } + EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)Dst)); + } + + // A packed-half operand: register, swizzle (0 H1_H0, 1 F32, 2 H0_H0, 3 H1_H1, as Maxwell's), abs, neg. + // A: reg 24, swizzle 74-75, abs 73, neg 72. B in bits 32-39 (swizzle 60-61, abs 62, neg 63) or, for forms 2/3/7, + // 64-71 (swizzle 81-82, abs 83, neg 84). C in 64-71 (81-82, 83, 84). Immediates are a half pair as stored; + // constants take swizzle/abs/neg from 60-63. + (int reg, uint swz, bool abs, bool neg) HalfSrc(int which, int scratch) + { + if (which == 0) return (Gpr(_i.Bits(24, 8)), _i.Bits(74, 2), _i.Bit(73), _i.Bit(72)); + uint form = Form; + bool wide = which == 1 ? form is 4 or 5 : form is 2 or 3; + if (wide) + { + if (form is 4 or 2) { Mov32i(scratch, _i.W1); return (scratch, 0, false, false); } + var (bank, off) = Cbuf(); MovC(scratch, bank, off); + return (scratch, _i.Bits(60, 2), _i.Bit(62), _i.Bit(63)); + } + if (form is 6 or 7) throw Fail("half operand in a uniform register"); + bool upper = which == 2 || form is 2 or 3; + return upper ? (Gpr(_i.Bits(64, 8)), _i.Bits(81, 2), _i.Bit(83), _i.Bit(84)) + : (Gpr(_i.Bits(32, 8)), _i.Bits(60, 2), _i.Bit(62), _i.Bit(63)); + } + + // HADD2 / HMUL2 / HFMA2 with packed-half results: saturate 77, FTZ 80, bit 76 the FMZ precision of HMUL2/HFMA2. + // HFMA2.RELU (79) clamps each lane at zero where the predicate in bits 87-90 holds; only PT is lowered: the sum + // goes to TexOut, PRMT replicates each lane's sign bit over that lane (selector 0xbb99) and LOP.AND with the + // inverted mask clears the negative lanes (negative NaN becomes 0 too). F32 results (78), the H0_NH1 source (86) + // and absolute values where Maxwell has none are not lowered. HADD2's second operand is B, or C for forms 2/3. + void Half(ulong maxwell) + { + uint op = _i.Bits(0, 12) & 0x1ff; + if (op == 0x030 && _i.Bit(78)) + { + // HADD2.F32 Rd, -RZ, b: the compiler's half-to-float conversion of one lane of b + var a0 = HalfSrc(0, SA); + var bh = HalfSrc(Form is 2 or 3 ? 2 : 1, SB); + if (a0.reg != RZ || bh.swz is 0 or 1 || _i.Bit(77) || _i.Bits(81, 10) != 0) throw Fail("HADD2.F32 mode"); + EmitG(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)bh.reg << 20) | (bh.swz == 3 ? 1UL << 41 : 0) + | (bh.neg ? 1UL << 45 : 0) | (bh.abs ? 1UL << 49 : 0) | (uint)Gpr((uint)Dst)); + return; + } + bool relu = op == 0x031 && _i.Bit(79); + if (relu && (_i.Bit(77) || _i.Bits(85, 2) != 0 || _i.Bits(87, 4) != 7)) throw Fail("HFMA2.RELU mode"); + if (_i.Bit(78) || (_i.Bit(79) && !relu) || _i.Bit(91) + || (op == 0x031 ? (relu ? _i.Bits(85, 2) : _i.Bits(85, 6)) != 0 : _i.Bits(81, 10) != 0)) + throw Fail("half arithmetic mode"); + if (op == 0x030 && _i.Bit(76)) throw Fail("HADD2 mode"); + var a = HalfSrc(0, SA); + var b = HalfSrc(op == 0x030 && Form is 2 or 3 ? 2 : 1, SB); + uint precision = _i.Bit(76) ? 2u : _i.Bits(80, 1); + ulong w = maxwell | ((ulong)a.reg << 8) | ((ulong)b.reg << 20) | ((ulong)b.swz << 28) | ((ulong)a.swz << 47) + | ((ulong)_i.Bits(77, 1) << 32) | (uint)(relu ? TexOut : Gpr((uint)Dst)); + if (op == 0x030) + w |= (b.abs ? 1UL << 30 : 0) | (b.neg ? 1UL << 31 : 0) | (a.neg ? 1UL << 43 : 0) | (a.abs ? 1UL << 44 : 0) + | ((ulong)_i.Bits(80, 1) << 39); + else if (op == 0x032) + w |= (b.abs ? 1UL << 30 : 0) | (a.neg != b.neg ? 1UL << 31 : 0) | (a.abs ? 1UL << 44 : 0) | ((ulong)precision << 39); + else + { + var c = HalfSrc(2, SC); + if (a.abs || b.abs || c.abs) throw Fail("HFMA2 absolute operand"); + w |= (c.neg ? 1UL << 30 : 0) | (a.neg != b.neg ? 1UL << 31 : 0) | ((ulong)c.swz << 35) | ((ulong)precision << 37) + | ((ulong)c.reg << 39); + } + if (!relu) { EmitG(w); return; } + Emit(Unguarded(w)); + Mov32i(TexOut + 1, 0xbb99); // PRMT selector: bytes 0-1 the sign of byte 1, bytes 2-3 the sign of byte 3 + Emit(Unguarded(0x5bc0000000000000UL | ((ulong)RZ << 39) | ((ulong)(TexOut + 1) << 20) | ((ulong)TexOut << 8) | (uint)(TexOut + 2))); + // LOP.AND Rd, sum, ~mask (InvB bit 40), no predicate output (PT at 48) + EmitG(0x5c40000000000000UL | (7UL << 48) | (1UL << 40) | ((ulong)(TexOut + 2) << 20) | ((ulong)TexOut << 8) | (uint)Gpr((uint)Dst)); + } + + // MOVM.16.MT88 Rd, Ra (0x23a): an 8x8 matrix of 16-bit elements spread over the warp (lane t holds row t/4, + // columns 2(t%4) in the low half and 2(t%4)+1 in the high half) is transposed into the same layout. Element (r, c) + // lives in lane 4r + c/2, half c%2, so lane t takes both halves from lanes 8(t%4) + t/8 and that + 4, half + // (t/4)%2. Maxwell has no matrix move: two SHFL.IDX over the whole warp, the half picked by a shift, ISCADD to pack. + // Scratch is written unguarded so every lane takes part in the shuffles; only Rd is guarded. + void Movm() + { + if (_i.W1 != 0 || _i.W2 != 0) throw Fail("MOVM mode"); + int src = Gpr(_i.Bits(24, 8)), d = Gpr((uint)Dst); + int lane = TexCoord, shift = TexCoord + 1, lo = TexMeta, hi = TexMeta + 1; + void shl(int r, int a, uint n) => Emit(Unguarded(0x3848000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)r)); + void shr(int r, int a, uint n) => Emit(Unguarded(0x3828000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)r)); + // SHFL.IDX r, value, lane register, clamp 31 (whole warp), no predicate output + void shfl(int r, int value, int from) => Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) + | ((ulong)from << 20) | ((ulong)value << 8) | (uint)r)); + Emit(Unguarded(0xf0c8000000000000UL | (uint)SX)); // S2R SX, SR_LANEID + Lop32i(lane, SX, 3, 0); + shl(lane, lane, 3); // 8(t%4) + shr(shift, SX, 3); // t/8 + Emit(Unguarded(0x5c10000000000000UL | ((ulong)shift << 20) | ((ulong)lane << 8) | (uint)lane)); // IADD + shfl(lo, src, lane); + Emit(Unguarded(0x1c00000000000000UL | (4UL << 20) | ((ulong)lane << 8) | (uint)lane)); // IADD32I +4: next row + shfl(hi, src, lane); + shr(shift, SX, 2); + Lop32i(shift, shift, 1, 0); + shl(shift, shift, 4); // 16 * ((t/4) % 2) + Emit(Unguarded(0x5c28000000000000UL | ((ulong)shift << 20) | ((ulong)lo << 8) | (uint)lo)); // SHR by register + Lop32i(lo, lo, 0xffff, 0); + Emit(Unguarded(0x5c28000000000000UL | ((ulong)shift << 20) | ((ulong)hi << 8) | (uint)hi)); + EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)lo << 20) | ((ulong)hi << 8) | (uint)d); // ISCADD (hi << 16) + lo + } + + // POPC Rd, b (bit 63: of ~b). + void Popc() + { + if ((_i.W2 & ~0x08000000u) != 0) throw Fail("POPC mode"); + var (b, _, inv) = Raw(1, SB); + EmitG(0x5c08000000000000UL | ((ulong)b << 20) | (inv ? 1UL << 40 : 0) | (uint)Gpr((uint)Dst)); + } + + // F2FP.PACK_AB Rd, a, b: half(a) in the high 16 bits, half(b) in the low (round to nearest). With .MERGE_C + // (bit 78) the high half is the low 16 bits of the C operand instead. + void F2fp() + { + if (_i.Bits(62, 2) != 0 || _i.Bits(73, 5) != 0 || _i.Bits(79, 2) != 0 || _i.Bits(81, 10) != 0 || _i.Bit(91) || _i.Bit(72)) + throw Fail("F2FP mode"); + bool merge = _i.Bit(78); + int a = merge ? Plain(2, SC) : Plain(0, SA), b = Plain(1, SB); + if (merge) Mov(247, a, false); + else Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)a << 20) | 247)); + Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)b << 20) | 246)); + EmitG(0x5c18000000000000UL | (16UL << 39) | (246UL << 20) | (247UL << 8) | (uint)Gpr((uint)Dst)); + } + + // SUST.P / SULD.P (formatted, bound descriptor): coordinates from Ra, data (stores) at 32-39 or destination + // (loads), channel mask 72-75, dimension 61-63 in Maxwell's surface numbering (1D, BUFFER_1D, ARRAY_1D, 2D, + // ARRAY_2D, 3D), descriptor offset 40-53 in bank 54-58. Strong stores (0x0010a000) are taken as plain ones. + void SurfaceAccess(bool store) + { + uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4), handle = Handle(true), bank = _i.Bits(54, 5); + uint mode = _i.W2 & ~0xf00u; + bool okMode = store ? mode is 0x00100000 or 0x0010a000 : mode == 0x001e0000; + if (!okMode || mask == 0 || dim > 5 || _i.Bits(59, 2) != 0 || handle > 0x1fff || !TexBank(bank)) + throw Fail("surface access mode"); + int coord = Gpr(_i.Bits(24, 8)); + int data = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); + EmitG((store ? 0xeb28000000000000UL : 0xeb08000000000000UL) | (uint)data | ((ulong)coord << 8) | ((ulong)mask << 20) + | ((ulong)dim << 33) | ((ulong)handle << 36)); + } + + // Descriptor offset (bits 40-53, in words) of a bound texture or image. NVN2 compute programs keep their handles + // in one table in the driver bank: texture slot k at start + 8k, image slot j at start + 8 * (textures + j), the + // start and counts from the control section (job "nvn2_table", runtime/gfx_shader.c). It is renumbered into the + // NVN1 layout the translator's host binds by: textures at 0x20 + 8k, images at 0x120 + 8j. + uint Handle(bool image) + { + uint h = _i.Bits(40, 14); + if (!_compute || _table == null) return h; + int entry = ((int)h * 4 - _table[0]) / 8; + if ((int)h * 4 < _table[0] || ((int)h * 4 - _table[0]) % 8 != 0 || entry >= _table[1] + _table[2]) + throw Fail($"handle word {h} outside the binding table (start 0x{_table[0]:x}, {_table[1]} textures, {_table[2]} images)"); + if (image != (entry >= _table[1])) throw Fail($"handle word {h} is a {(image ? "texture" : "image")} entry"); + return image ? (uint)(0x120 + 8 * (entry - _table[1])) / 4 : (uint)(0x20 + 8 * entry) / 4; + } + + // Bound texture descriptors: bank 2 for graphics stages, bank 0 for compute (NVN2 keeps the compute handles in + // the driver bank). The translator leaves the bank to the host, which reads the handle table as NVN1's c2. + bool TexBank(uint bank) => bank == 2 || (bank == 0 && _compute); + + // TLD4 (gather), 2D or ARRAY_2D: component 87-88, offset mode 76-77 (1: one offset register, 2: per-tap offsets + // in two registers), depth compare 78, FP16 results 79, operands packed as TEX's. Maxwell TLD4: component 56-57, + // offset mode 54-55 (same numbering), DC 50. + void Tld4() + { + uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4), offMode = _i.Bits(76, 2); + bool dc = _i.Bit(78); + if (dim is not 1 and not 5 || offMode == 3 || _i.Bits(81, 3) != 7 || _i.Bit(59) || mask == 0 || _i.Bit(89)) + throw Fail("TLD4 mode"); + uint handle = Handle(false), bank = _i.Bits(54, 5); + if (handle > 0x1fff || !TexBank(bank)) throw Fail($"TLD4 descriptor c[{bank}] 0x{handle:x}"); + int coords = TexCoords[dim]; + int count = coords + (int)offMode + (dc ? 1 : 0); + int inA = count <= 4 ? (count + 1) / 2 : 4; + int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); + int Value(int k) => k < inA ? (ra == RZ ? RZ : Gpr((uint)(ra + k))) : (rb == RZ ? RZ : Gpr((uint)(rb + k - inA))); + for (int k = 0; k < coords; k++) Mov(TexCoord + k, Value(k), false); + int next = coords, meta = TexMeta; + for (int k = 0; k < offMode; k++) Mov(meta++, Value(next++), false); + if (dc) Mov(meta++, Value(next++), false); + EmitG(0xc800000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)(meta == TexMeta ? RZ : TexMeta) << 20) + | ((ulong)TexMaxwellDim[dim] << 28) | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) + | (dc ? 1UL << 50 : 0) | (7UL << 51) | ((ulong)offMode << 54) | ((ulong)_i.Bits(87, 2) << 56)); + TexResults(mask, _i.Bit(79)); + } + + // PLOP3.LUT Pu, Pv, a, b, c: table in bits 64-66 and 72-76, inputs a 87-90, b 77-80, c 68-71 (index | negate << 3), + // outputs 81-83 and 84-86 (the second with the table in bits 16-23). Each input becomes 0 or ~0 (SEL), LOP3 combines + // them and ISETP.NE sets the output. Bit 67 (c a uniform predicate) is not lowered for the vector form. + void Plop3() + { + uint lut = _i.Bits(64, 3) | _i.Bits(72, 5) << 3, t1 = _i.Bits(81, 3), t2 = _i.Bits(84, 3), lut2 = _i.Bits(16, 8); + const int ones = 240, ra = 241, rb = 242, rc = 243, r1 = 244, r2 = 245; + Mov32i(ones, 0xffffffff); + Sel(ra, ones, RZ, _i.Bits(87, 4), false); + Sel(rb, ones, RZ, _i.Bits(77, 4), false); + uint c = _i.Bits(68, 4); + if (_i.Bit(67) && !_uniform) + { + // c is UPn: 0 - ((RUP >> n) & 1) gives 0 or ~0 straight from the uniform predicate state; UPT is ~0. + if ((c & 7) == 7) Mov(rc, ones, false); + else + { + Emit(Unguarded(0x3828000000000000UL | ((ulong)(c & 7) << 20) | ((ulong)RUP << 8) | rc)); // SHR imm + Lop32i(rc, rc, 1, 0); + Iadd3(rc, RZ, rc, RZ, false, negB: true); + } + if ((c & 8) != 0) Lop32i(rc, rc, 0xffffffff, 2); + } + else Sel(rc, ones, RZ, c, false); + ulong lop3(int dst, uint table) => Unguarded(0x5be0000000000000UL | ((ulong)rc << 39) | ((ulong)rb << 20) | ((ulong)ra << 8) | (uint)dst | ((ulong)table << 28) | (7UL << 48)); + Emit(lop3(r1, lut)); + if (t2 != 7) Emit(lop3(r2, lut2)); + ulong ne(uint p, int r) => 0x5b60000000000000UL | (5UL << 49) | ((ulong)PT << 39) | ((ulong)RZ << 20) | ((ulong)r << 8) | ((ulong)p << 3) | PT; + if (t1 != 7) EmitG(ne(t1, r1)); + if (t2 != 7) EmitG(ne(t2, r2)); + } + + // LDS / STS: [Ra << (stride ? stride + 1 : 0) (+ a uniform register when bit 91 is set: bits 32-39 for loads, + // 64-71 for stores) + offset 40-63], size 73-75 (as Maxwell's), stride 78-79. STS data at 32-39. + void SharedAccess(uint op) + { + bool store = op != 0x984; + uint width = _i.Bits(73, 3), stride = _i.Bits(78, 2); + bool uniformBase = op is 0x984 or 0x988 && _i.Bit(91); + uint ignored = uniformBase ? (op == 0x988 ? 0x080000ffu : 0x08000000u) : 0u; + if (width > 6 || (_i.W2 & ~ignored) != (width << 9 | stride << 14) || (op == 0x984 && !uniformBase && _i.Bits(32, 8) != 0) + || (op == 0x988 && !uniformBase)) + throw Fail("shared memory addressing mode"); + int addr = Gpr(_i.Bits(24, 8)); + if (stride != 0 && addr != RZ) + { + Emit(Unguarded(0x3848000000000000UL | ((ulong)(stride + 1) << 20) | ((ulong)addr << 8) | SA)); // SHL imm + addr = SA; + } + if (uniformBase) + { + int u = Ur(_i.Bits(op == 0x988 ? 64 : 32, 8)); + if (u != RZ) { Iadd3(SA, addr, u, RZ, false); addr = SA; } + } + int data = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); + EmitG((store ? 0xef58000000000000UL : 0xef48000000000000UL) | (uint)data | ((ulong)addr << 8) + | ((ulong)_i.Bits(40, 24) << 20) | ((ulong)width << 48)); + } + + // BAR.SYNC 0 for the whole workgroup (bit 80, .DEFER_BLOCKING, needs nothing here). + void Bar() + { + if (_i.W1 != 0 || (_i.W2 & ~0x10000u) != 0 || _i.Bits(12, 4) != 7) throw Fail("BAR mode"); + Emit(Unguarded(0xf0a8000000000000UL | (1UL << 43) | (1UL << 44) | (7UL << 39))); + } + + // ISETP.EX: the high-word half of a 64-bit compare. The low words' result is the predicate of bits 68-71; EQ is + // (high equal AND low), NE (high different OR low), an ordering (high strictly ordered OR (high equal AND low)), + // signed with bit 73. The result is combined with Pc (87-90) by the operation in 74-75 into P (81-83) and Q + // (84-86), as ISETP does: Maxwell ISETPs into a saved scratch predicate (P6, or the highest one the instruction + // does not name), then PSETP. + void SetpExtended() + { + uint cmp = _i.Bits(76, 3), combine = _i.Bits(74, 2), low = _i.Bits(68, 4); + if (cmp is 0 or 7 || combine == 3) throw Fail("ISETP.EX mode"); + var (a, _, _) = Raw(0, SA); + var (b, _, _) = Raw(1, SB); + // the scratch predicate: P6 unless the instruction names it (low result, Pc, guard or a destination) + int t = 6; + while (t >= 0 && (t == (int)(low & 7) || t == (int)_i.Bits(87, 3) || t == (int)_i.Bits(12, 3) || + t == (int)_i.Bits(81, 3) || t == (int)_i.Bits(84, 3))) + t--; + if (t < 0) throw Fail("ISETP.EX: no free scratch predicate"); + Emit(Unguarded(0x38e8000000000000UL | (1UL << (20 + t)) | ((ulong)RZ << 8) | SX)); // P2R: save P6 + ulong isetp(uint c, uint input, uint bop, bool signedCmp) => + 0x5b60000000000000UL | ((ulong)t << 3) | PT | ((ulong)a << 8) | ((ulong)b << 20) | ((ulong)(input & 7) << 39) + | ((ulong)(input >> 3) << 42) | ((ulong)bop << 45) | (signedCmp ? 1UL << 48 : 0) | ((ulong)c << 49); + bool sg = _i.Bit(73); + const uint And = 0, Or = 1, Lt = 1, Eq = 2, Gt = 4, Ne = 5; + if (cmp == Eq) EmitG(isetp(Eq, low, And, sg)); + else if (cmp == Ne) EmitG(isetp(Ne, low, Or, sg)); + else { EmitG(isetp(Eq, low, And, sg)); EmitG(isetp(cmp <= 3 ? Lt : Gt, (uint)t, Or, sg)); } + // PSETP P, Q = (P6 AND PT) combine Pc + EmitG(0x5090000000000000UL | _i.Bits(84, 3) | ((ulong)_i.Bits(81, 3) << 3) | ((ulong)t << 12) | ((ulong)PT << 29) + | ((ulong)_i.Bits(87, 3) << 39) | ((ulong)_i.Bits(90, 1) << 42) | ((ulong)combine << 45)); + Emit(Unguarded(0x38f0000000000000UL | (1UL << (20 + t)) | ((ulong)SX << 8))); // R2P: restore P6 + } + + // UBREV URd, URb: BREV on uniform registers (bit 91 marks the uniform datapath). + void Ubrev() + { + if ((_i.W2 & ~0x08000000u) != 0 || _i.W1 > 255) throw Fail("UBREV mode"); + int d = Ur((uint)Dst); + if (d == RZ) return; + EmitG(0x3828000000000000UL | ((ulong)Ur(_i.Bits(32, 8)) << 8) | (1UL << 39) | (1UL << 40) | (uint)d); // SHR.BREV by 0 + } + + // UPOPC URd, URb: POPC on uniform registers, lowered through POPC (0x309) with register fields naming uniform + // registers (as UniformAlu does). + void Upopc() + { + if ((_i.W2 & ~0x08000000u) != 0 || _i.W1 > 255) throw Fail("UPOPC mode"); + Insn saved = _i; + _i.Lo = (_i.Lo & ~0xfffUL) | 0x309; + _i.Hi &= ~(1UL << 27); + _uniform = true; + try { Popc(); } + finally { _uniform = false; _i = saved; } + } + + // UP2UR URd, UPR, URa, mask: P2R over the uniform predicates, byte 0 only: URd = (URa & ~mask) | (UP & mask), + // the mask in bits 32-38. RUP holds UPk in bit k. + void Up2ur() + { + if (_i.W1 > 0x7f || (_i.W2 & ~0x08000000u) != 0) throw Fail("UP2UR mode"); + int d = Ur((uint)Dst), a = Ur(_i.Bits(24, 8)); + if (d == RZ) return; + uint mask = _i.W1; + Lop32i(SX, RUP, mask, 0); // AND + if (a == RZ) { Mov(d, SX); return; } + Lop32i(SA, a, ~mask, 0); + Iadd3(d, SA, SX, RZ, true); // disjoint bits: the sum is the OR + } + + // F2IP.U8.F32 Rd, Ra, Rb, RZ (opcode 0x243): two floats converted to unsigned bytes and packed into the low + // half of Rd. Rb's byte goes in bits 0-7, Ra's byte in bits 8-15, and bits 16-31 are zero. With Ra = RZ only + // Rb's byte is written. W1 holds only Rb (bits 32-39), and the C operand (bits 64-71) must be RZ. Rounding is + // in bits 78-79, using the same codes as Maxwell's F2I RoundMode (bits 39-40): + // W2 0x0000c4ff: bits 78-79 = 3, round toward zero; + // W2 0x000004ff: bits 78-79 = 0, round to nearest even. The compiler uses this form for unorm values it has + // already multiplied by 255. + // Each byte goes through NaN -> 0, a float clamp to [0, 255], then F2I.U32.F32 with that rounding. The clamp + // has to be explicit. The translator's F2I (Ryujinx Instructions/InstEmitConversion.cs, EmitF2I) clamps only + // U16/S16 destinations, and IDstFmt has no U8, so a plain F2I.U8 lets values above 255 through. It also feeds + // NaN to FMax, whose SPIR-V result is undefined. Behaviour as read from the system (lower_to_maxwell.h, + // LowerToMaxwellWith; GPL-3.0, read for behaviour only). + void F2ipByte() + { + if ((_i.W1 & ~0xffu) != 0 || (_i.W2 != 0x0000c4ff && _i.W2 != 0x000004ff)) throw Fail("packed float-to-byte mode"); + int a = Gpr(_i.Bits(24, 8)), b = Gpr(_i.Bits(32, 8)), d = Gpr((uint)Dst); + if (d == RZ) return; + uint round = _i.Bits(78, 2); + + // Convert src to an integer 0-255. Steps 1-4 work in scratch register t without a guard. Step 5 writes dst + // and is guarded only when dst is the guest register. + void ToByte(int src, int t, int dst, bool guarded) + { + // FSET.NUM.AND t = (src is not NaN) ? 0xffffffff : 0 (FComp 7 in 48-51, source predicate PT in 39-41, + // BVal clear so the result is an integer mask) + Emit(Unguarded(0x5800000000000000UL | (7UL << 48) | ((ulong)PT << 39) | ((ulong)src << 20) | ((ulong)src << 8) | (uint)t)); + // LOP.AND t = src & t: NaN becomes +0.0 (destination predicate PT in 48-50) + Emit(Unguarded(0x5c40000000000000UL | ((ulong)PT << 48) | ((ulong)t << 20) | ((ulong)src << 8) | (uint)t)); + // FMNMX t = max(t, 0.0): immediate form, predicate !PT (39-42) selects the maximum + Emit(Unguarded(0x3860000000000000UL | ((ulong)PT << 39) | (1UL << 42) | ((ulong)t << 8) | (uint)t)); + // FMNMX t = min(t, 255.0): 255.0f is 0x437f0000, whose top 20 bits 0x437f0 fit the immediate field + Emit(Unguarded(0x3860000000000000UL | (0x437f0UL << 20) | ((ulong)PT << 39) | ((ulong)t << 8) | (uint)t)); + // F2I.U32.F32 dst = t, with the instruction's rounding (IDstFmt U32 in 8-9, SrcFmt F32 in 10-11, FTZ 44) + Put(0x5cb0000000000000UL | (2UL << 8) | (2UL << 10) | ((ulong)round << 39) | (1UL << 44) | ((ulong)t << 20) | (uint)dst, guarded); + } + + if (a == RZ) + { + // Single byte: convert straight into Rd. Steps 1-4 read b and write only SA, so this is safe when d == b. + ToByte(b, SA, d, true); + return; + } + // Both bytes go to scratch before Rd is written, so d may alias a or b. + ToByte(b, SA, SA, false); + ToByte(a, SB, SB, false); + // ISCADD Rd = (SB << 8) + SA. Both values are 0-255, so the sum equals the OR of the two bytes. + EmitG(0x5c18000000000000UL | (8UL << 39) | ((ulong)SA << 20) | ((ulong)SB << 8) | (uint)d); + } + + // TXD (0xb6c): a sample with explicit derivatives, 2D or ARRAY_2D, bound descriptor. Fields as TEX's: Rd 16-23, + // Ra 24-31 (x, y, then for ARRAY_2D the layer as an integer in its low 16 bits), Rb 32-39 (the four derivatives, + // interleaved: dx.x, dy.x, dx.y, dy.y), descriptor word offset 40-53 in bank 54-58, dimension 61-63, second + // destination 64-71, mask 72-75, bit 90 no-dependency. Offsets (76), FP16 results (79) and the bindless forms + // (59-60) are not lowered. Bits 81-84 are set in every TXD seen (81-83: no sparse predicate). + // Maxwell TXD (0xde00) takes the coordinates and the layer the same way in Ra.., but Ryujinx's EmitTxd reads Rb.. + // as dPdx (2 values) then dPdy (2 values), so the derivatives are reordered on the way into TexMeta. + // With no second destination the third and fourth channels are dropped (SM70+ register pairs), so the mask + // handed to Maxwell keeps only its two lowest channels and TexResults writes Rd, Rd+1; otherwise TexResults + // writes Rd, Rd+1 and then the second destination, which need not follow Rd. + void Txd() + { + uint dim = _i.Bits(61, 3), mask = _i.Bits(72, 4); + if ((_i.W2 & ~(0xfffu | 1u << 26)) != 0x001e0000 || _i.Bits(59, 2) != 0 || dim is not 1 and not 5 || mask == 0) + throw Fail("TXD mode"); + uint handle = Handle(false), bank = _i.Bits(54, 5); + if (handle > 0x1fff) throw Fail("TXD descriptor offset"); + if (!TexBank(bank)) throw Fail($"TXD descriptor bank {bank}"); + int d2 = (int)_i.Bits(64, 8); + if (Dst == RZ && d2 == RZ) return; // nothing is written, and sampling has no side effects + if (d2 == RZ) + { + uint upper = mask & (mask - 1); // drop the lowest channel... + upper &= upper - 1; // ...and the next: what is left has no destination + mask ^= upper; + } + int ra = (int)_i.Bits(24, 8), rb = (int)_i.Bits(32, 8); + int coords = dim == 5 ? 3 : 2; + for (int k = 0; k < coords; k++) Mov(TexCoord + k, ra == RZ ? RZ : Gpr((uint)(ra + k)), false); + // SM86 order dx.x, dy.x, dx.y, dy.y; Ryujinx's Maxwell order dx.x, dx.y, dy.x, dy.y. + int[] order = { 0, 2, 1, 3 }; + for (int k = 0; k < 4; k++) Mov(TexMeta + k, rb == RZ ? RZ : Gpr((uint)(rb + order[k])), false); + EmitG(0xde00000000000000UL | TexOut | ((ulong)TexCoord << 8) | ((ulong)TexMeta << 20) | ((ulong)TexMaxwellDim[dim] << 28) + | ((ulong)mask << 31) | ((ulong)handle << 36) | ((ulong)_i.Bits(90, 1) << 49) | (7UL << 51)); + TexResults(mask, false); + } + + // LDSM.16.M88.{1,2,4} and LDSM.16.MT88.{1,2,4} (ldmatrix): count = 1 << bits 72-73 (1, 2 or 4) 8x8 matrices of + // 16-bit elements from shared memory into Rd..Rd+count-1 (Rd aligned to count). Lanes 8m..8m+7 hold, in Ra, the + // byte addresses of rows 0-7 of matrix m; bit 91 adds a uniform register (bits 32-39, as in SharedAccess) and every + // row adds the immediate offset 40-63. Without transpose (bit 78 clear) lane t receives in Rd+m the 32 bits at row + // t/4, bytes 4(t%4)..4(t%4)+3 of matrix m. With transpose (.MT88) it receives column t/4 of row 2(t%4) in the low + // half and of row 2(t%4)+1 in the high half. + // Maxwell has no matrix load, so each lane fetches the row address it needs from the lane that holds it + // (SHFL.IDX across the whole warp), adds its own column and loads with LDS: one LDS.32 per matrix, or for .MT88 two + // LDS.U16 per matrix joined by ISCADD (high << 16) + low. Address arithmetic and the shuffles are unguarded + // scratch writes so every lane takes part in the shuffles (the hardware needs the whole warp too); the loads and + // the writes to Rd are guarded by the instruction's predicate. + // Scratch: SX lane id, then the column bytes (non-transposed); TexCoord the source lane of row 0 (or of row + // 2(t%4)), TexCoord+1 the lane of later matrices (or the column bytes for .MT88), TexCoord+2 the lane for .MT88; + // SA the fetched row address; SC the base address when it needs a copy; TexMeta/TexMeta+1 the two halves. + void Ldsm() + { + uint sizeCode = _i.Bits(72, 2); + bool transpose = _i.Bit(78), uniformOffset = _i.Bit(91); + // Allowed in word 2: size 72-73 (0x300), transpose 78 (0x4000), uniform offset 91 (0x08000000). + if (sizeCode == 3 || (_i.W2 & ~0x08004300u) != 0 || (!uniformOffset && _i.Bits(32, 8) != 0)) + throw Fail("LDSM mode"); + int count = 1 << (int)sizeCode; + if (Dst == RZ) return; // a load into RZ has no visible effect + if (Dst % count != 0 || Dst + count > RZ) throw Fail("LDSM destination alignment"); + var dst = new int[count]; + for (int m = 0; m < count; m++) dst[m] = Gpr((uint)(Dst + m)); + + int addr = Gpr(_i.Bits(24, 8)); + if (uniformOffset) + { + int u = Ur(_i.Bits(32, 8)); + if (u != RZ) { Iadd3(SC, addr, u, RZ, false); addr = SC; } + } + // A destination that is also the base would be overwritten before the later matrices fetch their rows. + if (addr != RZ && Array.IndexOf(dst, addr) >= 0) { Mov(SC, addr, false); addr = SC; } + ulong offset = (ulong)_i.Bits(40, 24) << 20; + + // Unguarded Maxwell helpers: SHR/SHL by an immediate (unsigned), IADD32I, and SHFL.IDX of the base address + // from a lane held in a register, clamp 31 with segment mask 0 (the whole warp), valid predicate to PT. + void shr(int d, int a, int n) => Emit(Unguarded(0x3828000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)d)); + void shl(int d, int a, int n) => Emit(Unguarded(0x3848000000000000UL | ((ulong)n << 20) | ((ulong)a << 8) | (uint)d)); + void addImm(int d, int a, uint imm) => Emit(Unguarded(0x1c00000000000000UL | ((ulong)imm << 20) | ((ulong)a << 8) | (uint)d)); + void fetchRow(int d, int lane) => Emit(Unguarded(0xef10000000000000UL | (uint)d | ((ulong)addr << 8) | ((ulong)lane << 20) + | (0x1fUL << 34) | (1UL << 29) | (7UL << 48))); + // LDS d, [a + offset], size as Maxwell's LsSize2 (2 U16, 4 B32), guarded. + void lds(int d, int a, ulong size) => EmitG(0xef48000000000000UL | (uint)d | ((ulong)a << 8) | offset | (size << 48)); + + Emit(Unguarded(0xf0c8000000000000UL | SX)); // S2R SX, SR_LANEID (0) + if (!transpose) + { + shr(TexCoord, SX, 2); // lane holding row t/4 of matrix 0 + Lop32i(SX, SX, 3, 0); // t & 3 (LOP32I.AND) + shl(SX, SX, 2); // column bytes 4(t%4) + for (int m = 0; m < count; m++) + { + int lane = TexCoord; + if (m != 0) { addImm(TexCoord + 1, TexCoord, (uint)(8 * m)); lane = TexCoord + 1; } + fetchRow(SA, lane); + Iadd3(SA, SA, SX, RZ, false); + lds(dst[m], SA, 4); // LDS.32 + } + return; + } + Lop32i(TexCoord, SX, 3, 0); + shl(TexCoord, TexCoord, 1); // lane holding row 2(t%4) of matrix 0 + shr(TexCoord + 1, SX, 2); + shl(TexCoord + 1, TexCoord + 1, 1); // column bytes 2(t/4) + for (int m = 0; m < count; m++) + { + for (int half = 0; half < 2; half++) + { + int lane = TexCoord; + uint step = (uint)(8 * m + half); + if (step != 0) { addImm(TexCoord + 2, TexCoord, step); lane = TexCoord + 2; } + fetchRow(SA, lane); + Iadd3(SA, SA, TexCoord + 1, RZ, false); + lds(TexMeta + half, SA, 2); // LDS.U16 (zero-extended) + } + // ISCADD Rd+m = (high << 16) + low + EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)TexMeta << 20) | ((ulong)(TexMeta + 1) << 8) | (uint)dst[m]); + } + } + + // HMMA.1688.F16 / HMMA.16816.F16 Rd, Ra, Rb, Rc: a warp-wide matrix multiply-accumulate D(16x8) = A(16xK) * + // B(Kx8) + C(16x8) on F16 elements, K = 8, or 16 when bit 75 is set. Fields: D 16-23 (a register pair), A 24-31 + // (K/4 registers), B 32-39 (K/8 registers), C 64-71 (a pair; RZ means a zero accumulator). With g = lane / 4 and + // q = lane % 4, each lane holds (the lower-numbered column or k in the low half of every register): + // A register 2j + r: row g + 8r, k 8j + 2q and 8j + 2q + 1 (j is 0 only when K = 8) + // B register j: k 8j + 2q and 8j + 2q + 1, column g + // C / D register r: row g + 8r, columns 2q and 2q + 1 + // Maxwell has no tensor instructions, so the product is expanded per lane. For each k pair (2p, 2p + 1) the lane + // fetches the A word of its own rows from lane 4g + p % 4 and, for each of its two columns n = 2q + c, the B word + // from lane 4n + p % 4 (SHFL.IDX), widens the halves to F32 (F2F.F32.F16, exact) and accumulates with FFMA (an + // F16 x F16 product is exact in F32). The accumulators are rounded to F16 after every eight k (one 1688 step of + // the hardware) and widened again when more follow; the final F16 values are packed as F2fp in this file does + // (F2F.F16 writes the low half and zeroes the high half, ISCADD merges the column pair). + // The work runs unguarded because every lane must supply its fragments to the shuffles; only the two result + // writes take the instruction's guard. Scratch R239-R252 (this file's scratch range, see the header); the guest + // operands are never there (guest and uniform registers end below R237, spill temporaries are R204-R235). + void Hmma() + { + bool k16 = _i.Bit(75); + uint d = (uint)Dst, a = _i.Bits(24, 8), b = _i.Bits(32, 8), c = _i.Bits(64, 8); + int aRegs = k16 ? 4 : 2, bRegs = k16 ? 2 : 1; + // A fragment base aligned to its size and the whole fragment below RZ (RZ itself reads as zeros). + bool fits(uint reg, int n) => reg == RZ || (reg % (uint)n == 0 && reg + (uint)n <= RZ); + // Word 1 above B, word 2 other than C and the shape bit, and word 3 bits 96-104 / 126-127 select modes + // (F32 accumulators and others) that are not lowered. + if ((_i.W1 & ~0xffu) != 0 || (_i.W2 & ~0x8ffu) != 0 || ((uint)(_i.Hi >> 32) & 0xc00001ffu) != 0 + || !fits(d, 2) || !fits(a, aRegs) || !fits(b, bRegs) || !fits(c, 2)) + throw Fail("HMMA mode"); + if (d == RZ) return; // no result and no other effect + + // Register k of a fragment (RZ stays RZ); Gpr maps spilled guest registers to this instruction's temporaries. + int part(uint reg, int k) => reg == RZ ? RZ : Gpr(reg + (uint)k); + + const int laneA = 239; // 4g: the lane of A's row group for k pair 0 + const int laneB = 240; // 8q: the lane holding B column 2q for k pair 0 + const int lane = 241; // the lane of the current shuffle + const int word = 242; // the shuffled packed-half word + const int aw = 243; // 243-246: A element (row g + 8r, k 2p + h) at aw + 2r + h, F32 + const int bw = 247; // 247-248: B element (k 2p + h, current column) at bw + h, F32 + const int acc = 249; // 249-252: accumulator (row g + 8r, column 2q + c) at acc + 2r + c, F32 + + void widen(int target, int src, bool high) => // F2F.F32.F16 target, src.H0 / .H1 + Emit(Unguarded(0x5ca8000000000000UL | (2UL << 8) | (1UL << 10) | ((ulong)src << 20) | (high ? 1UL << 41 : 0) | (uint)target)); + void narrow(int t) => // F2F.F16.F32 t, t (round to nearest; high half zero) + Emit(Unguarded(0x5ca8000000000000UL | (1UL << 8) | (2UL << 10) | ((ulong)t << 20) | (uint)t)); + void shuffle(int target, int src, int laneReg) => // SHFL.IDX PT, target, src, laneReg, 0x1f (as Redux) + Emit(Unguarded(0xef10000000000000UL | (7UL << 48) | (31UL << 34) | (1UL << 29) | ((ulong)laneReg << 20) | ((ulong)src << 8) | (uint)target)); + int laneAt(int baseLane, uint offset) // baseLane + offset in `lane` (IADD32I), or baseLane itself + { + if (offset == 0) return baseLane; + Emit(Unguarded(0x1c00000000000000UL | ((ulong)offset << 20) | ((ulong)baseLane << 8) | lane)); + return lane; + } + + // Accumulators from C (both halves of each row register), or zero. + for (int r = 0; r < 2; r++) + for (int col = 0; col < 2; col++) + { + if (c == RZ) Mov32i(acc + 2 * r + col, 0); + else widen(acc + 2 * r + col, part(c, r), col == 1); + } + + // Lane bases: 8q first (it needs the raw lane id), then 4g. + Put(0xf0c8000000000000UL | laneA, false); // S2R SR_LANEID + Lop32i(laneB, laneA, 3, 0); // q + Emit(Unguarded(0x3848000000000000UL | (3UL << 20) | ((ulong)laneB << 8) | laneB)); // SHL imm: 8q + Lop32i(laneA, laneA, 0x1c, 0); // 4g + + int pairs = k16 ? 8 : 4; + for (int p = 0; p < pairs; p++) + { + uint pq = (uint)(p & 3); // the source lanes' q for this k pair + int j = p >> 2; // which 8-k block of the fragments + int srcA = laneAt(laneA, pq); + for (int r = 0; r < 2; r++) + { + shuffle(word, part(a, 2 * j + r), srcA); + widen(aw + 2 * r, word, false); + widen(aw + 2 * r + 1, word, true); + } + for (int col = 0; col < 2; col++) + { + shuffle(word, part(b, j), laneAt(laneB, 4u * (uint)col + pq)); + widen(bw, word, false); + widen(bw + 1, word, true); + for (int r = 0; r < 2; r++) + for (int h = 0; h < 2; h++) + { + int t = acc + 2 * r + col; // FFMA t = A(r, 2p + h) * B(2p + h, col) + t + Emit(Unguarded(0x5980000000000000UL | ((ulong)t << 39) | ((ulong)(bw + h) << 20) | ((ulong)(aw + 2 * r + h) << 8) | (uint)t)); + } + } + if ((p & 3) == 3) // end of an eight-k step: round to F16, widen again if another step follows + for (int t = acc; t < acc + 4; t++) + { + narrow(t); + if (p + 1 < pairs) widen(t, t, false); + } + } + + // D register r = column 2q + 1 in the high half, column 2q in the low half (ISCADD hi << 16 + lo). + for (int r = 0; r < 2; r++) + EmitG(0x5c18000000000000UL | (16UL << 39) | ((ulong)(acc + 2 * r) << 20) | ((ulong)(acc + 2 * r + 1) << 8) | (uint)part(d, r)); + } + + + // UFLO URd, URb: FLO on uniform registers (bit 74 .SH, bit 63 of ~b). + void Uflo() + { + if ((_i.W2 & ~(0x400u | 0x08000000u)) != 0x000e0000) throw Fail("UFLO mode"); + int d = Ur((uint)Dst); + if (d == RZ) return; + EmitG(0x5c30000000000000UL | ((ulong)Ur(_i.Bits(32, 8)) << 20) | (_i.Bit(63) ? 1UL << 40 : 0) | ((ulong)_i.Bits(74, 1) << 41) | (uint)d); + } + + // I2F: source signed (bit 74) and size (84-85: 8/16/32 bits, its byte or half in 60-61), destination size + // (75-76: F16, F32, F64), rounding 78-79. Maxwell I2F: ISrcFmt (size code, +4 signed), ByteSel, DstFmt. + void I2f(uint op) + { + const uint known = 1u << 10 | 3u << 11 | 3u << 14 | 3u << 20; + uint w2 = _i.W2 & ~(op == 0xd06 ? 0x08000000u : 0u); + uint src = _i.Bits(84, 2), dst = _i.Bits(75, 2), sel = _i.Bits(60, 2); + if ((w2 & ~known) != 0 || src == 3 || dst == 0 || (op != 0xb06 && (_i.W1 & 0xcfffff00u) != 0)) throw Fail("I2F format"); + int b = op == 0xb06 ? Plain(1, SB) : op == 0xd06 ? Ur(_i.Bits(32, 8)) : Gpr(_i.Bits(32, 8)); + uint bytesel = src == 0 ? sel : src == 1 ? sel * 2 : 0; + EmitG(0x5cb8000000000000UL | ((ulong)dst << 8) | ((ulong)(src | (_i.Bit(74) ? 4u : 0u)) << 10) | ((ulong)b << 20) + | ((ulong)bytesel << 41) | ((ulong)_i.Bits(78, 2) << 39) | (uint)Gpr((uint)Dst)); + } + + // STL / LDL (local memory) at [Ra + offset 40-63], size 73-75 (U8 S8 U16 S16 32 64 128); 64 and 128 bits go as + // 32-bit words. STL data at 32-39. A uniform address (bit 91) is not lowered. + void LocalAccess(bool store) + { + uint size = _i.Bits(73, 3), allowed = store ? 0xe00u : 0x200e00u; + if (size > 6 || (_i.W2 & ~allowed) != 0x00100000 || (!store && _i.Bits(32, 8) != 0) || (store && _i.Bits(16, 8) != 0)) + throw Fail("local memory mode"); + int addr = Gpr(_i.Bits(24, 8)), value = store ? Gpr(_i.Bits(32, 8)) : Gpr((uint)Dst); + uint off = _i.Bits(40, 24), words = size == 6 ? 4u : size == 5 ? 2u : 1u; + if (!store && words > 1 && addr != RZ && addr >= value && addr < value + words) { Mov(SX, addr, false); addr = SX; } + for (uint k = 0; k < words; k++) + EmitG((store ? 0xef50000000000000UL : 0xef40000000000000UL) | (uint)(value == RZ ? RZ : value + (int)k) | ((ulong)addr << 8) + | ((ulong)((off + 4 * k) & 0xffffff) << 20) | ((ulong)(size < 4 ? size : 4) << 48)); + } + + // IPA with the attribute indexed by a uniform register (bits 24-31): Maxwell IPA.IDX.PASS with the whole byte + // address in Ra. As Ipa: the value times 1/w, which the program multiplies by w itself. + void IpaIndexed() + { + if (_i.Bits(32, 8) != RZ || (_i.Bits(76, 4) != 0 && _i.Bits(76, 4) != 4) || _i.Bits(81, 3) != 7) throw Fail("indexed IPA mode"); + int idx = Ur(_i.Bits(24, 8)), d = Gpr((uint)Dst); + uint attr = _i.Bits(64, 8); + if (idx == RZ) + { + EmitG(0xe000000000000000UL | ((ulong)RZ << 8) | ((ulong)RZ << 20) | ((ulong)attr << 30) | ((ulong)_i.Bits(78, 1) << 55) | ((ulong)PT << 47) | (uint)d); + return; + } + Emit(Unguarded(0x1c00000000000000UL | ((ulong)(attr * 4) << 20) | ((ulong)idx << 8) | SX)); // IADD32I + EmitG(0xe000000000000000UL | ((ulong)SX << 8) | ((ulong)RZ << 20) | (1UL << 38) | ((ulong)_i.Bits(78, 1) << 55) + | ((ulong)PT << 47) | (uint)d); + } + + // ULDC URd, c[bank][URa + offset]: Maxwell LDC with a register index; width 73-75 as LDC's. + void UldcIndexed() + { + uint width = _i.Bits(73, 3); + int d = Ur((uint)Dst), idx = Ur(_i.Bits(24, 8)), bank = Bank(_i.Bits(54, 5)); + if (width > 5 || (_i.W2 & ~0xe00u) != 0x08000000 || bank >= 18 || _i.Bits(62, 2) != 0) throw Fail("indexed ULDC mode"); + if (d == RZ) return; + EmitG(0xef90000000000000UL | (uint)d | ((ulong)idx << 8) | ((ulong)_i.Bits(38, 16) << 20) | ((ulong)bank << 36) | ((ulong)width << 48)); + } + + // BRX Ra / BRXU URa with the program start as base (the offset makes the target Ra itself): a switch. The + // compiler loads Ra from a jump table in c1 of native byte addresses, LDC Ra, c[0x1][Rx + base], after + // clamping the index (IMNMX.U32 Ry, Ry, bound; SHL Rx, Ry, 2). The translator's BRX compares the register with + // the lowered addresses, so it cannot take native ones; each table entry becomes a compare and branch instead: + // IADD32I.CC RZ, Ra, -target; BRA CC.EQ to the target's lowered code, the last entry also unconditionally. + void Brx(bool uniform) + { + if (_i.Bits(12, 4) != 7) throw Fail("predicated BRX"); + if ((_i.W2 & ~0x3ffffu) != (uniform ? 0x0b800000u : 0x03800000u)) throw Fail("BRX mode"); + long raw = (long)(_i.W1 | ((ulong)(_i.W2 & 0x3ffff) << 32)); + if ((long)_index * 16 + 16 + ((raw << 14) >> 14) != 0) throw Fail("BRX relative to a base other than the program start"); + uint src = _i.Bits(24, 8); + var targets = BrxTargets(src, uniform); + int reg = uniform ? Ur(src) : Gpr(src); + foreach (uint t in targets) + { + Emit(Unguarded(0x1c00000000000000UL | (1UL << 52) | ((ulong)(uint)(-(int)t) << 20) | ((ulong)reg << 8) | RZ)); + _branches.Add((_out.Count, (int)(t / 16))); + Emit(0xe240000000000000UL | ((ulong)PT << 16) | 2); // BRA CC.EQ + } + _branches.Add((_out.Count, (int)(targets[^1] / 16))); + Emit(0xe24000000000000fUL | ((ulong)PT << 16)); + } + + // The jump table behind a BRX register: the LDC (or, for BRXU, indexed ULDC) of bank 1 that last wrote it in + // the straight-line code before the BRX, and its entry count from the IMNMX bound of the index feeding it. + List BrxTargets(uint src, bool uniform) + { + if (C1 == null || C1.Length == 0) throw Fail("BRX without the program's c1 constants"); + int Writer(int from, uint reg, bool ur) + { + for (int k = from; k >= 0 && k >= from - 64; k--) + { + var p = _code[k]; + uint op = p.Bits(0, 12), b = op & 0x1ff; + if (b is >= 0x140 and <= 0x15f) return -1; // control flow: not straight-line + // uniform datapath: ULDC, UMOV and the U* ALU ops (base 0x080-0x0bf; UISETP/UPLOP3 write predicates) + bool isUr = op is 0xab9 or 0xabb or 0x882 or 0xc82 || (b is >= 0x080 and < 0x0c0 && b != 0x08c && b != 0x09c); + if (p.Bits(16, 8) == reg && isUr == ur) return k; + } + return -1; + } + int ldc = Writer(_index - 1, src, uniform); + if (ldc < 0) throw Fail("BRX register not loaded from a jump table"); + var l = _code[ldc]; + uint lop = l.Bits(0, 12); + if (!(lop == 0xb82 && !uniform || lop == 0xabb && uniform) || l.Bits(54, 5) != 1) + throw Fail($"BRX register loaded by op {lop:x3}, not from c1"); + uint tableBase = l.Bits(38, 16); + // the bound: the index register of the load, written by SHL x, 2 from an IMNMX.U32 with an immediate + int count = 0; + int shl = Writer(ldc - 1, l.Bits(24, 8), uniform); + if (shl >= 0) + { + var sh = _code[shl]; + int mn = Writer(shl - 1, sh.Bits(24, 8), uniform); + // IMNMX / UIMNMX with an immediate bound, or SEL / USEL Rx, Rx, bound under the compare that clamps it + // (the uniform switches: UISETP.GT P0, URx, bound; USEL URx, URx, bound, P0) + uint mop = mn >= 0 ? _code[mn].Bits(0, 12) : 0, mb = mop & 0x7f; + if (mn >= 0 && (mb == 0x17 || mb == 0x07) && _code[mn].Bits(9, 3) == 4) count = (int)_code[mn].W1 + 1; + } + if (count <= 0 || count > 1024) throw Fail("BRX jump table without a bound"); + var t = new List(); + for (int k = 0; k < count; k++) + { + int at = (int)tableBase + 4 * k; + if (at + 4 > C1.Length) throw Fail("BRX jump table outside c1"); + uint v = BitConverter.ToUInt32(C1, at); + if (v % 16 != 0 || v / 16 > _code.Length) throw Fail($"BRX jump table entry 0x{v:x}"); + if (!t.Contains(v)) t.Add(v); + } + return t; + } + + // CALL.REL.NOINC (relative, the offset as BRA's) pushes no return address: it is a jump. The compiler makes a + // call of it by putting the address after the CALL in a register first (MOV Rn, imm, which the scheduler may + // hoist dozens of instructions, within the straight-line code before the CALL) + // for the callee's RET.REL.NODEC Rn; such a CALL becomes Maxwell CAL and its RET Maxwell RET (the translator + // keeps the call stack, so the link register is not written). Without that MOV it is a plain, possibly + // predicated, jump (`@P0 CALL.NOINC` over a BRA) and becomes BRA: as CAL it made functions that call each + // other, a recursive call graph Vulkan forbids and NVIDIA's compiler inlines until its stack overflows. + bool SetsReturnAddress() + { + uint ret = (uint)(_index + 1) * 16; + for (int k = _index - 1; k >= 0 && k >= _index - 64; k--) + { + var p = _code[k]; + uint op = p.Bits(0, 12); + if ((op & 0x1ff) >= 0x140 && (op & 0x1ff) <= 0x15f) break; // control flow: the MOV would not reach the CALL + if (op == 0x802 && p.W1 == ret && p.Bits(12, 4) == 7) return true; + } + return false; + } + + void Call() + { + if (_i.W2 >> 18 != 0x03c00000 >> 18) throw Fail("CALL mode"); + long raw = (long)(_i.W1 | ((ulong)(_i.W2 & 0x3ffff) << 32)); + long target = (long)_index * 16 + 16 + ((raw << 14) >> 14); + if (target < 0 || target % 16 != 0 || target / 16 > _code.Length) throw Fail("CALL target"); + if (!SetsReturnAddress()) + { + _branches.Add((_out.Count, (int)(target / 16))); + Emit(0xe24000000000000fUL | ((ulong)_i.Bits(12, 4) << 16)); // BRA under the CALL's own predicate + return; + } + if (_i.Bits(12, 4) != 7) + { + // Maxwell CAL has no predicate: skip the call unless the guard holds + _branches.Add((_out.Count, _index + 1)); + Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); + } + _branches.Add((_out.Count, (int)(target / 16))); + // Predicate field 0, as Maxwell CAL encodes it (it has no predicate): Ryujinx's Decoder reads a CAL whose + // field says PT as an unconditional branch and decodes nothing after it (Decoder.IsUnconditionalBranch). + Emit(0xe260000000000000UL); + } + + void Ret() + { + if ((_i.W2 & ~0x3ffffu) != 0x03c00000) throw Fail("RET mode"); + EmitG(0xe32000000000000fUL); + } + + // ---- control flow ------------------------------------------------------------------------------------------ + void Kill() + { + if (_i.W1 != 0 || _i.W2 != 0x03800000) throw Fail("KILL mode"); + EmitG(0xe33000000000000fUL); + } + + // EXIT, with an optional second condition Pc in bits 87-90: exits when the guard and Pc hold. + void Exit() + { + if (_i.Bits(84, 2) != 0) throw Fail("conditional/special EXIT"); + uint pc = _i.Bits(87, 4); + if (pc == 7) { EmitG(0xe30000000000000fUL); return; } + if (_i.Bits(12, 4) != 7) + { + _branches.Add((_out.Count, _index + 1)); + Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); // skip unless the guard holds + } + Emit(0xe30000000000000fUL | ((ulong)pc << 16)); + } + + // BRA: byte offset from the next instruction in bits 34-81 (sign-extended), mode in bits 32-33 (.DIV 2: taken + // only on a diverged warp, never here; .CONV 3: taken on a converged warp, always here), an extra condition + // predicate in bits 87-90. + void Bra() + { + uint mode = _i.Bits(32, 2); + if (mode <= 1 && _i.Bit(91)) throw Fail("BRA uniform condition"); + if (mode == 2) return; + long raw = (long)((_i.W1 & ~3u) | ((ulong)(_i.W2 & 0x3ffff) << 32)); + long rel = (raw << 14) >> 14; + long target = (long)_index * 16 + 16 + rel; + if (target < 0 || target % 16 != 0 || target / 16 > _code.Length) throw Fail("BRA target"); + ulong pred = _guard; + uint extra = _i.Bits(87, 4); + if (extra != 7) + { + if (_i.Bits(12, 4) != 7) + { + // both conditions: skip the branch unless the guard holds + _branches.Add((_out.Count, _index + 1)); + Emit(0xe24000000000000fUL | ((ulong)(_i.Bits(12, 4) ^ 8) << 16)); + } + pred = (ulong)extra << 16; + } + _branches.Add((_out.Count, (int)(target / 16))); + Emit(0xe24000000000000fUL | pred); + } + } + + // code: the NVN2 program as stored (graphics: 0x80-byte header then instructions; compute: instructions only), size its + // byte length from the control section (+0x38). Returns the Maxwell program in the translator's input layout. + // mesh: a task or mesh program (graphics false): its 0x80-byte header is read for the output map and dropped, and the + // program is lowered as a compute program. + public static byte[] Lower(byte[] code, int size, bool graphics, int[] table = null, MeshInfo mesh = null) => + Lower(code, size, graphics, out _, table, mesh); + + // c1: the program's c1 constants (BRX jump tables); set before Lower by the caller. + [ThreadStatic] public static byte[] C1Bytes; + + // localBytes: the local memory size the lowered program needs when that changed (spill mode), else 0. Graphics + // programs carry it in their header already; compute and task/mesh programs need it from the translator's host + // (job "local_mem"). + public static byte[] Lower(byte[] code, int size, bool graphics, out int localBytes, int[] table = null, MeshInfo mesh = null) + { + localBytes = 0; + int header = graphics || mesh != null ? 0x80 : 0; + if (size <= header || size > code.Length || (size - header) % 16 != 0) + throw new LoweringException($"sm86 program size 0x{size:x} (file 0x{code.Length:x})"); + var insns = new Insn[(size - header) / 16]; + for (int k = 0; k < insns.Length; k++) + insns[k] = new Insn { Lo = BitConverter.ToUInt64(code, header + k * 16), Hi = BitConverter.ToUInt64(code, header + k * 16 + 8) }; + byte[] sph = null; + if (graphics && ((BitConverter.ToUInt32(code, 0) >> 10) & 15) == 5) { sph = new byte[0x50]; Buffer.BlockCopy(code, 0, sph, 0, 0x50); } + if (mesh != null) { var h = new byte[0x80]; Buffer.BlockCopy(code, 0, h, 0, 0x80); mesh.Layout(h); } + List lowered; + List<(int output, int targetSource)> branches; + int[] first; + int spillAt = -1; + try { lowered = new Lowerer(insns, sph, !graphics, table, mesh) { C1 = C1Bytes }.Run(out branches, out first); } + catch (RegisterPressureException) + { + // Local memory after the program's own: the header's sizes (Ryujinx: low + high + CRS / 32), or the + // translator's compute default (0x1000) for compute programs. + spillAt = 0x1000; + if (header != 0) + spillAt = ((int)(BitConverter.ToUInt32(code, 4) & 0xffffff) + (int)(BitConverter.ToUInt32(code, 8) & 0xffffff) + + (int)(BitConverter.ToUInt32(code, 12) & 0xffffff) / 32 + 15) & ~15; + lowered = new Lowerer(insns, sph, !graphics, table, mesh, spillAt) { C1 = C1Bytes }.Run(out branches, out first); + localBytes = spillAt + SpillBytes; + } + + int start = graphics ? 0x50 : 0; + var address = new int[lowered.Count + 1]; + int pos = start; + for (int k = 0; k <= lowered.Count; k++) + { + if (((pos - start) & 0x1f) == 0) pos += 8; + address[k] = pos; + pos += 8; + } + foreach (var (output, targetSource) in branches) + { + long rel = address[first[targetSource]] - (address[output] + 8); + if (rel < -0x800000 || rel > 0x7fffff) throw new LoweringException("sm86 lowered branch distance"); + lowered[output] |= ((ulong)rel & 0xffffff) << 20; + } + var result = new byte[pos + 16]; + if (graphics) Buffer.BlockCopy(code, 0, result, 0, 0x50); + if (graphics && spillAt >= 0) // the header's low local memory size covers the spill area too + BitConverter.GetBytes((BitConverter.ToUInt32(result, 4) & 0xff000000u) | (uint)localBytes).CopyTo(result, 4); + for (int k = 0; k < lowered.Count; k++) BitConverter.GetBytes(lowered[k]).CopyTo(result, address[k]); + // a closing EXIT after the last instruction, so a branch to the end lands somewhere + BitConverter.GetBytes(0xe30000000007000fUL).CopyTo(result, address[lowered.Count]); + return result; + } +} diff --git a/tools/xlate/MeshSpirv.cs b/tools/xlate/MeshSpirv.cs new file mode 100644 index 00000000..db507d2a --- /dev/null +++ b/tools/xlate/MeshSpirv.cs @@ -0,0 +1,367 @@ +// Turns the compute SPIR-V the translator made from a lowered NVN2 task or mesh program (AmpereLowering.MeshInfo) into a +// TaskEXT or MeshEXT module (SPV_EXT_mesh_shader, VK_EXT_mesh_shader). +// +// The translator's main function is kept as it is and called from a new entry function: +// mesh: [after a task stage: the task payload copied into the shared memory payload area, barrier] main; barrier; +// SetMeshOutputsEXT(vertex limit, primitive count from shared memory); every thread copies vertices +// i, i + threads, ... and primitives likewise from the shared memory stage buffer into the outputs: position +// and point size built-ins, generic vector g at location g, the primitive's vertex indices. +// task: main; barrier; the payload area copied out to the TaskPayloadWorkgroupEXT variable; EmitMeshTasksEXT with +// the mesh workgroup count the program wrote. +// An EXIT in main returns to the entry function, so threads that leave early still reach the barrier. SPV_EXT_mesh_shader +// needs SPIR-V 1.4, where BufferBlock is gone: the translator's storage buffers (Uniform + BufferBlock, SPIR-V 1.3) move +// to the StorageBuffer storage class with Block, and the entry point lists every global variable. +// +// Opcode and enum numbers are the SPIR-V 1.6 specification's and SPV_EXT_mesh_shader's. +using System; +using System.Collections.Generic; +using System.Linq; + +static class MeshSpirv +{ + // Generic vectors the mesh program writes (output locations). + public static IEnumerable GenericLocations(AmpereLowering.MeshInfo m) + { + var seen = new SortedSet(); + foreach (int a in m.Lines) if (a >= 0x80) seen.Add((a - 0x80) / 16); + return seen; + } + + const int OpExtension = 10, OpExtInstImport = 11, OpExtInst = 12, OpMemoryModel = 14, OpEntryPoint = 15, + OpExecutionMode = 16, OpCapability = 17, OpTypeVoid = 19, OpTypeBool = 20, OpTypeInt = 21, OpTypeFloat = 22, + OpTypeVector = 23, OpTypeArray = 28, OpTypePointer = 32, OpTypeFunction = 33, OpConstant = 43, + OpFunction = 54, OpFunctionEnd = 56, OpFunctionCall = 57, OpVariable = 59, OpLoad = 61, OpStore = 62, + OpAccessChain = 65, OpInBoundsAccessChain = 66, OpPtrAccessChain = 67, OpInBoundsPtrAccessChain = 70, + OpDecorate = 71, OpCompositeConstruct = 80, OpIAdd = 128, OpIMul = 132, OpULessThan = 176, + OpShiftRightLogical = 194, OpBitwiseAnd = 199, OpBitcast = 124, OpControlBarrier = 224, OpSelectionMerge = 247, + OpLabel = 248, OpBranch = 249, OpBranchConditional = 250, OpReturn = 253, + OpEmitMeshTasksEXT = 5294, OpSetMeshOutputsEXT = 5295; + const int ScUniform = 2, ScOutput = 3, ScWorkgroup = 4, ScInput = 1, ScFunction = 7, ScStorageBuffer = 12, ScTaskPayload = 5402; + const int DecBlock = 2, DecBufferBlock = 3, DecBuiltIn = 11, DecLocation = 30; + const int BiPosition = 0, BiPointSize = 1, BiLocalInvocationIndex = 29, + BiPrimitivePointIndices = 5294, BiPrimitiveLineIndices = 5295, BiPrimitiveTriangleIndices = 5296; + const int CapMeshShading = 5283, ModelTask = 5364, ModelMesh = 5365, ModelGLCompute = 5; + const int ModeOutputVertices = 26, ModeOutputPoints = 27, ModeOutputLines = 5269, ModeOutputPrimitives = 5270, + ModeOutputTriangles = 5298; + const int GlslUMin = 38; + + sealed class Insn + { + public int Op; + public List W; // operands, without the opcode word + public Insn(int op, IEnumerable w) { Op = op; W = w.ToList(); } + } + + static List Str(string s) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(s + "\0"); + var w = new List(); + for (int i = 0; i < bytes.Length; i += 4) + { + uint v = 0; + for (int k = 0; k < 4 && i + k < bytes.Length; k++) v |= (uint)bytes[i + k] << (8 * k); + w.Add(v); + } + return w; + } + + // The first function that reaches itself through OpFunctionCall, or -1. Vulkan SPIR-V forbids recursion + // (VUID-StandaloneSpirv-None-04634); NVIDIA's compiler inlines such a cycle until its thread's stack overflows. + public static long CallCycle(byte[] spirv) + { + var w = new uint[spirv.Length / 4]; + Buffer.BlockCopy(spirv, 0, w, 0, w.Length * 4); + var calls = new Dictionary>(); + uint fn = 0; + for (int p = 5; p < w.Length;) + { + int n = (int)(w[p] >> 16), op = (int)(w[p] & 0xffff); + if (n == 0) break; + if (op == OpFunction) fn = w[p + 2]; + else if (op == OpFunctionCall) { if (!calls.TryGetValue(fn, out var l)) calls[fn] = l = new(); l.Add(w[p + 3]); } + p += n; + } + var state = new Dictionary(); // 1 on the path, 2 done + long found = -1; + bool Visit(uint f) + { + state[f] = 1; + if (calls.TryGetValue(f, out var l)) + foreach (uint g in l) + { + if (state.TryGetValue(g, out int st) ? st == 1 : Visit(g)) { if (found < 0) found = g; return true; } + } + state[f] = 2; + return false; + } + foreach (uint f in calls.Keys.ToList()) if (!state.ContainsKey(f) && Visit(f)) break; + return found; + } + + public static byte[] Convert(byte[] spirv, AmpereLowering.MeshInfo m) + { + if (spirv.Length < 20 || BitConverter.ToUInt32(spirv, 0) != 0x07230203) throw new Exception("mesh: not a SPIR-V module"); + var words = new uint[spirv.Length / 4]; + Buffer.BlockCopy(spirv, 0, words, 0, words.Length * 4); + uint bound = words[3]; + var ins = new List(); + for (int p = 5; p < words.Length;) + { + int n = (int)(words[p] >> 16), op = (int)(words[p] & 0xffff); + if (n == 0 || p + n > words.Length) throw new Exception("mesh: malformed SPIR-V"); + ins.Add(new Insn(op, new ArraySegment(words, p + 1, n - 1))); + p += n; + } + uint Id() => bound++; + + // ---- storage buffers: Uniform + BufferBlock -> StorageBuffer + Block + var bufferBlocks = new HashSet(); + foreach (var i in ins) + if (i.Op == OpDecorate && i.W[1] == DecBufferBlock) { bufferBlocks.Add(i.W[0]); i.W[1] = DecBlock; } + var pointee = new Dictionary(); // pointer type -> storage class, pointee + var ssboPointerTypes = new HashSet(); + foreach (var i in ins) + if (i.Op == OpTypePointer) + { + if (i.W[1] == ScUniform && bufferBlocks.Contains(i.W[2])) { i.W[1] = ScStorageBuffer; ssboPointerTypes.Add(i.W[0]); } + pointee[i.W[0]] = (i.W[1], i.W[2]); + } + var ssboPointers = new HashSet(); + var newTypes = new List(); // inserted before the first function + var sbPointerOf = new Dictionary(); // pointee -> StorageBuffer pointer type + uint StoragePointer(uint type) + { + if (sbPointerOf.TryGetValue(type, out uint t)) return t; + foreach (var (pt, (sc, pe)) in pointee) if (sc == ScStorageBuffer && pe == type) { sbPointerOf[type] = pt; return pt; } + t = Id(); + newTypes.Add(new Insn(OpTypePointer, new[] { t, (uint)ScStorageBuffer, type })); + pointee[t] = (ScStorageBuffer, type); + sbPointerOf[type] = t; + return t; + } + foreach (var i in ins) + { + if (i.Op == OpVariable && ssboPointerTypes.Contains(i.W[0])) { i.W[2] = ScStorageBuffer; ssboPointers.Add(i.W[1]); } + else if (i.Op is OpAccessChain or OpInBoundsAccessChain or OpPtrAccessChain or OpInBoundsPtrAccessChain && ssboPointers.Contains(i.W[2])) + { + var (sc, pe) = pointee[i.W[0]]; + if (sc != ScStorageBuffer) i.W[0] = StoragePointer(pe); + ssboPointers.Add(i.W[1]); + } + } + + // ---- existing types and ids + uint Find(Func f) { var i = ins.FirstOrDefault(f); return i == null ? 0 : i.W[0]; } + int firstFunction = ins.FindIndex(i => i.Op == OpFunction); + var globals = new List(); // new types, constants, variables, inserted before the first function + uint Ty(int op, params uint[] operands) + { + uint t = Find(i => i.Op == op && i.W.Skip(1).SequenceEqual(operands)); + if (t == 0) t = newTypes.Concat(globals).FirstOrDefault(i => i.Op == op && i.W.Skip(1).SequenceEqual(operands))?.W[0] ?? 0; + if (t != 0) return t; + t = Id(); + globals.Add(new Insn(op, new[] { t }.Concat(operands))); + return t; + } + uint tVoid = Ty(OpTypeVoid), tBool = Ty(OpTypeBool), tUint = Ty(OpTypeInt, 32, 0), tFloat = Ty(OpTypeFloat, 32); + uint tFnVoid = Ty(OpTypeFunction, tVoid); + var constants = new Dictionary(); + uint C(uint v) + { + if (constants.TryGetValue(v, out uint id)) return id; + id = Id(); + globals.Add(new Insn(OpConstant, new[] { tUint, id, v })); + constants[v] = id; + return id; + } + uint Ptr(int sc, uint type) { uint t = Id(); globals.Add(new Insn(OpTypePointer, new[] { t, (uint)sc, type })); return t; } + uint Arr(uint elem, uint n) { uint t = Id(); globals.Add(new Insn(OpTypeArray, new[] { t, elem, C(n) })); return t; } + var decorations = new List(); + var interfaceVars = new List(); + uint Var(uint ptrType, int sc, int builtin = -1, int location = -1) + { + uint v = Id(); + globals.Add(new Insn(OpVariable, new[] { ptrType, v, (uint)sc })); + if (builtin >= 0) decorations.Add(new Insn(OpDecorate, new[] { v, (uint)DecBuiltIn, (uint)builtin })); + if (location >= 0) decorations.Add(new Insn(OpDecorate, new[] { v, (uint)DecLocation, (uint)location })); + return v; + } + + var shared = ins.Where(i => i.Op == OpVariable && i.W[2] == ScWorkgroup).ToList(); + if (shared.Count != 1) throw new Exception($"mesh: {shared.Count} workgroup variables (one shared memory array expected)"); + uint sharedVar = shared[0].W[1]; + uint pSharedUint = Ptr(ScWorkgroup, tUint); + var entry = ins.Single(i => i.Op == OpEntryPoint); + if (entry.W[0] != ModelGLCompute) throw new Exception("mesh: the translator's entry point is not a compute one"); + uint mainFn = entry.W[1]; + uint glsl = Find(i => i.Op == OpExtInstImport && Str("GLSL.std.450").SequenceEqual(i.W.Skip(1))); + var preamble = new List(); // capability, extension, import: inserted after the last capability + if (glsl == 0) { glsl = Id(); preamble.Add(new Insn(OpExtInstImport, new[] { glsl }.Concat(Str("GLSL.std.450")))); } + + uint pInputUint = Ptr(ScInput, tUint); + uint lii = Var(pInputUint, ScInput, BiLocalInvocationIndex); + int payloadWords = m.PayloadBytes / 4; + uint payloadVar = 0, pPayloadUint = 0; + if (m.Stage == 6 || m.HasTask) + { + if (payloadWords == 0) throw new Exception("mesh: a task payload of size 0"); + payloadVar = Var(Ptr(ScTaskPayload, Arr(tUint, (uint)payloadWords)), ScTaskPayload); + pPayloadUint = Ptr(ScTaskPayload, tUint); + } + + // ---- the entry function body + var body = new List(); + uint Emit(int op, uint type, params uint[] operands) { uint r = Id(); body.Add(new Insn(op, new[] { type, r }.Concat(operands))); return r; } + void Do(int op, params uint[] operands) => body.Add(new Insn(op, operands)); + uint Label() { uint l = Id(); Do(OpLabel, l); return l; } + uint Add(uint a, uint b) => Emit(OpIAdd, tUint, a, b); + uint SharedWord(uint index) => Emit(OpLoad, tUint, Emit(OpAccessChain, pSharedUint, sharedVar, index)); + void Barrier() => Do(OpControlBarrier, C(2), C(2), C(0x108)); // Workgroup, Workgroup, AcquireRelease | WorkgroupMemory + uint UMin(uint a, uint b) => Emit(OpExtInst, tUint, glsl, GlslUMin, a, b); + // if (a < b) { then() } + void If(uint a, uint b, Action then) + { + uint c = Emit(OpULessThan, tBool, a, b), t = Id(), merge = Id(); + Do(OpSelectionMerge, merge, 0); + Do(OpBranchConditional, c, t, merge); + Do(OpLabel, t); + then(); + Do(OpBranch, merge); + Do(OpLabel, merge); + } + // Words i, i + threads, ... below words of shared memory from word sharedAt <-> the payload variable. + void CopyPayload(uint tid, bool toShared) + { + int threads = Math.Max(1, m.Threads); + for (int k = 0; k * threads < payloadWords; k++) + { + uint idx = k == 0 ? tid : Add(tid, C((uint)(k * threads))); + void copy() + { + uint pp = Emit(OpAccessChain, pPayloadUint, payloadVar, idx); + uint sp = Emit(OpAccessChain, pSharedUint, sharedVar, Add(idx, C((uint)(m.Pay / 4)))); + if (toShared) Do(OpStore, sp, Emit(OpLoad, tUint, pp)); + else Do(OpStore, pp, Emit(OpLoad, tUint, sp)); + } + if ((k + 1) * threads > payloadWords) If(idx, C((uint)payloadWords), copy); + else copy(); + } + } + + uint entryFn = Id(); + body.Add(new Insn(OpFunction, new[] { tVoid, entryFn, 0u, tFnVoid })); + Label(); + uint tid = Emit(OpLoad, tUint, lii); + if (m.Stage == 7 && m.HasTask) { CopyPayload(tid, true); Barrier(); } + Emit(OpFunctionCall, tVoid, mainFn); + Barrier(); + var modes = new List(); + if (m.Stage == 6) + { + uint count = SharedWord(C((uint)(m.Pay + 4) / 4)); + CopyPayload(tid, false); + Do(OpEmitMeshTasksEXT, count, C(1), C(1), payloadVar); + } + else + { + int maxV = m.MaxVertices, maxP = m.MaxPrimitives, ipp = m.IndicesPerPrimitive, threads = Math.Max(1, m.Threads); + if (maxV <= 0 || maxP <= 0) throw new Exception("mesh: no vertex or primitive limit"); + // outputs + uint tV4 = Ty(OpTypeVector, tFloat, 4); + uint pOutFloat = Ptr(ScOutput, tFloat); + uint posVar = 0, sizeVar = 0; + var generic = new Dictionary(); + foreach (int a in m.Lines) + { + if (a >= 0x70 && a < 0x80 && posVar == 0) posVar = Var(Ptr(ScOutput, Arr(tV4, (uint)maxV)), ScOutput, BiPosition); + else if (a == 0x6c && sizeVar == 0) sizeVar = Var(Ptr(ScOutput, Arr(tFloat, (uint)maxV)), ScOutput, BiPointSize); + else if (a >= 0x80 && !generic.ContainsKey((a - 0x80) / 16)) + generic[(a - 0x80) / 16] = Var(Ptr(ScOutput, Arr(tV4, (uint)maxV)), ScOutput, -1, (a - 0x80) / 16); + } + uint tIdx = ipp == 1 ? tUint : Ty(OpTypeVector, tUint, (uint)ipp); + int idxBuiltin = ipp == 1 ? BiPrimitivePointIndices : ipp == 2 ? BiPrimitiveLineIndices : BiPrimitiveTriangleIndices; + uint idxVar = Var(Ptr(ScOutput, Arr(tIdx, (uint)maxP)), ScOutput, idxBuiltin); + uint pOutIdx = Ptr(ScOutput, tIdx); + + uint count = UMin(SharedWord(C((uint)m.Map / 4)), C((uint)maxP)); + Do(OpSetMeshOutputsEXT, C((uint)maxV), count); + int lines = m.Lines.Count; + for (int k = 0; k * threads < maxV; k++) + { + uint v = k == 0 ? tid : Add(tid, C((uint)(k * threads))); + If(v, C((uint)maxV), () => + { + // word ((v >> 5) * lines + line) * 32 + (v & 31) of the vertex area + uint group = Emit(OpShiftRightLogical, tUint, v, C(5)), lane = Emit(OpBitwiseAnd, tUint, v, C(31)); + uint at = Add(Add(Emit(OpIMul, tUint, group, C((uint)(lines * 32))), lane), C((uint)m.Attr / 4)); + for (int l = 0; l < lines; l++) + { + int a = m.Lines[l]; + uint var = 0, comp = uint.MaxValue; + if (a >= 0x70 && a < 0x80) (var, comp) = (posVar, (uint)(a - 0x70) / 4); + else if (a == 0x6c) var = sizeVar; + else if (a >= 0x80) (var, comp) = (generic[(a - 0x80) / 16], (uint)((a - 0x80) % 16) / 4); + if (var == 0) continue; // layer, viewport index: per primitive in Vulkan, not written + uint value = Emit(OpBitcast, tFloat, SharedWord(Add(at, C((uint)(l * 32))))); + uint ptr = comp == uint.MaxValue ? Emit(OpAccessChain, pOutFloat, var, v) : Emit(OpAccessChain, pOutFloat, var, v, C(comp)); + Do(OpStore, ptr, value); + } + }); + } + for (int k = 0; k * threads < maxP; k++) + { + uint p = k == 0 ? tid : Add(tid, C((uint)(k * threads))); + If(p, count, () => + { + // byte 4 + ipp * p + j of the primitive area + uint first = Add(Emit(OpIMul, tUint, p, C((uint)ipp)), C((uint)m.Map + 4)); + var idx = new uint[ipp]; + for (int j = 0; j < ipp; j++) + { + uint b = j == 0 ? first : Add(first, C((uint)j)); + uint word = SharedWord(Emit(OpShiftRightLogical, tUint, b, C(2))); + uint shift = Emit(OpIMul, tUint, Emit(OpBitwiseAnd, tUint, b, C(3)), C(8)); + uint value = Emit(OpBitwiseAnd, tUint, Emit(OpShiftRightLogical, tUint, word, shift), C(0xff)); + idx[j] = UMin(value, C((uint)maxV - 1)); + } + uint composite = ipp == 1 ? idx[0] : Emit(OpCompositeConstruct, tIdx, idx); + Do(OpStore, Emit(OpAccessChain, pOutIdx, idxVar, p), composite); + }); + } + Do(OpReturn); + modes.Add(new Insn(OpExecutionMode, new[] { entryFn, (uint)ModeOutputVertices, (uint)maxV })); + modes.Add(new Insn(OpExecutionMode, new[] { entryFn, (uint)ModeOutputPrimitives, (uint)maxP })); + modes.Add(new Insn(OpExecutionMode, new[] { entryFn, (uint)(ipp == 1 ? ModeOutputPoints : ipp == 2 ? ModeOutputLines : ModeOutputTriangles) })); + } + body.Add(new Insn(OpFunctionEnd, Array.Empty())); + + // ---- reassemble + foreach (var i in ins.Where(i => i.Op == OpExecutionMode && i.W[0] == mainFn)) i.W[0] = entryFn; + preamble.Insert(0, new Insn(OpExtension, Str("SPV_EXT_mesh_shader"))); + preamble.Insert(0, new Insn(OpCapability, new[] { (uint)CapMeshShading })); + int lastCap = ins.FindLastIndex(i => i.Op == OpCapability); + // the entry point: the new model and function, every global variable as its interface (SPIR-V 1.4) + var name = Str("main"); + var globalVars = ins.Take(firstFunction).Where(i => i.Op == OpVariable && i.W[2] != ScFunction).Select(i => i.W[1]) + .Concat(globals.Where(i => i.Op == OpVariable).Select(i => i.W[1])).ToList(); + entry.W = new List { (uint)(m.Stage == 6 ? ModelTask : ModelMesh), entryFn }.Concat(name).Concat(globalVars).ToList(); + int lastMode = ins.FindLastIndex(i => i.Op == OpExecutionMode || i.Op == OpEntryPoint); + int firstType = ins.FindIndex(i => i.Op >= OpTypeVoid && i.Op <= 39 || i.Op == OpConstant || i.Op == OpVariable); + var result = new List(); + for (int k = 0; k < ins.Count; k++) + { + if (k == firstType) result.AddRange(decorations); + if (k == firstFunction) { result.AddRange(newTypes); result.AddRange(globals); } + result.Add(ins[k]); + if (k == lastCap) result.AddRange(preamble); + if (k == lastMode) result.AddRange(modes); + } + result.AddRange(body); + + var output = new List { 0x07230203, 0x00010400, words[2], bound, 0 }; + foreach (var i in result) { output.Add((uint)((i.W.Count + 1) << 16 | i.Op)); output.AddRange(i.W); } + var bytes = new byte[output.Count * 4]; + Buffer.BlockCopy(output.ToArray(), 0, bytes, 0, bytes.Length); + return bytes; + } +} diff --git a/tools/xlate/Program.cs b/tools/xlate/Program.cs index 98712447..3777a6a0 100644 --- a/tools/xlate/Program.cs +++ b/tools/xlate/Program.cs @@ -7,6 +7,9 @@ // "ok " or "fail "; a malformed line gets "fail ? " and the loop continues. // nx2navi_xlate batch mode: a JSON array of jobs, prints a summary, exit 1 if any failed. // job: { "key", "stage": "vs|ps|cs", "code": path (graphics = 0x50-byte SPH + instructions, compute = instructions), "c1": hex, +// "sm86": bool + "size": bytes (an NVN2 program, lowered by AmpereLowering.cs), "nvn2_table": [start, textures, +// images] (NVN2 compute binding table), "lowered_out": path, "mesh": {"stage": 6|7, "threads", "maxv", "maxp", +// "topo", "payload", "task"} (stage "ts" / "ms": an NVN2 task or mesh program, AmpereLowering.MeshInfo), // "tex": {slot: type}, "img": {slot: {type, fmt}}, "attr": {loc: float|sint|uint}, "fout": {idx: float|sint|uint}, // "local": [x, y, z], "shared": bytes, "debug": bool, "reduced": bool, "softfloat64": bool, "zero_locals": bool, // "sbuf_align": int, "subgroup": int } @@ -37,13 +40,21 @@ static class SwshXlate // 全部资源放进第 0 组,各占不同绑定号(spirv-cross 出成一个参数缓冲);常量段号 index 的缓冲由翻译器给 binding = index+1 的惯例不依赖 // All resources go into set 0, each with a distinct binding number (spirv-cross emits them as one argument buffer); the translator's convention of giving the buffer of constant segment number index binding = index+1 is not relied on public SetBindingPair CreateConstantBufferBinding(int index) => new(0, _binding++); - public SetBindingPair CreateImageBinding(int count, bool isBuffer) { int b = _binding; _binding += count; return new(0, b); } + // A pool array (bindless) takes one binding number: the runtime moves it to its own set (vk_pipeline.c + // spirv_pool_rebind), and 32768 skipped numbers pushed the bindings after it past the runtime's 16-bit field. + static bool PoolArray(int count) => count == TexturePoolLength || count == SamplerPoolLength; + public SetBindingPair CreateImageBinding(int count, bool isBuffer) { int b = _binding; _binding += PoolArray(count) ? 1 : count; return new(0, b); } public SetBindingPair CreateStorageBufferBinding(int index) => new(0, _binding++); - public SetBindingPair CreateTextureBinding(int count, bool isBuffer) { int b = _binding; _binding += count; return new(0, b); } + public SetBindingPair CreateTextureBinding(int count, bool isBuffer) { int b = _binding; _binding += PoolArray(count) ? 1 : count; return new(0, b); } public ReadOnlySpan GetCode(ulong address, int minimumSize) => MemoryMarshal.Cast(new ReadOnlySpan(_code)[(int)address..]); - public int QuerySamplerArrayLengthFromPool() => 32; + // Bindless handles loaded from memory index the whole texture and sampler pools (Ryujinx BindlessElimination + // GenerateBindlessAccess): arrays of TexturePoolLength textures (handle bits 0-19) and SamplerPoolLength samplers + // (bits 20-31; NVN sampler ids are below 4096). res.json marks them "pool": 1 (textures) / 2 (samplers); the + // runtime binds them from its global pool set (vk_exec_draw.c, texture and sampler pools). + public const int TexturePoolLength = 32768, SamplerPoolLength = 4096; + public int QuerySamplerArrayLengthFromPool() => SamplerPoolLength; public int QueryTextureArrayLengthFromBuffer(int slot) => 32; - public int QueryTextureArrayLengthFromPool() => 32; + public int QueryTextureArrayLengthFromPool() => TexturePoolLength; string Get(string table, int slot) => _job[table] is JsonObject t && t[slot.ToString()] is JsonNode n ? (n is JsonObject o ? (string)o["type"] : (string)n) : null; int TexSlot(int handle, int cbufSlot) @@ -68,9 +79,19 @@ static class SwshXlate _ => SamplerType.Texture2D, }; } + // Image handles: NVN1 c[2] 0x120 + 8 * slot (NVN2 compute handles are renumbered there by AmpereLowering.Handle); + // TexSlot knows only the separate texture range, so image formats were never answered (an R32UI view of an + // R11G11B10F target was declared float, and the raw words the shader packs were converted on store). + int ImgSlot(int handle, int cbufSlot) + { + (int off0, _, _) = TextureHandle.UnpackOffsets(handle); + int bytes = off0 * 4; + return (cbufSlot < 0 || (cbufSlot & 0xffff) == 2) && bytes >= 0x120 && bytes < TexBase && (bytes - 0x120) % 8 == 0 ? (bytes - 0x120) / 8 : -1; + } public TextureFormat QueryTextureFormat(int handle, int cbufSlot = -1) { - int slot = TexSlot(handle, cbufSlot); + int slot = ImgSlot(handle, cbufSlot); + if (slot < 0) slot = TexSlot(handle, cbufSlot); if (slot >= 0) ImgQueried.Add(slot); string f = _job["img"] is JsonObject t && slot >= 0 && t[slot.ToString()] is JsonObject o ? (string)o["fmt"] : null; if (f == null) { Notes.Add($"图像句柄 {handle:X} 槽 {slot} 无 PC 格式,按 R8G8B8A8Unorm"); return TextureFormat.R8G8B8A8Unorm; } @@ -80,6 +101,8 @@ static class SwshXlate public int QueryComputeLocalSizeX() => Local(0); public int QueryComputeLocalSizeY() => Local(1); public int QueryComputeLocalSizeZ() => Local(2); + // "local_mem": local memory bytes of a compute program (the lowering's spill area adds to it); else Ryujinx's 0x1000. + public int QueryComputeLocalMemorySize() => _job["local_mem"] is JsonNode n ? (int)n : 0x1000; public int QueryComputeSharedMemorySize() => _job["shared"] is JsonNode n ? (int)n : 0; // 配对 PC 的 groupshared 声明之和 | sum of the paired PC shader's groupshared declarations static AttributeType Attr(string s) => s switch { "sint" => AttributeType.Sint, "uint" => AttributeType.Uint, _ => AttributeType.Float }; public GpuGraphicsState QueryGraphicsState() @@ -134,13 +157,58 @@ static class SwshXlate try { byte[] code = File.ReadAllBytes((string)job["code"]); + AmpereLowering.MeshInfo mesh = null; + // "sm86": true: the code is an NVN2 (Switch 2) program as stored, "size" bytes long (control section +0x38); + // it is lowered to Maxwell first (AmpereLowering.cs). "lowered_out": a path that receives the Maxwell code. + if (job["sm86"] is JsonValue sm && sm.TryGetValue(out bool isSm86) && isSm86) + { + int size = job["size"] is JsonNode sz ? (int)sz : code.Length; + AmpereLowering.C1Bytes = job["c1"] is JsonNode c1h ? Convert.FromHexString((string)c1h) : null; + int[] table = job["nvn2_table"] is JsonArray t && t.Count == 3 ? new[] { (int)t[0], (int)t[1], (int)t[2] } : null; + if (job["mesh"] is JsonObject mj) + { + // A task or mesh program: lowered and translated as a compute program whose shared memory holds + // the stage buffer, then rewritten into a TaskEXT / MeshEXT module (MeshSpirv.cs). + mesh = new AmpereLowering.MeshInfo + { + Stage = (int)mj["stage"], Threads = (int)mj["threads"], MaxVertices = (int)mj["maxv"], + MaxPrimitives = (int)mj["maxp"], Topology = (int)mj["topo"], PayloadBytes = (int)mj["payload"], + HasTask = mj["task"] is JsonValue tv && tv.TryGetValue(out bool tb) && tb, + }; + code = AmpereLowering.Lower(code, size, false, out int meshLocal, null, mesh); + if (meshLocal != 0) job["local_mem"] = meshLocal; + job["local"] = new JsonArray(mesh.Threads, 1, 1); + job["shared"] = mesh.SharedBytes; + } + else + { + code = AmpereLowering.Lower(code, size, (string)job["stage"] != "cs", out int local, table); + if (local != 0) job["local_mem"] = local; + } + if (job["lowered_out"] is JsonNode lo) File.WriteAllBytes((string)lo, code); + } var acc = new Accessor(code, job); // 作业 debug = false:关 DebugMode(DebugMode 会跳过 AST 优化器,产物满是临时变量;不改语义) // Job debug = false: turn off DebugMode (DebugMode skips the AST optimizer and the output is full of temporaries; semantics unchanged) - var flags = (job["debug"] is JsonNode dbg && !(bool)dbg ? 0 : TranslationFlags.DebugMode) | ((string)job["stage"] == "cs" ? TranslationFlags.Compute : 0); + var flags = (job["debug"] is JsonNode dbg && !(bool)dbg ? 0 : TranslationFlags.DebugMode) | ((string)job["stage"] is "cs" or "ts" or "ms" ? TranslationFlags.Compute : 0); var ctx = Translator.CreateContext(0, acc, new TranslationOptions(TargetLanguage.Spirv, TargetApi.Vulkan, flags)); ShaderProgram p = ctx.Translate(); - File.WriteAllBytes(Path.Combine(outDir, key + ".spv"), p.BinaryCode); + // "glsl_out": a path that receives the same program as GLSL, for reading (diagnosis only). + if (job["glsl_out"] is JsonNode go) + { + var gctx = Translator.CreateContext(0, new Accessor(code, job), new TranslationOptions(TargetLanguage.Glsl, TargetApi.OpenGL, flags)); + File.WriteAllText((string)go, gctx.Translate().Code); + } + byte[] spirv = p.BinaryCode; + // A recursive call graph (subroutines the translator could not tell apart) is refused: Vulkan forbids it + // and NVIDIA's compiler overflows its stack on it (MeshSpirv.CallCycle). + if (MeshSpirv.CallCycle(spirv) is long cyc && cyc >= 0) throw new Exception($"recursive call graph (function %{cyc})"); + if (mesh != null) + { + if (job["compute_out"] is JsonNode co) File.WriteAllBytes((string)co, spirv); + spirv = MeshSpirv.Convert(spirv, mesh); + } + File.WriteAllBytes(Path.Combine(outDir, key + ".spv"), spirv); var res = new JsonObject { ["key"] = key, ["stage"] = (string)job["stage"], ["notes"] = new JsonArray(acc.Notes.ConvertAll(n => (JsonNode)n).ToArray()), @@ -149,10 +217,14 @@ static class SwshXlate ["tex_queried"] = new JsonArray(acc.TexQueried.Select(x => (JsonNode)x).ToArray()), ["img_queried"] = new JsonArray(acc.ImgQueried.Select(x => (JsonNode)x).ToArray()), }; if (acc.QueryMotionPrevious()) (res["mvprev"], res["mvprev_banks"]) = (acc.MotionPreviousLocation, acc.MotionPreviousBanks); + if (mesh != null) + res["mesh"] = new JsonObject { ["lines"] = mesh.Lines.Count, ["shared"] = mesh.SharedBytes, + ["outputs"] = new JsonArray(MeshSpirv.GenericLocations(mesh).Select(x => (JsonNode)x).ToArray()) }; foreach (var b in p.Info.CBuffers) res["cbufs"].AsArray().Add((JsonNode)new JsonObject { ["binding"] = b.Binding, ["slot"] = b.Slot }); foreach (var b in p.Info.SBuffers) res["sbufs"].AsArray().Add((JsonNode)new JsonObject { ["binding"] = b.Binding, ["slot"] = b.Slot, ["cb_slot"] = b.SbCbSlot, ["cb_offset"] = b.SbCbOffset, ["flags"] = (int)b.Flags }); foreach (var (list, name) in new[] { (p.Info.Textures, "textures"), (p.Info.Images, "images") }) - foreach (var t in list) res[name].AsArray().Add((JsonNode)new JsonObject { ["binding"] = t.Binding, ["type"] = (int)t.Type, ["format"] = t.Format.ToString(), ["cbuf_slot"] = t.CbufSlot, ["handle"] = t.HandleIndex, ["array"] = t.ArrayLength, ["separate"] = t.Separate, ["flags"] = (int)t.Flags }); + foreach (var t in list) res[name].AsArray().Add((JsonNode)new JsonObject { ["binding"] = t.Binding, ["type"] = (int)t.Type, ["format"] = t.Format.ToString(), ["cbuf_slot"] = t.CbufSlot, ["handle"] = t.HandleIndex, ["array"] = t.ArrayLength, ["separate"] = t.Separate, ["flags"] = (int)t.Flags, + ["pool"] = t.ArrayLength == Accessor.TexturePoolLength ? 1 : t.ArrayLength == Accessor.SamplerPoolLength ? 2 : 0 }); File.WriteAllText(Path.Combine(outDir, key + ".res.json"), res.ToJsonString(new JsonSerializerOptions { WriteIndented = true })); return null; } diff --git a/tools/xlate/patches/0017-goto-elimination-outer-loop-exits.patch b/tools/xlate/patches/0017-goto-elimination-outer-loop-exits.patch new file mode 100644 index 00000000..703c2694 --- /dev/null +++ b/tools/xlate/patches/0017-goto-elimination-outer-loop-exits.patch @@ -0,0 +1,118 @@ +--- a/src/Ryujinx.Graphics.Shader/StructuredIr/GotoElimination.cs ++++ b/src/Ryujinx.Graphics.Shader/StructuredIr/GotoElimination.cs +@@ -1,4 +1,5 @@ + using Ryujinx.Graphics.Shader.IntermediateRepresentation; ++using Ryujinx.Graphics.Shader.Translation; + using System; + using System.Collections.Generic; + using static Ryujinx.Graphics.Shader.StructuredIr.AstHelper; +@@ -9,8 +10,12 @@ + { + // This is a modified version of the algorithm presented on the paper + // "Taming Control Flow: A Structured Approach to Eliminating Goto Statements". +- public static void Eliminate(GotoStatement[] gotos) ++ // NX2NAVI: makes the flags RouteOuterExits needs (StructuredProgramContext.NewTemp). ++ [ThreadStatic] private static Func _newTemp; ++ ++ public static void Eliminate(GotoStatement[] gotos, Func newTemp) + { ++ _newTemp = newTemp; + for (int index = gotos.Length - 1; index >= 0; index--) + { + GotoStatement stmt = gotos[index]; +@@ -381,9 +386,84 @@ + first = next; + } + ++ if (type == AstBlockType.DoWhile) ++ { ++ RouteOuterExits(newBlock); ++ } ++ + return newBlock; + } + ++ // NX2NAVI: a loop made here around statements that already hold a break or continue (made by MoveOutward or ++ // EncloseDoWhile for an enclosing loop) would take them as its own: the break then left only the new loop and ++ // the enclosing one never ended (a GPU hang in fragment shaders whose light loop holds a BRX switch, lowered to ++ // compares and branches by AmpereLowering). Each such exit now sets a flag and breaks the new loop; after it the ++ // flag repeats the exit for the enclosing loop. ++ private static void RouteOuterExits(AstBlock loop) ++ { ++ List exits = []; ++ ++ CollectExits(loop, exits); ++ ++ if (exits.Count == 0 || _newTemp == null) ++ { ++ return; ++ } ++ ++ AstBlock parent = loop.Parent; ++ AstOperand breakFlag = null, continueFlag = null; ++ ++ foreach (AstOperation exit in exits) ++ { ++ bool isBreak = exit.Inst == Instruction.LoopBreak; ++ AstOperand flag = isBreak ? (breakFlag ??= _newTemp(AggregateType.Bool)) : (continueFlag ??= _newTemp(AggregateType.Bool)); ++ AstBlock block = ParentBlock(exit); ++ ++ block.AddBefore(exit, Assign(flag, Const(IrConsts.True))); ++ ++ if (!isBreak) ++ { ++ block.AddBefore(exit, new AstOperation(Instruction.LoopBreak)); ++ block.Remove(exit); ++ } ++ } ++ ++ foreach (AstOperand flag in new[] { continueFlag, breakFlag }) ++ { ++ if (flag == null) ++ { ++ continue; ++ } ++ ++ parent.AddBefore(loop, Assign(flag, Const(IrConsts.False))); ++ ++ AstBlock check = new(AstBlockType.If, flag); ++ ++ check.AddFirst(new AstOperation(flag == breakFlag ? Instruction.LoopBreak : Instruction.LoopContinue)); ++ ++ parent.AddAfter(loop, check); ++ } ++ } ++ ++ // The breaks and continues in block that belong to a loop around it (those inside a nested loop are its own). ++ private static void CollectExits(AstBlock block, List exits) ++ { ++ foreach (IAstNode node in block) ++ { ++ if (node is AstBlock child) ++ { ++ if (child.Type != AstBlockType.DoWhile) ++ { ++ CollectExits(child, exits); ++ } ++ } ++ else if (node is AstOperation op && (op.Inst == Instruction.LoopBreak || op.Inst == Instruction.LoopContinue)) ++ { ++ exits.Add(op); ++ } ++ } ++ } ++ + private static bool BlockMatches(IAstNode node, AstBlockType type, IAstNode cond) + { + if (node is not AstBlock block) +--- a/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs ++++ b/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs +@@ -68,7 +68,7 @@ + } + } + +- GotoElimination.Eliminate(context.GetGotos()); ++ GotoElimination.Eliminate(context.GetGotos(), context.NewTemp); + + AstOptimizer.Optimize(context); + diff --git a/tools/xlate/patches/0018-integer-image-formats.patch b/tools/xlate/patches/0018-integer-image-formats.patch new file mode 100644 index 00000000..1c00ad40 --- /dev/null +++ b/tools/xlate/patches/0018-integer-image-formats.patch @@ -0,0 +1,22 @@ +--- a/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs ++++ b/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs +@@ -1,3 +1,4 @@ ++using Ryujinx.Graphics.Shader.Translation; + using System.Collections.Generic; + + namespace Ryujinx.Graphics.Shader.StructuredIr +@@ -70,7 +71,13 @@ + // specify a format, we can just declare it without a format and the GPU will handle it. + if (gpuAccessor.QueryHostSupportsImageLoadFormatted()) + { +- return TextureFormat.Unknown; ++ // NX2NAVI: an integer image still needs its format: without one it is declared float, and a store of raw ++ // words (Ocarina packs R11G11B10 itself and stores it through an R32UI view) converts them. ++ TextureFormat integer = gpuAccessor.QueryTextureFormat(handle, cbufSlot); ++ ++ AggregateType component = integer.GetComponentType(); ++ ++ return component == AggregateType.U32 || component == AggregateType.S32 ? integer : TextureFormat.Unknown; + } + + TextureFormat format = gpuAccessor.QueryTextureFormat(handle, cbufSlot); diff --git a/tools/xlate/patches/0019-fragment-depth-register.patch b/tools/xlate/patches/0019-fragment-depth-register.patch new file mode 100644 index 00000000..39cc20bb --- /dev/null +++ b/tools/xlate/patches/0019-fragment-depth-register.patch @@ -0,0 +1,27 @@ +--- a/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs +@@ -159,8 +159,22 @@ + + internal int GetDepthRegister() + { +- // The depth register is always two registers after the last color output. +- return BitOperations.PopCount((uint)Definitions.OmapTargets) + 1; ++ // NX2NAVI: every render target with any component enabled takes four registers (the colour outputs above), then ++ // the sample mask when written, then one register is skipped. The component count was used, which matches ++ // only when every enabled target writes all four: Ocarina's depth downsample (targets 0 and 2, .x each) read ++ // its depth from R3, never written, and wrote 0 (all its fog then sat at the near plane). As the hardware's ++ // ExitFragment counts. ++ int targets = 0; ++ ++ for (int rtIndex = 0; rtIndex < 8; rtIndex++) ++ { ++ if ((Definitions.OmapTargets & (0xf << (rtIndex * 4))) != 0) ++ { ++ targets++; ++ } ++ } ++ ++ return targets * 4 + (Definitions.OmapSampleMask ? 1 : 0) + 1; + } + + public void SetLayerOutputAttribute(int attr) diff --git a/tools/xlate/patches/0020-storage-base-not-from-c1.patch b/tools/xlate/patches/0020-storage-base-not-from-c1.patch new file mode 100644 index 00000000..f0ee3715 --- /dev/null +++ b/tools/xlate/patches/0020-storage-base-not-from-c1.patch @@ -0,0 +1,18 @@ +--- a/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs ++++ b/src/Ryujinx.Graphics.Shader/Translation/Optimizations/GlobalToStorage.cs +@@ -1046,6 +1046,15 @@ + // because constant buffer are always encoded as the second operand, + // so the second operand will always be the one from the last instruction. + ++ // NX2NAVI: c1 holds the compiler's constants (BRX jump tables, offsets), never a buffer address; an address ++ // that adds one of them to a pointer from another bank (Ocarina's NVN2 compute: a push-constant pointer in ++ // c0 plus the constant 12 from c1) has the pointer as its base. Taking c1 sent those stores to nothing. ++ if (src1.Type == OperandType.ConstantBuffer && src2.Type == OperandType.ConstantBuffer && ++ src2.GetCbufSlot() == 1 && src1.GetCbufSlot() != 1) ++ { ++ return GetBaseAddressCbWithOffset(src1, src2, constOffset); ++ } ++ + if (src1.Type != OperandType.ConstantBuffer || + (src1.Type == OperandType.ConstantBuffer && src2.Type == OperandType.ConstantBuffer) || + (src2.Type == OperandType.ConstantBuffer && src2.GetCbufSlot() == DriverReservedCb))