#include // Shaders: a draw's Maxwell programs (read from guest memory) through suyu's shader recompiler and the WGSL // backend, into WebGPU shader modules. The vertex and pixel programs are translated together: the vertex stage's // output interpolation and the pixel stage's inputs must agree. #include #include #include #include #include #include #include #include #include #include #include #include #include #include "shader_recompiler/backend/bindings.h" #include "shader_recompiler/backend/wgsl/emit_wgsl.h" #include "shader_recompiler/environment.h" #include "shader_recompiler/exception.h" #include "shader_recompiler/frontend/ampere/lower_to_maxwell.h" #include "shader_recompiler/frontend/ampere/program_address_map.h" #include "shader_recompiler/frontend/ampere/program_capture.h" #include "shader_recompiler/frontend/ir/basic_block.h" #include "shader_recompiler/frontend/maxwell/control_flow.h" #include "shader_recompiler/frontend/maxwell/translate_program.h" #include "shader_recompiler/host_translate_info.h" #include "shader_recompiler/object_pool.h" #include "shader_recompiler/profile.h" #include "shader_recompiler/program_header.h" #include "shader_recompiler/runtime_info.h" #include "video_core/texture_cache/format_lookup_table.h" #include "rx_internal.h" namespace rx { namespace { Shader::TextureType convert_texture_type(const Tegra::Texture::TICEntry& entry) { using T = Tegra::Texture::TextureType; switch (entry.texture_type) { case T::Texture1D: return Shader::TextureType::Color1D; case T::Texture2D: case T::Texture2DNoMipmap: return entry.normalized_coords ? Shader::TextureType::Color2D : Shader::TextureType::Color2DRect; case T::Texture3D: return Shader::TextureType::Color3D; case T::TextureCubemap: return Shader::TextureType::ColorCube; case T::Texture1DArray: return Shader::TextureType::ColorArray1D; case T::Texture2DArray: return Shader::TextureType::ColorArray2D; case T::Texture1DBuffer: return Shader::TextureType::Buffer; case T::TextureCubeArray: return Shader::TextureType::ColorArrayCube; default: return Shader::TextureType::Color2D; } } /// suyu's GraphicsEnvironment over guest memory: the program at program_region + offset, the channel's constant /// buffers, the texture headers of the bound pool. class GuestEnvironment final : public Shader::Environment { public: GuestEnvironment(const Regs& r, const Channel& ch, Shader::Stage stage_, uint32_t stage_index_, uint32_t offset) : regs{r}, channel{ch}, stage_index{stage_index_}, base{r.program_region.Address()} { stage = stage_; start_address = offset; code.resize(0x10000 / 8); gpu_read(base + offset, code.data(), code.size() * 8); std::memcpy(&sph, code.data(), sizeof(sph)); local_memory = static_cast(sph.LocalMemorySize()) + sph.common3.shader_local_memory_crs_size; texture_bound = r.bindless_texture_const_buffer_slot; is_proprietary_driver = texture_bound == 2; } u64 ReadInstruction(u32 address) override { const uint64_t rel = address - start_address; if (rel / 8 < code.size()) return code[rel / 8]; return gpu_get(base + address); } u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override { const ConstBuffer& cb = channel.cbufs[stage_index][cbuf_index % 18]; return cbuf_offset < cb.size ? gpu_get(cb.address + cbuf_offset) : 0; } Shader::TextureType ReadTextureType(u32 handle) override { return convert_texture_type(tic(handle)); } Shader::TexturePixelFormat ReadTexturePixelFormat(u32 handle) override { const auto e = tic(handle); return static_cast(VideoCommon::PixelFormatFromTextureInfo( e.format, e.r_type, e.g_type, e.b_type, e.a_type, e.srgb_conversion)); } bool IsTexturePixelFormatInteger(u32 handle) override { return VideoCore::Surface::IsPixelFormatInteger(static_cast(ReadTexturePixelFormat(handle))); } u32 ReadViewportTransformState() override { return regs.viewport_scale_offset_enabled; } u32 TextureBoundBuffer() const override { return texture_bound; } u32 LocalMemorySize() const override { return local_memory; } u32 SharedMemorySize() const override { return 0; } std::array WorkgroupSize() const override { return {1, 1, 1}; } bool HasHLEMacroState() const override { return false; } std::optional GetReplaceConstBuffer(u32, u32) override { return std::nullopt; } void Dump(u64, u64) override {} private: Tegra::Texture::TICEntry tic(u32 handle) { const bool via_header = regs.sampler_binding == Regs::SamplerBinding::ViaHeaderBinding; return read_tic(regs, Tegra::Texture::TexturePair(handle, via_header).first); } const Regs& regs; const Channel& channel; uint32_t stage_index; uint64_t base; std::vector code; uint32_t local_memory = 0, texture_bound = 0; }; /// suyu's ComputeEnvironment over guest memory: the program at code_loc + program_start (no program header), /// constant buffers from the launch descriptor. class ComputeGuestEnvironment final : public Shader::Environment { public: explicit ComputeGuestEnvironment(const ComputeLaunch& l) : launch{l} { stage = Shader::Stage::Compute; start_address = l.program_start; code.resize(0x10000 / 8); gpu_read(l.code_base + l.program_start, code.data(), code.size() * 8); is_proprietary_driver = l.texture_bound == 2; } u64 ReadInstruction(u32 address) override { const uint64_t rel = address - start_address; if (rel / 8 < code.size()) return code[rel / 8]; return gpu_get(launch.code_base + address); } u32 ReadCbufValue(u32 cbuf_index, u32 cbuf_offset) override { const auto& cb = launch.cbufs[cbuf_index % 8]; return cb.enabled && cbuf_offset < cb.size ? gpu_get(cb.address + cbuf_offset) : 0; } Shader::TextureType ReadTextureType(u32 handle) override { return convert_texture_type(tic(handle)); } Shader::TexturePixelFormat ReadTexturePixelFormat(u32 handle) override { const auto e = tic(handle); return static_cast(VideoCommon::PixelFormatFromTextureInfo( e.format, e.r_type, e.g_type, e.b_type, e.a_type, e.srgb_conversion)); } bool IsTexturePixelFormatInteger(u32 handle) override { return VideoCore::Surface::IsPixelFormatInteger(static_cast(ReadTexturePixelFormat(handle))); } u32 ReadViewportTransformState() override { return 0; } u32 TextureBoundBuffer() const override { return launch.texture_bound; } u32 LocalMemorySize() const override { return launch.local_size; } u32 SharedMemorySize() const override { return launch.shared_size; } std::array WorkgroupSize() const override { return {launch.block[0], launch.block[1], launch.block[2]}; } bool HasHLEMacroState() const override { return false; } std::optional GetReplaceConstBuffer(u32, u32) override { return std::nullopt; } void Dump(u64, u64) override {} private: Tegra::Texture::TICEntry tic(u32 handle) { return read_tic(*launch.tex_regs, Tegra::Texture::TexturePair(handle, launch.linked_tsc).first); } const ComputeLaunch& launch; std::vector code; }; struct Translated { Shader::ObjectPool flow_blocks; Shader::ObjectPool insts{8192}; Shader::ObjectPool blocks{32}; Shader::IR::Program program; }; thread_local bool on_translation_worker; // (the worker: no count(), no log_limited(): not thread-safe) /// Marks the thread as a translation worker while it lives (build_pair's helper threads: pool threads) struct AsWorker { bool was_worker = on_translation_worker, was_side = rx_side_thread; AsWorker() { on_translation_worker = rx_side_thread = true; } ~AsWorker() { on_translation_worker = was_worker; rx_side_thread = was_side; } }; std::unique_ptr translate(const Regs& r, const Channel& ch, Shader::Stage stage, uint32_t stage_index, uint32_t offset) { struct TranslateTimer { int64_t t0 = hos_now_ns(); ~TranslateTimer() { if (!on_translation_worker) count("t_translate", hos_now_ns() - t0); } } translate_timer; auto t = std::make_unique(); GuestEnvironment env{r, ch, stage, stage_index, offset}; Shader::Maxwell::Flow::CFG cfg{env, t->flow_blocks, env.StartAddress() + static_cast(sizeof(Shader::ProgramHeader))}; Shader::HostTranslateInfo host{}; host.support_float64 = false; host.support_int64 = false; host.min_ssbo_alignment = dev.limits.minStorageBufferOffsetAlignment; // RX_DUMP_IR= (debugging): each program's IR before constant folding, as /_
.pre.txt static const char* ir_dir = getenv("RX_DUMP_IR"); if (ir_dir) { char path[512]; std::snprintf(path, sizeof path, "%s/%s_%010llx", ir_dir, stage == Shader::Stage::Fragment ? "fs" : "vs", (unsigned long long)(r.program_region.Address() + offset)); Shader::Maxwell::ir_dump_path = path; } t->program = Shader::Maxwell::TranslateProgram(t->insts, t->blocks, env, cfg, host); if (ir_dir) Shader::Maxwell::ir_dump_path.clear(); return t; } /// What the stage programs at these addresses use, from their first translation (for the pair keys). struct StageInfo { std::bitset<32> loads, stores; bool failed = false; }; std::unordered_map stage_infos; // by StageKey hash std::unordered_map> pairs; std::mutex prog_mu; // stage_infos, pairs and the WGSL text ids: the translation workers add to them std::atomic wgsl_texts; std::atomic wgsl_bytes; struct InternedText { uint32_t id = 0; uint64_t check = 0, cache_hash = 0; std::string text; }; std::unordered_map interned_texts; uint64_t stage_hash(uint64_t base, uint32_t offset) { // (not cached by address and page writes: tried 2026-10-13, programs went stale somehow and gameplay broke. // Cached by address and validated by comparing the bytes themselves: a memcmp is several times cheaper than // hashing them, and exact) constexpr size_t kBytes = 0x50 + 0x400; struct Entry { uint8_t bytes[kBytes]; uint64_t hash; }; static std::unordered_map> cache; const uint64_t va = base + offset; static const bool no_cache = env_flag("RX_NO_STAGE_CACHE"); // (debugging) if (no_cache) { uint8_t b[kBytes]; gpu_read(va, b, kBytes); return hash_bytes(b, kBytes, va); } uint64_t avail = 0; const uint8_t* host = hos_gpu_ptr(va, &avail); uint8_t buf[kBytes]; const uint8_t* now = host; if (!host || avail < kBytes) { gpu_read(va, buf, kBytes); now = buf; } auto& e = cache[va]; if (e && std::memcmp(e->bytes, now, kBytes) == 0) { static const bool verify = env_flag("RX_VERIFY_STAGE_CACHE"); // (debugging) if (verify) { uint8_t b[kBytes]; gpu_read(va, b, kBytes); const uint64_t fresh = hash_bytes(b, kBytes, va); static int logged; if (fresh != e->hash && logged++ < 20) hos_log("rx: stage cache %#llx: cached %016llx fresh %016llx (memcmp of the copy %d)", (unsigned long long)va, (unsigned long long)e->hash, (unsigned long long)fresh, std::memcmp(b, e->bytes, kBytes)); } return e->hash; } if (!e) e = std::make_unique(); std::memcpy(e->bytes, now, kBytes); e->hash = hash_bytes(e->bytes, kBytes, va); // (the copy: the guest may be writing the live bytes) return e->hash; } Shader::AttributeType attribute_type(const Regs::VertexAttribute& a) { if (a.constant) return Shader::AttributeType::Float; switch (a.type) { case Regs::VertexAttribute::Type::SInt: return Shader::AttributeType::SignedInt; case Regs::VertexAttribute::Type::UInt: return Shader::AttributeType::UnsignedInt; case Regs::VertexAttribute::Type::SScaled: return Shader::AttributeType::SignedScaled; case Regs::VertexAttribute::Type::UScaled: return Shader::AttributeType::UnsignedScaled; default: return Shader::AttributeType::Float; } } Shader::AttributeType output_type(const Regs& r, uint32_t i) { if (i >= r.rt_control.count) return Shader::AttributeType::Float; const uint32_t target = r.rt_control.Map(i); if (target >= 8) return Shader::AttributeType::Float; const auto f = r.rt[target].format; if (f == Tegra::RenderTargetFormat::NONE) return Shader::AttributeType::Float; const PixelFormat pf = VideoCore::Surface::PixelFormatFromRenderTargetFormat(f); if (!VideoCore::Surface::IsPixelFormatInteger(pf)) return Shader::AttributeType::Float; return VideoCore::Surface::IsPixelFormatSignedInteger(pf) ? Shader::AttributeType::SignedInt : Shader::AttributeType::UnsignedInt; } bool create_module(ShaderModule& m, const char* what, uint64_t key) { WGPUShaderSourceWGSL src{}; src.chain.sType = WGPUSType_ShaderSourceWGSL; const std::string& wgsl = m.source(); src.code = WGPUStringView{wgsl.data(), wgsl.size()}; // the browser (Tint) enforces derivative uniformity, which wgpu (naga) does not: the translated programs sample // in control flow it cannot prove uniform std::string web_wgsl; if (RX_WEB) { web_wgsl = web_shader_source(wgsl); src.code = WGPUStringView{web_wgsl.data(), web_wgsl.size()}; } WGPUShaderModuleDescriptor d{}; d.nextInChain = &src.chain; static const bool times = env_flag("RX_MODULE_TIMES"); const auto t0 = times ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; scope_push(); m.module = program_module(&d); const std::string error = scope_pop(); if (times) { const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); std::fprintf(stderr, "rx: module %s %016llx %zu bytes %.1f ms\n", what, (unsigned long long)key, wgsl.size(), ms); if (const char* dir = getenv("RX_DUMP_SLOW_WGSL"); dir && ms > 40) { char path[512]; std::snprintf(path, sizeof path, "%s/slow_%s_%016llx.wgsl", dir, what, (unsigned long long)key); std::ofstream(path, std::ios::binary) << wgsl; } } if (error.empty()) return true; m.failed = true; log_limited("wgsl_error", 20, "rx: %s shader %016llx does not compile: %s", what, (unsigned long long)key, error.c_str()); if (const char* dir = getenv("RX_DUMP_WGSL")) { char path[512]; std::snprintf(path, sizeof path, "%s/bad_%s_%016llx.wgsl", dir, what, (unsigned long long)key); std::ofstream(path, std::ios::binary) << wgsl; } return false; } } // namespace ShaderModule* prewarm_stage_module(uint64_t h, const std::string& text, const char* what) { static std::deque modules; // (live as long as the program pairs: modules are never released) std::lock_guard lk(prog_mu); const uint64_t check = hash_bytes(text.data(), text.size(), 0xc0ffee5ull); if (hash_bytes(text.data(), text.size(), 0x1d7e57ull) != h) return nullptr; // (a damaged record) auto [it, inserted] = interned_texts.try_emplace(h); InternedText& t = it->second; if (inserted) { t.id = (uint32_t)interned_texts.size(); t.check = check; t.text = text; wgsl_texts++; wgsl_bytes += t.text.size(); } else if (t.check != check) { return nullptr; } ShaderModule& m = modules.emplace_back(); m.text_id = t.id; m.text_hash = h; m.text = &t.text; m.what = what; m.key = h; return &m; } void release_module_text(ShaderModule& m) { if (!m.text_id || !m.text) return; std::lock_guard g(prog_mu); wgsl_bytes -= m.text->size(); std::string().swap(*m.text); } bool ensure_module(ShaderModule& m) { // per module: WGSL parsing and validation (most of the compilers' time: 36 s of 42 at the Great Sky Island // spawn) runs on several compilers at once std::lock_guard lk(*m.creating); if (m.module || m.failed) return !m.failed; // and once per distinct WGSL: a program pair is per vertex + pixel program combination and vertex format, so // the same stage translates to the same text in many pairs (Great Sky Island spawn: 6954 modules, 773 distinct) struct Shared { std::mutex creating; bool done = false, ok = false; WGPUShaderModule module = nullptr; }; static std::mutex cache_mutex; static std::unordered_map> by_text_id; static std::unordered_map> by_text; static const bool no_share = env_flag("RX_NO_MODULE_SHARING"); if (no_share) return create_module(m, m.what, m.key); std::shared_ptr s; { std::lock_guard g(cache_mutex); auto& e = m.text_id ? by_text_id[m.text_id] : by_text[m.source()]; if (!e) e = std::make_shared(); s = e; } std::lock_guard sl(s->creating); if (!s->done) { s->ok = create_module(m, m.what, m.key); s->module = m.module; // (modules live as long as the program pairs: never released) s->done = true; if (!RX_WEB) release_module_text(m); return s->ok; } m.module = s->module; m.failed = !s->ok; return s->ok; } uint8_t vertex_unpack(const Regs::VertexAttribute& a) { using U = Shader::Backend::WGSL::VertexUnpack; using S = Regs::VertexAttribute::Size; using T = Regs::VertexAttribute::Type; if (a.constant) return (uint8_t)U::None; if (a.size == S::Size_B10_G11_R11) return (uint8_t)U::B10G11R11_FLOAT; if (a.size != S::Size_A2_B10_G10_R10) return (uint8_t)U::None; switch (a.type) { case T::SNorm: return (uint8_t)U::A2B10G10R10_SNORM; case T::UInt: return (uint8_t)U::A2B10G10R10_UINT; case T::SInt: return (uint8_t)U::A2B10G10R10_SINT; case T::UScaled: return (uint8_t)U::A2B10G10R10_USCALED; case T::SScaled: return (uint8_t)U::A2B10G10R10_SSCALED; default: return (uint8_t)U::None; // unorm: WebGPU's own format } } bool fetchable(const Regs::VertexAttribute& a, const Regs::VertexStream& vs); // rx_draw.cpp PullPlan pull_plan(const Regs& r, const std::bitset<32>& loads) { PullPlan plan; for (uint32_t i = 0; i < 32; i++) { if (!loads[i]) continue; const auto& a = r.vertex_attrib_format[i]; const auto& vs = r.vertex_streams[a.buffer % 16]; if (a.constant || !vs.IsEnabled() || fetchable(a, vs)) continue; uint32_t k = 0; while (k < plan.count && plan.streams[k] != a.buffer % 16) k++; if (k == plan.count) { if (plan.count == 3) continue; // (more: the CPU converts them) plan.streams[plan.count++] = a.buffer % 16; } auto& p = plan.attr[i]; p.enabled = true; p.stream = (uint8_t)k; p.instanced = r.vertex_stream_instances.IsInstancingEnabled(a.buffer % 16); p.attribute = a.hex; p.stride = vs.stride; } // more streams than vertex buffers (browsers allow 8; TOTK draws with 9): the last ones are pulled too // (RX_MAX_VERTEX_BUFFERS=: as if the device allowed n, to test the browser's limit natively) static const uint32_t forced = getenv("RX_MAX_VERTEX_BUFFERS") ? (uint32_t)atoi(getenv("RX_MAX_VERTEX_BUFFERS")) : 0; const uint32_t max_buffers = forced ? forced : dev.limits.maxVertexBuffers ? dev.limits.maxVertexBuffers : 16; uint32_t fetched = 0; bool constants = false; for (uint32_t i = 0; i < 32; i++) { if (!loads[i]) continue; const auto& a = r.vertex_attrib_format[i]; if (a.constant || !r.vertex_streams[a.buffer % 16].IsEnabled()) constants = true; else if (!plan.attr[i].enabled) fetched |= 1u << (a.buffer % 16); } uint32_t buffers = (uint32_t)std::popcount(fetched) + (constants ? 1u : 0u); for (int st = 15; st >= 0 && buffers > max_buffers && plan.count < 3; st--) { if (!(fetched >> st & 1)) continue; const uint32_t k = plan.count++; plan.streams[k] = (uint32_t)st; for (uint32_t i = 0; i < 32; i++) { const auto& a = r.vertex_attrib_format[i]; if (!loads[i] || a.constant || a.buffer % 16 != (uint32_t)st || plan.attr[i].enabled) continue; auto& p = plan.attr[i]; p.enabled = true; p.stream = (uint8_t)k; p.instanced = r.vertex_stream_instances.IsInstancingEnabled(st); p.attribute = a.hex; p.stride = r.vertex_streams[st].stride; } buffers--; } return plan; } ShaderModule* compute_module(const ComputeLaunch& l, const std::array* ssbo_alias, const std::array* ssbo_alias_words) { static std::unordered_map> modules; uint8_t head[0x400]; gpu_read(l.code_base + l.program_start, head, sizeof head); uint64_t key = hash_bytes(head, sizeof head, l.code_base + l.program_start); const uint32_t shape[5] = {l.block[0], l.block[1], l.block[2], l.shared_size, l.local_size}; key = hash_bytes(shape, sizeof shape, key); if (ssbo_alias) key = hash_bytes(ssbo_alias->data(), ssbo_alias->size(), key ^ 0xA11A5); if (ssbo_alias_words) key = hash_bytes(ssbo_alias_words->data(), ssbo_alias_words->size() * 4, key); auto it = modules.find(key); if (it != modules.end()) return it->second->failed ? nullptr : it->second.get(); auto m = std::make_unique(); try { struct TranslateTimer { int64_t t0 = hos_now_ns(); ~TranslateTimer() { count("t_translate", hos_now_ns() - t0); } } translate_timer; Translated t; ComputeGuestEnvironment env{l}; Shader::Maxwell::Flow::CFG cfg{env, t.flow_blocks, env.StartAddress()}; Shader::HostTranslateInfo host{}; host.support_float64 = false; host.support_int64 = false; host.min_ssbo_alignment = dev.limits.minStorageBufferOffsetAlignment; t.program = Shader::Maxwell::TranslateProgram(t.insts, t.blocks, env, cfg, host); Shader::Profile profile{}; Shader::RuntimeInfo rt{}; Shader::Backend::Bindings b{}; if (ssbo_alias) Shader::Backend::WGSL::ssbo_alias_next = *ssbo_alias; if (ssbo_alias_words) Shader::Backend::WGSL::ssbo_alias_words_next = *ssbo_alias_words; m->wgsl = Shader::Backend::WGSL::EmitWGSL(profile, rt, t.program, b); m->info = t.program.info; } catch (const std::exception& e) { log_limited("translate_cs", 30, "rx: compute program %#llx does not translate: %s", (unsigned long long)(l.code_base + l.program_start), e.what()); m->failed = true; } if (!m->failed) { if (const char* dir = getenv("RX_DUMP_WGSL")) { char path[512]; std::snprintf(path, sizeof path, "%s/cs_%010llx.wgsl", dir, (unsigned long long)(l.code_base + l.program_start)); std::ofstream(path, std::ios::binary) << m->wgsl; } // (made by the pipeline cache's thread when a session before compiled the same text) const uint64_t th = hash_bytes(m->wgsl.data(), m->wgsl.size(), 0x1d7e57ull); if (WGPUShaderModule cached = native_cache_compute_module(th)) { m->module = cached; count("cs_module_cached"); } else if (create_module(*m, "compute", key)) { native_cache_compute_text(th, m->wgsl); } } ShaderModule* raw = m.get(); modules[key] = std::move(m); return raw->failed ? nullptr : raw; } /// RX_DEBUG_SHADOWS_LIT=1 (debugging): every depth-compare sample of the pixel programs reads 1 (lit). void debug_compare_lit(std::string& wgsl) { for (const char* call : {"textureSampleCompareLevel(", "textureSampleCompare("}) { size_t at = 0; while ((at = wgsl.find(call, at)) != std::string::npos) { if (at >= 10 && wgsl.compare(at - 10, 10, "(0.0*(1.0)") == 0) { // (already wrapped) at += std::strlen(call); continue; } size_t i = at + std::strlen(call), depth = 1; while (i < wgsl.size() && depth) { if (wgsl[i] == '(') depth++; else if (wgsl[i] == ')') depth--; i++; } const std::string inner = wgsl.substr(at, i - at); const std::string repl = "(0.0*(1.0)*" + inner + "+1.0)"; wgsl.replace(at, i - at, repl); at += repl.size(); } } } /// A pair's key: its two programs, the types of the attributes the vertex program reads and of the targets the pixel /// program writes (known once each program was translated). Caller holds prog_mu. uint64_t pair_key(const Regs& r, uint64_t vs_key, uint64_t fs_key, bool has_fs, bool* complete, bool* failed) { auto vs_info = stage_infos.find(vs_key); auto fs_info = has_fs ? stage_infos.find(fs_key) : stage_infos.end(); *failed = (vs_info != stage_infos.end() && vs_info->second.failed) || (fs_info != stage_infos.end() && fs_info->second.failed); uint64_t key = hash_bytes(&vs_key, 8, fs_key); if (vs_info != stage_infos.end()) { for (uint32_t i = 0; i < 32; i++) if (vs_info->second.loads[i]) { const uint8_t t[2] = {(uint8_t)attribute_type(r.vertex_attrib_format[i]), vertex_unpack(r.vertex_attrib_format[i])}; key = hash_bytes(t, 2, key ^ i); } const PullPlan plan = pull_plan(r, vs_info->second.loads); for (uint32_t i = 0; i < 32; i++) if (plan.attr[i].enabled) { // (field by field: the struct's padding byte is whatever was on the stack, which differs between the // translation workers and the render thread: a pair translated on a worker was then never found) const auto& a = plan.attr[i]; const uint32_t f[3] = {a.stream | (a.instanced ? 0x100u : 0u), a.attribute, a.stride}; key = hash_bytes(f, sizeof f, key ^ (i + 200)); } } if (fs_info != stage_infos.end()) for (uint32_t i = 0; i < 8; i++) { const uint8_t t = (uint8_t)output_type(r, i); key = hash_bytes(&t, 1, key ^ (i + 100)); } *complete = vs_info != stage_infos.end() && (!has_fs || fs_info != stage_infos.end()); return key; } /// Translates a pair and makes its WGSL (prog_mu is held only to read and publish), once: null if it does not /// translate. Runs on a translation worker with copies of the registers and channel, or on the render thread /// (parallel: the pixel stage is translated and emitted on another thread meanwhile: a one-off draw waits for it; /// RX_SERIAL_STAGES=1 off). ProgramPair* build_pair(const Channel& ch, const Regs& r, uint64_t vs_key, uint64_t fs_key, bool parallel = false) { const uint64_t base = r.program_region.Address(); const uint32_t vs_offset = r.pipelines[1].offset; const bool has_fs = r.pipelines[5].enable != 0; const uint32_t fs_offset = r.pipelines[5].offset; auto pair = std::make_unique(); pair->has_fs = has_fs; uint64_t key = 0; try { static const bool serial_stages = env_flag("RX_SERIAL_STAGES"); // (not in the browser: a pthread started there needs the page's main thread, which may be the one waiting) const bool split = !RX_WEB && parallel && has_fs && !serial_stages; std::unique_ptr vs, fs; if (split) { // (the future's destructor waits for it: a vertex stage that throws leaves no thread behind) auto fs_job = std::async(std::launch::async, [&] { AsWorker as_worker; return translate(r, ch, Shader::Stage::Fragment, 4, fs_offset); }); vs = translate(r, ch, Shader::Stage::VertexB, 0, vs_offset); fs = fs_job.get(); } else { vs = translate(r, ch, Shader::Stage::VertexB, 0, vs_offset); if (has_fs) fs = translate(r, ch, Shader::Stage::Fragment, 4, fs_offset); } std::bitset<32> loads; { std::lock_guard lk(prog_mu); StageInfo& vi = stage_infos[vs_key]; for (uint32_t i = 0; i < 32; i++) { vi.loads[i] = vs->program.info.loads.Generic(i); vi.stores[i] = vs->program.info.stores.Generic(i); } if (fs) { StageInfo& fi = stage_infos[fs_key]; for (uint32_t i = 0; i < 32; i++) fi.loads[i] = fs->program.info.loads.Generic(i); } bool complete, failed; key = pair_key(r, vs_key, fs_key, has_fs, &complete, &failed); if (auto it = pairs.find(key); it != pairs.end()) return it->second->failed ? nullptr : it->second.get(); loads = vi.loads; } Shader::Profile profile{}; Shader::Backend::WGSL::Options options{}; Shader::RuntimeInfo vs_rt{}; // VertexB consumes vertex fetch, not an earlier shader. An empty mask makes the backend replace every // attribute read with (0,0,0,1), collapsing all vertices. Match suyu's first-stage runtime configuration. vs_rt.previous_stage_stores.mask.set(); const PullPlan plan = pull_plan(r, loads); for (uint32_t i = 0; i < 32; i++) { vs_rt.generic_input_types[i] = attribute_type(r.vertex_attrib_format[i]); options.vertex_unpack[i] = (Shader::Backend::WGSL::VertexUnpack)vertex_unpack(r.vertex_attrib_format[i]); auto& vp = options.vertex_pull[i]; vp.enabled = plan.attr[i].enabled; vp.stream = plan.attr[i].stream; vp.instanced = plan.attr[i].instanced; vp.attribute = plan.attr[i].attribute; vp.stride = plan.attr[i].stride; } if (fs) for (uint32_t i = 0; i < 32; i++) options.output_interpolation[i] = fs->program.info.interpolation[i]; auto emit_fs = [&] { Shader::RuntimeInfo fs_rt{}; fs_rt.previous_stage_stores = vs->program.info.stores; for (uint32_t i = 0; i < 8; i++) fs_rt.color_output_types[i] = output_type(r, i); Shader::Backend::Bindings fb{}; pair->fs.wgsl = Shader::Backend::WGSL::EmitWGSL(profile, fs_rt, fs->program, fb); static const bool shadows_lit = env_flag("RX_DEBUG_SHADOWS_LIT"); if (shadows_lit) debug_compare_lit(pair->fs.wgsl); pair->fs.info = fs->program.info; }; std::future fs_emit; if (fs && split) // (reads vs->program.info only) fs_emit = std::async(std::launch::async, [&] { AsWorker as_worker; emit_fs(); }); Shader::Backend::Bindings vb{}; pair->vs.wgsl = Shader::Backend::WGSL::EmitWGSL(profile, vs_rt, vs->program, vb, options); pair->vs.info = vs->program.info; if (fs_emit.valid()) fs_emit.get(); else if (fs) emit_fs(); } catch (const std::exception& e) { static std::atomic logged; if (logged++ < 40) hos_log("rx: programs vs %#llx / fs %#llx do not translate: %s", (unsigned long long)(base + vs_offset), (unsigned long long)(base + fs_offset), e.what()); std::lock_guard lk(prog_mu); stage_infos[vs_key].failed = true; // (whichever failed: both stop being tried) if (has_fs) stage_infos[fs_key].failed = true; return nullptr; } // RX_WGSL_OVERRIDE= (debugging): a program's WGSL is replaced by /vs_
.wgsl or fs_
.wgsl // when that file exists (e.g. an RX_DUMP_WGSL file edited to output an intermediate value) if (const char* dir = getenv("RX_WGSL_OVERRIDE")) { auto override_from = [&](const char* kind, uint64_t address, std::string& wgsl) { char path[512]; std::snprintf(path, sizeof path, "%s/%s_%010llx.wgsl", dir, kind, (unsigned long long)address); std::ifstream f(path, std::ios::binary); if (!f) return; wgsl.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); hos_log("rx: %s program %#llx: WGSL from %s", kind, (unsigned long long)address, path); }; override_from("vs", base + vs_offset, pair->vs.wgsl); if (has_fs) override_from("fs", base + fs_offset, pair->fs.wgsl); } if (const char* dir = getenv("RX_DUMP_WGSL")) { char path[512]; std::snprintf(path, sizeof path, "%s/vs_%010llx.wgsl", dir, (unsigned long long)(base + vs_offset)); std::ofstream(path, std::ios::binary) << pair->vs.wgsl; if (has_fs) { std::snprintf(path, sizeof path, "%s/fs_%010llx.wgsl", dir, (unsigned long long)(base + fs_offset)); std::ofstream(path, std::ios::binary) << pair->fs.wgsl; } } // (the modules are created by the pipeline compilers: ensure_module) pair->vs.what = "vertex"; pair->vs.key = vs_key; pair->fs.what = "pixel"; pair->fs.key = fs_key; pair->failed = false; std::lock_guard lk(prog_mu); auto intern = [](ShaderModule& m) { const uint64_t h = hash_bytes(m.wgsl.data(), m.wgsl.size(), 0x1d7e57ull); const uint64_t check = hash_bytes(m.wgsl.data(), m.wgsl.size(), 0xc0ffee5ull); auto [it, inserted] = interned_texts.try_emplace(h); InternedText& t = it->second; if (inserted) { t.id = (uint32_t)interned_texts.size(); t.check = check; t.text = std::move(m.wgsl); if (RX_WEB) t.cache_hash = pipecache_text_hash(t.text); else native_cache_text(h, t.text); wgsl_texts++; wgsl_bytes += t.text.size(); } else if (t.check != check) { return; } m.text_id = t.id; m.text_hash = h; m.cache_hash = t.cache_hash; m.text = &t.text; std::string().swap(m.wgsl); }; intern(pair->vs); if (has_fs) intern(pair->fs); if (auto it = pairs.find(key); it != pairs.end()) return it->second.get(); // (another worker was first) ProgramPair* raw = pair.get(); pairs[key] = std::move(pair); return raw; } /// New programs are translated by two workers while the draws that need them are skipped (as for pipelines still /// compiling): translating on the render thread was most of the stutter after a load. A job carries copies of the /// registers and the channel (constant buffer bindings). RX_SYNC_TRANSLATE=1: on the render thread, as before. struct TranslationJob { std::unique_ptr regs; Channel ch; uint64_t vs_key = 0, fs_key = 0, pending = 0; }; std::mutex job_mu; std::condition_variable job_cv; std::deque translation_jobs; std::unordered_set translations_pending; void translation_worker() { on_translation_worker = true; rx_side_thread = true; background_thread(); for (;;) { TranslationJob j; { std::unique_lock lk(job_mu); job_cv.wait(lk, [] { return !translation_jobs.empty(); }); j = std::move(translation_jobs.front()); translation_jobs.pop_front(); } build_pair(j.ch, *j.regs, j.vs_key, j.fs_key); std::lock_guard lk(job_mu); translations_pending.erase(j.pending); } } void queue_translation(const Channel& ch, const Regs& r, uint64_t vs_key, uint64_t fs_key) { // (one job for the same programs and formats at a time) uint32_t ingredients[32 + 16 + 9]; for (uint32_t i = 0; i < 32; i++) ingredients[i] = r.vertex_attrib_format[i].hex; for (uint32_t i = 0; i < 16; i++) ingredients[32 + i] = (r.vertex_streams[i].stride & 0xFFF) | (r.vertex_streams[i].IsEnabled() ? 1u << 12 : 0u) | (r.vertex_stream_instances.IsInstancingEnabled(i) ? 1u << 13 : 0u); for (uint32_t i = 0; i < 8; i++) ingredients[48 + i] = (uint32_t)r.rt[i].format; std::memcpy(&ingredients[56], &r.rt_control, 4); const uint64_t pending = hash_bytes(ingredients, sizeof ingredients, hash_bytes(&vs_key, 8, fs_key)); std::lock_guard lk(job_mu); if (!translations_pending.insert(pending).second) return; count("translation_jobs"); static bool started; if (!started) { started = true; for (int i = 0; i < 2; i++) std::thread(translation_worker).detach(); } TranslationJob j; j.regs = std::make_unique(r); j.ch = ch; j.vs_key = vs_key; j.fs_key = fs_key; j.pending = pending; translation_jobs.push_back(std::move(j)); job_cv.notify_one(); } ProgramPair* programs_for_draw_uncached(Channel& ch, const Regs& r) { if (r.pipelines[0].enable) { log_limited("vertex_a", 4, "rx: dual vertex programs (VertexA) not drawn yet"); return nullptr; } for (int i = 2; i <= 4; i++) if (r.pipelines[i].enable) { log_limited("tess_geom", 4, "rx: tessellation/geometry program (stage %d): WebGPU has none", i); return nullptr; } const uint64_t base = r.program_region.Address(); const bool has_fs = r.pipelines[5].enable != 0; const uint64_t vs_key = stage_hash(base, r.pipelines[1].offset); const uint64_t fs_key = has_fs ? stage_hash(base, r.pipelines[5].offset) : 0; { std::lock_guard lk(prog_mu); bool complete, failed; const uint64_t key = pair_key(r, vs_key, fs_key, has_fs, &complete, &failed); if (failed) return nullptr; if (complete) if (auto it = pairs.find(key); it != pairs.end()) return it->second->failed ? nullptr : it->second.get(); } static const bool sync = env_flag("RX_SYNC_TRANSLATE"); if (sync || frame_dump_active() || draw_must_run) { const int64_t t0 = hos_now_ns(); ProgramPair* p = build_pair(ch, r, vs_key, fs_key, /*parallel=*/true); spike.translate_ns += hos_now_ns() - t0; return p; } queue_translation(ch, r, vs_key, fs_key); spike.translations++; count("program_translating"); return nullptr; } /// Consecutive draws mostly use the same programs with the same vertex and target formats: a memo keyed by exactly /// the registers the pair's key is made from (program offsets, attribute formats, stream strides/enables/instancing, /// target formats and mapping), still checked against the programs' code bytes (RX_NO_PROGRAM_MEMO=1: off). ProgramPair* programs_for_draw(Channel& ch, const Regs& r) { static const bool no_memo = env_flag("RX_NO_PROGRAM_MEMO"); if (no_memo || frame_dump_active()) return programs_for_draw_uncached(ch, r); struct State { uint64_t base; uint32_t vs, fs, enables; uint32_t attribs[32]; uint32_t streams[16]; uint32_t targets[8]; uint32_t rt_control; } st; std::memset(&st, 0, sizeof st); st.base = r.program_region.Address(); st.vs = r.pipelines[1].offset; st.fs = r.pipelines[5].offset; for (uint32_t i = 0; i < 6; i++) st.enables |= (r.pipelines[i].enable ? 1u : 0u) << i; for (uint32_t i = 0; i < 32; i++) st.attribs[i] = r.vertex_attrib_format[i].hex; for (uint32_t i = 0; i < 16; i++) { const auto& vs = r.vertex_streams[i]; st.streams[i] = (vs.stride & 0xFFF) | (vs.IsEnabled() ? 1u << 12 : 0u) | (r.vertex_stream_instances.IsInstancingEnabled(i) ? 1u << 13 : 0u); } for (uint32_t i = 0; i < 8; i++) st.targets[i] = (uint32_t)r.rt[i].format; std::memcpy(&st.rt_control, &r.rt_control, sizeof st.rt_control); constexpr size_t kCode = 0x50 + 0x400; // (as stage_hash) struct Memo { State st; ProgramPair* pair = nullptr; uint64_t seq = 0; // memory_seq() when the code was last compared uint8_t code[2][kCode]; }; static std::unique_ptr memo(new Memo[2048]); const uint64_t h = hash_bytes(&st, sizeof st); Memo& m = memo[h & 2047]; const bool has_fs = (st.enables >> 5) & 1; auto code_now = [&](uint32_t offset, uint8_t* scratch) -> const uint8_t* { uint64_t avail = 0; const uint8_t* p = hos_gpu_ptr(st.base + offset, &avail); if (p && avail >= kCode) return p; gpu_read(st.base + offset, scratch, kCode); return scratch; }; // the code bytes need no compare when no page of them was written since they last matched (CPU writes, applied // at each submission, and the GPU's own writes both bump the pages' sequence numbers): the compares were ~3% of // the render thread static const bool always_compare = env_flag("RX_PROGRAM_MEMO_COMPARE"); const uint64_t seq_now = memory_seq(); if (m.pair && m.seq && seq_now && !always_compare && !std::memcmp(&m.st, &st, sizeof st)) { const uint64_t gvs = hos_gpu_translate(st.base + st.vs), gfs = has_fs ? hos_gpu_translate(st.base + st.fs) : 0; if (gvs && !memory_written_since(gvs, kCode, m.seq) && (!has_fs || (gfs && !memory_written_since(gfs, kCode, m.seq)))) { count("program_memo_hits"); return m.pair; } } uint8_t scratch[2][kCode]; const uint8_t* vs_code = code_now(st.vs, scratch[0]); const uint8_t* fs_code = has_fs ? code_now(st.fs, scratch[1]) : nullptr; if (m.pair && !std::memcmp(&m.st, &st, sizeof st) && !std::memcmp(m.code[0], vs_code, kCode) && (!has_fs || !std::memcmp(m.code[1], fs_code, kCode))) { count("program_memo_hits"); m.seq = seq_now; return m.pair; } ProgramPair* pair = programs_for_draw_uncached(ch, r); if (pair) { // (failures are not kept: they are rare and retried) m.st = st; m.pair = pair; m.seq = seq_now; std::memcpy(m.code[0], vs_code, kCode); if (has_fs) std::memcpy(m.code[1], fs_code, kCode); } return pair; } void wgsl_text_stats(size_t* n, uint64_t* bytes) { *n = wgsl_texts.load(); *bytes = wgsl_bytes.load(); } // ---- the native NVN layer: programs from NVN shader packages ---- // (mkw-recomp: NVN2, the Switch 2's) A stage of an NVN2 program is a control section (CPU memory, 0xd8 bytes: magic // 0x98761234 and version 0xc9 at +0, the code length at +0x38, the c1 constants' offset in the code area at +0x40 and // the area's end at +0x44, the register count at +0x4c, the NVN stage as a 16-bit value at +0x56, a compute stage's // workgroup size at +0x58) and a code area (GPU memory): the Ampere program (a graphics stage: its 128-byte header and // its instructions), then the c1 constants (indirect branch tables). The program is lowered to Maxwell (Eglantine's // frontend/ampere) and translated with the same recompiler as any Maxwell program, answering its questions from NVN // state: c0 / c2 from the driver banks (nvn_driver_bank; texture handles at c2 0x20 + 8 slot), c1 from the code area, // user uniform buffers (c3-c16: slots 0-13) from their memory, texture types from the registered textures. namespace { Shader::TextureType nvn_texture_type(int target) { switch (target) { case NVN_TEXTURE_TARGET_1D: return Shader::TextureType::Color1D; case NVN_TEXTURE_TARGET_3D: return Shader::TextureType::Color3D; case NVN_TEXTURE_TARGET_1D_ARRAY: return Shader::TextureType::ColorArray1D; case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: return Shader::TextureType::ColorArray2D; case NVN_TEXTURE_TARGET_RECTANGLE: return Shader::TextureType::Color2DRect; case NVN_TEXTURE_TARGET_CUBEMAP: return Shader::TextureType::ColorCube; case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: return Shader::TextureType::ColorArrayCube; case NVN_TEXTURE_TARGET_BUFFER: return Shader::TextureType::Buffer; default: return Shader::TextureType::Color2D; } } /// The texture type an NVN texture handle names (its registration's view target, else its texture's) Shader::TextureType nvn_handle_type(uint32_t handle) { nvn_tex t; nvn_view v; bool has_view = false; if (!handle || !nvn_texture_of((int)(handle & 0xFFFFF), &t, &v, &has_view)) return Shader::TextureType::Color2D; return nvn_texture_type(has_view && v.set_target ? v.target : t.d.target); } PixelFormat nvn_handle_format(uint32_t handle) { nvn_tex t; nvn_view v; bool has_view = false; if (!handle || !nvn_texture_of((int)(handle & 0xFFFFF), &t, &v, &has_view)) return PixelFormat::A8B8G8R8_UNORM; const PixelFormat pf = nvn_pixel_format(has_view && v.set_format ? v.format : t.d.format); return pf == PixelFormat::Invalid ? PixelFormat::A8B8G8R8_UNORM : pf; } /// A stage's package, read from guest memory and lowered to Maxwell struct NvnCode { int stage = -1; // NVN stage uint64_t data = 0, key = 0; std::vector code; // the lowered program as the Maxwell frontend reads it: a graphics stage's header // (0x50 bytes; zeros for compute), the instructions from 0x50 with their scheduling // words at each 32 bytes, then the native address map (indirect branch targets) std::vector c1; uint32_t local[3]{}; uint32_t shared = 0, local_memory = 0; // compute: workgroup shared memory, local memory bytes bool ok = false; }; constexpr uint32_t kNvn2ControlMagic = 0x98761234; constexpr uint32_t kNvn2ControlSize = 0xd8; constexpr uint32_t kAmpereHeader = 128; // (Turing and later: the graphics program header is 128 bytes) NvnCode read_nvn_code(const nvn_stage_data& sd) { NvnCode c; uint8_t ctl[kNvn2ControlSize]; if (!sd.control || !sd.data || !gpu_read(sd.control, ctl, sizeof ctl)) return c; uint32_t magic, len, c1o, end; uint16_t st; std::memcpy(&magic, ctl, 4); std::memcpy(&len, ctl + 0x38, 4); std::memcpy(&c1o, ctl + 0x40, 4); std::memcpy(&end, ctl + 0x44, 4); std::memcpy(&st, ctl + 0x56, 2); if (magic != kNvn2ControlMagic || !len || end > (16u << 20) || c1o > end || len > end || st > 5) { log_limited("nvn_code", 8, "rx: NVN2 shader package at %#llx: magic %#x, code %#x, c1 at %#x..%#x, stage %u", (unsigned long long)sd.data, magic, len, c1o, end, st); return c; } std::vector area(end); if (!gpu_read(sd.data, area.data(), end)) return c; c.key = hash_bytes(area.data(), end, hash_bytes(ctl, sizeof ctl)); c.stage = (int)st; c.data = sd.data; const bool graphics = st != 5; const uint32_t hdr = graphics ? kAmpereHeader : 0; // the native instructions, to the program's end (the capture stops at its final branch to itself or exit) using Shader::Ampere::Instruction; std::vector native; Shader::Ampere::ProgramCaptureState capture; for (uint32_t i = 0; hdr + (i + 1) * sizeof(Instruction) <= len; i++) { Instruction& insn = native.emplace_back(); std::memcpy(&insn, area.data() + hdr + i * sizeof(Instruction), sizeof insn); const auto r = capture.Observe(insn, i); if (r == Shader::Ampere::ProgramCaptureState::Result::InvalidTarget) { log_limited("nvn_capture", 8, "rx: NVN2 program %016llx: branch outside the program at +%#x", (unsigned long long)c.key, i * 16); return c; } if (r == Shader::Ampere::ProgramCaptureState::Result::Complete) break; } const auto lowered = Shader::Ampere::LowerToMaxwell(native); if (!lowered) { const uint32_t at = lowered.error_offset / 16; const uint64_t lo = at < native.size() ? (uint64_t)native[at].words[0] | (uint64_t)native[at].words[1] << 32 : 0, hi = at < native.size() ? (uint64_t)native[at].words[2] | (uint64_t)native[at].words[3] << 32 : 0; log_limited("nvn_lower", 30, "rx: NVN2 program %016llx (stage %u, at %#llx) does not lower to Maxwell at +%#x: " "%s (instruction %016llx %016llx, opcode %#x)", (unsigned long long)c.key, st, (unsigned long long)sd.data, lowered.error_offset, std::string(lowered.error).c_str(), (unsigned long long)hi, (unsigned long long)lo, (unsigned)(lo & 0xfff)); return c; } c.code.assign(sizeof(Shader::ProgramHeader) / 8, 0); if (graphics) std::memcpy(c.code.data(), area.data(), sizeof(Shader::ProgramHeader)); for (const uint64_t instruction : lowered.instructions) { if (c.code.size() * 8 % 32 == 0) c.code.push_back(0); // (the Maxwell scheduling word) c.code.push_back(instruction); } c.code.push_back(0); Shader::Ampere::AppendNativeAddressMap(c.code, lowered.source_instruction_offsets); c.c1.assign(area.begin() + c1o, area.end()); if (!graphics) { std::memcpy(c.local, ctl + 0x58, 12); // (the shared memory size is not found in the control section yet: what the device allows a workgroup) c.shared = 16u << 10; } // RX_NVN_CODE_LOG=1 (debugging): the first packages if (static const bool log_code = env_flag("RX_NVN_CODE_LOG"); log_code) { static int n; if (n++ < 24) hos_log("rx: NVN2 package %016llx stage %u code %#x c1 %#x..%#x at %#llx: %zu native -> %zu Maxwell words", (unsigned long long)c.key, st, len, c1o, end, (unsigned long long)sd.data, native.size(), lowered.instructions.size()); } c.ok = true; return c; } /// suyu's Environment over an NVN stage: what the recompiler asks, answered from the NVN state of the draw class NvnEnvironment final : public Shader::Environment { public: NvnEnvironment(const NvnState& state, const NvnCode& c, Shader::Stage st) : s{state}, nvn{c} { stage = st; start_address = st == Shader::Stage::Compute ? static_cast(sizeof(Shader::ProgramHeader)) : 0; if (st != Shader::Stage::Compute) { std::memcpy(&sph, reinterpret_cast(nvn.code.data()), sizeof sph); local_memory = static_cast(sph.LocalMemorySize()) + sph.common3.shader_local_memory_crs_size; } else { local_memory = nvn.local_memory; } is_proprietary_driver = true; } uint64_t key() const { return nvn.key; } u64 ReadInstruction(u32 address) override { return address / 8 < nvn.code.size() ? nvn.code[address / 8] : 0; } u32 TranslateProgramAddress(u32 address) const override { if (const auto t = Shader::Ampere::NativeProgramAddress(nvn.code, address)) return *t; throw Shader::RuntimeError("indirect branch to native address {:#x}, not in the program", address); } u32 ReadCbufValue(u32 index, u32 offset) override { if (index == 1) { uint32_t v = 0; if ((size_t)offset + 4 <= nvn.c1.size()) std::memcpy(&v, nvn.c1.data() + offset, 4); return v; } if (index == 0 || index == 2) { // (the handle words the translation read: c2 for graphics, c0 for NVN2 compute) if ((index == 2 && offset < 0x568) || (index == 0 && nvn.stage == 5 && offset >= 0x10)) c2_reads.push_back(offset); return nvn_driver_bank_word(s, (int)index, nvn.stage, offset); } if (index >= 3 && index < 3 + 16) { const nvn_p_bind_buf& u = s.ubo[nvn.stage][index - 3]; return u.addr && (uint64_t)offset + 4 <= nvn_ubo_size(u.size) ? gpu_get(u.addr + offset) : 0; } return 0; } Shader::TextureType ReadTextureType(u32 handle) override { return nvn_handle_type(handle); } Shader::TexturePixelFormat ReadTexturePixelFormat(u32 handle) override { return static_cast(nvn_handle_format(handle)); } bool IsTexturePixelFormatInteger(u32 handle) override { return VideoCore::Surface::IsPixelFormatInteger(nvn_handle_format(handle)); } u32 ReadViewportTransformState() override { return 1; } u32 TextureBoundBuffer() const override { return stage == Shader::Stage::Compute ? 0 : 2; // (NVN2: compute handles in c0, see nvn2_cs_layout) } u32 LocalMemorySize() const override { return local_memory; } u32 SharedMemorySize() const override { return nvn.shared; } std::array WorkgroupSize() const override { return {std::max(1u, nvn.local[0]), std::max(1u, nvn.local[1]), std::max(1u, nvn.local[2])}; } bool HasHLEMacroState() const override { return false; } std::optional GetReplaceConstBuffer(u32, u32) override { return std::nullopt; } void Dump(u64, u64) override {} std::vector c2_reads; // texture handle words the translation read (what its answers depend on) private: const NvnState& s; const NvnCode& nvn; uint32_t local_memory = 0; }; std::unique_ptr nvn_translate(NvnEnvironment& env, bool graphics) { struct TranslateTimer { int64_t t0 = hos_now_ns(); ~TranslateTimer() { if (!on_translation_worker) count("t_translate", hos_now_ns() - t0); } } translate_timer; auto t = std::make_unique(); Shader::Maxwell::Flow::CFG cfg{env, t->flow_blocks, env.StartAddress() + (graphics ? static_cast(sizeof(Shader::ProgramHeader)) : 0u)}; Shader::HostTranslateInfo host{}; host.support_float64 = false; host.support_int64 = false; host.min_ssbo_alignment = dev.limits.minStorageBufferOffsetAlignment; t->program = Shader::Maxwell::TranslateProgram(t->insts, t->blocks, env, cfg, host); // RX_WARP_LOG=1: programs using the warp's lane id, shuffles, votes or lane masks (the WGSL backend has no warps) static const bool warp_log = env_flag("RX_WARP_LOG"); if (const auto& i = t->program.info; warp_log && (i.uses_subgroup_invocation_id || i.uses_subgroup_shuffles || i.uses_subgroup_vote || i.uses_subgroup_mask)) hos_log("rx: %s program %016llx uses the warp: lane id %d shuffles %d vote %d masks %d", graphics ? "graphics" : "compute", (unsigned long long)env.key(), i.uses_subgroup_invocation_id, i.uses_subgroup_shuffles, i.uses_subgroup_vote, i.uses_subgroup_mask); return t; } uint8_t nvn_type_at(const NvnState& s, int stage, uint32_t offset) { return (uint8_t)nvn_handle_type(nvn_driver_bank_word(s, stage == 5 ? 0 : 2, stage, offset)); // (NVN2 compute: c0) } /// A translated pair and what it was translated for struct NvnVariant { std::unique_ptr pair; bool failed = false; // (does not translate whatever the state: not tried again) std::bitset<32> loads; uint64_t attr_sig = 0, plan_sig = 0; uint8_t out_types[8]{}; std::vector> answers[2]; }; struct NvnProgCodes { NvnCode vs, fs; bool has_fs = false, ok = false; }; std::unordered_map> nvn_codes; // by program serial std::unordered_map>> nvn_variants; // by the stages' code keys uint64_t nvn_attr_sig(const NvnState& s, const std::bitset<32>& loads) { uint64_t h = 0x5eed; for (uint32_t i = 0; i < 16; i++) if (loads[i]) { const Regs::VertexAttribute a = nvn_attribute(s.vattr[i]); const uint8_t t[2] = {(uint8_t)attribute_type(a), vertex_unpack(a)}; h = hash_bytes(t, 2, h ^ i); } return h; } uint64_t nvn_plan_sig(const PullPlan& plan) { uint64_t h = 0x91a7; for (uint32_t i = 0; i < 32; i++) if (plan.attr[i].enabled) { const auto& a = plan.attr[i]; const uint32_t f[3] = {a.stream | (a.instanced ? 0x100u : 0u), a.attribute, a.stride}; h = hash_bytes(f, sizeof f, h ^ (i + 200)); } return h; } void intern_module(ShaderModule& m) { // (prog_mu held) the same WGSL text: one copy, one module const uint64_t h = hash_bytes(m.wgsl.data(), m.wgsl.size(), 0x1d7e57ull); const uint64_t check = hash_bytes(m.wgsl.data(), m.wgsl.size(), 0xc0ffee5ull); auto [it, inserted] = interned_texts.try_emplace(h); InternedText& t = it->second; if (inserted) { t.id = (uint32_t)interned_texts.size(); t.check = check; t.text = std::move(m.wgsl); if (RX_WEB) t.cache_hash = pipecache_text_hash(t.text); else native_cache_text(h, t.text); wgsl_texts++; wgsl_bytes += t.text.size(); } else if (t.check != check) { return; } m.text_id = t.id; m.text_hash = h; m.cache_hash = t.cache_hash; m.text = &t.text; std::string().swap(m.wgsl); } /// An NVN program's compute stage: its package (once per program object) and a module per answer to the texture /// types it reads struct NvnComputeVariant { std::unique_ptr module; std::vector> answers; std::array alias{}; // (web: storage buffers read through another's binding, see compute_module) std::array alias_words{}; std::array images{}; // (mkw-recomp) the bound images' WebGPU formats it was translated for }; std::unordered_map> nvn_cs_codes; // by program serial std::unordered_map>> nvn_cs_variants; // by code key const NvnCode* nvn_cs_code(const NvnState& s) { auto& c = nvn_cs_codes[s.prog.serial]; if (!c) { c = std::make_unique(); for (uint32_t i = 0; i < s.prog.nstage && i < 6; i++) { NvnCode k = read_nvn_code(s.prog.sd[i]); if (k.ok && k.stage == 5) { *c = std::move(k); break; } } } return c->ok ? c.get() : nullptr; } } // namespace uint64_t nvn_cs_key(const NvnState& s) { const NvnCode* code = s.has_prog ? nvn_cs_code(s) : nullptr; return code ? code->key : 0; } ShaderModule* nvn_compute_module(const NvnState& s, const std::array* ssbo_alias, const std::array* ssbo_alias_words, const std::array* image_formats) { if (!s.has_prog) return nullptr; const NvnCode* code = nvn_cs_code(s); if (!code) return nullptr; static const std::array no_alias{}; static const std::array no_words{}; const auto& alias = ssbo_alias ? *ssbo_alias : no_alias; const auto& alias_words = ssbo_alias_words ? *ssbo_alias_words : no_words; static const std::array no_images{}; const auto& images = image_formats ? *image_formats : no_images; auto& list = nvn_cs_variants[code->key]; for (const auto& v : list) { bool same = v->alias == alias && v->alias_words == alias_words && v->images == images; for (const auto& [off, type] : v->answers) if (nvn_type_at(s, 5, off) != type) { same = false; break; } if (same) return v->module->failed ? nullptr : v->module.get(); } auto var = std::make_unique(); var->module = std::make_unique(); var->alias = alias; var->alias_words = alias_words; var->images = images; ShaderModule& m = *var->module; try { NvnEnvironment env{s, *code, Shader::Stage::Compute}; auto t = nvn_translate(env, false); for (uint32_t off : env.c2_reads) var->answers.push_back({off, nvn_type_at(s, 5, off)}); Shader::Profile profile{}; Shader::RuntimeInfo rt{}; Shader::Backend::Bindings b{}; Shader::Backend::WGSL::ssbo_alias_next = alias; Shader::Backend::WGSL::ssbo_alias_words_next = alias_words; for (size_t i = 0; i < images.size(); i++) if (images[i]) Shader::Backend::WGSL::image_formats_next[i] = wgsl_storage_format((WGPUTextureFormat)images[i]); m.wgsl = Shader::Backend::WGSL::EmitWGSL(profile, rt, t->program, b); m.info = t->program.info; } catch (const std::exception& e) { log_limited("nvn_translate_cs", 30, "rx: NVN compute program %016llx does not translate: %s", (unsigned long long)code->key, e.what()); m.failed = true; } if (!m.failed) { if (const char* dir = getenv("RX_DUMP_WGSL")) { char path[512]; std::snprintf(path, sizeof path, "%s/nvn_cs_%016llx.wgsl", dir, (unsigned long long)code->key); std::ofstream(path, std::ios::binary) << m.wgsl; } const uint64_t th = hash_bytes(m.wgsl.data(), m.wgsl.size(), 0x1d7e57ull); if (WGPUShaderModule cached = native_cache_compute_module(th)) { m.module = cached; count("cs_module_cached"); } else if (create_module(m, "compute", code->key)) { native_cache_compute_text(th, m.wgsl); } else { m.failed = true; } if (!m.failed) count("nvn_compute_programs"); } ShaderModule* out = m.failed ? nullptr : &m; list.push_back(std::move(var)); return out; } namespace { } // namespace ProgramPair* nvn_programs_uncached(const NvnState& s, const uint8_t out_types[8], const NvnVariant** chosen); uint32_t nvn_ub_limit() { static const uint32_t limit = getenv("RX_NVN_UB_LIMIT") ? (uint32_t)atoi(getenv("RX_NVN_UB_LIMIT")) : 512; return limit; } const std::vector* nvn_c1(const NvnState& s, int stage) { if (stage == 5) { auto cs = nvn_cs_codes.find(s.prog.serial); return cs != nvn_cs_codes.end() && cs->second && cs->second->ok ? &cs->second->c1 : nullptr; } auto it = nvn_codes.find(s.prog.serial); if (it == nvn_codes.end() || !it->second) return nullptr; const NvnCode& c = stage == 0 ? it->second->vs : it->second->fs; return c.ok ? &c.c1 : nullptr; } ProgramPair* nvn_programs(const NvnState& s, const uint8_t out_types[8]) { if (!s.has_prog) return nullptr; // an answer for this program, vertex state and targets while nothing that picks the variant changed (the variant // checks below cost ~8% of the replay thread; draws of different materials take turns, so one entry each). // Per frame: texture pool entries can change a handle's type. struct Memo { const NvnState* s = nullptr; uint64_t epoch = 0, vtx = 0, serial = 0; uint32_t vbufs = 0; int64_t frame = -1; uint8_t out[8]{}; ProgramPair* pair = nullptr; const NvnVariant* var = nullptr; }; static Memo memos[512]; static const bool no_memo = env_flag("RX_NO_NVN_PROGRAM_MEMO"); uint32_t vbufs = 0; // (which streams have a buffer: the pull plan depends on it) for (int i = 0; i < 16; i++) vbufs |= (s.vbuf[i].address != 0 ? 1u : 0u) << i; uint64_t out_bits; std::memcpy(&out_bits, out_types, 8); Memo& memo = memos[hash_bytes(&out_bits, 8, s.prog.serial * 0x9e3779b97f4a7c15ull ^ s.vtx_hash ^ vbufs) & 511]; if (!no_memo && memo.pair && memo.s == &s && memo.serial == s.prog.serial && memo.vtx == s.vtx_hash && memo.vbufs == vbufs && memo.frame == frame_number && !std::memcmp(memo.out, out_types, 8)) { if (memo.epoch == s.prog_epoch) return memo.pair; // only handles changed (or another program was bound between): the variant holds while the texture types it // was translated for do if (memo.var) { bool same = true; for (int st = 0; st < 2 && same; st++) for (const auto& [off, type] : memo.var->answers[st]) if (nvn_type_at(s, st, off) != type) { same = false; break; } if (same) { memo.epoch = s.prog_epoch; count("nvn_program_memo_rechecked"); return memo.pair; } } } const NvnVariant* var = nullptr; ProgramPair* found = nvn_programs_uncached(s, out_types, &var); memo.s = &s, memo.epoch = s.prog_epoch, memo.vtx = s.vtx_hash, memo.serial = s.prog.serial; memo.vbufs = vbufs, memo.frame = frame_number; std::memcpy(memo.out, out_types, 8); memo.pair = found; memo.var = var; return found; } ProgramPair* nvn_programs_uncached(const NvnState& s, const uint8_t out_types[8], const NvnVariant** chosen) { *chosen = nullptr; if (!s.has_prog) return nullptr; // the stages' packages (once per program object's shaders) auto& codes = nvn_codes[s.prog.serial]; if (!codes) { codes = std::make_unique(); for (uint32_t i = 0; i < s.prog.nstage && i < 6; i++) { NvnCode c = read_nvn_code(s.prog.sd[i]); if (!c.ok) continue; if (c.stage == 0) codes->vs = std::move(c); else if (c.stage == 1) codes->fs = std::move(c), codes->has_fs = true; else log_limited("nvn_stage", 8, "rx: NVN program with a stage %d: not drawn", c.stage); } codes->ok = codes->vs.ok; } if (!codes->ok) return nullptr; const uint64_t vkey = codes->vs.key, fkey = codes->has_fs ? codes->fs.key : 0; auto& list = nvn_variants[hash_bytes(&vkey, 8, fkey)]; // a variant translated for what the state answers now for (const auto& v : list) { if (v->failed) return nullptr; if (std::memcmp(v->out_types, out_types, 8)) continue; if (v->attr_sig != nvn_attr_sig(s, v->loads)) continue; if (v->plan_sig != nvn_plan_sig(nvn_pull_plan(s, v->loads))) continue; bool same = true; for (int st = 0; st < 2 && same; st++) for (const auto& [off, type] : v->answers[st]) if (nvn_type_at(s, st, off) != type) { same = false; break; } if (same) { *chosen = v.get(); return v->pair ? v->pair.get() : nullptr; } } // a new one auto var = std::make_unique(); std::memcpy(var->out_types, out_types, 8); try { NvnEnvironment venv{s, codes->vs, Shader::Stage::VertexB}; auto vs = nvn_translate(venv, true); std::unique_ptr fs; std::unique_ptr fenv; if (codes->has_fs) { fenv = std::make_unique(s, codes->fs, Shader::Stage::Fragment); fs = nvn_translate(*fenv, true); } for (uint32_t i = 0; i < 32; i++) var->loads[i] = vs->program.info.loads.Generic(i); for (uint32_t off : venv.c2_reads) var->answers[0].push_back({off, nvn_type_at(s, 0, off)}); if (fenv) for (uint32_t off : fenv->c2_reads) var->answers[1].push_back({off, nvn_type_at(s, 1, off)}); var->attr_sig = nvn_attr_sig(s, var->loads); const PullPlan plan = nvn_pull_plan(s, var->loads); var->plan_sig = nvn_plan_sig(plan); auto pair = std::make_unique(); pair->has_fs = fs != nullptr; Shader::Profile profile{}; Shader::Backend::WGSL::Options options{}; Shader::RuntimeInfo vs_rt{}; vs_rt.previous_stage_stores.mask.set(); for (uint32_t i = 0; i < 32; i++) { const Regs::VertexAttribute a = i < 16 ? nvn_attribute(s.vattr[i]) : Regs::VertexAttribute{}; vs_rt.generic_input_types[i] = attribute_type(a); options.vertex_unpack[i] = (Shader::Backend::WGSL::VertexUnpack)vertex_unpack(a); auto& vp = options.vertex_pull[i]; vp.enabled = plan.attr[i].enabled; vp.stream = plan.attr[i].stream; vp.instanced = plan.attr[i].instanced; vp.attribute = plan.attr[i].attribute; vp.stride = plan.attr[i].stride; } if (fs) for (uint32_t i = 0; i < 32; i++) options.output_interpolation[i] = fs->program.info.interpolation[i]; vs_rt.cbuf_ub_limit = nvn_ub_limit(); Shader::Backend::Bindings vb{}; pair->vs.wgsl = Shader::Backend::WGSL::EmitWGSL(profile, vs_rt, vs->program, vb, options); pair->vs.info = vs->program.info; // a vertex program that stores to memory: also as a compute program (nvn_vs_compute), translated again (the // emission changes the IR). TOTK's Ultrahand reads the object under the reticle in a one-point draw whose // vertex program stores it; the vertex stage's stores were dropped and Ultrahand never locked on (orange // outline, A missed: SkyboundTotk issue 9). RX_NO_VS_COMPUTE=1: off. bool vs_stores = false; for (const auto& sd : vs->program.info.storage_buffers_descriptors) vs_stores |= sd.is_written; if (static const bool no_vs_compute = env_flag("RX_NO_VS_COMPUTE"); vs_stores && !no_vs_compute) { bool pulled = true; for (uint32_t i = 0; i < 32; i++) pulled &= !var->loads[i] || plan.attr[i].enabled; try { if (!pulled) throw std::runtime_error("vertex attributes not all pulled"); NvnEnvironment cenv{s, codes->vs, Shader::Stage::VertexB}; auto vc = nvn_translate(cenv, true); Shader::Backend::Bindings cb{}; Shader::Backend::WGSL::vs_as_compute_next = true; pair->vs_compute_wgsl = Shader::Backend::WGSL::EmitWGSL(profile, vs_rt, vc->program, cb, options); count("nvn_vs_compute_programs"); if (const char* dir = getenv("RX_DUMP_WGSL")) { char path[512]; std::snprintf(path, sizeof path, "%s/nvn_vscs_%016llx.wgsl", dir, (unsigned long long)vkey); std::ofstream(path, std::ios::binary) << pair->vs_compute_wgsl; } } catch (const std::exception& e) { Shader::Backend::WGSL::vs_as_compute_next = false; log_limited("nvn_vs_compute", 20, "rx: NVN vertex program %016llx stores to memory but cannot run as compute: %s", (unsigned long long)vkey, e.what()); } } if (fs) { Shader::RuntimeInfo fs_rt{}; fs_rt.previous_stage_stores = vs->program.info.stores; for (uint32_t i = 0; i < 8; i++) fs_rt.color_output_types[i] = (Shader::AttributeType)out_types[i]; fs_rt.cbuf_ub_limit = nvn_ub_limit(); // RX_EARLY_Z_STORERS=1 (experiment): pixel programs that store to memory test depth before they run if (static const bool early = env_flag("RX_EARLY_Z_STORERS"); early && dev.early_depth_test) for (const auto& d : fs->program.info.storage_buffers_descriptors) fs_rt.force_early_z |= d.is_written; Shader::Backend::Bindings fb{}; pair->fs.wgsl = Shader::Backend::WGSL::EmitWGSL(profile, fs_rt, fs->program, fb); pair->fs.info = fs->program.info; } // RX_WGSL_OVERRIDE= (debugging): a program's WGSL is replaced by /nvn_vs_.wgsl or // nvn_fs_.wgsl when that file exists (e.g. an RX_DUMP_WGSL file edited to output an intermediate value) if (const char* dir = getenv("RX_WGSL_OVERRIDE")) { auto override_from = [&](const char* kind, uint64_t key, std::string& wgsl) { char path[512]; std::snprintf(path, sizeof path, "%s/nvn_%s_%016llx.wgsl", dir, kind, (unsigned long long)key); std::ifstream f(path, std::ios::binary); if (!f) return; wgsl.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); hos_log("rx: %s program %016llx: WGSL from %s", kind, (unsigned long long)key, path); }; override_from("vs", vkey, pair->vs.wgsl); if (fs) override_from("fs", fkey, pair->fs.wgsl); } if (const char* dir = getenv("RX_DUMP_WGSL")) { char path[512]; auto banks = [](const Shader::Info& info) { // (the uniform block's banks, in order) std::string c; for (const auto& d : info.constant_buffer_descriptors) { char b[96]; std::snprintf(b, sizeof b, "// c%u used %#x\n", d.index, info.constant_buffer_used_sizes[d.index]); c += b; } return c; }; std::snprintf(path, sizeof path, "%s/nvn_vs_%016llx.wgsl", dir, (unsigned long long)vkey); std::ofstream(path, std::ios::binary) << banks(pair->vs.info) << pair->vs.wgsl; if (fs) { std::snprintf(path, sizeof path, "%s/nvn_fs_%016llx.wgsl", dir, (unsigned long long)fkey); std::ofstream(path, std::ios::binary) << banks(pair->fs.info) << pair->fs.wgsl; } } pair->vs.what = "vertex"; pair->vs.key = vkey; pair->fs.what = "pixel"; pair->fs.key = fkey; std::lock_guard lk(prog_mu); intern_module(pair->vs); if (pair->has_fs) intern_module(pair->fs); var->pair = std::move(pair); count("nvn_programs"); } catch (const std::exception& e) { log_limited("nvn_translate", 40, "rx: NVN program (vertex %016llx, pixel %016llx) does not translate: %s", (unsigned long long)vkey, (unsigned long long)fkey, e.what()); var->failed = true; } ProgramPair* out = var->pair ? var->pair.get() : nullptr; *chosen = var.get(); list.push_back(std::move(var)); return out; } } // namespace rx