mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
4709 lines
211 KiB
C++
4709 lines
211 KiB
C++
// The renderer's core: the WebGPU device and the window's surface, batches (one command encoder and its upload
|
|
// memory per GPU submission), render targets, clears, the 3D engine's draw methods (suyu's DrawManager), present.
|
|
#include <intrin.h>
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <atomic>
|
|
#include <cstdarg>
|
|
#include <condition_variable>
|
|
#include <cstdio>
|
|
#include <deque>
|
|
#include <set>
|
|
#include <functional>
|
|
#include <mutex>
|
|
#include <thread>
|
|
#include <cstdlib>
|
|
|
|
#define WIN32_LEAN_AND_MEAN
|
|
#define NOMINMAX
|
|
#include <windows.h>
|
|
|
|
#include "rx_internal.h"
|
|
#include "video_core/textures/decoders.h"
|
|
#ifdef __EMSCRIPTEN__
|
|
#include <emscripten.h>
|
|
#include <emscripten/threading.h>
|
|
extern "C" void rx_web_canvas_init(int w, int h); // hos/web/library.js
|
|
extern "C" void rx_web_present(void);
|
|
namespace rx { extern Device dev; }
|
|
static thread_local bool on_render_thread; // (the thread that makes every WebGPU call)
|
|
// Back to the worker's event loop, where WebGPU's callbacks run and the canvas is shown. Not emscripten_sleep: a
|
|
// timer, which the browser holds to 4 ms once timers nest, and every present, readback wait and idle turn paid it.
|
|
// (one channel a worker, kept: making and closing one at each yield was ~2.5% of the render worker at the castle)
|
|
EM_ASYNC_JS(void, rx_web_yield, (), {
|
|
if (!globalThis.rxYield) {
|
|
const c = new MessageChannel(), waiting = [];
|
|
c.port1.onmessage = () => { const resolve = waiting.shift(); if (resolve) resolve(); };
|
|
globalThis.rxYield = (resolve) => { waiting.push(resolve); c.port2.postMessage(0); };
|
|
}
|
|
await new Promise(globalThis.rxYield);
|
|
});
|
|
// Until the word at addr is no longer value (Atomics.notify: emscripten_futex_wake), or ms pass.
|
|
EM_ASYNC_JS(void, rx_web_wait_word, (uint32_t addr, uint32_t value, double ms), {
|
|
const r = Atomics.waitAsync(HEAP32, addr >>> 2, value | 0, ms);
|
|
if (r.async) await r.value;
|
|
});
|
|
bool wgpuDevicePoll(WGPUDevice device, bool wait, const void* submission) {
|
|
(void)device;
|
|
(void)submission;
|
|
wgpuInstanceProcessEvents(rx::dev.instance);
|
|
if (wait) rx_web_yield(); // (the callbacks run on this worker's event loop)
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
namespace rx {
|
|
bool draw_must_run = false;
|
|
bool rt_history_on() { static const bool on = getenv("RX_RT_HISTORY") != nullptr; return on; }
|
|
|
|
/// A short wait for WebGPU's callbacks: on the web the worker's event loop must run for them (JSPI suspends us).
|
|
void wait_step() {
|
|
#ifdef __EMSCRIPTEN__
|
|
emscripten_sleep(1);
|
|
#else
|
|
Sleep(1);
|
|
#endif
|
|
}
|
|
|
|
Device dev;
|
|
RecursiveLock mu;
|
|
void RecursiveLock::lock() {
|
|
const unsigned long me = GetCurrentThreadId();
|
|
if (owner_.load(std::memory_order_relaxed) == me) {
|
|
depth_++;
|
|
return;
|
|
}
|
|
AcquireSRWLockExclusive(reinterpret_cast<PSRWLOCK>(&srw_));
|
|
owner_.store(me, std::memory_order_relaxed);
|
|
depth_ = 1;
|
|
}
|
|
void RecursiveLock::unlock() {
|
|
if (--depth_) return;
|
|
owner_.store(0, std::memory_order_relaxed);
|
|
ReleaseSRWLockExclusive(reinterpret_cast<PSRWLOCK>(&srw_));
|
|
}
|
|
int64_t frame_number;
|
|
int64_t st_draws, st_skipped, st_pipelines, st_clears;
|
|
Stream stream_ro, stream_rw;
|
|
|
|
namespace {
|
|
bool started;
|
|
WGPUSurface surface;
|
|
WGPUTextureFormat surface_format = WGPUTextureFormat_BGRA8Unorm;
|
|
WGPUPresentMode present_mode = WGPUPresentMode_Fifo;
|
|
uint32_t surface_w, surface_h;
|
|
volatile LONG want_w, want_h;
|
|
WGPURenderPipeline blit_pipeline;
|
|
WGPUBindGroupLayout blit_layout;
|
|
WGPUSampler blit_sampler;
|
|
|
|
WGPUCommandEncoder enc;
|
|
WGPURenderPassEncoder pass;
|
|
Attachments pass_at;
|
|
WGPUQuerySet pass_occlusion; // the open pass's occlusion query set (occlusion counts, below)
|
|
|
|
std::unordered_map<uint64_t, std::unique_ptr<Image>> images; // by guest address
|
|
// Surfaces the game drew at an address before the one there now (it reuses memory for different transient targets
|
|
// within a frame): kept for when that surface comes back, instead of a texture created and destroyed every frame.
|
|
std::unordered_multimap<uint64_t, std::unique_ptr<Image>> spare_images;
|
|
uint64_t image_versions; // render target versions come from one counter: no two images share one
|
|
std::atomic<int64_t> count_parts, st_finish_ns, st_write_ns, st_submit_ns, st_after_ns, st_wait_ns; // the submit thread's
|
|
std::atomic<int64_t> st_writeback_words;
|
|
uint64_t images_changed = 1; // bumped whenever which image is at an address changes (see images_generation)
|
|
struct ImageChange {
|
|
uint64_t generation, guest;
|
|
};
|
|
ImageChange image_changes[256]; // the last changes, by generation (images_changed_in)
|
|
void note_image_change(uint64_t guest) {
|
|
images_changed++;
|
|
image_changes[images_changed & 255] = {images_changed, guest};
|
|
}
|
|
// (by the key's address: the keys are literals; a string key was built and hashed per call, every draw for some)
|
|
std::unordered_map<const char*, int> log_counts;
|
|
|
|
void on_error(WGPUDevice const*, WGPUErrorType type, WGPUStringView message, void*, void*) {
|
|
static int n;
|
|
if (n++ < 60) {
|
|
std::string m = str(message);
|
|
if (m.size() > 1500) m.resize(1500);
|
|
hos_log("rx: WebGPU error (type %d): %s", (int)type, m.c_str());
|
|
}
|
|
}
|
|
void on_lost(WGPUDevice const*, WGPUDeviceLostReason reason, WGPUStringView message, void*, void*) {
|
|
hos_log("rx: WebGPU device lost (%d): %s", (int)reason, str(message).c_str());
|
|
}
|
|
|
|
const char* backend_name(WGPUBackendType t) {
|
|
switch (t) {
|
|
case WGPUBackendType_Vulkan: return "Vulkan";
|
|
case WGPUBackendType_D3D12: return "D3D12";
|
|
case WGPUBackendType_D3D11: return "D3D11";
|
|
case WGPUBackendType_OpenGL: return "OpenGL";
|
|
case WGPUBackendType_Metal: return "Metal";
|
|
case WGPUBackendType_WebGPU: return "WebGPU";
|
|
default: return "?";
|
|
}
|
|
}
|
|
|
|
void configure_surface(uint32_t w, uint32_t h) {
|
|
if (!surface || !w || !h) return;
|
|
WGPUSurfaceConfiguration c{};
|
|
c.device = dev.device;
|
|
c.format = surface_format;
|
|
c.usage = WGPUTextureUsage_RenderAttachment;
|
|
c.width = w;
|
|
c.height = h;
|
|
c.alphaMode = WGPUCompositeAlphaMode_Auto;
|
|
c.presentMode = present_mode;
|
|
wgpuSurfaceConfigure(surface, &c);
|
|
surface_w = w;
|
|
surface_h = h;
|
|
}
|
|
|
|
const char* kBlitWgsl = R"(
|
|
@group(0) @binding(0) var src: texture_2d<f32>;
|
|
@group(0) @binding(1) var samp: sampler;
|
|
struct VO { @builtin(position) pos: vec4<f32>, @location(0) uv: vec2<f32> };
|
|
@vertex fn vs(@builtin(vertex_index) i: u32) -> VO {
|
|
var o: VO;
|
|
let uv = vec2<f32>(f32((i << 1u) & 2u), f32(i & 2u));
|
|
o.pos = vec4<f32>(uv.x * 2.0 - 1.0, 1.0 - uv.y * 2.0, 0.0, 1.0);
|
|
o.uv = uv;
|
|
return o;
|
|
}
|
|
@fragment fn fs(v: VO) -> @location(0) vec4<f32> {
|
|
return vec4<f32>(textureSample(src, samp, v.uv).rgb, 1.0);
|
|
}
|
|
)";
|
|
|
|
void create_blit() {
|
|
WGPUShaderSourceWGSL src{};
|
|
src.chain.sType = WGPUSType_ShaderSourceWGSL;
|
|
src.code = sv(kBlitWgsl);
|
|
WGPUShaderModuleDescriptor md{};
|
|
md.nextInChain = &src.chain;
|
|
WGPUShaderModule mod = wgpuDeviceCreateShaderModule(dev.device, &md);
|
|
|
|
WGPUBindGroupLayoutEntry e[2]{};
|
|
e[0].binding = 0;
|
|
e[0].visibility = WGPUShaderStage_Fragment;
|
|
e[0].texture.sampleType = WGPUTextureSampleType_Float;
|
|
e[0].texture.viewDimension = WGPUTextureViewDimension_2D;
|
|
e[1].binding = 1;
|
|
e[1].visibility = WGPUShaderStage_Fragment;
|
|
e[1].sampler.type = WGPUSamplerBindingType_Filtering;
|
|
WGPUBindGroupLayoutDescriptor bd{};
|
|
bd.entryCount = 2;
|
|
bd.entries = e;
|
|
blit_layout = wgpuDeviceCreateBindGroupLayout(dev.device, &bd);
|
|
WGPUPipelineLayoutDescriptor pd{};
|
|
pd.bindGroupLayoutCount = 1;
|
|
pd.bindGroupLayouts = &blit_layout;
|
|
WGPUPipelineLayout pl = wgpuDeviceCreatePipelineLayout(dev.device, &pd);
|
|
|
|
WGPUColorTargetState target{};
|
|
target.format = surface_format;
|
|
target.writeMask = WGPUColorWriteMask_All;
|
|
WGPUFragmentState fs{};
|
|
fs.module = mod;
|
|
fs.entryPoint = sv("fs");
|
|
fs.targetCount = 1;
|
|
fs.targets = ⌖
|
|
WGPURenderPipelineDescriptor rd{};
|
|
rd.layout = pl;
|
|
rd.vertex.module = mod;
|
|
rd.vertex.entryPoint = sv("vs");
|
|
rd.primitive.topology = WGPUPrimitiveTopology_TriangleList;
|
|
rd.primitive.cullMode = WGPUCullMode_None;
|
|
rd.primitive.frontFace = WGPUFrontFace_CCW;
|
|
rd.multisample.count = 1;
|
|
rd.multisample.mask = 0xFFFFFFFF;
|
|
rd.fragment = &fs;
|
|
blit_pipeline = wgpuDeviceCreateRenderPipeline(dev.device, &rd);
|
|
wgpuPipelineLayoutRelease(pl);
|
|
wgpuShaderModuleRelease(mod);
|
|
|
|
WGPUSamplerDescriptor sd{};
|
|
sd.addressModeU = sd.addressModeV = sd.addressModeW = WGPUAddressMode_ClampToEdge;
|
|
sd.magFilter = sd.minFilter = WGPUFilterMode_Linear;
|
|
sd.mipmapFilter = WGPUMipmapFilterMode_Nearest;
|
|
sd.lodMaxClamp = 32.0f;
|
|
sd.maxAnisotropy = 1;
|
|
blit_sampler = wgpuDeviceCreateSampler(dev.device, &sd);
|
|
}
|
|
|
|
/// CPU memory for one batch's stream contents: recycled (pages are touched only as far as batches fill them).
|
|
std::mutex stream_pool_mu;
|
|
struct PooledStream {
|
|
uint64_t capacity;
|
|
uint8_t* memory;
|
|
int64_t since;
|
|
bool reset;
|
|
};
|
|
std::vector<PooledStream> stream_pool;
|
|
uint8_t* take_stream_memory(uint64_t size) {
|
|
{
|
|
std::lock_guard lk(stream_pool_mu);
|
|
static const bool oldest = env_flag("RX_STREAM_POOL_FIFO");
|
|
for (size_t k = 0; k < stream_pool.size(); k++) {
|
|
const size_t i = oldest ? k : stream_pool.size() - 1 - k;
|
|
if (stream_pool[i].capacity == size) {
|
|
uint8_t* m = stream_pool[i].memory;
|
|
stream_pool.erase(stream_pool.begin() + (ptrdiff_t)i);
|
|
return m;
|
|
}
|
|
}
|
|
}
|
|
return static_cast<uint8_t*>(VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
|
}
|
|
void give_stream_memory(uint64_t size, uint8_t* m, uint64_t used) {
|
|
#if !RX_WEB
|
|
constexpr uint64_t keep = 16ull << 20;
|
|
if (used > keep && size > keep) {
|
|
const uint64_t end = std::min<uint64_t>(size, (used + 0xFFFF) & ~0xFFFFull);
|
|
if (VirtualFree(m + keep, end - keep, MEM_DECOMMIT) &&
|
|
!VirtualAlloc(m + keep, end - keep, MEM_COMMIT, PAGE_READWRITE)) {
|
|
VirtualFree(m, 0, MEM_RELEASE);
|
|
return;
|
|
}
|
|
}
|
|
#endif
|
|
std::lock_guard lk(stream_pool_mu);
|
|
#if !RX_WEB
|
|
int same = 0;
|
|
for (const auto& e : stream_pool) same += e.capacity == size;
|
|
if (same >= 10) {
|
|
VirtualFree(m, 0, MEM_RELEASE);
|
|
return;
|
|
}
|
|
#endif
|
|
stream_pool.push_back({size, m, hos_now_ns(), false});
|
|
}
|
|
|
|
void reset_idle_stream_memory() {
|
|
#if !RX_WEB
|
|
static const bool off = env_flag("RX_STREAM_POOL_FIFO");
|
|
if (off) return;
|
|
constexpr uint64_t keep = 16ull << 20;
|
|
const int64_t now = hos_now_ns();
|
|
std::lock_guard lk(stream_pool_mu);
|
|
int n = 0;
|
|
for (size_t i = 0; i < stream_pool.size();) {
|
|
PooledStream& e = stream_pool[i];
|
|
if (e.reset || now - e.since < 10000000000LL) {
|
|
i++;
|
|
continue;
|
|
}
|
|
const uint64_t len = std::min<uint64_t>(e.capacity, keep);
|
|
if (VirtualFree(e.memory, len, MEM_DECOMMIT) && !VirtualAlloc(e.memory, len, MEM_COMMIT, PAGE_READWRITE)) {
|
|
VirtualFree(e.memory, 0, MEM_RELEASE);
|
|
stream_pool.erase(stream_pool.begin() + (ptrdiff_t)i);
|
|
continue;
|
|
}
|
|
e.reset = true;
|
|
n++;
|
|
i++;
|
|
}
|
|
if (n) count("stream_blocks_reset", n);
|
|
#endif
|
|
}
|
|
|
|
void create_stream(Stream& s, const char* name, uint64_t size, WGPUBufferUsage usage) {
|
|
WGPUBufferDescriptor d{};
|
|
d.label = sv(name);
|
|
d.usage = usage | WGPUBufferUsage_CopyDst;
|
|
d.size = size;
|
|
s.buffer = wgpuDeviceCreateBuffer(dev.device, &d);
|
|
s.cpu = take_stream_memory(size);
|
|
s.capacity = size;
|
|
s.used = 0;
|
|
s.usage = usage;
|
|
s.name = name;
|
|
}
|
|
|
|
bool init_device(HWND hwnd) {
|
|
#ifdef __EMSCRIPTEN__
|
|
(void)hwnd;
|
|
dev.instance = wgpuCreateInstance(nullptr);
|
|
if (!dev.instance) {
|
|
hos_log("rx: no WebGPU in this browser");
|
|
return false;
|
|
}
|
|
{ // (the page's canvas, transferred to this thread: hos/gpu.c)
|
|
WGPUEmscriptenSurfaceSourceCanvasHTMLSelector src{};
|
|
src.chain.sType = WGPUSType_EmscriptenSurfaceSourceCanvasHTMLSelector;
|
|
src.selector = sv("#screen");
|
|
WGPUSurfaceDescriptor sd{};
|
|
sd.nextInChain = &src.chain;
|
|
surface = wgpuInstanceCreateSurface(dev.instance, &sd);
|
|
}
|
|
#else
|
|
WGPUInstanceExtras extras{};
|
|
extras.chain.sType = (WGPUSType)WGPUSType_InstanceExtras;
|
|
const char* be = getenv("RX_BACKEND");
|
|
extras.backends = !be ? WGPUInstanceBackend_Primary
|
|
: !_stricmp(be, "vulkan") ? WGPUInstanceBackend_Vulkan
|
|
: !_stricmp(be, "dx12") ? WGPUInstanceBackend_DX12
|
|
: !_stricmp(be, "gl") ? WGPUInstanceBackend_GL
|
|
: WGPUInstanceBackend_Primary;
|
|
if (getenv("RX_VALIDATION")) extras.flags = WGPUInstanceFlag_Validation | WGPUInstanceFlag_Debug;
|
|
// RX_SHADER_NAMES=1: the debug flag alone (no validation layers): naga names SPIR-V variables after the WGSL ones,
|
|
// so RenderDoc's shader debugger shows f_0, u_12, in_attr3... of the dumped WGSL (RX_DUMP_WGSL)
|
|
if (getenv("RX_SHADER_NAMES")) extras.flags = (WGPUInstanceFlag)(extras.flags | WGPUInstanceFlag_Debug);
|
|
// wgpu's own log (why a device was lost, driver warnings) into ours: warnings and errors, 400 different ones;
|
|
// RX_WGPU_LOG=info|debug|trace for more, off for none
|
|
{
|
|
const char* lv = getenv("RX_WGPU_LOG");
|
|
const WGPULogLevel level = !lv ? WGPULogLevel_Warn
|
|
: !_stricmp(lv, "off") ? WGPULogLevel_Off
|
|
: !_stricmp(lv, "error") ? WGPULogLevel_Error
|
|
: !_stricmp(lv, "info") ? WGPULogLevel_Info
|
|
: !_stricmp(lv, "debug") ? WGPULogLevel_Debug
|
|
: !_stricmp(lv, "trace") ? WGPULogLevel_Trace
|
|
: WGPULogLevel_Warn;
|
|
wgpuSetLogCallback(
|
|
[](WGPULogLevel l, WGPUStringView message, void*) {
|
|
static std::mutex m; // (each message once: some warnings come for every pipeline)
|
|
static std::set<std::string> seen;
|
|
std::string t = str(message);
|
|
{
|
|
std::lock_guard g(m);
|
|
if (seen.size() >= 400 || !seen.insert(t).second) return;
|
|
}
|
|
hos_log("wgpu %s: %s", l <= WGPULogLevel_Error ? "error" : l == WGPULogLevel_Warn ? "warning" : "log", t.c_str());
|
|
},
|
|
nullptr);
|
|
wgpuSetLogLevel(level);
|
|
}
|
|
WGPUInstanceDescriptor id{};
|
|
id.nextInChain = &extras.chain;
|
|
dev.instance = wgpuCreateInstance(&id);
|
|
if (!dev.instance) {
|
|
hos_log("rx: no WebGPU instance");
|
|
return false;
|
|
}
|
|
if (hwnd) {
|
|
WGPUSurfaceSourceWindowsHWND src{};
|
|
src.chain.sType = WGPUSType_SurfaceSourceWindowsHWND;
|
|
src.hinstance = GetModuleHandleA(nullptr);
|
|
src.hwnd = hwnd;
|
|
WGPUSurfaceDescriptor sd{};
|
|
sd.nextInChain = &src.chain;
|
|
surface = wgpuInstanceCreateSurface(dev.instance, &sd);
|
|
}
|
|
#endif
|
|
|
|
#if !RX_WEB
|
|
// RX_ADAPTER=<part of a GPU's name> (e.g. "AMD", "Radeon"): that GPU instead of the high-performance one (a PC
|
|
// with two GPUs; testing on another vendor's driver). The log lists the GPUs when it is set.
|
|
if (const char* want = getenv("RX_ADAPTER")) {
|
|
WGPUInstanceEnumerateAdapterOptions eo{};
|
|
eo.backends = extras.backends;
|
|
std::vector<WGPUAdapter> all(wgpuInstanceEnumerateAdapters(dev.instance, &eo, nullptr));
|
|
wgpuInstanceEnumerateAdapters(dev.instance, &eo, all.data());
|
|
std::string w = want;
|
|
for (char& c : w) c = (char)tolower((unsigned char)c);
|
|
for (WGPUAdapter a : all) {
|
|
WGPUAdapterInfo ai{};
|
|
wgpuAdapterGetInfo(a, &ai);
|
|
std::string name = str(ai.device), lower = name;
|
|
for (char& c : lower) c = (char)tolower((unsigned char)c);
|
|
const bool pick = !dev.adapter && lower.find(w) != std::string::npos;
|
|
hos_log("rx: GPU %s (%s)%s", name.c_str(), backend_name(ai.backendType), pick ? ": chosen (RX_ADAPTER)" : "");
|
|
wgpuAdapterInfoFreeMembers(ai);
|
|
if (pick) dev.adapter = a;
|
|
else wgpuAdapterRelease(a);
|
|
}
|
|
if (!dev.adapter) hos_log("rx: no GPU's name contains \"%s\" (RX_ADAPTER): the default one", want);
|
|
}
|
|
#endif
|
|
WGPURequestAdapterOptions ao{};
|
|
ao.featureLevel = WGPUFeatureLevel_Core;
|
|
ao.powerPreference = WGPUPowerPreference_HighPerformance;
|
|
ao.compatibleSurface = surface;
|
|
WGPURequestAdapterCallbackInfo acb{};
|
|
acb.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
acb.callback = [](WGPURequestAdapterStatus status, WGPUAdapter adapter, WGPUStringView message, void* u1, void*) {
|
|
if (status == WGPURequestAdapterStatus_Success) *(WGPUAdapter*)u1 = adapter;
|
|
else hos_log("rx: no adapter: %s", str(message).c_str());
|
|
};
|
|
acb.userdata1 = &dev.adapter;
|
|
if (!dev.adapter) wgpuInstanceRequestAdapter(dev.instance, &ao, acb);
|
|
for (int i = 0; i < 5000 && !dev.adapter; i++) {
|
|
wgpuInstanceProcessEvents(dev.instance);
|
|
wait_step();
|
|
}
|
|
if (!dev.adapter) return false;
|
|
|
|
WGPUAdapterInfo info{};
|
|
wgpuAdapterGetInfo(dev.adapter, &info);
|
|
char desc[256];
|
|
std::snprintf(desc, sizeof desc, "%s (%s)", str(info.device).c_str(), backend_name(info.backendType));
|
|
dev.description = desc;
|
|
wgpuAdapterInfoFreeMembers(info);
|
|
|
|
// what the adapter has of what we use
|
|
std::vector<WGPUFeatureName> want = {
|
|
WGPUFeatureName_TextureCompressionBC, WGPUFeatureName_Float32Filterable,
|
|
WGPUFeatureName_Depth32FloatStencil8, WGPUFeatureName_RG11B10UfloatRenderable,
|
|
WGPUFeatureName_DepthClipControl, WGPUFeatureName_IndirectFirstInstance,
|
|
WGPUFeatureName_Float32Blendable, WGPUFeatureName_TextureFormatsTier1,
|
|
WGPUFeatureName_DualSourceBlending, WGPUFeatureName_ClipDistances,
|
|
};
|
|
std::vector<WGPUFeatureName> have;
|
|
for (auto f : want)
|
|
if (wgpuAdapterHasFeature(dev.adapter, f)) have.push_back(f);
|
|
// native (wgpu) only; the web gets the same formats from texture-formats-tier1
|
|
const auto norm16 = (WGPUFeatureName)WGPUNativeFeature_TextureFormat16bitNorm;
|
|
if (wgpuAdapterHasFeature(dev.adapter, norm16)) have.push_back(norm16);
|
|
#if !RX_WEB
|
|
// (mkw-recomp) the adapter's own format capabilities (storage R16F, RG16F, R8...: NVN2 compute writes them as
|
|
// images), where it does not report texture-formats-tier1
|
|
const auto specific = (WGPUFeatureName)WGPUNativeFeature_TextureAdapterSpecificFormatFeatures;
|
|
dev.adapter_formats = wgpuAdapterHasFeature(dev.adapter, specific);
|
|
if (dev.adapter_formats) have.push_back(specific);
|
|
const auto mdi_count = (WGPUFeatureName)WGPUNativeFeature_MultiDrawIndirectCount; // (NVN's indirect count draws)
|
|
dev.multi_draw_count = wgpuAdapterHasFeature(dev.adapter, mdi_count);
|
|
if (dev.multi_draw_count) have.push_back(mdi_count);
|
|
const auto tier2 = WGPUFeatureName_TextureFormatsTier2; // (read-write storage of more formats)
|
|
if (wgpuAdapterHasFeature(dev.adapter, tier2)) have.push_back(tier2);
|
|
#endif
|
|
#if !RX_WEB
|
|
// (@early_depth_test(force): pixel programs whose stores the depth test must stop)
|
|
const auto early_z = (WGPUFeatureName)WGPUNativeFeature_ShaderEarlyDepthTest;
|
|
dev.early_depth_test = wgpuAdapterHasFeature(dev.adapter, early_z);
|
|
if (dev.early_depth_test) have.push_back(early_z);
|
|
#endif
|
|
dev.norm16 = wgpuAdapterHasFeature(dev.adapter, norm16) ||
|
|
// (the web: r16unorm, rg16unorm, rgba16unorm and the snorm ones come with texture-formats-tier1)
|
|
(RX_WEB && wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_TextureFormatsTier1));
|
|
dev.bc = wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_TextureCompressionBC);
|
|
dev.tier1 = wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_TextureFormatsTier1);
|
|
dev.depth_clip_control = wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_DepthClipControl);
|
|
dev.float32_filterable = wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_Float32Filterable);
|
|
dev.float32_blendable = wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_Float32Blendable);
|
|
dev.depth32_stencil8 = wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_Depth32FloatStencil8);
|
|
dev.timestamps = env_flag("RX_GPU_TIMING") && wgpuAdapterHasFeature(dev.adapter, WGPUFeatureName_TimestampQuery);
|
|
if (dev.timestamps) have.push_back(WGPUFeatureName_TimestampQuery);
|
|
WGPULimits limits{};
|
|
wgpuAdapterGetLimits(dev.adapter, &limits);
|
|
|
|
WGPUDeviceDescriptor dd{};
|
|
dd.label = sv("rexcomp");
|
|
dd.requiredFeatureCount = have.size();
|
|
dd.requiredFeatures = have.data();
|
|
dd.requiredLimits = &limits;
|
|
dd.deviceLostCallbackInfo.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
dd.deviceLostCallbackInfo.callback = on_lost;
|
|
dd.uncapturedErrorCallbackInfo.callback = on_error;
|
|
WGPURequestDeviceCallbackInfo dcb{};
|
|
dcb.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
dcb.callback = [](WGPURequestDeviceStatus status, WGPUDevice device, WGPUStringView message, void* u1, void*) {
|
|
if (status == WGPURequestDeviceStatus_Success) *(WGPUDevice*)u1 = device;
|
|
else hos_log("rx: no device: %s", str(message).c_str());
|
|
};
|
|
dcb.userdata1 = &dev.device;
|
|
wgpuAdapterRequestDevice(dev.adapter, &dd, dcb);
|
|
for (int i = 0; i < 5000 && !dev.device; i++) {
|
|
wgpuInstanceProcessEvents(dev.instance);
|
|
wait_step();
|
|
}
|
|
if (!dev.device) return false;
|
|
dev.queue = wgpuDeviceGetQueue(dev.device);
|
|
wgpuDeviceGetLimits(dev.device, &dev.limits);
|
|
hos_log("rx: limits: per stage %u storage, %u uniform buffers; dynamic %u storage, %u uniform; %u bindings a group; "
|
|
"offset alignment storage %u uniform %u", dev.limits.maxStorageBuffersPerShaderStage,
|
|
dev.limits.maxUniformBuffersPerShaderStage, dev.limits.maxDynamicStorageBuffersPerPipelineLayout,
|
|
dev.limits.maxDynamicUniformBuffersPerPipelineLayout, dev.limits.maxBindingsPerBindGroup,
|
|
dev.limits.minStorageBufferOffsetAlignment, dev.limits.minUniformBufferOffsetAlignment);
|
|
// RX_SSBO_ALIGN=<bytes>: a coarser storage offset alignment (the browser's 256, to test its paths natively)
|
|
if (const char* a = getenv("RX_SSBO_ALIGN"); a && atoi(a) > (int)dev.limits.minStorageBufferOffsetAlignment)
|
|
dev.limits.minStorageBufferOffsetAlignment = (uint32_t)atoi(a);
|
|
hos_log("rx: storage buffer offset alignment %u, uniform %u, vertex buffers %u", dev.limits.minStorageBufferOffsetAlignment,
|
|
dev.limits.minUniformBufferOffsetAlignment, dev.limits.maxVertexBuffers);
|
|
|
|
if (!surface) { // headless
|
|
create_blit();
|
|
create_stream(stream_ro, "rx stream", (RX_WEB ? 32ull : 96ull) << 20,
|
|
WGPUBufferUsage_Uniform | WGPUBufferUsage_Storage | WGPUBufferUsage_Vertex |
|
|
WGPUBufferUsage_Index);
|
|
create_stream(stream_rw, "rx storage", (RX_WEB ? 16ull : 32ull) << 20, WGPUBufferUsage_Storage | WGPUBufferUsage_CopySrc);
|
|
astc_gpu_on();
|
|
unswizzle_gpu_on();
|
|
hos_log("rx: WebGPU on %s; %zu optional features; no window (offscreen)", desc, have.size());
|
|
return true;
|
|
}
|
|
WGPUSurfaceCapabilities caps{};
|
|
wgpuSurfaceGetCapabilities(surface, dev.adapter, &caps);
|
|
surface_format = caps.formatCount ? caps.formats[0] : WGPUTextureFormat_BGRA8Unorm;
|
|
for (size_t i = 0; i < caps.formatCount; i++)
|
|
if (caps.formats[i] == WGPUTextureFormat_BGRA8Unorm || caps.formats[i] == WGPUTextureFormat_RGBA8Unorm) {
|
|
surface_format = caps.formats[i]; // the window buffer holds encoded values: no sRGB conversion
|
|
break;
|
|
}
|
|
bool mailbox = false, immediate = false;
|
|
for (size_t i = 0; i < caps.presentModeCount; i++) {
|
|
mailbox |= caps.presentModes[i] == WGPUPresentMode_Mailbox;
|
|
immediate |= caps.presentModes[i] == WGPUPresentMode_Immediate;
|
|
}
|
|
// the game paces itself (nn_vi's buffer queue): presenting must not wait for a vblank
|
|
present_mode = mailbox ? WGPUPresentMode_Mailbox : immediate ? WGPUPresentMode_Immediate : WGPUPresentMode_Fifo;
|
|
if (const char* want = getenv("RX_PRESENT")) {
|
|
if (!std::strcmp(want, "fifo")) present_mode = WGPUPresentMode_Fifo;
|
|
else if (!std::strcmp(want, "mailbox") && mailbox) present_mode = WGPUPresentMode_Mailbox;
|
|
else if (!std::strcmp(want, "immediate") && immediate) present_mode = WGPUPresentMode_Immediate;
|
|
hos_log("rx: present mode %s (RX_PRESENT=%s)",
|
|
present_mode == WGPUPresentMode_Fifo ? "fifo" : present_mode == WGPUPresentMode_Mailbox ? "mailbox" : "immediate",
|
|
want);
|
|
}
|
|
wgpuSurfaceCapabilitiesFreeMembers(caps);
|
|
#ifdef __EMSCRIPTEN__
|
|
present_mode = WGPUPresentMode_Fifo;
|
|
configure_surface(1280, 720);
|
|
#else
|
|
RECT rc;
|
|
GetClientRect(hwnd, &rc);
|
|
configure_surface((uint32_t)std::max<LONG>(rc.right, 1), (uint32_t)std::max<LONG>(rc.bottom, 1));
|
|
present_window_size(rc.right, rc.bottom); // (ultrawide's aspect before the game makes its cameras)
|
|
#endif
|
|
|
|
create_blit();
|
|
create_stream(stream_ro, "rx stream", (RX_WEB ? 32ull : 96ull) << 20,
|
|
WGPUBufferUsage_Uniform | WGPUBufferUsage_Storage | WGPUBufferUsage_Vertex | WGPUBufferUsage_Index);
|
|
create_stream(stream_rw, "rx storage", (RX_WEB ? 16ull : 32ull) << 20, WGPUBufferUsage_Storage | WGPUBufferUsage_CopySrc);
|
|
pipecache_start(); // (the browser: the pipelines of earlier sessions, made again in the background)
|
|
native_cache_start(); // (natively: the same, rxcache/pipelines.bin next to the exe)
|
|
astc_gpu_on(); // (the compute passes that are on made here: error scopes belong to the device, not to a thread)
|
|
unswizzle_gpu_on();
|
|
hos_log("rx: WebGPU on %s; %zu optional features; surface %ux%u format %#x present mode %d", desc, have.size(),
|
|
surface_w, surface_h, (unsigned)surface_format, (int)present_mode);
|
|
return true;
|
|
}
|
|
|
|
Image* rt_image(const Regs& r, uint32_t i) {
|
|
const auto& rt = r.rt[i];
|
|
const uint64_t va = rt.Address();
|
|
if (!va || rt.format == Tegra::RenderTargetFormat::NONE) return nullptr;
|
|
const PixelFormat pf = VideoCore::Surface::PixelFormatFromRenderTargetFormat(rt.format);
|
|
uint32_t w = rt.width;
|
|
if (rt.tile_mode.is_pitch_linear) w = rt.width / std::max(1u, VideoCore::Surface::BytesPerBlock(pf));
|
|
// Maxwell permits padded backing strides; the surface clip is the logical render area. WebGPU requires
|
|
// equal attachment extents, so remove only the row padding, never enlarge an undersized target.
|
|
const uint32_t clip_width = r.surface_clip.x + r.surface_clip.width;
|
|
const uint32_t clip_height = r.surface_clip.y + r.surface_clip.height;
|
|
if (clip_width && clip_width < w && (rt.tile_mode.is_pitch_linear || w - clip_width < 64)) w = clip_width;
|
|
const uint32_t h = clip_height && clip_height < rt.height && rt.height - clip_height < 8 ? clip_height : rt.height;
|
|
Image* im = find_render_target(va, w, h, pf);
|
|
if (im) {
|
|
im->pitch_linear = rt.tile_mode.is_pitch_linear;
|
|
im->bl_height = rt.tile_mode.block_height;
|
|
im->bl_depth = rt.tile_mode.block_depth;
|
|
im->reg_width = rt.width; // (pitch linear: the row pitch in bytes)
|
|
}
|
|
return im;
|
|
}
|
|
|
|
Image* zeta_image(const Regs& r) {
|
|
if (!r.zeta_enable) return nullptr;
|
|
const uint64_t va = r.zeta.Address();
|
|
if (!va) return nullptr;
|
|
const PixelFormat pf = VideoCore::Surface::PixelFormatFromDepthFormat(r.zeta.format);
|
|
uint32_t w = r.zeta_size.width, h = r.zeta_size.height;
|
|
const uint32_t clip_width = r.surface_clip.x + r.surface_clip.width;
|
|
const uint32_t clip_height = r.surface_clip.y + r.surface_clip.height;
|
|
if (clip_width && clip_width < w && w - clip_width < 64) w = clip_width;
|
|
if (clip_height && clip_height < h && h - clip_height < 8) h = clip_height;
|
|
return find_render_target(va, w, h, pf);
|
|
}
|
|
|
|
uint64_t passes_begun;
|
|
|
|
// ---- occlusion counts: NVN's SAMPLES_PASSED from occlusion queries ----
|
|
// TOTK's occlusion culling draws a 256x144 occluder depth, then a bounding-volume proxy for each candidate object
|
|
// between a SAMPLES_PASSED reset and a report, and does not draw an object whose report is 0 (MacNativeGames'
|
|
// nvn_mtl.m says the same). Each draw between a reset and its report gets an occlusion query in the batch's query
|
|
// set; at the batch's end the set is resolved and read back, and each report's counter word is written with the sum
|
|
// of its queries once the GPU is done (rx_nvn.cpp writes the timestamp word at the replay). Draws left without a
|
|
// query (skipped, or the set full) add the estimate (`extra`); a report whose draws span two batches is the estimate.
|
|
namespace {
|
|
std::vector<std::function<void(bool)>> open_afters; // (flush_batch's: after its submit)
|
|
struct OcclusionSet {
|
|
static constexpr uint32_t kQueries = 1024;
|
|
WGPUQuerySet set = nullptr;
|
|
WGPUBuffer resolve = nullptr, read = nullptr;
|
|
uint32_t used = 0;
|
|
struct Report {
|
|
uint64_t addr, extra;
|
|
uint32_t first, n;
|
|
};
|
|
std::vector<Report> reports;
|
|
};
|
|
OcclusionSet* occ_open; // the open batch's (render thread)
|
|
bool occ_counting; // between a reset and its report
|
|
uint32_t occ_first; // the reset's first query in occ_open
|
|
uint64_t occ_gen, occ_reset_gen, occ_queried; // occ_gen: batches flushed
|
|
std::mutex occ_mu;
|
|
std::vector<OcclusionSet*> occ_pool;
|
|
|
|
OcclusionSet* occ_set() {
|
|
if (occ_open) return occ_open;
|
|
{
|
|
std::lock_guard lk(occ_mu);
|
|
if (!occ_pool.empty()) {
|
|
occ_open = occ_pool.back();
|
|
occ_pool.pop_back();
|
|
return occ_open;
|
|
}
|
|
}
|
|
auto* o = new OcclusionSet();
|
|
WGPUQuerySetDescriptor qd{};
|
|
qd.type = WGPUQueryType_Occlusion;
|
|
qd.count = OcclusionSet::kQueries;
|
|
o->set = wgpuDeviceCreateQuerySet(dev.device, &qd);
|
|
WGPUBufferDescriptor bd{};
|
|
bd.size = OcclusionSet::kQueries * 8;
|
|
bd.usage = WGPUBufferUsage_QueryResolve | WGPUBufferUsage_CopySrc;
|
|
o->resolve = wgpuDeviceCreateBuffer(dev.device, &bd);
|
|
bd.usage = WGPUBufferUsage_MapRead | WGPUBufferUsage_CopyDst;
|
|
o->read = wgpuDeviceCreateBuffer(dev.device, &bd);
|
|
return occ_open = o;
|
|
}
|
|
|
|
// RX_OCCLUSION_LOG=1: every 5 s, the reports and how many were 0 / estimated; =2 also each report (the first 2000)
|
|
void occ_done(OcclusionSet* o, const uint64_t* counts) {
|
|
static const int log = [] {
|
|
const char* e = getenv("RX_OCCLUSION_LOG");
|
|
return e ? std::atoi(e) : 0;
|
|
}();
|
|
static int logged;
|
|
static uint64_t reports, zeros, estimated, unread;
|
|
static int64_t since;
|
|
for (const OcclusionSet::Report& r : o->reports) {
|
|
uint64_t v = r.extra;
|
|
for (uint32_t i = 0; i < r.n; i++) v += counts ? counts[r.first + i] : 16;
|
|
nvn_gpu_write(r.addr, &v, 8);
|
|
reports++, zeros += v == 0, estimated += r.extra != 0, unread += !counts;
|
|
if (log >= 2 && logged < 2000) {
|
|
logged++;
|
|
hos_log("rx: occlusion report %llx = %llu (%u queries, %llu estimated)%s", (unsigned long long)r.addr,
|
|
(unsigned long long)v, r.n, (unsigned long long)r.extra, counts ? "" : " (not read)");
|
|
}
|
|
}
|
|
if (log && hos_now_ns() - since > 5000000000) {
|
|
if (since)
|
|
hos_log("rx: occlusion: %llu reports in 5 s, %llu of them 0, %llu with estimated draws, %llu not read",
|
|
(unsigned long long)reports, (unsigned long long)zeros, (unsigned long long)estimated,
|
|
(unsigned long long)unread);
|
|
reports = zeros = estimated = unread = 0;
|
|
since = hos_now_ns();
|
|
}
|
|
o->reports.clear();
|
|
o->used = 0;
|
|
std::lock_guard lk(occ_mu);
|
|
occ_pool.push_back(o);
|
|
}
|
|
|
|
WGPUQuerySet occlusion_pass_set() { return occ_counting ? occ_set()->set : nullptr; }
|
|
|
|
/// At a batch's end: its query set is resolved into the batch and read back after the submit.
|
|
void occlusion_flush() {
|
|
OcclusionSet* o = occ_open;
|
|
if (!o) return;
|
|
occ_open = nullptr;
|
|
occ_gen++;
|
|
if (o->used) {
|
|
ce_resolve_query_set(encoder(), o->set, 0, o->used, o->resolve, 0);
|
|
ce_copy_buffer_to_buffer(encoder(), o->resolve, 0, o->read, 0, uint64_t{o->used} * 8);
|
|
}
|
|
open_afters.push_back([o](bool submitted) { // (the submit thread, after the batch's submit)
|
|
if (!submitted || !o->used) return occ_done(o, nullptr);
|
|
WGPUBufferMapCallbackInfo mc{};
|
|
mc.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
|
|
auto* o = static_cast<OcclusionSet*>(u1);
|
|
const bool ok = status == WGPUMapAsyncStatus_Success;
|
|
occ_done(o, ok ? static_cast<const uint64_t*>(wgpuBufferGetConstMappedRange(o->read, 0, uint64_t{o->used} * 8))
|
|
: nullptr);
|
|
if (ok) wgpuBufferUnmap(o->read);
|
|
};
|
|
mc.userdata1 = o;
|
|
wgpuBufferMapAsync(o->read, WGPUMapMode_Read, 0, uint64_t{o->used} * 8, mc);
|
|
});
|
|
}
|
|
} // namespace
|
|
} // namespace (the file's)
|
|
|
|
bool occlusion_queries() {
|
|
static const bool on = [] {
|
|
const char* e = getenv("RX_OCCLUSION_QUERIES");
|
|
return e ? std::atoi(e) != 0 : !RX_WEB; // (the web: not tried yet)
|
|
}();
|
|
return on;
|
|
}
|
|
|
|
void occlusion_reset() {
|
|
if (!occlusion_queries()) return;
|
|
occ_counting = true;
|
|
OcclusionSet* o = occ_set();
|
|
if (pass && pass_occlusion != o->set) end_render_pass("pe_occlusion"); // (the next pass counts)
|
|
occ_first = o->used;
|
|
occ_reset_gen = occ_gen;
|
|
}
|
|
|
|
bool occlusion_draw_begin() {
|
|
if (!occ_counting || !pass || !occ_open || pass_occlusion != occ_open->set ||
|
|
occ_open->used >= OcclusionSet::kQueries)
|
|
return false;
|
|
rp_begin_occlusion_query(pass, occ_open->used++);
|
|
return true;
|
|
}
|
|
|
|
void occlusion_draw_end() {
|
|
rp_end_occlusion_query(pass);
|
|
occ_queried++;
|
|
}
|
|
|
|
uint64_t occlusion_queried() { return occ_queried; }
|
|
|
|
bool occlusion_report(uint64_t addr, uint64_t extra) {
|
|
if (!occlusion_queries()) return false;
|
|
const bool ok = occ_counting && occ_open && occ_reset_gen == occ_gen;
|
|
occ_counting = false;
|
|
if (!ok) return false;
|
|
occ_open->reports.push_back({addr, extra, occ_first, occ_open->used - occ_first});
|
|
return true;
|
|
}
|
|
|
|
namespace {
|
|
|
|
struct PendingClear {
|
|
Image* im;
|
|
bool color, depth, stencil;
|
|
WGPUColor value;
|
|
float depth_value;
|
|
uint32_t stencil_value;
|
|
};
|
|
std::vector<PendingClear> pending_clears;
|
|
void flush_pending_clears();
|
|
|
|
void begin_pass(const Attachments& at, const WGPURenderPassColorAttachment* colors,
|
|
const WGPURenderPassDepthStencilAttachment* depth) {
|
|
end_render_pass("pe_begin");
|
|
std::array<WGPURenderPassColorAttachment, 8> folded_colors;
|
|
WGPURenderPassDepthStencilAttachment folded_depth;
|
|
if (!pending_clears.empty()) {
|
|
for (auto it = pending_clears.begin(); it != pending_clears.end();) {
|
|
int slot = -1;
|
|
if (it->color && colors)
|
|
for (int i = 0; i < 8; i++)
|
|
if (at.color[i] == it->im && colors[i].loadOp == WGPULoadOp_Load) slot = i;
|
|
const bool depth_here = !it->color && depth && at.depth == it->im &&
|
|
(!it->depth || depth->depthLoadOp == WGPULoadOp_Load) &&
|
|
(!it->stencil || depth->stencilLoadOp == WGPULoadOp_Load);
|
|
if (slot >= 0) {
|
|
if (colors != folded_colors.data()) {
|
|
std::copy(colors, colors + 8, folded_colors.begin());
|
|
colors = folded_colors.data();
|
|
}
|
|
folded_colors[slot].loadOp = WGPULoadOp_Clear;
|
|
folded_colors[slot].clearValue = it->value;
|
|
} else if (depth_here) {
|
|
if (depth != &folded_depth) {
|
|
folded_depth = *depth;
|
|
depth = &folded_depth;
|
|
}
|
|
if (it->depth) {
|
|
folded_depth.depthLoadOp = WGPULoadOp_Clear;
|
|
folded_depth.depthClearValue = it->depth_value;
|
|
}
|
|
if (it->stencil) {
|
|
folded_depth.stencilLoadOp = WGPULoadOp_Clear;
|
|
folded_depth.stencilClearValue = it->stencil_value;
|
|
}
|
|
} else {
|
|
++it;
|
|
continue;
|
|
}
|
|
count("clears_folded");
|
|
it = pending_clears.erase(it);
|
|
}
|
|
flush_pending_clears();
|
|
}
|
|
passes_begun++;
|
|
count("passes");
|
|
static const bool old_versions = env_flag("RX_OLD_VERSIONS"); // (debugging: per-image counters)
|
|
for (Image* im : at.color)
|
|
if (im) im->version = old_versions ? im->version + 1 : ++image_versions;
|
|
if (at.depth) at.depth->version = old_versions ? at.depth->version + 1 : ++image_versions;
|
|
uint32_t n = 0;
|
|
for (uint32_t i = 0; i < 8; i++)
|
|
if (at.color[i]) n = i + 1;
|
|
WGPURenderPassDescriptor pd{};
|
|
pd.colorAttachmentCount = n;
|
|
pd.colorAttachments = colors;
|
|
pd.depthStencilAttachment = at.depth ? depth : nullptr;
|
|
if (gpu_timing()) {
|
|
const Image* first = nullptr;
|
|
for (uint32_t i = 0; !first && i < n; i++) first = at.color[i];
|
|
if (!first) first = at.depth;
|
|
const uint32_t cf = at.color[0] ? static_cast<uint32_t>(at.color[0]->format) : 0;
|
|
const uint32_t df = at.depth ? static_cast<uint32_t>(at.depth->format) : 0;
|
|
ce_pass_tag(first ? first->gpu_va : 0, uint64_t{at.width & 0x7FFF} << 48 | uint64_t{at.height & 0xFFFF} << 32 |
|
|
uint64_t{n} << 24 | uint64_t{cf & 0xFFFF} << 8 | (df & 0xFF));
|
|
}
|
|
pd.occlusionQuerySet = pass_occlusion = occlusion_pass_set();
|
|
pass = ce_begin_render_pass(encoder(), &pd);
|
|
pass_at = at;
|
|
}
|
|
|
|
WGPURenderPassColorAttachment color_attachment(Image* im, WGPULoadOp load, WGPUColor clear) {
|
|
WGPURenderPassColorAttachment c{};
|
|
c.view = im ? im->attachment : nullptr;
|
|
c.depthSlice = WGPU_DEPTH_SLICE_UNDEFINED;
|
|
c.loadOp = im ? load : WGPULoadOp_Undefined;
|
|
c.storeOp = im ? WGPUStoreOp_Store : WGPUStoreOp_Undefined;
|
|
c.clearValue = clear;
|
|
return c;
|
|
}
|
|
|
|
/// A depth clear value; on a 16-bit target, on the step draws put that depth on (d16q_ in the WGSL backend), so that
|
|
/// draws testing EQUAL against cleared pixels match them.
|
|
float depth_clear_value(const Image* im, float v) {
|
|
v = std::clamp(v, 0.0f, 1.0f);
|
|
static const bool no_d16q = env_flag("RX_NO_D16Q");
|
|
if (!im || im->format != WGPUTextureFormat_Depth16Unorm || no_d16q) return v;
|
|
return std::min((std::floor(v * 65535.0f + 0.5f) + 0.25f) / 65535.0f, 1.0f);
|
|
}
|
|
|
|
WGPURenderPassDepthStencilAttachment depth_attachment(Image* im, WGPULoadOp depth_load, WGPULoadOp stencil_load,
|
|
float depth_clear, uint32_t stencil_clear) {
|
|
WGPURenderPassDepthStencilAttachment d{};
|
|
d.view = im->attachment;
|
|
const bool has_depth = im->format != WGPUTextureFormat_Stencil8;
|
|
d.depthLoadOp = has_depth ? depth_load : WGPULoadOp_Undefined;
|
|
d.depthStoreOp = has_depth ? WGPUStoreOp_Store : WGPUStoreOp_Undefined;
|
|
d.depthClearValue = depth_clear;
|
|
d.stencilLoadOp = has_stencil(im->format) ? stencil_load : WGPULoadOp_Undefined;
|
|
d.stencilStoreOp = has_stencil(im->format) ? WGPUStoreOp_Store : WGPUStoreOp_Undefined;
|
|
d.stencilClearValue = stencil_clear;
|
|
return d;
|
|
}
|
|
|
|
uint32_t rt_byte_cost(WGPUTextureFormat f) {
|
|
switch (f) {
|
|
case WGPUTextureFormat_R8Unorm: case WGPUTextureFormat_R8Snorm: case WGPUTextureFormat_R8Uint:
|
|
case WGPUTextureFormat_R8Sint:
|
|
return 1;
|
|
case WGPUTextureFormat_RG8Unorm: case WGPUTextureFormat_RG8Snorm: case WGPUTextureFormat_RG8Uint:
|
|
case WGPUTextureFormat_RG8Sint: case WGPUTextureFormat_R16Float: case WGPUTextureFormat_R16Uint:
|
|
case WGPUTextureFormat_R16Sint: case WGPUTextureFormat_R16Unorm:
|
|
return 2;
|
|
case WGPUTextureFormat_RG16Float: case WGPUTextureFormat_RG16Uint: case WGPUTextureFormat_RG16Sint:
|
|
case WGPUTextureFormat_R32Float: case WGPUTextureFormat_R32Uint: case WGPUTextureFormat_R32Sint:
|
|
return 4;
|
|
case WGPUTextureFormat_RGBA32Float: case WGPUTextureFormat_RGBA32Uint: case WGPUTextureFormat_RGBA32Sint:
|
|
return 16;
|
|
default:
|
|
return 8;
|
|
}
|
|
}
|
|
|
|
void flush_pending_clears() {
|
|
if (pending_clears.empty()) return;
|
|
static std::vector<PendingClear> list;
|
|
list.clear();
|
|
list.swap(pending_clears);
|
|
Attachments at;
|
|
std::array<WGPURenderPassColorAttachment, 8> c{};
|
|
WGPURenderPassDepthStencilAttachment d{};
|
|
uint32_t n = 0;
|
|
for (const PendingClear& p : list) {
|
|
at.width = p.im->width;
|
|
at.height = p.im->height;
|
|
if (p.color) {
|
|
at.color[n] = p.im;
|
|
c[n++] = color_attachment(p.im, WGPULoadOp_Clear, p.value);
|
|
} else {
|
|
at.depth = p.im;
|
|
d = depth_attachment(p.im, p.depth ? WGPULoadOp_Clear : WGPULoadOp_Load,
|
|
p.stencil ? WGPULoadOp_Clear : WGPULoadOp_Load, p.depth_value, p.stencil_value);
|
|
}
|
|
}
|
|
begin_pass(at, n ? c.data() : nullptr, at.depth ? &d : nullptr);
|
|
end_render_pass("pe_clear");
|
|
}
|
|
|
|
bool clears_deferred() {
|
|
static const bool off = env_flag("RX_NO_DEFERRED_CLEARS");
|
|
return !off && !frame_dump_active();
|
|
}
|
|
|
|
void add_pending_clear(const PendingClear& p) {
|
|
end_render_pass("pe_begin");
|
|
p.im->version = ++image_versions;
|
|
for (PendingClear& q : pending_clears)
|
|
if (q.im == p.im) {
|
|
if (p.color) q.color = true, q.value = p.value;
|
|
if (p.depth) q.depth = true, q.depth_value = p.depth_value;
|
|
if (p.stencil) q.stencil = true, q.stencil_value = p.stencil_value;
|
|
return;
|
|
}
|
|
uint32_t colors = 0, bytes = 0;
|
|
bool has_depth = false;
|
|
for (const PendingClear& q : pending_clears) {
|
|
colors += q.color;
|
|
if (q.color) bytes += rt_byte_cost(q.im->format);
|
|
has_depth |= !q.color;
|
|
}
|
|
const bool fits = pending_clears.empty() ||
|
|
(pending_clears[0].im->width == p.im->width && pending_clears[0].im->height == p.im->height &&
|
|
(p.color ? colors < 8 && bytes + rt_byte_cost(p.im->format) <= dev.limits.maxColorAttachmentBytesPerSample
|
|
: !has_depth));
|
|
if (!fits) flush_pending_clears();
|
|
pending_clears.push_back(p);
|
|
}
|
|
|
|
// ---- draws: suyu's Maxwell3D::DrawManager ----
|
|
using Topology = Regs::PrimitiveTopology;
|
|
|
|
/// Whether none of the draw's targets was drawn in the last 64 frames (each target's last two frames drawn, by GPU
|
|
/// address; the answer is kept while consecutive draws name the same targets in a frame)
|
|
bool one_off_targets(const Regs& r) {
|
|
static const bool off = env_flag("RX_SKIP_ONE_OFF");
|
|
if (off) return false;
|
|
uint64_t addr[9];
|
|
int n = 0;
|
|
for (uint32_t i = 0; i < 8; i++)
|
|
if (r.rt[i].format != Tegra::RenderTargetFormat::NONE && r.rt[i].Address()) addr[n++] = r.rt[i].Address();
|
|
if (r.zeta_enable && r.zeta.Address()) addr[n++] = r.zeta.Address();
|
|
if (!n) return false;
|
|
const uint64_t sig = hash_bytes(addr, n * sizeof addr[0]);
|
|
static uint64_t last_sig;
|
|
static int64_t last_frame = -1;
|
|
static bool last_answer;
|
|
if (sig == last_sig && last_frame == frame_number) return last_answer;
|
|
struct Seen { int64_t last = -100, prev = -100; };
|
|
static std::unordered_map<uint64_t, Seen> seen;
|
|
bool recent = false;
|
|
for (int i = 0; i < n; i++) {
|
|
Seen& s = seen[addr[i]];
|
|
if (s.last != frame_number) s.prev = s.last, s.last = frame_number;
|
|
recent |= s.prev >= frame_number - 64; // (some targets are drawn every 9 frames)
|
|
}
|
|
last_sig = sig;
|
|
last_frame = frame_number;
|
|
last_answer = !recent;
|
|
return last_answer;
|
|
}
|
|
|
|
void process_draw(Channel& ch, const Regs& r, bool indexed, uint32_t instance_count, uint32_t base_instance,
|
|
int32_t base_vertex, uint32_t first, uint32_t count) {
|
|
DrawParams p;
|
|
p.indexed = indexed;
|
|
p.topology = ch.topology;
|
|
if (r.primitive_topology_control == Regs::PrimitiveTopologyControl::UseSeparateState) {
|
|
switch (r.topology_override) {
|
|
case Regs::PrimitiveTopologyOverride::None: break;
|
|
case Regs::PrimitiveTopologyOverride::Points: p.topology = (uint32_t)Topology::Points; break;
|
|
case Regs::PrimitiveTopologyOverride::Lines: p.topology = (uint32_t)Topology::Lines; break;
|
|
case Regs::PrimitiveTopologyOverride::LineStrip: p.topology = (uint32_t)Topology::LineStrip; break;
|
|
default: p.topology = (uint32_t)r.topology_override; break;
|
|
}
|
|
}
|
|
p.first = first;
|
|
p.count = count;
|
|
p.instance_count = std::max(1u, instance_count);
|
|
p.base_instance = base_instance;
|
|
p.base_vertex = base_vertex;
|
|
if (indexed) {
|
|
if (ch.draw_mode == Channel::DrawMode::InlineIndex) {
|
|
p.inline_index = &ch.inline_index;
|
|
p.index_size = 4;
|
|
} else {
|
|
p.index_va = r.index_buffer.StartAddress();
|
|
p.index_size = r.index_buffer.format == Regs::IndexFormat::UnsignedByte ? 1
|
|
: r.index_buffer.format == Regs::IndexFormat::UnsignedShort ? 2
|
|
: 4;
|
|
}
|
|
}
|
|
if (count && r.zpass_pixel_count_enable) ch.zpass += (uint64_t)count * p.instance_count * 64; // (estimate)
|
|
if (count) {
|
|
draw_must_run = one_off_targets(r);
|
|
draw(ch, r, p);
|
|
draw_must_run = false;
|
|
} else { // (nothing to draw: counted, and logged in a frame dump, where indirect arguments came out empty)
|
|
rx::count("draws_empty");
|
|
if (frame_dump_active())
|
|
hos_log("rx-frame: draw with no vertices: vs %llx fs %llx x%u %s",
|
|
(unsigned long long)(r.program_region.Address() + r.pipelines[1].offset),
|
|
(unsigned long long)(r.program_region.Address() + r.pipelines[5].offset), instance_count,
|
|
indexed ? "indexed" : "");
|
|
}
|
|
}
|
|
|
|
void draw_end(Channel& ch, const Regs& r, uint32_t instance_count, bool force) {
|
|
switch (ch.draw_mode) {
|
|
case Channel::DrawMode::Instance:
|
|
if (!force) break;
|
|
[[fallthrough]];
|
|
case Channel::DrawMode::General:
|
|
if (ch.draw_indexed)
|
|
process_draw(ch, r, true, instance_count, r.global_base_instance_index,
|
|
(int32_t)r.global_base_vertex_index, r.index_buffer.first, r.index_buffer.count);
|
|
else
|
|
process_draw(ch, r, false, instance_count, r.global_base_instance_index, 0, r.vertex_buffer.first,
|
|
r.vertex_buffer.count);
|
|
ch.draw_indexed = false;
|
|
break;
|
|
case Channel::DrawMode::InlineIndex:
|
|
process_draw(ch, r, true, instance_count, r.global_base_instance_index, (int32_t)r.global_base_vertex_index,
|
|
0, (uint32_t)(ch.inline_index.size() / 4));
|
|
ch.inline_index.clear();
|
|
break;
|
|
}
|
|
}
|
|
|
|
void draw_deferred(Channel& ch, const Regs& r) {
|
|
if (ch.draw_mode != Channel::DrawMode::Instance || ch.instance_count == 0) return;
|
|
draw_end(ch, r, ch.instance_count + 1, true);
|
|
ch.instance_count = 0;
|
|
}
|
|
|
|
// RX_RT_HISTORY=1 (debugging): every color target address a draw or clear named, with its last frame (what
|
|
// rt_history_find answers for a frame dump's texinfo)
|
|
struct RtSeen { int64_t frame; uint32_t width, height, format; bool on; };
|
|
std::unordered_map<uint64_t, RtSeen> rt_history;
|
|
void rt_history_note(const Regs& r, bool on) {
|
|
for (uint32_t i = 0; i < 8; i++)
|
|
if (r.rt[i].format != Tegra::RenderTargetFormat::NONE && r.rt[i].Address()) {
|
|
// (RX_RT_HISTORY_LOG=<width>: targets that wide logged when first named)
|
|
static const uint32_t log_w = getenv("RX_RT_HISTORY_LOG") ? atoi(getenv("RX_RT_HISTORY_LOG")) : 0;
|
|
// (RX_RT_HISTORY_RANGE=<lo>-<hi> GPU hex: targets there logged when first named)
|
|
static uint64_t range_lo = 1, range_hi;
|
|
if (range_lo == 1) {
|
|
range_lo = 0;
|
|
if (const char* v = getenv("RX_RT_HISTORY_RANGE")) {
|
|
char* e;
|
|
range_lo = strtoull(v, &e, 16);
|
|
range_hi = *e == '-' ? strtoull(e + 1, nullptr, 16) : 0;
|
|
}
|
|
}
|
|
const bool in_range = (range_hi && r.rt[i].Address() >= range_lo && r.rt[i].Address() < range_hi) ||
|
|
guest_watch_hit(r.rt[i].Address(), 1);
|
|
if (((log_w && r.rt[i].width == log_w) || in_range) && !rt_history.count(r.rt[i].Address()))
|
|
hos_log("rx: target %#llx %ux%u format %#x first drawn, frame %lld", (unsigned long long)r.rt[i].Address(),
|
|
r.rt[i].width, r.rt[i].height, (unsigned)r.rt[i].format, (long long)frame_number);
|
|
rt_history[r.rt[i].Address()] = {frame_number, r.rt[i].width, r.rt[i].height, (uint32_t)r.rt[i].format, on};
|
|
}
|
|
}
|
|
|
|
void draw_begin(Channel& ch, const Regs& r) {
|
|
if (r.draw.instance_id == Regs::Draw::InstanceId::First) {
|
|
draw_deferred(ch, r);
|
|
ch.instance_count = 0;
|
|
ch.draw_mode = Channel::DrawMode::General;
|
|
} else if (r.draw.instance_id == Regs::Draw::InstanceId::Subsequent) {
|
|
ch.instance_count++;
|
|
ch.draw_mode = Channel::DrawMode::Instance;
|
|
}
|
|
ch.topology = (uint32_t)r.draw.topology.Value();
|
|
}
|
|
|
|
void push_inline_index(Channel& ch, uint32_t index) {
|
|
ch.inline_index.insert(ch.inline_index.end(), {uint8_t(index), uint8_t(index >> 8), uint8_t(index >> 16),
|
|
uint8_t(index >> 24)});
|
|
ch.draw_mode = Channel::DrawMode::InlineIndex;
|
|
}
|
|
|
|
void draw_array_instanced(Channel& ch, const Regs& r, const Regs::VertexArray& va, bool subsequent) {
|
|
ch.topology = (uint32_t)va.topology.Value();
|
|
if (!subsequent) ch.instance_count = 1;
|
|
const uint32_t base_instance = ch.instance_count - 1;
|
|
ch.draw_mode = Channel::DrawMode::Instance;
|
|
ch.instance_count++;
|
|
process_draw(ch, r, false, 1, base_instance, 0, va.start, va.count);
|
|
}
|
|
|
|
void draw_index_small(Channel& ch, const Regs& r, uint32_t arg) {
|
|
Regs::IndexBufferSmall s{};
|
|
s.raw = arg;
|
|
ch.topology = (uint32_t)s.topology.Value();
|
|
const auto mode = ch.draw_mode;
|
|
ch.draw_mode = Channel::DrawMode::General; // the index buffer, not inline indices
|
|
process_draw(ch, r, true, 1, r.global_base_instance_index, (int32_t)r.global_base_vertex_index, s.first,
|
|
s.count);
|
|
ch.draw_mode = mode;
|
|
}
|
|
|
|
std::array<uint8_t, 0x1000> method_table() {
|
|
std::array<uint8_t, 0x1000> t{};
|
|
for (uint32_t m : {RX_REG(clear_surface), RX_REG(draw.begin), RX_REG(draw.end), RX_REG(index_buffer.count),
|
|
RX_REG(index_buffer32_first), RX_REG(index_buffer16_first), RX_REG(index_buffer8_first),
|
|
RX_REG(index_buffer32_subsequent), RX_REG(index_buffer16_subsequent),
|
|
RX_REG(index_buffer8_subsequent), RX_REG(draw_inline_index), RX_REG(inline_index_2x16.even),
|
|
RX_REG(inline_index_4x8.index0), RX_REG(vertex_array_instance_first),
|
|
RX_REG(vertex_array_instance_subsequent), RX_REG(draw_texture.src_y0),
|
|
RX_REG(invalidate_texture_data_cache), RX_REG(tiled_cache_barrier),
|
|
RX_REG(invalidate_texture_cache_no_wfi), RX_REG(invalidate_texture_data_cache_lines),
|
|
RX_REG(render_enable.mode), RX_REG(render_enable_override), RX_REG(clear_report_value)})
|
|
t[m] = 1;
|
|
for (uint32_t s = 0; s < 5; s++) t[RX_REG(bind_groups) + s * 8 + 4] = 1;
|
|
return t;
|
|
}
|
|
} // namespace
|
|
|
|
WGPUShaderModule program_module(const WGPUShaderModuleDescriptor* d) {
|
|
#if !RX_WEB
|
|
static const char* checks = getenv("RX_SHADER_CHECKS");
|
|
if (checks)
|
|
return wgpuDeviceCreateShaderModuleTrusted(dev.device, d, (WGPUShaderRuntimeChecks)std::strtoul(checks, 0, 0));
|
|
#endif
|
|
return wgpuDeviceCreateShaderModule(dev.device, d);
|
|
}
|
|
|
|
/// The render targets a draw writes: rt_control maps each fragment output to a physical color target.
|
|
/// WebGPU receives one attachment per fragment output, restricted to one size per pass.
|
|
Attachments draw_attachments_uncached(const Regs& r);
|
|
|
|
/// Draws of a pass have the same attachments: the last answer holds while the target registers, the frame and the
|
|
/// set of images are the same (it was ~4.5% of the GPU thread, an image lookup per target per draw).
|
|
Attachments draw_attachments(const Regs& r) {
|
|
struct Key {
|
|
decltype(r.rt) rt;
|
|
decltype(r.rt_control) rt_control;
|
|
decltype(r.zeta) zeta;
|
|
decltype(r.zeta_size) zeta_size;
|
|
decltype(r.surface_clip) surface_clip;
|
|
uint32_t zeta_enable;
|
|
};
|
|
static Key last_key;
|
|
static Attachments last;
|
|
static int64_t last_frame = -1;
|
|
static uint64_t last_images = 0;
|
|
Key k;
|
|
std::memset(&k, 0, sizeof k);
|
|
std::memcpy(&k.rt, &r.rt, sizeof k.rt);
|
|
std::memcpy(&k.rt_control, &r.rt_control, sizeof k.rt_control);
|
|
std::memcpy(&k.zeta, &r.zeta, sizeof k.zeta);
|
|
std::memcpy(&k.zeta_size, &r.zeta_size, sizeof k.zeta_size);
|
|
std::memcpy(&k.surface_clip, &r.surface_clip, sizeof k.surface_clip);
|
|
k.zeta_enable = r.zeta_enable;
|
|
if (last_frame == frame_number && last_images == images_changed && !std::memcmp(&k, &last_key, sizeof k)) {
|
|
count("attachments_memo_hits");
|
|
return last;
|
|
}
|
|
last = draw_attachments_uncached(r);
|
|
last_key = k;
|
|
last_frame = frame_number;
|
|
last_images = images_changed; // (the lookup itself can make or replace images)
|
|
return last;
|
|
}
|
|
|
|
Attachments draw_attachments_uncached(const Regs& r) {
|
|
Attachments at;
|
|
const uint32_t n = std::min<uint32_t>(r.rt_control.count, 8);
|
|
for (uint32_t i = 0; i < n; i++) {
|
|
const uint32_t m = r.rt_control.Map(i);
|
|
if (m >= 8) {
|
|
log_limited("rt_map", 4, "rx: render target output %u has invalid mapping %u", i, m);
|
|
continue;
|
|
}
|
|
at.color[i] = rt_image(r, m);
|
|
}
|
|
at.depth = zeta_image(r);
|
|
for (Image* im : at.color)
|
|
if (im && !at.width) {
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
}
|
|
if (!at.width && at.depth) {
|
|
at.width = at.depth->width;
|
|
at.height = at.depth->height;
|
|
}
|
|
for (auto& im : at.color)
|
|
if (im && (im->width != at.width || im->height != at.height)) {
|
|
log_limited("rt_size", 8, "rx: color target %ux%u dropped from a %ux%u pass", im->width, im->height,
|
|
at.width, at.height);
|
|
static int attachment_trace_count;
|
|
if (static const bool trace_att = env_flag("RX_TRACE_ATTACHMENTS"); trace_att && attachment_trace_count++ < 16) {
|
|
hos_log("rx: rt routing count %u maps %u/%u/%u/%u/%u/%u/%u/%u",
|
|
static_cast<uint32_t>(r.rt_control.count), r.rt_control.Map(0), r.rt_control.Map(1),
|
|
r.rt_control.Map(2), r.rt_control.Map(3), r.rt_control.Map(4), r.rt_control.Map(5),
|
|
r.rt_control.Map(6), r.rt_control.Map(7));
|
|
for (uint32_t i = 0; i < n; i++) {
|
|
const uint32_t target = r.rt_control.Map(i);
|
|
const auto& rt = r.rt[target];
|
|
hos_log("rx: rt output %u -> %u VA %#llx %ux%u format %u", i, target,
|
|
(unsigned long long)rt.Address(), rt.width, rt.height, (unsigned)rt.format);
|
|
}
|
|
}
|
|
im = nullptr;
|
|
}
|
|
if (at.depth && (at.depth->width != at.width || at.depth->height != at.height)) {
|
|
log_limited("zt_size", 8, "rx: depth target %ux%u dropped from a %ux%u pass", at.depth->width,
|
|
at.depth->height, at.width, at.height);
|
|
at.depth = nullptr;
|
|
}
|
|
return at;
|
|
}
|
|
|
|
// ---- shared helpers ----
|
|
thread_local std::vector<int> scope_counts; // (mkw-recomp) the scopes each scope_push opened, for its scope_pop
|
|
|
|
void scope_push(bool out_of_memory) {
|
|
if (RX_WEB) return; // (the browser reports errors in its console, and an invalid submit does not end the page)
|
|
scope_counts.push_back(out_of_memory ? 2 : 1);
|
|
// ((mkw-recomp) out-of-memory errors too: a filter of their own, which a validation scope does not catch. A
|
|
// buffer the GPU had no memory for was kept as if made, and the submit that used it aborted the process)
|
|
if (out_of_memory) wgpuDevicePushErrorScope(dev.device, WGPUErrorFilter_OutOfMemory);
|
|
wgpuDevicePushErrorScope(dev.device, WGPUErrorFilter_Validation);
|
|
}
|
|
|
|
std::string scope_pop() {
|
|
if (RX_WEB) return {};
|
|
struct ScopeResult {
|
|
std::mutex mutex;
|
|
std::condition_variable ready;
|
|
bool complete = false;
|
|
std::string error;
|
|
} result;
|
|
WGPUPopErrorScopeCallbackInfo cb{};
|
|
cb.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
cb.callback = [](WGPUPopErrorScopeStatus, WGPUErrorType type, WGPUStringView message, void* u1, void*) {
|
|
auto& result = *static_cast<ScopeResult*>(u1);
|
|
{
|
|
std::lock_guard lock(result.mutex);
|
|
if (type != WGPUErrorType_NoError) result.error = str(message);
|
|
result.complete = true;
|
|
}
|
|
result.ready.notify_one();
|
|
};
|
|
cb.userdata1 = &result;
|
|
const int scopes = scope_counts.empty() ? 1 : scope_counts.back();
|
|
if (!scope_counts.empty()) scope_counts.pop_back();
|
|
for (int scope = 0; scope < scopes; scope++) { // (validation, then out of memory: scope_push's)
|
|
{
|
|
std::lock_guard lock(result.mutex);
|
|
result.complete = false;
|
|
}
|
|
wgpuDevicePopErrorScope(dev.device, cb);
|
|
std::unique_lock lock(result.mutex);
|
|
result.ready.wait(lock, [&result] { return result.complete; });
|
|
}
|
|
if (result.error.size() > 1500) result.error.resize(1500);
|
|
return result.error;
|
|
}
|
|
|
|
/// (mkw-recomp) A bind group checked at creation: null when it is invalid (a view of a texture that was not made, a
|
|
/// buffer the GPU had no memory for). wgpu-native aborts the process at a submit that uses an invalid group, so the
|
|
/// draw or dispatch it was for is left out instead (logged).
|
|
WGPUBindGroup checked_bind_group(const WGPUBindGroupDescriptor& d, const char* what) {
|
|
scope_push(false); // (one scope: made for each dispatch, and a scope's pop waits for its answer)
|
|
WGPUBindGroup g = wgpuDeviceCreateBindGroup(dev.device, &d);
|
|
if (const std::string error = scope_pop(); !error.empty()) {
|
|
log_limited("bind_group_invalid", 24, "rx: %s bind group invalid, left out: %s", what, error.c_str());
|
|
count("bind_groups_invalid");
|
|
if (g) wgpuBindGroupRelease(g);
|
|
return nullptr;
|
|
}
|
|
return g;
|
|
}
|
|
|
|
/// Conditional rendering (suyu's Maxwell3D::ProcessQueryCondition): draws and clears run only while the render
|
|
/// enable condition holds (Channel::render_on, evaluated when the mode or the override is written); its modes compare
|
|
/// 64-bit values the GPU released, at the address and 16 bytes on. TOTK
|
|
/// skips the later clears of its half-resolution depth target this way, keeping the scene's depth in it for the
|
|
/// cloud dome (done, they let the clouds cover the terrain above the horizon). RX_NO_CONDITIONAL=1: always render.
|
|
bool render_enabled(const Regs& r) {
|
|
using RE = Regs::RenderEnable;
|
|
static const bool off = env_flag("RX_NO_CONDITIONAL");
|
|
if (off) return true;
|
|
if (r.render_enable_override == RE::Override::AlwaysRender) return true;
|
|
if (r.render_enable_override == RE::Override::NeverRender) return false;
|
|
switch (r.render_enable.mode) {
|
|
case RE::Mode::False: return false;
|
|
case RE::Mode::True: return true;
|
|
default: break;
|
|
}
|
|
uint64_t v[3]{};
|
|
gpu_read(r.render_enable.Address(), v, sizeof v);
|
|
switch (r.render_enable.mode) {
|
|
case RE::Mode::Conditional: return v[0] != 0;
|
|
case RE::Mode::IfEqual: return v[0] == v[2];
|
|
case RE::Mode::IfNotEqual: return v[0] != v[2];
|
|
default: return true;
|
|
}
|
|
}
|
|
|
|
extern "C" volatile long maps_generation; // hos/nvdrv.c: bumped by every mapping change
|
|
|
|
// (the counters and the timers only with RX_STATS: ~10 timestamps and as many counts a draw were ~3% of the render
|
|
// thread)
|
|
// (the web's environment comes after static initialization: read once the renderer runs, in rx_start)
|
|
bool stats_on = false;
|
|
|
|
int64_t clock_ns() {
|
|
if (!stats_on) return 0;
|
|
static const double ns_per_tick = [] {
|
|
const int64_t t0 = hos_now_ns();
|
|
const uint64_t c0 = __rdtsc();
|
|
Sleep(20);
|
|
const int64_t t1 = hos_now_ns();
|
|
const uint64_t c1 = __rdtsc();
|
|
return c1 > c0 ? double(t1 - t0) / double(c1 - c0) : 0.0;
|
|
}();
|
|
return (int64_t)(double(__rdtsc()) * ns_per_tick);
|
|
}
|
|
|
|
bool gpu_read(uint64_t va, void* dst, uint64_t n) {
|
|
auto* d = static_cast<uint8_t*>(dst);
|
|
// the last mapping this thread read from (most reads are small and land in the one before: TIC/TSC entries,
|
|
// constant buffers; hos_gpu_ptr, out of line, was ~7% of the GPU thread)
|
|
thread_local struct {
|
|
uint64_t lo = 1, hi = 0;
|
|
const uint8_t* host = nullptr;
|
|
long generation = 0;
|
|
} last;
|
|
const long generation = maps_generation;
|
|
if (va >= last.lo && va + n <= last.hi && last.generation == generation && n) {
|
|
std::memcpy(d, last.host + (va - last.lo), n);
|
|
return true;
|
|
}
|
|
{
|
|
uint64_t avail = 0;
|
|
if (const uint8_t* p = hos_gpu_ptr(va, &avail); p && avail >= n) {
|
|
last.lo = va;
|
|
last.hi = va + avail;
|
|
last.host = p;
|
|
last.generation = generation;
|
|
if (n) std::memcpy(d, p, n);
|
|
return true;
|
|
}
|
|
}
|
|
bool ok = true;
|
|
while (n) {
|
|
uint64_t avail = 0;
|
|
const uint8_t* p = hos_gpu_ptr(va, &avail);
|
|
if (!p) {
|
|
const uint64_t step = std::min<uint64_t>(n, 0x1000 - (va & 0xFFF));
|
|
std::memset(d, 0, step);
|
|
d += step;
|
|
va += step;
|
|
n -= step;
|
|
ok = false;
|
|
continue;
|
|
}
|
|
const uint64_t k = std::min(n, avail);
|
|
std::memcpy(d, p, k);
|
|
d += k;
|
|
va += k;
|
|
n -= k;
|
|
}
|
|
return ok;
|
|
}
|
|
|
|
const uint8_t* gpu_span(uint64_t va, uint64_t n) {
|
|
uint64_t avail = 0;
|
|
const uint8_t* p = hos_gpu_ptr(va, &avail);
|
|
return p && avail >= n ? p : nullptr;
|
|
}
|
|
|
|
/// A PNG of RGBA8 rows (stored deflate blocks: no compression library needed).
|
|
void write_png(const char* path, const uint8_t* rgba, uint32_t w, uint32_t h, size_t stride) {
|
|
std::vector<uint8_t> raw;
|
|
raw.reserve((size_t)(w * 4 + 1) * h);
|
|
for (uint32_t y = 0; y < h; y++) {
|
|
raw.push_back(0);
|
|
raw.insert(raw.end(), rgba + y * stride, rgba + y * stride + (size_t)w * 4);
|
|
}
|
|
std::vector<uint8_t> z = {0x78, 0x01};
|
|
for (size_t i = 0; i < raw.size(); i += 65535) {
|
|
const size_t n = std::min<size_t>(65535, raw.size() - i);
|
|
z.push_back(i + n == raw.size());
|
|
z.insert(z.end(), {uint8_t(n), uint8_t(n >> 8), uint8_t(~n), uint8_t(~n >> 8)});
|
|
z.insert(z.end(), raw.begin() + i, raw.begin() + i + n);
|
|
}
|
|
uint32_t a = 1, b = 0;
|
|
for (uint8_t c : raw) {
|
|
a = (a + c) % 65521;
|
|
b = (b + a) % 65521;
|
|
}
|
|
const uint32_t adler = b << 16 | a;
|
|
z.insert(z.end(), {uint8_t(adler >> 24), uint8_t(adler >> 16), uint8_t(adler >> 8), uint8_t(adler)});
|
|
static uint32_t table[256];
|
|
if (!table[1])
|
|
for (uint32_t n = 0; n < 256; n++) {
|
|
uint32_t c = n;
|
|
for (int k = 0; k < 8; k++) c = c & 1 ? 0xEDB88320u ^ (c >> 1) : c >> 1;
|
|
table[n] = c;
|
|
}
|
|
FILE* f = std::fopen(path, "wb");
|
|
if (!f) return;
|
|
auto chunk = [&](const char* type, const std::vector<uint8_t>& data) {
|
|
const uint32_t n = (uint32_t)data.size();
|
|
const uint8_t len[4] = {uint8_t(n >> 24), uint8_t(n >> 16), uint8_t(n >> 8), uint8_t(n)};
|
|
std::fwrite(len, 1, 4, f);
|
|
uint32_t crc = 0xFFFFFFFFu;
|
|
for (int i = 0; i < 4; i++) crc = table[(crc ^ (uint8_t)type[i]) & 0xFF] ^ (crc >> 8);
|
|
for (uint8_t c : data) crc = table[(crc ^ c) & 0xFF] ^ (crc >> 8);
|
|
crc ^= 0xFFFFFFFFu;
|
|
std::fwrite(type, 1, 4, f);
|
|
std::fwrite(data.data(), 1, data.size(), f);
|
|
const uint8_t c4[4] = {uint8_t(crc >> 24), uint8_t(crc >> 16), uint8_t(crc >> 8), uint8_t(crc)};
|
|
std::fwrite(c4, 1, 4, f);
|
|
};
|
|
static const uint8_t signature[8] = {0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A};
|
|
std::fwrite(signature, 1, 8, f);
|
|
chunk("IHDR", {uint8_t(w >> 24), uint8_t(w >> 16), uint8_t(w >> 8), uint8_t(w), uint8_t(h >> 24), uint8_t(h >> 16),
|
|
uint8_t(h >> 8), uint8_t(h), 8, 6, 0, 0, 0});
|
|
chunk("IDAT", z);
|
|
chunk("IEND", {});
|
|
std::fclose(f);
|
|
}
|
|
|
|
void memory_report() {
|
|
uint64_t rt_bytes = 0, spare_bytes = 0, copy_bytes = 0;
|
|
for (const auto& [g, im] : images)
|
|
if (im) {
|
|
rt_bytes += image_bytes(*im);
|
|
if (im->sampled_copy) copy_bytes += image_bytes(*im->sampled_copy);
|
|
}
|
|
for (const auto& [g, im] : spare_images)
|
|
if (im) spare_bytes += image_bytes(*im);
|
|
size_t ntex = 0, ntext = 0;
|
|
uint64_t tex_bytes = 0, text_bytes = 0, priv = 0, ws = 0;
|
|
decoded_texture_stats(&ntex, &tex_bytes);
|
|
wgsl_text_stats(&ntext, &text_bytes);
|
|
hos_process_memory(&priv, &ws);
|
|
hos_log("mem: process %llu MB private, %llu MB working set | GPU: decoded textures %llu MB (%zu), render targets "
|
|
"%llu MB (%zu) + %llu MB spare (%zu) + %llu MB feedback copies, guest-memory buffers %zu x 16 MB | "
|
|
"%zu pipelines, %zu WGSL texts %llu MB",
|
|
(unsigned long long)(priv >> 20), (unsigned long long)(ws >> 20), (unsigned long long)(tex_bytes >> 20),
|
|
ntex, (unsigned long long)(rt_bytes >> 20), images.size(), (unsigned long long)(spare_bytes >> 20),
|
|
spare_images.size(), (unsigned long long)(copy_bytes >> 20), buffer_segments(), pipeline_count(), ntext,
|
|
(unsigned long long)(text_bytes >> 20));
|
|
hos_memory_map_log();
|
|
}
|
|
|
|
void background_thread() {
|
|
#if !RX_WEB && defined(_WIN32)
|
|
static const bool off = getenv("RX_NO_BACKGROUND_PRIORITY") != nullptr;
|
|
if (!off) SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL);
|
|
#endif
|
|
}
|
|
|
|
#if !RX_WEB
|
|
extern "C" int hos_ab(void);
|
|
#endif
|
|
bool ab_named(const char* name) {
|
|
const char* sw = getenv("HOS_AB_SWITCH");
|
|
return !RX_WEB && sw && !std::strcmp(sw, name);
|
|
}
|
|
|
|
bool ab_flipped() {
|
|
#if RX_WEB
|
|
return false;
|
|
#else
|
|
return hos_ab() != 0;
|
|
#endif
|
|
}
|
|
|
|
bool env_flag(const char* name) {
|
|
// keyed by the literal's address (callers pass literals): the hot paths ask several times per draw, and a
|
|
// locked string map was ~3% of the GPU thread
|
|
struct Entry {
|
|
std::atomic<const char*> key{nullptr};
|
|
bool value = false;
|
|
};
|
|
static Entry table[64];
|
|
for (Entry& e : table) {
|
|
const char* k = e.key.load(std::memory_order_acquire);
|
|
if (k == name) return e.value;
|
|
if (!k) break;
|
|
}
|
|
static std::mutex m;
|
|
std::lock_guard lk(m);
|
|
for (Entry& e : table) {
|
|
const char* k = e.key.load(std::memory_order_relaxed);
|
|
if (k == name) return e.value;
|
|
if (!k) {
|
|
e.value = getenv(name) != nullptr;
|
|
e.key.store(name, std::memory_order_release);
|
|
return e.value;
|
|
}
|
|
}
|
|
return getenv(name) != nullptr;
|
|
}
|
|
|
|
namespace {
|
|
std::unordered_map<const char*, int64_t> counters; // this period's, by key (string literals)
|
|
uint64_t counters_generation; // bumped when the counters are cleared (count's slot cache)
|
|
int64_t period_start, period_frames;
|
|
|
|
/// Every 5 s with RX_STATS: where the renderer's time went, and why draws were skipped.
|
|
void report_period() {
|
|
const int64_t now = hos_now_ns();
|
|
if (!period_start) period_start = now;
|
|
period_frames++;
|
|
if (now - period_start < 5000000000LL) return;
|
|
if (env_flag("RX_STATS")) {
|
|
std::unordered_map<std::string, int64_t> merged;
|
|
for (auto& [k, v] : counters) merged[k] += v;
|
|
const double frames = (double)std::max<int64_t>(period_frames, 1);
|
|
auto ms = [&](const char* k) { return merged.count(k) ? merged[k] / 1e6 / frames : 0.0; };
|
|
auto per = [&](const char* k) { return merged.count(k) ? merged[k] / frames : 0.0; };
|
|
hos_log("rx: %.1f fps; per frame: %.0f draws %.1f ms, %.0f skipped, flush %.1f ms, present %.1f ms, "
|
|
"translate %.1f ms, decode %.1f ms, pipelines %.1f ms (%.1f new), bind groups %.0f, upload %.1f MB",
|
|
period_frames * 1e9 / (now - period_start), per("draws"), ms("t_draw"), per("skipped"),
|
|
ms("t_flush"), ms("t_present"), ms("t_translate"), ms("t_decode"), ms("t_pipeline"),
|
|
per("pipelines"), per("bind_groups"), per("upload_bytes") / 1048576.0);
|
|
std::string reasons;
|
|
for (auto& [k, v] : merged)
|
|
if (k.rfind("t_", 0) && k != "draws" && k != "skipped" && k != "pipelines" && k != "bind_groups" &&
|
|
k != "upload_bytes") {
|
|
char b[96];
|
|
std::snprintf(b, sizeof b, "%s%s %.0f", reasons.empty() ? "" : ", ", k.c_str(), v / frames);
|
|
reasons += b;
|
|
}
|
|
if (!reasons.empty()) hos_log("rx: per frame: %s", reasons.c_str());
|
|
std::string timers;
|
|
for (auto& [k, v] : merged)
|
|
if (!k.rfind("t_", 0)) {
|
|
char b[96];
|
|
std::snprintf(b, sizeof b, "%s%s %.2f ms", timers.empty() ? "" : ", ", k.c_str() + 2, v / 1e6 / frames);
|
|
timers += b;
|
|
}
|
|
if (!timers.empty()) hos_log("rx: time per frame: %s", timers.c_str());
|
|
hos_log("rx: submit thread per frame: %.1f encoders, replay+finish %.2f ms, writes %.2f ms, submit %.2f ms, present %.2f ms; the "
|
|
"GPU thread waited for it %.2f ms",
|
|
(double)count_parts.exchange(0) / frames, st_finish_ns.exchange(0) / 1e6 / frames,
|
|
st_write_ns.exchange(0) / 1e6 / frames,
|
|
st_submit_ns.exchange(0) / 1e6 / frames, st_after_ns.exchange(0) / 1e6 / frames,
|
|
st_wait_ns.exchange(0) / 1e6 / frames);
|
|
if (const int64_t w = st_writeback_words.exchange(0))
|
|
hos_log("rx: shader writes copied back to guest memory: %.0f words per frame", w / frames);
|
|
}
|
|
counters.clear();
|
|
counters_generation++;
|
|
period_start = now;
|
|
period_frames = 0;
|
|
}
|
|
/// RX_SCREENSHOT=<dir>: the presented image every 5 s as <dir>/frame_<n>.png (synchronous readback: debugging only).
|
|
std::atomic<int64_t> screenshot_interval{-1}; // (rx_set_screenshot_ms at the end of this file)
|
|
void screenshot(Image* im) {
|
|
static const char* dir = getenv("RX_SCREENSHOT");
|
|
static int64_t last;
|
|
if (!dir || !im || (im->format != WGPUTextureFormat_RGBA8Unorm && im->format != WGPUTextureFormat_RGBA8UnormSrgb))
|
|
return;
|
|
// RX_SCREENSHOT_MS=<ms>: another interval; RX_SCREENSHOT_AFTER=<s>: none before that
|
|
if (screenshot_interval < 0)
|
|
screenshot_interval = getenv("RX_SCREENSHOT_MS") ? atoll(getenv("RX_SCREENSHOT_MS")) * 1000000LL : 5000000000LL;
|
|
static const int64_t after = getenv("RX_SCREENSHOT_AFTER") ? atoll(getenv("RX_SCREENSHOT_AFTER")) * 1000000000LL : 0;
|
|
static const int64_t start = hos_now_ns();
|
|
const int64_t now = hos_now_ns();
|
|
if (now - start < after) return;
|
|
if (last && now - last < screenshot_interval) return;
|
|
last = now;
|
|
wait_submitted(); // (the frame's batches are on the submit thread)
|
|
const size_t stride = ((size_t)im->width * 4 + 255) & ~size_t(255), size = stride * im->height;
|
|
WGPUBufferDescriptor bd{};
|
|
bd.size = size;
|
|
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
|
|
WGPUBuffer buffer = wgpuDeviceCreateBuffer(dev.device, &bd);
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
WGPUTexelCopyTextureInfo src{};
|
|
src.texture = im->tex;
|
|
WGPUTexelCopyBufferInfo dst{};
|
|
dst.buffer = buffer;
|
|
dst.layout.bytesPerRow = (uint32_t)stride;
|
|
dst.layout.rowsPerImage = im->height;
|
|
const WGPUExtent3D extent{im->width, im->height, 1};
|
|
wgpuCommandEncoderCopyTextureToBuffer(e, &src, &dst, &extent);
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(e);
|
|
struct Done {
|
|
bool complete = false;
|
|
WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error;
|
|
} done;
|
|
WGPUBufferMapCallbackInfo mc{};
|
|
mc.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
|
|
auto* d = static_cast<Done*>(u1);
|
|
d->status = status;
|
|
d->complete = true;
|
|
};
|
|
mc.userdata1 = &done;
|
|
wgpuBufferMapAsync(buffer, WGPUMapMode_Read, 0, size, mc);
|
|
while (!done.complete) wgpuDevicePoll(dev.device, true, nullptr);
|
|
if (done.status == WGPUMapAsyncStatus_Success) {
|
|
const auto* px = static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(buffer, 0, size));
|
|
static int n;
|
|
char path[512];
|
|
std::snprintf(path, sizeof path, "%s/frame_%03d.png", dir, n++);
|
|
// the image is drawn with alpha that the display ignores: store it opaque
|
|
std::vector<uint8_t> copy(px, px + size);
|
|
for (size_t i = 3; i < copy.size(); i += 4) copy[i] = 255;
|
|
write_png(path, copy.data(), im->width, im->height, stride);
|
|
hos_log("rx: screenshot %s (frame %lld)", path, (long long)frame_number);
|
|
wgpuBufferUnmap(buffer);
|
|
}
|
|
wgpuBufferRelease(buffer);
|
|
}
|
|
} // namespace
|
|
|
|
thread_local bool rx_side_thread;
|
|
|
|
void count(const char* key, int64_t n) {
|
|
if (!stats_on || rx_side_thread) return;
|
|
// (a slot cache by the literal's address: a map lookup per call was ~2% of the GPU thread)
|
|
struct Slot {
|
|
const char* key;
|
|
int64_t* value;
|
|
uint64_t generation;
|
|
};
|
|
static Slot cache[256];
|
|
Slot& s = cache[((uintptr_t)key >> 3) & 255];
|
|
if (s.key != key || s.generation != counters_generation) s = {key, &counters[key], counters_generation};
|
|
*s.value += n;
|
|
}
|
|
|
|
void log_limited(const char* key, int limit, const char* fmt, ...) {
|
|
if (rx_side_thread) return;
|
|
int& n = log_counts[key];
|
|
if (n >= limit) return;
|
|
n++;
|
|
char buf[1024];
|
|
va_list ap;
|
|
va_start(ap, fmt);
|
|
std::vsnprintf(buf, sizeof buf, fmt, ap);
|
|
va_end(ap);
|
|
hos_log("%s", buf);
|
|
}
|
|
|
|
uint64_t hash_bytes(const void* p, size_t n, uint64_t h) {
|
|
// the seed is mixed first: used as is, (seed ^ first word) made hash(A, 0) == hash(B, 1) for data differing in the
|
|
// low bit of its first word: sampler_for's mode seeds then handed a filtering sampler to an unfilterable texture
|
|
// (and could hand a comparison sampler to a colour one)
|
|
h = (h + 0x9e3779b97f4a7c15ull) * 0xbf58476d1ce4e5b9ull;
|
|
h ^= h >> 31;
|
|
const auto* b = static_cast<const uint8_t*>(p);
|
|
size_t i = 0;
|
|
if (n >= 64) { // four independent lanes (the multiply's latency, not its throughput, bounded the one-lane loop)
|
|
uint64_t l[4] = {h, h ^ 0x9e3779b97f4a7c15ull, h ^ 0xc2b2ae3d27d4eb4full, h ^ 0x165667b19e3779f9ull};
|
|
for (; i + 32 <= n; i += 32)
|
|
for (int k = 0; k < 4; k++) {
|
|
uint64_t v;
|
|
std::memcpy(&v, b + i + 8 * k, 8);
|
|
l[k] = (l[k] ^ v) * 0x100000001b3ull;
|
|
l[k] ^= l[k] >> 29;
|
|
}
|
|
h = l[0];
|
|
for (int k = 1; k < 4; k++) {
|
|
h = (h ^ l[k]) * 0x100000001b3ull;
|
|
h ^= h >> 29;
|
|
}
|
|
}
|
|
for (; i + 8 <= n; i += 8) {
|
|
uint64_t v;
|
|
std::memcpy(&v, b + i, 8);
|
|
h = (h ^ v) * 0x100000001b3ull;
|
|
h ^= h >> 29;
|
|
}
|
|
for (; i < n; i++) h = (h ^ b[i]) * 0x100000001b3ull;
|
|
return h;
|
|
}
|
|
|
|
uint64_t Stream::alloc(uint64_t n, uint64_t align) {
|
|
const uint64_t off = (used + align - 1) & ~(align - 1);
|
|
if (off + n > capacity) return UINT64_MAX;
|
|
used = off + n;
|
|
return off;
|
|
}
|
|
|
|
uint64_t stream_alloc(Stream& s, uint64_t n, uint64_t align) {
|
|
uint64_t off = s.alloc(n, align);
|
|
if (off == UINT64_MAX && n <= s.capacity) {
|
|
flush_batch();
|
|
off = s.alloc(n, align);
|
|
}
|
|
return off;
|
|
}
|
|
|
|
WGPUCommandEncoder encoder() {
|
|
if (!pending_clears.empty()) flush_pending_clears();
|
|
if (!enc) enc = encoding_deferred() ? fake_encoder() : wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
return enc;
|
|
}
|
|
|
|
namespace {
|
|
struct PassResult {
|
|
uint64_t rw_offset, size, gpu_va;
|
|
};
|
|
std::vector<PassResult> pass_results; // storage the open pass's pixel programs write (copies in stream_rw)
|
|
} // namespace
|
|
|
|
/// Whether a pixel program of the open pass writes storage overlapping this range (into its stream_rw copy: the
|
|
/// buffer cache gets it when the pass ends)
|
|
bool storage_written_this_pass(uint64_t gpu_va, uint64_t size) {
|
|
for (const PassResult& r : pass_results)
|
|
if (r.gpu_va < gpu_va + size && r.gpu_va + r.size > gpu_va) return true;
|
|
return false;
|
|
}
|
|
|
|
bool pass_wrote_storage() { return !pass_results.empty(); }
|
|
|
|
void storage_written_in_pass(uint64_t rw_offset, uint64_t size, uint64_t gpu_va) {
|
|
for (const PassResult& r : pass_results)
|
|
if (r.gpu_va == gpu_va && r.rw_offset == rw_offset) return;
|
|
pass_results.push_back({rw_offset, size, gpu_va});
|
|
}
|
|
|
|
void split_long_pass() {
|
|
static const uint32_t limit = [] {
|
|
const char* e = getenv("RX_PASS_SPLIT");
|
|
return e ? (uint32_t)std::atoi(e) : 256u;
|
|
}();
|
|
static uint64_t seen = ~0ull;
|
|
static uint32_t draws = 0;
|
|
if (!limit || !pass) return;
|
|
if (passes_begun != seen) {
|
|
seen = passes_begun;
|
|
draws = 0;
|
|
}
|
|
// (the pass's storage results pending, as in TOTK's G-buffer pass, its biggest: the copy at the split is an early
|
|
// one; the draws after it write the same copy, draw_storage's rw_copies are per batch, and the final end copies
|
|
// it all again. A read-only binding of that memory after the split still breaks the pass at its first write.)
|
|
if (++draws < limit) return;
|
|
end_render_pass("pe_split");
|
|
}
|
|
|
|
void end_render_pass(const char* why) {
|
|
if (!pass) return;
|
|
if (why) count(why);
|
|
rp_end(pass);
|
|
pass = nullptr;
|
|
pass_at = {};
|
|
pass_occlusion = nullptr;
|
|
// what the pass's pixel programs wrote goes to the buffer cache, for the draws and dispatches after it (TOTK's
|
|
// G-buffer shaders count each tile's materials in a buffer the lighting reads)
|
|
if (!pass_results.empty()) {
|
|
std::vector<PassResult> results;
|
|
results.swap(pass_results);
|
|
for (const PassResult& r : results) {
|
|
const BufferRef ref = guest_buffer(r.gpu_va, r.size);
|
|
if (!ref.buffer) continue;
|
|
if (frame_dump_active()) {
|
|
char b[128];
|
|
std::snprintf(b, sizeof b, "pass result %llx+%llx -> buffer cache", (unsigned long long)r.gpu_va,
|
|
(unsigned long long)r.size);
|
|
dump_draw(b);
|
|
}
|
|
ce_copy_buffer_to_buffer(encoder(), stream_rw.buffer, r.rw_offset, ref.buffer, ref.offset,
|
|
(r.size + 3) & ~3ull);
|
|
buffer_cache_gpu_written(ref);
|
|
count("pass_results_cached", (int64_t)r.size);
|
|
}
|
|
}
|
|
}
|
|
|
|
uint64_t pass_count() {
|
|
return passes_begun;
|
|
}
|
|
|
|
const Attachments& current_attachments() {
|
|
return pass_at;
|
|
}
|
|
|
|
WGPURenderPassEncoder render_pass(const Attachments& at) {
|
|
if (pass && pass_at == at) return pass;
|
|
if (pass) count("pe_switch");
|
|
dump_pass(at, "draw");
|
|
std::array<WGPURenderPassColorAttachment, 8> colors{};
|
|
for (uint32_t i = 0; i < 8; i++) colors[i] = color_attachment(at.color[i], WGPULoadOp_Load, {});
|
|
WGPURenderPassDepthStencilAttachment depth{};
|
|
if (at.depth) depth = depth_attachment(at.depth, WGPULoadOp_Load, WGPULoadOp_Load, 0.0f, 0);
|
|
begin_pass(at, colors.data(), &depth);
|
|
return pass;
|
|
}
|
|
|
|
// ---- the submit thread: wgpu turns a command encoder into Vulkan commands at its finish (with the queue writes
|
|
// and the submit, a third of the GPU thread's time), so batches are finished and submitted there, in order, while
|
|
// the GPU thread goes on recording the next ----
|
|
namespace {
|
|
/// Bytes for a buffer (guest memory for the buffer cache, ...), at `src` in its batch's upload_bytes.
|
|
struct Upload {
|
|
WGPUBuffer buffer;
|
|
uint64_t offset, src, size;
|
|
};
|
|
struct Writeback {
|
|
uint64_t rw_offset, size, gpu_va;
|
|
};
|
|
/// GPU results on their way back to guest memory: a staging copy of the written ranges, and what they held before.
|
|
struct Readback {
|
|
WGPUBuffer staging = nullptr;
|
|
uint64_t total = 0, serial = 0; // serial: its batch's
|
|
std::vector<Writeback> ranges;
|
|
std::vector<uint8_t> before; // the ranges' contents as uploaded, concatenated (4-byte aligned each)
|
|
};
|
|
struct Batch {
|
|
WGPUCommandEncoder enc = nullptr;
|
|
uint64_t serial = 0; // flush order (the native NVN layer's fences name one: wait_batch)
|
|
bool deferred = false; // its commands are a recording, replayed into a new encoder here
|
|
Recording* rec = nullptr;
|
|
std::vector<Writeback> writebacks;
|
|
std::vector<Upload> uploads; // before the stream writes and the submit
|
|
std::vector<uint8_t> upload_bytes;
|
|
uint8_t *ro = nullptr, *rw = nullptr;
|
|
uint64_t ro_used = 0, rw_used = 0;
|
|
std::function<void()> after; // after the submit (a present)
|
|
bool present = false;
|
|
std::vector<WGPUBindGroup> group_releases; // after the replay (release_bind_group_later)
|
|
std::vector<WGPUBuffer> buffer_releases;
|
|
std::vector<WGPURenderPipeline> pipeline_releases;
|
|
std::vector<std::function<void(bool)>> afters; // after the submit (true: it was submitted): writebacks' readbacks
|
|
};
|
|
std::vector<Upload> open_uploads;
|
|
std::vector<uint8_t> open_upload_bytes;
|
|
std::mutex upload_pool_mu;
|
|
std::vector<std::vector<uint8_t>> upload_pool; // (upload_bytes given back by run_batch: no regrowing each batch)
|
|
std::vector<WGPUBindGroup> open_group_releases;
|
|
std::vector<WGPUBuffer> open_buffer_releases;
|
|
std::vector<WGPURenderPipeline> open_pipeline_releases;
|
|
std::vector<Writeback> open_writebacks;
|
|
/// Ranges shaders wrote in submitted batches whose results are not in guest memory yet (rx_gpu_writes_pending).
|
|
std::mutex inflight_mu;
|
|
std::vector<Writeback> inflight;
|
|
std::atomic<int> readbacks_pending;
|
|
std::multiset<uint64_t> readback_serials; // the batches of the readbacks not applied yet (under inflight_mu)
|
|
|
|
// Batch serials: the native NVN layer's fences wait for one batch (and the batches before it) to have run on the
|
|
// GPU and its readbacks to be in guest memory, not for everything submitted since (wait_batch)
|
|
std::atomic<uint64_t> batch_serial{0}; // the last batch flushed (flush_batch, under mu)
|
|
std::atomic<uint64_t> batch_serial_run{0}; // the last one the submit thread submitted
|
|
std::atomic<uint64_t> gpu_done_serial{0}; // the last one the GPU finished (its queue's work-done callback)
|
|
std::mutex index_mu;
|
|
struct SerialIndex {
|
|
uint64_t serial = 0, index = 0;
|
|
};
|
|
SerialIndex serial_index[1024]; // a batch's last submission (the queue's index: wgpuDevicePoll waits for one)
|
|
uint64_t last_index = 0; // (the submit thread's)
|
|
|
|
void retire_inflight(const std::vector<Writeback>& done) {
|
|
std::lock_guard lk(inflight_mu);
|
|
for (const Writeback& w : done)
|
|
for (size_t i = 0; i < inflight.size(); i++)
|
|
if (inflight[i].gpu_va == w.gpu_va && inflight[i].size == w.size) {
|
|
inflight[i] = inflight.back();
|
|
inflight.pop_back();
|
|
break;
|
|
}
|
|
}
|
|
std::mutex submit_mu;
|
|
std::condition_variable submit_cv, submitted_cv;
|
|
std::deque<std::unique_ptr<Batch>> submit_queue;
|
|
int submit_busy; // batches popped and not done yet
|
|
int presents_queued; // presents in the queue (frames the GPU thread is ahead by)
|
|
|
|
/// The words a batch's shaders changed in the ranges, written into guest memory (the map callback).
|
|
void apply_readback(Readback* rb, WGPUMapAsyncStatus status) {
|
|
if (status == WGPUMapAsyncStatus_Success) {
|
|
const auto* gpu = static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(rb->staging, 0, rb->total));
|
|
uint64_t pos = 0;
|
|
int64_t words = 0;
|
|
for (const Writeback& w : rb->ranges) {
|
|
const uint64_t n = (w.size + 3) & ~3ull;
|
|
uint64_t avail = 0;
|
|
uint8_t* guest = hos_gpu_ptr(w.gpu_va, &avail);
|
|
if (gpu && guest) words += readback_to_guest(guest, gpu + pos, rb->before.data() + pos, std::min(n, avail) & ~3ull);
|
|
pos += n;
|
|
}
|
|
st_writeback_words += words;
|
|
wgpuBufferUnmap(rb->staging);
|
|
}
|
|
retire_inflight(rb->ranges);
|
|
{
|
|
std::lock_guard lk(inflight_mu);
|
|
if (auto it = readback_serials.find(rb->serial); it != readback_serials.end()) readback_serials.erase(it);
|
|
}
|
|
readbacks_pending--;
|
|
wgpuBufferRelease(rb->staging);
|
|
delete rb;
|
|
}
|
|
|
|
/// After a submitted batch: its writable storage ranges are copied out of stream_rw (before the next batch's writes
|
|
/// replace it) and mapped; apply_readback writes the changes back.
|
|
void start_readback(Batch& b) {
|
|
auto* rb = new Readback();
|
|
rb->serial = b.serial;
|
|
{
|
|
std::lock_guard lk(inflight_mu);
|
|
readback_serials.insert(b.serial);
|
|
}
|
|
rb->ranges = std::move(b.writebacks);
|
|
for (const Writeback& w : rb->ranges) {
|
|
const uint64_t n = (w.size + 3) & ~3ull;
|
|
rb->before.insert(rb->before.end(), b.rw + w.rw_offset, b.rw + w.rw_offset + n);
|
|
rb->total += n;
|
|
}
|
|
WGPUBufferDescriptor d{};
|
|
d.label = sv("rx readback");
|
|
d.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
|
|
d.size = rb->total;
|
|
rb->staging = wgpuDeviceCreateBuffer(dev.device, &d);
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
uint64_t pos = 0;
|
|
for (const Writeback& w : rb->ranges) {
|
|
const uint64_t n = (w.size + 3) & ~3ull;
|
|
wgpuCommandEncoderCopyBufferToBuffer(e, stream_rw.buffer, w.rw_offset, rb->staging, pos, n);
|
|
pos += n;
|
|
}
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
#if RX_WEB
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
#else
|
|
last_index = wgpuQueueSubmitForIndex(dev.queue, 1, &cb);
|
|
#endif
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(e);
|
|
WGPUBufferMapCallbackInfo mc{};
|
|
mc.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
|
|
apply_readback(static_cast<Readback*>(u1), status);
|
|
};
|
|
mc.userdata1 = rb;
|
|
readbacks_pending++;
|
|
wgpuBufferMapAsync(rb->staging, WGPUMapMode_Read, 0, rb->total, mc);
|
|
}
|
|
|
|
/// wgpu turns a command encoder into Vulkan commands at its finish (most of the submit thread: 9 ms a frame in the
|
|
/// rainy open world), so a large batch is replayed into several encoders, split where passes end, and finished on
|
|
/// helper threads at once; the command buffers go to the queue in one submit, in order. RX_FINISH_THREADS=<n>
|
|
/// (default 3; 1: one encoder, as before).
|
|
struct FinishPart {
|
|
Recording* rec = nullptr;
|
|
size_t from = 0, to = 0;
|
|
WGPUCommandBuffer cb = nullptr;
|
|
std::string error;
|
|
GpuTimer* timer = nullptr; // RX_GPU_TIMING
|
|
int64_t ns = 0; // (its replay and finish)
|
|
};
|
|
std::mutex finish_mu;
|
|
std::condition_variable finish_cv, finish_done_cv;
|
|
std::deque<FinishPart*> finish_jobs;
|
|
int finish_left;
|
|
|
|
void finish_part(FinishPart& p) {
|
|
const int64_t t0 = hos_now_ns();
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
if (p.rec && gpu_timing()) p.timer = gpu_timer_take();
|
|
if (p.rec) recording_replay(p.rec, e, p.from, p.to, p.timer);
|
|
gpu_timer_resolve(p.timer, e);
|
|
// an invalid command buffer must not reach the queue (wgpu-native aborts the process on a failed submit)
|
|
scope_push();
|
|
p.cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
p.error = scope_pop();
|
|
wgpuCommandEncoderRelease(e);
|
|
p.ns = hos_now_ns() - t0;
|
|
}
|
|
|
|
void finisher_main() {
|
|
hos_host_thread_register_self("GPU finish");
|
|
hos_thread_boost(1);
|
|
for (;;) {
|
|
FinishPart* p;
|
|
{
|
|
std::unique_lock lk(finish_mu);
|
|
finish_cv.wait(lk, [] { return !finish_jobs.empty(); });
|
|
p = finish_jobs.front();
|
|
finish_jobs.pop_front();
|
|
}
|
|
finish_part(*p);
|
|
std::lock_guard lk(finish_mu);
|
|
if (--finish_left == 0) finish_done_cv.notify_one();
|
|
}
|
|
}
|
|
|
|
uint32_t finish_threads() {
|
|
static const uint32_t n = [] {
|
|
const char* e = getenv("RX_FINISH_THREADS");
|
|
const int v = RX_WEB ? 1 : e ? std::atoi(e) : 3;
|
|
return (uint32_t)std::clamp(v, 1, 8);
|
|
}();
|
|
return n;
|
|
}
|
|
|
|
void run_batch(Batch& b) {
|
|
// (one queue write each: copying them from one written buffer at the start of the batch's first command buffer
|
|
// was slower, renderer CPU +3% and -7% fps in an in-run A/B at the castle, 2026-10-25)
|
|
for (const Upload& u : b.uploads) {
|
|
wgpuQueueWriteBuffer(dev.queue, u.buffer, u.offset, b.upload_bytes.data() + u.src, u.size);
|
|
#if RX_WEB_MT
|
|
serve_gpu_calls_now_if_any();
|
|
#endif
|
|
}
|
|
const int64_t t0 = hos_now_ns();
|
|
bool submitted = false;
|
|
std::vector<FinishPart> parts;
|
|
if (b.deferred) {
|
|
const auto ranges = b.rec ? recording_parts(b.rec, finish_threads(), 4096)
|
|
: std::vector<std::pair<size_t, size_t>>{{0, 0}};
|
|
parts.resize(ranges.size());
|
|
for (size_t i = 0; i < ranges.size(); i++) parts[i] = {b.rec, ranges[i].first, ranges[i].second};
|
|
if (parts.size() > 1) {
|
|
static bool started;
|
|
std::unique_lock lk(finish_mu);
|
|
if (!started) {
|
|
started = true;
|
|
for (uint32_t i = 1; i < finish_threads(); i++) std::thread(finisher_main).detach();
|
|
}
|
|
finish_left = (int)parts.size() - 1;
|
|
for (size_t i = 1; i < parts.size(); i++) finish_jobs.push_back(&parts[i]);
|
|
lk.unlock();
|
|
finish_cv.notify_all();
|
|
}
|
|
finish_part(parts[0]);
|
|
if (parts.size() > 1) {
|
|
std::unique_lock lk(finish_mu);
|
|
finish_done_cv.wait(lk, [] { return finish_left == 0; });
|
|
}
|
|
} else if (b.enc) {
|
|
parts.resize(1);
|
|
scope_push();
|
|
parts[0].cb = wgpuCommandEncoderFinish(b.enc, nullptr);
|
|
parts[0].error = scope_pop();
|
|
wgpuCommandEncoderRelease(b.enc);
|
|
}
|
|
if (!parts.empty()) {
|
|
const int64_t t1 = hos_now_ns();
|
|
st_finish_ns += t1 - t0;
|
|
if (parts.size() > 1) count_parts += (int64_t)parts.size();
|
|
static const char* const part_keys[8] = {"t_part0", "t_part1", "t_part2", "t_part3", "t_part4", "t_part5",
|
|
"t_part6", "t_part7"};
|
|
for (size_t i = 0; i < parts.size() && i < 8; i++) count(part_keys[i], parts[i].ns); // (balance)
|
|
if (static const bool plog = env_flag("RX_PART_LOG"); plog && frame_number % 600 == 0) { // (balance, a frame now and then)
|
|
std::string line;
|
|
for (const FinishPart& p : parts) {
|
|
char b[64];
|
|
std::snprintf(b, sizeof b, " %zu words %.2f ms;", p.to - p.from, p.ns / 1e6);
|
|
line += b;
|
|
}
|
|
hos_log("rx: batch at frame %lld: %zu parts:%s", (long long)frame_number, parts.size(), line.c_str());
|
|
}
|
|
std::string error;
|
|
for (const FinishPart& p : parts)
|
|
if (!p.error.empty() && error.empty()) error = p.error;
|
|
if (error.empty()) {
|
|
if (b.ro_used) wgpuQueueWriteBuffer(dev.queue, stream_ro.buffer, 0, b.ro, (b.ro_used + 3) & ~3ull);
|
|
if (b.rw_used) wgpuQueueWriteBuffer(dev.queue, stream_rw.buffer, 0, b.rw, (b.rw_used + 3) & ~3ull);
|
|
const int64_t t2 = hos_now_ns();
|
|
st_write_ns += t2 - t1;
|
|
WGPUCommandBuffer cbs[8];
|
|
for (size_t i = 0; i < parts.size(); i++) cbs[i] = parts[i].cb;
|
|
#if RX_WEB
|
|
wgpuQueueSubmit(dev.queue, parts.size(), cbs);
|
|
#else
|
|
last_index = wgpuQueueSubmitForIndex(dev.queue, parts.size(), cbs);
|
|
#endif
|
|
st_submit_ns += hos_now_ns() - t2;
|
|
submitted = true;
|
|
if (!b.writebacks.empty() && b.rw) start_readback(b);
|
|
} else {
|
|
static int dropped;
|
|
if (dropped++ < 20) hos_log("rx: batch dropped: %s", error.c_str());
|
|
}
|
|
for (const FinishPart& p : parts) {
|
|
if (p.cb) wgpuCommandBufferRelease(p.cb);
|
|
gpu_timer_read(p.timer);
|
|
}
|
|
}
|
|
for (auto& f : b.afters) f(submitted);
|
|
recording_free(b.rec); // (what it held: the encoders have their own references now)
|
|
b.rec = nullptr;
|
|
for (WGPUBindGroup g : b.group_releases) wgpuBindGroupRelease(g);
|
|
for (WGPUBuffer x : b.buffer_releases) wgpuBufferRelease(x);
|
|
for (WGPURenderPipeline x : b.pipeline_releases) wgpuRenderPipelineRelease(x);
|
|
if (!b.writebacks.empty()) retire_inflight(b.writebacks); // (no readback started)
|
|
if (RX_WEB_MT) { // (copies: flush_batch)
|
|
std::free(b.ro);
|
|
std::free(b.rw);
|
|
} else {
|
|
if (b.ro) give_stream_memory(stream_ro.capacity, b.ro, b.ro_used);
|
|
if (b.rw) give_stream_memory(stream_rw.capacity, b.rw, b.rw_used);
|
|
}
|
|
if (b.upload_bytes.capacity() && b.upload_bytes.capacity() <= (4u << 20)) { // (a load's big ones not kept)
|
|
b.upload_bytes.clear();
|
|
std::lock_guard lk(upload_pool_mu);
|
|
if (upload_pool.size() < 8) upload_pool.push_back(std::move(b.upload_bytes));
|
|
}
|
|
if (b.after) {
|
|
const int64_t t0 = hos_now_ns();
|
|
b.after();
|
|
st_after_ns += hos_now_ns() - t0;
|
|
}
|
|
if (b.serial) {
|
|
{
|
|
std::lock_guard lk(index_mu);
|
|
serial_index[b.serial % 1024] = {b.serial, last_index};
|
|
}
|
|
WGPUQueueWorkDoneCallbackInfo wd{};
|
|
wd.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
wd.callback = [](WGPUQueueWorkDoneStatus, WGPUStringView, void* u1, void*) {
|
|
const uint64_t serial = (uint64_t)(uintptr_t)u1;
|
|
uint64_t cur = gpu_done_serial.load();
|
|
while (cur < serial && !gpu_done_serial.compare_exchange_weak(cur, serial)) {}
|
|
};
|
|
wd.userdata1 = (void*)(uintptr_t)b.serial;
|
|
wgpuQueueOnSubmittedWorkDone(dev.queue, wd);
|
|
}
|
|
wgpuDevicePoll(dev.device, false, nullptr);
|
|
}
|
|
|
|
/// After a frame's present (the submit thread): wait until the GPU has finished the frame RX_GPU_FRAMES_AHEAD back
|
|
/// (default 3; 0: off). Only the batches not yet submitted were capped (presents_queued); what was submitted could
|
|
/// pile up without bound when the GPU is slower than the game: with another game on the GPU, committed memory grew
|
|
/// ~2-3 GB a minute (each submission holds its staging) and so did the latency. A window's swapchain throttled
|
|
/// most of it; headless runs not at all. A fast GPU is never 3 frames behind: no wait.
|
|
void throttle_gpu_frames() {
|
|
#if !RX_WEB
|
|
static const int ahead = getenv("RX_GPU_FRAMES_AHEAD") ? atoi(getenv("RX_GPU_FRAMES_AHEAD")) : 3;
|
|
static std::deque<uint64_t> frames; // (the presents' last submissions, oldest first)
|
|
if (ahead <= 0 || !last_index) return;
|
|
if (frames.empty() || frames.back() != last_index) frames.push_back(last_index);
|
|
if ((int)frames.size() <= ahead) return;
|
|
const WGPUSubmissionIndex i = frames.front();
|
|
frames.pop_front();
|
|
const int64_t t0 = hos_now_ns();
|
|
wgpuDevicePoll(dev.device, true, &i);
|
|
const int64_t waited = hos_now_ns() - t0;
|
|
// (RX_LOG_GPU_AHEAD=1: every 300 frames, how long the submit thread waited and how many submissions ran ahead)
|
|
static const bool log_ahead = env_flag("RX_LOG_GPU_AHEAD");
|
|
static int64_t sum, most, nframes;
|
|
sum += waited, most = std::max(most, waited);
|
|
if (log_ahead && ++nframes % 300 == 0) {
|
|
hos_log("rx: GPU frames ahead: waited %.1f ms in 300 frames (longest %.1f ms); submissions %llu, frame %d back "
|
|
"is %llu", sum / 1e6, most / 1e6, (unsigned long long)last_index, ahead, (unsigned long long)i);
|
|
sum = most = 0;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
void submit_main() {
|
|
hos_host_thread_register_self("GPU submit");
|
|
hos_thread_boost(1);
|
|
for (;;) {
|
|
std::unique_ptr<Batch> b;
|
|
{
|
|
std::unique_lock lk(submit_mu);
|
|
submit_cv.wait(lk, [] { return !submit_queue.empty(); });
|
|
b = std::move(submit_queue.front());
|
|
submit_queue.pop_front();
|
|
submit_busy++;
|
|
}
|
|
run_batch(*b);
|
|
{
|
|
std::lock_guard lk(submit_mu);
|
|
submit_busy--;
|
|
if (b->present) presents_queued--;
|
|
if (b->serial > batch_serial_run) batch_serial_run = b->serial;
|
|
}
|
|
submitted_cv.notify_all();
|
|
if (b->present) throttle_gpu_frames();
|
|
}
|
|
}
|
|
|
|
#if RX_WEB_MT
|
|
} // namespace
|
|
void post_to_gpu_thread(void (*fn)(void*), void* arg); // rx_web_mt.cpp
|
|
/// The threaded web build: one queued batch run on the WebGPU thread (rx_web_mt.cpp), as the native submit thread
|
|
/// runs them. False when none is queued.
|
|
bool gpu_thread_batch() {
|
|
std::unique_ptr<Batch> b;
|
|
{
|
|
std::lock_guard lk(submit_mu);
|
|
if (submit_queue.empty()) return false;
|
|
b = std::move(submit_queue.front());
|
|
submit_queue.pop_front();
|
|
submit_busy++;
|
|
}
|
|
run_batch(*b);
|
|
{
|
|
std::lock_guard lk(submit_mu);
|
|
submit_busy--;
|
|
if (b->present) presents_queued--;
|
|
if (b->serial > batch_serial_run) batch_serial_run = b->serial;
|
|
}
|
|
submitted_cv.notify_all();
|
|
return true;
|
|
}
|
|
namespace {
|
|
#endif
|
|
|
|
void push_batch(std::unique_ptr<Batch> b) {
|
|
if (RX_WEB && !RX_WEB_MT) { // (one thread makes every WebGPU call: the batch is finished and submitted here)
|
|
run_batch(*b);
|
|
if (b->serial > batch_serial_run) batch_serial_run = b->serial;
|
|
return;
|
|
}
|
|
static bool started_thread;
|
|
if (!started_thread && !RX_WEB_MT) { // (the threaded web build: the WebGPU thread takes the batches)
|
|
started_thread = true;
|
|
std::thread(submit_main).detach();
|
|
}
|
|
const int64_t t0 = hos_now_ns();
|
|
std::unique_lock lk(submit_mu);
|
|
// the GPU thread stays at most 2 frames (and 64 batches) ahead of the queue
|
|
// (the threaded web build: at most 8 batches, each holds a copy of the stream bytes it used (flush_batch); the heap is
|
|
// ~570 MB)
|
|
submitted_cv.wait(lk, [] { return submit_queue.size() < (RX_WEB_MT ? 8u : 64u) && presents_queued < 2; });
|
|
st_wait_ns += hos_now_ns() - t0;
|
|
if (b->present) presents_queued++;
|
|
submit_queue.push_back(std::move(b));
|
|
lk.unlock();
|
|
submit_cv.notify_one();
|
|
#if RX_WEB_MT
|
|
// (a job in the WebGPU thread's queue, so the batch runs in order with the calls this thread made before it:
|
|
// reference counts, writes, creations. Run apart, a batch released objects before AddRefs queued ahead of it.)
|
|
post_to_gpu_thread([](void*) { gpu_thread_batch(); }, nullptr);
|
|
#endif
|
|
}
|
|
} // namespace
|
|
|
|
void queue_upload(WGPUBuffer buffer, uint64_t offset, const void* data, uint64_t size) {
|
|
if (!open_upload_bytes.capacity()) {
|
|
std::lock_guard lk(upload_pool_mu);
|
|
if (!upload_pool.empty()) {
|
|
open_upload_bytes.swap(upload_pool.back());
|
|
upload_pool.pop_back();
|
|
}
|
|
}
|
|
const uint64_t src = open_upload_bytes.size();
|
|
const auto* p = static_cast<const uint8_t*>(data);
|
|
open_upload_bytes.insert(open_upload_bytes.end(), p, p + size);
|
|
open_upload_bytes.resize((open_upload_bytes.size() + 3) & ~size_t{3});
|
|
open_uploads.push_back({buffer, offset, src, size});
|
|
}
|
|
|
|
void writeback_after_batch(uint64_t rw_offset, uint64_t size, uint64_t gpu_va) {
|
|
open_writebacks.push_back({rw_offset, size, gpu_va});
|
|
}
|
|
|
|
void wait_submitted() {
|
|
const int64_t t0 = hos_now_ns();
|
|
std::unique_lock lk(submit_mu);
|
|
submitted_cv.wait(lk, [] { return submit_queue.empty() && !submit_busy; });
|
|
spike.wait_ns += hos_now_ns() - t0;
|
|
}
|
|
|
|
FrameSpike spike;
|
|
|
|
void flush_batch() {
|
|
flush_pending_clears();
|
|
if (!enc && open_uploads.empty() && open_afters.empty()) return;
|
|
const int64_t t0 = hos_now_ns();
|
|
count("upload_bytes", (int64_t)(stream_ro.used + stream_rw.used));
|
|
end_render_pass("pe_flush");
|
|
occlusion_flush();
|
|
auto b = std::make_unique<Batch>();
|
|
b->serial = ++batch_serial;
|
|
const bool has_commands = enc != nullptr;
|
|
if (encoding_deferred()) {
|
|
b->deferred = has_commands;
|
|
b->rec = recording_take();
|
|
} else {
|
|
b->enc = enc;
|
|
}
|
|
enc = nullptr;
|
|
b->uploads.swap(open_uploads);
|
|
b->upload_bytes.swap(open_upload_bytes);
|
|
b->writebacks.swap(open_writebacks);
|
|
b->group_releases.swap(open_group_releases);
|
|
b->buffer_releases.swap(open_buffer_releases);
|
|
b->pipeline_releases.swap(open_pipeline_releases);
|
|
b->afters.swap(open_afters);
|
|
if (!b->writebacks.empty()) {
|
|
std::lock_guard lk(inflight_mu);
|
|
inflight.insert(inflight.end(), b->writebacks.begin(), b->writebacks.end());
|
|
}
|
|
if (has_commands) {
|
|
b->ro = stream_ro.cpu;
|
|
b->ro_used = stream_ro.used;
|
|
b->rw = stream_rw.cpu;
|
|
b->rw_used = stream_rw.used;
|
|
if (RX_WEB_MT) {
|
|
// (the threaded web build: the batch takes a copy of what it used, a few MB, and the blocks (48 MB) stay
|
|
// here: a block for each batch in flight ran the web's host heap out during a load)
|
|
auto copy_used = [](const uint8_t* m, uint64_t n) -> uint8_t* {
|
|
if (!n) return nullptr;
|
|
auto* c = static_cast<uint8_t*>(std::malloc((n + 3) & ~3ull));
|
|
std::memcpy(c, m, n);
|
|
return c;
|
|
};
|
|
b->ro = copy_used(stream_ro.cpu, stream_ro.used);
|
|
b->rw = copy_used(stream_rw.cpu, stream_rw.used);
|
|
} else if (RX_WEB) {
|
|
stream_ro.cpu = stream_rw.cpu = nullptr;
|
|
} else {
|
|
stream_ro.cpu = take_stream_memory(stream_ro.capacity);
|
|
stream_rw.cpu = take_stream_memory(stream_rw.capacity);
|
|
}
|
|
}
|
|
stream_ro.used = stream_rw.used = 0;
|
|
push_batch(std::move(b));
|
|
if (!stream_ro.cpu) stream_ro.cpu = take_stream_memory(stream_ro.capacity);
|
|
if (!stream_rw.cpu) stream_rw.cpu = take_stream_memory(stream_rw.capacity);
|
|
buffer_cache_new_batch();
|
|
count("t_flush", hos_now_ns() - t0);
|
|
}
|
|
|
|
/// The window's present (the submit thread): the frame's image blitted into the surface texture.
|
|
void present_pacing() {
|
|
static int64_t first, last, window_start;
|
|
static std::vector<int64_t> gaps;
|
|
const int64_t now = hos_now_ns();
|
|
if (!first) first = window_start = now;
|
|
if (last) gaps.push_back(now - last);
|
|
last = now;
|
|
if (now - window_start < 5000000000LL || gaps.empty()) return;
|
|
const double avg = (double)(now - window_start) / (double)gaps.size();
|
|
const double limit = (avg < 22e6 ? 1e9 / 60 : 1e9 / 30) * 1.5;
|
|
int stutters = 0;
|
|
int64_t longest = 0;
|
|
for (int64_t g : gaps) {
|
|
stutters += g > limit;
|
|
longest = std::max(longest, g);
|
|
}
|
|
if (stutters)
|
|
hos_log("rx: presents at %.0f s: %.1f fps, %d stutters (gaps over %.0f ms), longest %.1f ms", (now - first) / 1e9,
|
|
(double)gaps.size() * 1e9 / (double)(now - window_start), stutters, limit / 1e6, longest / 1e6);
|
|
gaps.clear();
|
|
window_start = now;
|
|
}
|
|
|
|
/// The frame into the window and presented (the submit thread, or with frame interpolation the display thread:
|
|
/// rx_framegen.cpp); `view` is released. One present at a time.
|
|
void surface_present(WGPUTextureView view, uint32_t sw, uint32_t sh, int32_t ww, int32_t wh, bool debug_present) {
|
|
static std::mutex present_mu;
|
|
std::lock_guard present_lk(present_mu);
|
|
if (ww > 0 && wh > 0 && ((uint32_t)ww != surface_w || (uint32_t)wh != surface_h))
|
|
configure_surface((uint32_t)ww, (uint32_t)wh);
|
|
WGPUSurfaceTexture st{};
|
|
wgpuSurfaceGetCurrentTexture(surface, &st);
|
|
if (st.status != WGPUSurfaceGetCurrentTextureStatus_SuccessOptimal &&
|
|
st.status != WGPUSurfaceGetCurrentTextureStatus_SuccessSuboptimal) {
|
|
static int logged;
|
|
if (logged++ < 8) hos_log("rx: no surface texture (status %d)", (int)st.status);
|
|
if (st.texture) wgpuTextureRelease(st.texture);
|
|
configure_surface(surface_w, surface_h);
|
|
if (view) wgpuTextureViewRelease(view);
|
|
return;
|
|
}
|
|
WGPUTextureView target = wgpuTextureCreateView(st.texture, nullptr);
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
WGPUBindGroup bg = nullptr;
|
|
if (view && !debug_present) { // (the frame, its aspect kept, upscaled with FSR 1: rx_present_fx.cpp)
|
|
present_fx_draw(e, view, sw, sh, target, surface_format, surface_w, surface_h, WGPUColor{0, 0, 0, 1});
|
|
} else {
|
|
WGPURenderPassColorAttachment ca{};
|
|
ca.view = target;
|
|
ca.depthSlice = WGPU_DEPTH_SLICE_UNDEFINED;
|
|
ca.loadOp = WGPULoadOp_Clear;
|
|
ca.storeOp = WGPUStoreOp_Store;
|
|
ca.clearValue = debug_present ? WGPUColor{1.0, 0.0, 1.0, 1.0} : WGPUColor{0.15, 0.0, 0.15, 1.0};
|
|
WGPURenderPassDescriptor pd{};
|
|
pd.colorAttachmentCount = 1;
|
|
pd.colorAttachments = &ca;
|
|
WGPURenderPassEncoder p = wgpuCommandEncoderBeginRenderPass(e, &pd);
|
|
if (view) {
|
|
WGPUBindGroupEntry be[2]{};
|
|
be[0].binding = 0;
|
|
be[0].textureView = view;
|
|
be[1].binding = 1;
|
|
be[1].sampler = blit_sampler;
|
|
WGPUBindGroupDescriptor bd{};
|
|
bd.layout = blit_layout;
|
|
bd.entryCount = 2;
|
|
bd.entries = be;
|
|
bg = wgpuDeviceCreateBindGroup(dev.device, &bd);
|
|
wgpuRenderPassEncoderSetPipeline(p, blit_pipeline);
|
|
wgpuRenderPassEncoderSetBindGroup(p, 0, bg, 0, nullptr);
|
|
wgpuRenderPassEncoderDraw(p, 3, 1, 0, 0);
|
|
}
|
|
wgpuRenderPassEncoderEnd(p);
|
|
wgpuRenderPassEncoderRelease(p);
|
|
}
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(e);
|
|
if (bg) wgpuBindGroupRelease(bg);
|
|
#ifdef __EMSCRIPTEN__
|
|
rx_web_present(); // (the page counts frames)
|
|
rx_web_yield(); // (back to the event loop: the browser shows the canvas and runs WebGPU's callbacks)
|
|
#else
|
|
wgpuSurfacePresent(surface);
|
|
#endif
|
|
present_pacing();
|
|
wgpuTextureViewRelease(target);
|
|
wgpuTextureRelease(st.texture);
|
|
if (view) wgpuTextureViewRelease(view);
|
|
}
|
|
|
|
void present_on_submit_thread(WGPUTextureView view, uint32_t sw, uint32_t sh, LONG ww, LONG wh, bool debug_present) {
|
|
// frame interpolation (F7): the frame and the ones made between it and the last go to the display thread
|
|
if (view && !debug_present && present_framegen() && framegen_frame(view, sw, sh, ww, wh)) return;
|
|
surface_present(view, sw, sh, ww, wh, debug_present);
|
|
}
|
|
|
|
/// Queues a present of the frame (after its batches): runs on the submit thread.
|
|
void queue_present(std::function<void()> fn) {
|
|
flush_batch();
|
|
auto b = std::make_unique<Batch>();
|
|
b->after = std::move(fn);
|
|
b->present = true;
|
|
push_batch(std::move(b));
|
|
}
|
|
|
|
/// RX_FLASH_LOG=<dir> (diagnostic): every presented frame is read back (asynchronously, a ring of staging buffers)
|
|
/// and reduced to a 16x9 grid of mean luma on a worker. A frame whose cells jump away from both neighbours while the
|
|
/// neighbours agree is a one-frame flash: logged ("rx: flash ...") and the three frames saved as PNGs (the first 40).
|
|
/// RX_FLASH_DELTA=<0-255>: the cell jump that counts (default 48); RX_FLASH_CELLS: how many cells must jump (default 8:
|
|
/// a small bright thing moving fast, such as a fairy, jumps in 2-5); frames are dropped when the ring is full.
|
|
namespace {
|
|
struct FlashFrame {
|
|
int64_t frame = 0, t_ns = 0;
|
|
float cell[16 * 9]{};
|
|
float mean = 0;
|
|
std::vector<uint8_t> px; // (kept for the saved PNGs)
|
|
uint32_t w = 0, h = 0;
|
|
size_t stride = 0;
|
|
};
|
|
struct FlashSlot {
|
|
WGPUBuffer buf = nullptr;
|
|
size_t size = 0;
|
|
uint32_t w = 0, h = 0;
|
|
size_t stride = 0;
|
|
int64_t frame = 0, t_ns = 0;
|
|
std::atomic<int> state{0}; // 0 free, 1 copying/mapping, 2 mapped (the worker's)
|
|
};
|
|
FlashSlot flash_slots[6];
|
|
std::mutex flash_mu;
|
|
std::condition_variable flash_cv;
|
|
std::deque<int> flash_ready;
|
|
|
|
extern "C" void rx_request_frame_dump(void); // rx_debug.cpp
|
|
extern "C" void rx_request_rdoc_capture(int frames); // rx_debug.cpp
|
|
std::atomic<bool> flash_dark_armed{false}; // (the live input's "!flasharm": RX_FLASH_DARK_AFTER=-1)
|
|
void flash_worker() {
|
|
hos_host_thread_register_self("flash probe");
|
|
static const char* dir = getenv("RX_FLASH_LOG");
|
|
static const float delta = getenv("RX_FLASH_DELTA") ? (float)atof(getenv("RX_FLASH_DELTA")) : 48.0f;
|
|
static const int min_cells = getenv("RX_FLASH_CELLS") ? atoi(getenv("RX_FLASH_CELLS")) : 8;
|
|
static const int64_t t0 = hos_now_ns();
|
|
std::deque<FlashFrame> hist; // the last three frames
|
|
int saved = 0, flashes = 0;
|
|
int64_t frames = 0, last_report = 0;
|
|
for (;;) {
|
|
int i;
|
|
{
|
|
std::unique_lock lk(flash_mu);
|
|
flash_cv.wait(lk, [] { return !flash_ready.empty(); });
|
|
i = flash_ready.front();
|
|
flash_ready.pop_front();
|
|
}
|
|
FlashSlot& s = flash_slots[i];
|
|
FlashFrame f;
|
|
f.frame = s.frame, f.t_ns = s.t_ns, f.w = s.w, f.h = s.h, f.stride = s.stride;
|
|
const auto* p = static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(s.buf, 0, s.size));
|
|
if (p) {
|
|
double sum[16 * 9]{};
|
|
int n[16 * 9]{};
|
|
for (uint32_t y = 0; y < s.h; y += 4)
|
|
for (uint32_t x = 0; x < s.w; x += 4) {
|
|
const uint8_t* q = p + (size_t)y * s.stride + (size_t)x * 4;
|
|
const int c = (int)(y * 9 / s.h) * 16 + (int)(x * 16 / s.w);
|
|
sum[c] += 0.2126 * q[0] + 0.7152 * q[1] + 0.0722 * q[2];
|
|
n[c]++;
|
|
}
|
|
double m = 0;
|
|
for (int c = 0; c < 16 * 9; c++) {
|
|
f.cell[c] = n[c] ? (float)(sum[c] / n[c]) : 0.0f;
|
|
m += f.cell[c];
|
|
}
|
|
f.mean = (float)(m / (16 * 9));
|
|
if (saved < 40) f.px.assign(p, p + s.size);
|
|
}
|
|
wgpuBufferUnmap(s.buf);
|
|
s.state.store(0);
|
|
frames++;
|
|
// RX_FLASH_DARK_DUMP=<ratio> (with RX_FLASH_LOG): a frame darker than ratio x the mean of the last 16 normal
|
|
// frames requests one frame dump (the next frame: a dark flash that lasts a few frames is caught in it)
|
|
{
|
|
static const float dark = getenv("RX_FLASH_DARK_DUMP") ? (float)atof(getenv("RX_FLASH_DARK_DUMP")) : 0.0f;
|
|
static std::deque<float> recent;
|
|
static bool dumped;
|
|
if (dark > 0 && !dumped && recent.size() == 16) {
|
|
float avg = 0;
|
|
for (float v : recent) avg += v;
|
|
avg /= 16;
|
|
static const int64_t after = getenv("RX_FLASH_DARK_AFTER") ? atoll(getenv("RX_FLASH_DARK_AFTER")) : 0;
|
|
if (f.mean < dark * avg && avg >= 35.0f && (after >= 0 ? f.frame >= after : flash_dark_armed.load())) { // (not on loading screens: RX_FLASH_DARK_AFTER=<frame>)
|
|
dumped = true;
|
|
// (RX_FLASH_DARK_RDOC=<frames>: a RenderDoc capture instead, run with RenderDoc's layer)
|
|
static const int rdoc = getenv("RX_FLASH_DARK_RDOC") ? atoi(getenv("RX_FLASH_DARK_RDOC")) : 0;
|
|
if (rdoc > 0) rx_request_rdoc_capture(rdoc);
|
|
else rx_request_frame_dump();
|
|
hos_log("rx: dark frame %lld (mean luma %.0f, recent %.0f): frame dump requested", (long long)f.frame,
|
|
f.mean, avg);
|
|
}
|
|
}
|
|
if (dark > 0 && (recent.size() < 16 || f.mean >= dark * recent.back() * 0.9f)) {
|
|
recent.push_back(f.mean);
|
|
if (recent.size() > 16) recent.pop_front();
|
|
}
|
|
}
|
|
hist.push_back(std::move(f));
|
|
if (hist.size() > 3) hist.pop_front();
|
|
if (hist.size() == 3 && hist[0].frame + 2 == hist[2].frame) { // (consecutive frames only)
|
|
const FlashFrame &a = hist[0], &b = hist[1], &c = hist[2];
|
|
int cells = 0;
|
|
float worst = 0;
|
|
for (int k = 0; k < 16 * 9; k++) {
|
|
const float up = std::min(std::fabs(b.cell[k] - a.cell[k]), std::fabs(b.cell[k] - c.cell[k]));
|
|
const float spike = up - std::fabs(c.cell[k] - a.cell[k]);
|
|
if (spike > delta) cells++;
|
|
worst = std::max(worst, spike);
|
|
}
|
|
if (cells >= min_cells) {
|
|
flashes++;
|
|
hos_log("rx: flash at frame %lld (%.1f s into the probe): %d of 144 cells, jump %.0f; mean luma %.0f -> %.0f -> %.0f",
|
|
(long long)b.frame, (b.t_ns - t0) / 1e9, cells, worst, a.mean, b.mean, c.mean);
|
|
if (dir && saved < 40 && !a.px.empty() && !b.px.empty() && !c.px.empty()) {
|
|
for (const FlashFrame* g : {&a, &b, &c}) {
|
|
std::vector<uint8_t> copy = g->px;
|
|
for (size_t j = 3; j < copy.size(); j += 4) copy[j] = 255;
|
|
char path[512];
|
|
std::snprintf(path, sizeof path, "%s/flash_%06lld_f%lld.png", dir, (long long)b.frame,
|
|
(long long)g->frame);
|
|
write_png(path, copy.data(), g->w, g->h, g->stride);
|
|
}
|
|
saved++;
|
|
}
|
|
}
|
|
}
|
|
if (frames - last_report >= 600) {
|
|
last_report = frames;
|
|
hos_log("rx: flash probe: %lld frames read, %d flashes", (long long)frames, flashes);
|
|
}
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
void flash_probe(Image* im) {
|
|
static const char* dir = getenv("RX_FLASH_LOG");
|
|
if (!dir || !im || (im->format != WGPUTextureFormat_RGBA8Unorm && im->format != WGPUTextureFormat_RGBA8UnormSrgb))
|
|
return;
|
|
static std::once_flag started;
|
|
std::call_once(started, [] { std::thread(flash_worker).detach(); });
|
|
int i = 0;
|
|
while (i < 6 && flash_slots[i].state.load() != 0) i++;
|
|
if (i == 6) return count("flash_probe_dropped");
|
|
FlashSlot& s = flash_slots[i];
|
|
const size_t stride = ((size_t)im->width * 4 + 255) & ~size_t(255), size = stride * im->height;
|
|
if (s.size != size) {
|
|
if (s.buf) wgpuBufferRelease(s.buf);
|
|
WGPUBufferDescriptor bd{};
|
|
bd.size = size;
|
|
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
|
|
s.buf = wgpuDeviceCreateBuffer(dev.device, &bd);
|
|
s.size = size;
|
|
}
|
|
s.w = im->width, s.h = im->height, s.stride = stride, s.frame = frame_number, s.t_ns = hos_now_ns();
|
|
s.state.store(1);
|
|
WGPUTexture tex = im->tex;
|
|
wgpuTextureAddRef(tex);
|
|
// after the frame's batches, on the submit thread (queue order)
|
|
flush_batch();
|
|
auto b = std::make_unique<Batch>();
|
|
b->after = [i, tex] {
|
|
FlashSlot& s = flash_slots[i];
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
WGPUTexelCopyTextureInfo src{};
|
|
src.texture = tex;
|
|
WGPUTexelCopyBufferInfo dst{};
|
|
dst.buffer = s.buf;
|
|
dst.layout.bytesPerRow = (uint32_t)s.stride;
|
|
dst.layout.rowsPerImage = s.h;
|
|
const WGPUExtent3D extent{s.w, s.h, 1};
|
|
wgpuCommandEncoderCopyTextureToBuffer(e, &src, &dst, &extent);
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(e);
|
|
wgpuTextureRelease(tex);
|
|
WGPUBufferMapCallbackInfo mc{};
|
|
mc.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
|
|
const int i = (int)(intptr_t)u1;
|
|
if (status != WGPUMapAsyncStatus_Success) {
|
|
flash_slots[i].state.store(0);
|
|
return;
|
|
}
|
|
flash_slots[i].state.store(2);
|
|
{
|
|
std::lock_guard lk(flash_mu);
|
|
flash_ready.push_back(i);
|
|
}
|
|
flash_cv.notify_one();
|
|
};
|
|
mc.userdata1 = (void*)(intptr_t)i;
|
|
wgpuBufferMapAsync(s.buf, WGPUMapMode_Read, 0, s.size, mc);
|
|
};
|
|
push_batch(std::move(b));
|
|
}
|
|
|
|
// ---- images ----
|
|
namespace {
|
|
std::vector<WGPUTextureView> retired;
|
|
}
|
|
|
|
void retire_view(WGPUTextureView view) {
|
|
retired.push_back(view);
|
|
}
|
|
|
|
bool retired_views_pending() {
|
|
return retired.size() > 256;
|
|
}
|
|
|
|
void release_retired_views() {
|
|
for (WGPUTextureView v : retired) wgpuTextureViewRelease(v);
|
|
retired.clear();
|
|
}
|
|
|
|
const std::vector<WGPUTextureView>& retired_view_list() {
|
|
return retired;
|
|
}
|
|
|
|
void release_buffer_later(WGPUBuffer b) {
|
|
if (encoding_deferred()) open_buffer_releases.push_back(b);
|
|
else wgpuBufferRelease(b);
|
|
}
|
|
|
|
void release_bind_group_later(WGPUBindGroup g) {
|
|
if (encoding_deferred()) open_group_releases.push_back(g);
|
|
else wgpuBindGroupRelease(g);
|
|
}
|
|
|
|
void release_pipeline_later(WGPURenderPipeline p) {
|
|
if (encoding_deferred()) open_pipeline_releases.push_back(p);
|
|
else wgpuRenderPipelineRelease(p);
|
|
}
|
|
|
|
Image::~Image() {
|
|
if (rendered)
|
|
for (auto it = pending_clears.begin(); it != pending_clears.end();)
|
|
it = it->im == this ? pending_clears.erase(it) : std::next(it);
|
|
for (auto& v : views) retire_view(v.second);
|
|
if (attachment) retire_view(attachment);
|
|
if (tex) wgpuTextureRelease(tex);
|
|
}
|
|
|
|
const std::unordered_map<uint64_t, std::unique_ptr<Image>>& all_images() {
|
|
return images;
|
|
}
|
|
|
|
uint64_t images_generation() {
|
|
return images_changed;
|
|
}
|
|
|
|
bool images_changed_in(uint64_t since, uint64_t lo, uint64_t hi) {
|
|
if (images_changed - since >= 256) return true; // (older than the log)
|
|
for (uint64_t g = since + 1; g <= images_changed; g++) {
|
|
const ImageChange& c = image_changes[g & 255];
|
|
if (c.generation != g || (c.guest >= lo && c.guest < hi)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
Image* image_at(uint64_t guest) {
|
|
// a direct-mapped cache in front of the map, valid while no image was added or dropped (the map's lookup was
|
|
// ~4% of the GPU thread: per texture binding, subresource and attachment)
|
|
struct Slot {
|
|
uint64_t guest = ~0ull, generation = 0;
|
|
Image* image = nullptr;
|
|
};
|
|
thread_local Slot slots[1024];
|
|
Slot& s = slots[(guest >> 9 ^ guest >> 19) & 1023];
|
|
if (s.guest == guest && s.generation == images_changed) return s.image;
|
|
auto it = images.find(guest);
|
|
s = {guest, images_changed, it == images.end() ? nullptr : it->second.get()};
|
|
return s.image;
|
|
}
|
|
|
|
namespace {
|
|
uint64_t barrier_epoch = 1; // texture barriers so far (feedback snapshots taken since the last one stay valid)
|
|
} // namespace
|
|
|
|
void texture_barrier(uint32_t method, uint32_t arg) {
|
|
barrier_epoch++;
|
|
count("texture_barriers");
|
|
if (frame_dump_active()) {
|
|
char b[64];
|
|
std::snprintf(b, sizeof b, "barrier %#x = %#x", method, arg);
|
|
dump_draw(b);
|
|
}
|
|
}
|
|
|
|
Image* snapshot_for_feedback(Image& image) {
|
|
if (!image.sampled_copy) {
|
|
auto copy = std::make_unique<Image>();
|
|
copy->width = image.width;
|
|
copy->height = image.height;
|
|
copy->layers = image.layers;
|
|
copy->levels = image.levels;
|
|
copy->pixel_format = image.pixel_format;
|
|
copy->format = image.format;
|
|
copy->dimension = image.dimension;
|
|
WGPUTextureDescriptor d{};
|
|
d.label = sv("rx attachment feedback snapshot");
|
|
d.usage = WGPUTextureUsage_TextureBinding | WGPUTextureUsage_CopyDst;
|
|
d.dimension = image.dimension;
|
|
d.size = {image.width, image.height, image.layers};
|
|
d.format = image.format;
|
|
d.mipLevelCount = image.levels;
|
|
d.sampleCount = 1;
|
|
scope_push();
|
|
copy->tex = wgpuDeviceCreateTexture(dev.device, &d);
|
|
const std::string error = scope_pop();
|
|
if (!error.empty()) {
|
|
log_limited("feedback_create", 8, "rx: attachment feedback snapshot not created: %s", error.c_str());
|
|
return nullptr;
|
|
}
|
|
image.sampled_copy = std::move(copy);
|
|
}
|
|
// Refreshed once per texture barrier: the guest GPU sees its own render target writes through the texture
|
|
// cache only after a barrier, and TOTK puts one before a group of feedback draws (its characters' G-buffer
|
|
// draws read the G-buffer: 34 per frame), not between them. A copy before each draw split the G-buffer pass
|
|
// that many times (each copy ~20 MB). RX_SNAPSHOT_EVERY_DRAW=1: the old behavior.
|
|
static const bool every_draw = env_flag("RX_SNAPSHOT_EVERY_DRAW");
|
|
if (!every_draw && image.snapshot_epoch == barrier_epoch) {
|
|
count("feedback_snapshot_reused");
|
|
return image.sampled_copy.get();
|
|
}
|
|
image.snapshot_epoch = barrier_epoch;
|
|
count("feedback_snapshots");
|
|
end_render_pass("pe_feedback");
|
|
for (uint32_t level = 0; level < image.levels; ++level) {
|
|
WGPUTexelCopyTextureInfo source{}, destination{};
|
|
source.texture = image.tex;
|
|
destination.texture = image.sampled_copy->tex;
|
|
source.mipLevel = destination.mipLevel = level;
|
|
const WGPUExtent3D extent{std::max(1u, image.width >> level), std::max(1u, image.height >> level),
|
|
image.dimension == WGPUTextureDimension_3D
|
|
? std::max(1u, image.layers >> level) : image.layers};
|
|
ce_copy_texture_to_texture(encoder(), &source, &destination, &extent);
|
|
}
|
|
return image.sampled_copy.get();
|
|
}
|
|
|
|
Image* find_render_target(uint64_t gpu_va, uint32_t width, uint32_t height, PixelFormat pf, uint64_t key) {
|
|
const uint64_t guest = key ? key : hos_gpu_translate(gpu_va);
|
|
if (!guest || !width || !height || width > dev.limits.maxTextureDimension2D ||
|
|
height > dev.limits.maxTextureDimension2D)
|
|
return nullptr;
|
|
// (mkw-recomp: a layered or 3D storage image filed here is not a 2D target: nvn_storage_image)
|
|
auto flat = [](const Image* im) { return im->dimension == WGPUTextureDimension_2D && im->layers == 1; };
|
|
if (Image* im = image_at(guest); im && im->width == width && im->height == height && im->pixel_format == pf &&
|
|
flat(im)) {
|
|
im->last_use = frame_number; // (the usual case: the same surface again)
|
|
return im;
|
|
}
|
|
auto it = images.find(guest);
|
|
if (it != images.end()) {
|
|
Image* im = it->second.get();
|
|
if (im->width == width && im->height == height && im->pixel_format == pf && flat(im)) {
|
|
im->last_use = frame_number;
|
|
return im;
|
|
}
|
|
// the memory now holds another surface: the old image steps aside (ending a pass that draws into it), and
|
|
// a spare of this shape takes its place
|
|
bool attached = pass_at.depth == im;
|
|
for (Image* c : pass_at.color) attached |= c == im;
|
|
if (attached) end_render_pass("pe_alias");
|
|
std::unique_ptr<Image> old = std::move(it->second);
|
|
images.erase(it);
|
|
note_image_change(guest);
|
|
if (env_flag("RX_NO_SPARES")) old.reset(); // (debugging: every new surface is a new, cleared texture)
|
|
auto [a, b] = spare_images.equal_range(guest);
|
|
for (auto s = a; s != b; ++s) {
|
|
Image* sp = s->second.get();
|
|
if (sp->width == width && sp->height == height && sp->pixel_format == pf) {
|
|
std::unique_ptr<Image> back = std::move(s->second);
|
|
spare_images.erase(s);
|
|
if (old) spare_images.emplace(guest, std::move(old));
|
|
back->last_use = frame_number;
|
|
Image* raw = back.get();
|
|
images[guest] = std::move(back);
|
|
count("rt_spare_reused");
|
|
return raw;
|
|
}
|
|
}
|
|
if (old) spare_images.emplace(guest, std::move(old));
|
|
}
|
|
WGPUTextureFormat format = wgpu_format(pf);
|
|
// (mkw-recomp) 8-bit SNORM formats cannot be render attachments in WebGPU: half floats hold their [-1, 1] values
|
|
switch (pf) {
|
|
case PixelFormat::A8B8G8R8_SNORM: format = WGPUTextureFormat_RGBA16Float; break;
|
|
case PixelFormat::R8G8_SNORM: format = WGPUTextureFormat_RG16Float; break;
|
|
case PixelFormat::R8_SNORM: format = WGPUTextureFormat_R16Float; break;
|
|
default: break;
|
|
}
|
|
static const bool norm16_rt = env_flag("RX_NORM16_RT");
|
|
if (!norm16_rt || !dev.norm16) switch (pf) {
|
|
case PixelFormat::R16_UNORM:
|
|
case PixelFormat::R16_SNORM: format = WGPUTextureFormat_R16Float; break;
|
|
case PixelFormat::R16G16_UNORM:
|
|
case PixelFormat::R16G16_SNORM: format = WGPUTextureFormat_RG16Float; break;
|
|
case PixelFormat::R16G16B16A16_UNORM:
|
|
case PixelFormat::R16G16B16A16_SNORM: {
|
|
// (float32: exact for every 16-bit value. TOTK's object-id target, which Ultrahand reads the object under the
|
|
// reticle from, is RGBA16 UNORM: as float16 an id came back a few off, the pick named no object and Ultrahand
|
|
// never locked on: SkyboundTotk issue 9. RX_NORM16_HALF=1: float16, as before)
|
|
static const bool half = env_flag("RX_NORM16_HALF");
|
|
format = dev.float32_blendable && !half ? WGPUTextureFormat_RGBA32Float : WGPUTextureFormat_RGBA16Float;
|
|
break;
|
|
}
|
|
default: break;
|
|
}
|
|
if (format == WGPUTextureFormat_Undefined) {
|
|
log_limited("rt_format", 16, "rx: render target format %d has no WebGPU format", (int)pf);
|
|
return nullptr;
|
|
}
|
|
auto im = std::make_unique<Image>();
|
|
im->gpu_va = gpu_va;
|
|
im->guest = guest;
|
|
im->width = width;
|
|
im->height = height;
|
|
im->pixel_format = pf;
|
|
im->format = format;
|
|
im->rendered = true;
|
|
im->last_use = frame_number;
|
|
WGPUTextureDescriptor d{};
|
|
d.usage = WGPUTextureUsage_RenderAttachment | WGPUTextureUsage_TextureBinding | WGPUTextureUsage_CopySrc |
|
|
WGPUTextureUsage_CopyDst;
|
|
// (mkw-recomp) compute programs write NVN2 targets as images: storage textures where the format allows
|
|
if (wgsl_storage_format(format)) d.usage |= WGPUTextureUsage_StorageBinding;
|
|
d.dimension = WGPUTextureDimension_2D;
|
|
d.size = {width, height, 1};
|
|
d.format = format;
|
|
d.mipLevelCount = 1;
|
|
d.sampleCount = 1;
|
|
const WGPUTextureFormat pair = srgb_pair(format);
|
|
if (pair != WGPUTextureFormat_Undefined) {
|
|
d.viewFormatCount = 1;
|
|
d.viewFormats = &pair;
|
|
}
|
|
scope_push();
|
|
im->tex = wgpuDeviceCreateTexture(dev.device, &d);
|
|
im->attachment = wgpuTextureCreateView(im->tex, nullptr);
|
|
if (const std::string error = scope_pop(); !error.empty()) {
|
|
log_limited("rt_create", 16, "rx: render target %ux%u format %d not created: %s", width, height, (int)pf,
|
|
error.c_str());
|
|
return nullptr;
|
|
}
|
|
Image* raw = im.get();
|
|
images[guest] = std::move(im);
|
|
note_image_change(guest);
|
|
static int logged;
|
|
if (logged++ < 64)
|
|
hos_log("rx: render target %ux%u format %d (WebGPU %#x) at GPU %#llx (guest %#llx)", width, height, (int)pf,
|
|
(unsigned)format, (unsigned long long)gpu_va, (unsigned long long)guest);
|
|
return raw;
|
|
}
|
|
|
|
WGPUTextureFormat srgb_pair(WGPUTextureFormat f) {
|
|
switch (f) {
|
|
case WGPUTextureFormat_RGBA8Unorm: return WGPUTextureFormat_RGBA8UnormSrgb;
|
|
case WGPUTextureFormat_RGBA8UnormSrgb: return WGPUTextureFormat_RGBA8Unorm;
|
|
case WGPUTextureFormat_BGRA8Unorm: return WGPUTextureFormat_BGRA8UnormSrgb;
|
|
case WGPUTextureFormat_BGRA8UnormSrgb: return WGPUTextureFormat_BGRA8Unorm;
|
|
case WGPUTextureFormat_BC1RGBAUnorm: return WGPUTextureFormat_BC1RGBAUnormSrgb;
|
|
case WGPUTextureFormat_BC1RGBAUnormSrgb: return WGPUTextureFormat_BC1RGBAUnorm;
|
|
case WGPUTextureFormat_BC2RGBAUnorm: return WGPUTextureFormat_BC2RGBAUnormSrgb;
|
|
case WGPUTextureFormat_BC2RGBAUnormSrgb: return WGPUTextureFormat_BC2RGBAUnorm;
|
|
case WGPUTextureFormat_BC3RGBAUnorm: return WGPUTextureFormat_BC3RGBAUnormSrgb;
|
|
case WGPUTextureFormat_BC3RGBAUnormSrgb: return WGPUTextureFormat_BC3RGBAUnorm;
|
|
case WGPUTextureFormat_BC7RGBAUnorm: return WGPUTextureFormat_BC7RGBAUnormSrgb;
|
|
case WGPUTextureFormat_BC7RGBAUnormSrgb: return WGPUTextureFormat_BC7RGBAUnorm;
|
|
default: return WGPUTextureFormat_Undefined;
|
|
}
|
|
}
|
|
|
|
bool is_depth_format(WGPUTextureFormat f) {
|
|
switch (f) {
|
|
case WGPUTextureFormat_Depth16Unorm:
|
|
case WGPUTextureFormat_Depth24Plus:
|
|
case WGPUTextureFormat_Depth24PlusStencil8:
|
|
case WGPUTextureFormat_Depth32Float:
|
|
case WGPUTextureFormat_Depth32FloatStencil8:
|
|
case WGPUTextureFormat_Stencil8:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool has_stencil(WGPUTextureFormat f) {
|
|
return f == WGPUTextureFormat_Depth24PlusStencil8 || f == WGPUTextureFormat_Depth32FloatStencil8 ||
|
|
f == WGPUTextureFormat_Stencil8;
|
|
}
|
|
|
|
/// (mkw-recomp) WGSL's names of the formats a storage texture can have (core WebGPU and texture-formats-tier1)
|
|
const char* wgsl_storage_format(WGPUTextureFormat f) {
|
|
switch (f) {
|
|
case WGPUTextureFormat_RGBA8Unorm: return "rgba8unorm";
|
|
case WGPUTextureFormat_RGBA8Snorm: return "rgba8snorm";
|
|
case WGPUTextureFormat_RGBA8Uint: return "rgba8uint";
|
|
case WGPUTextureFormat_RGBA8Sint: return "rgba8sint";
|
|
case WGPUTextureFormat_RGBA16Float: return "rgba16float";
|
|
case WGPUTextureFormat_RGBA16Uint: return "rgba16uint";
|
|
case WGPUTextureFormat_RGBA16Sint: return "rgba16sint";
|
|
case WGPUTextureFormat_R32Float: return "r32float";
|
|
case WGPUTextureFormat_R32Uint: return "r32uint";
|
|
case WGPUTextureFormat_R32Sint: return "r32sint";
|
|
case WGPUTextureFormat_RG32Float: return "rg32float";
|
|
case WGPUTextureFormat_RG32Uint: return "rg32uint";
|
|
case WGPUTextureFormat_RG32Sint: return "rg32sint";
|
|
case WGPUTextureFormat_RGBA32Float: return "rgba32float";
|
|
case WGPUTextureFormat_RGBA32Uint: return "rgba32uint";
|
|
case WGPUTextureFormat_RGBA32Sint: return "rgba32sint";
|
|
default: break;
|
|
}
|
|
if (!dev.tier1 && !dev.adapter_formats) return nullptr;
|
|
switch (f) {
|
|
case WGPUTextureFormat_R8Unorm: return "r8unorm";
|
|
case WGPUTextureFormat_R8Uint: return "r8uint";
|
|
case WGPUTextureFormat_R8Sint: return "r8sint";
|
|
case WGPUTextureFormat_RG8Unorm: return "rg8unorm";
|
|
case WGPUTextureFormat_RG8Uint: return "rg8uint";
|
|
case WGPUTextureFormat_RG8Sint: return "rg8sint";
|
|
case WGPUTextureFormat_R16Float: return "r16float";
|
|
case WGPUTextureFormat_R16Uint: return "r16uint";
|
|
case WGPUTextureFormat_R16Sint: return "r16sint";
|
|
case WGPUTextureFormat_RG16Float: return "rg16float";
|
|
case WGPUTextureFormat_RG16Uint: return "rg16uint";
|
|
case WGPUTextureFormat_RG16Sint: return "rg16sint";
|
|
case WGPUTextureFormat_RGB10A2Unorm: return "rgb10a2unorm";
|
|
case WGPUTextureFormat_RG11B10Ufloat: return "rg11b10ufloat";
|
|
default: return nullptr;
|
|
}
|
|
}
|
|
|
|
WGPUTextureFormat wgpu_format(PixelFormat pf) {
|
|
using P = PixelFormat;
|
|
switch (pf) {
|
|
case P::A8B8G8R8_UNORM: return WGPUTextureFormat_RGBA8Unorm;
|
|
case P::A8B8G8R8_SNORM: return WGPUTextureFormat_RGBA8Snorm;
|
|
case P::A8B8G8R8_SINT: return WGPUTextureFormat_RGBA8Sint;
|
|
case P::A8B8G8R8_UINT: return WGPUTextureFormat_RGBA8Uint;
|
|
case P::A8B8G8R8_SRGB: return WGPUTextureFormat_RGBA8UnormSrgb;
|
|
case P::B8G8R8A8_UNORM: return WGPUTextureFormat_BGRA8Unorm;
|
|
case P::B8G8R8A8_SRGB: return WGPUTextureFormat_BGRA8UnormSrgb;
|
|
// 16-bit packed formats: no WebGPU equivalent; drawn as RGBA8 (textures are converted on upload)
|
|
case P::R5G6B5_UNORM:
|
|
case P::B5G6R5_UNORM:
|
|
case P::A1R5G5B5_UNORM:
|
|
case P::A1B5G5R5_UNORM:
|
|
case P::A5B5G5R1_UNORM:
|
|
case P::A4B4G4R4_UNORM:
|
|
case P::G4R4_UNORM:
|
|
return WGPUTextureFormat_RGBA8Unorm;
|
|
case P::A2B10G10R10_UNORM: return WGPUTextureFormat_RGB10A2Unorm;
|
|
case P::A2B10G10R10_UINT: return WGPUTextureFormat_RGB10A2Uint;
|
|
case P::A2R10G10B10_UNORM: return WGPUTextureFormat_RGB10A2Unorm;
|
|
case P::R8_UNORM: return WGPUTextureFormat_R8Unorm;
|
|
case P::R8_SNORM: return WGPUTextureFormat_R8Snorm;
|
|
case P::R8_SINT: return WGPUTextureFormat_R8Sint;
|
|
case P::R8_UINT: return WGPUTextureFormat_R8Uint;
|
|
case P::R8G8_UNORM: return WGPUTextureFormat_RG8Unorm;
|
|
case P::R8G8_SNORM: return WGPUTextureFormat_RG8Snorm;
|
|
case P::R8G8_SINT: return WGPUTextureFormat_RG8Sint;
|
|
case P::R8G8_UINT: return WGPUTextureFormat_RG8Uint;
|
|
case P::R16G16B16A16_FLOAT: return WGPUTextureFormat_RGBA16Float;
|
|
case P::R16G16B16X16_FLOAT: return WGPUTextureFormat_RGBA16Float;
|
|
case P::R16G16B16A16_UNORM: return dev.norm16 ? WGPUTextureFormat_RGBA16Unorm : WGPUTextureFormat_RGBA16Float;
|
|
case P::R16G16B16A16_SNORM: return dev.norm16 ? WGPUTextureFormat_RGBA16Snorm : WGPUTextureFormat_RGBA16Float;
|
|
case P::R16G16B16A16_SINT: return WGPUTextureFormat_RGBA16Sint;
|
|
case P::R16G16B16A16_UINT: return WGPUTextureFormat_RGBA16Uint;
|
|
case P::B10G11R11_FLOAT: return WGPUTextureFormat_RG11B10Ufloat;
|
|
case P::E5B9G9R9_FLOAT: return WGPUTextureFormat_RGB9E5Ufloat;
|
|
case P::R32G32B32A32_UINT: return WGPUTextureFormat_RGBA32Uint;
|
|
case P::R32G32B32A32_FLOAT: return WGPUTextureFormat_RGBA32Float;
|
|
case P::R32G32B32A32_SINT: return WGPUTextureFormat_RGBA32Sint;
|
|
case P::R32G32B32_FLOAT: return WGPUTextureFormat_RGBA32Float; // converted on upload
|
|
case P::R32G32_FLOAT: return WGPUTextureFormat_RG32Float;
|
|
case P::R32G32_SINT: return WGPUTextureFormat_RG32Sint;
|
|
case P::R32G32_UINT: return WGPUTextureFormat_RG32Uint;
|
|
case P::R32_FLOAT: return WGPUTextureFormat_R32Float;
|
|
case P::R32_UINT: return WGPUTextureFormat_R32Uint;
|
|
case P::R32_SINT: return WGPUTextureFormat_R32Sint;
|
|
case P::R16_FLOAT: return WGPUTextureFormat_R16Float;
|
|
case P::R16_UNORM: return dev.norm16 ? WGPUTextureFormat_R16Unorm : WGPUTextureFormat_R16Float;
|
|
case P::R16_SNORM: return dev.norm16 ? WGPUTextureFormat_R16Snorm : WGPUTextureFormat_R16Float;
|
|
case P::R16_UINT: return WGPUTextureFormat_R16Uint;
|
|
case P::R16_SINT: return WGPUTextureFormat_R16Sint;
|
|
case P::R16G16_UNORM: return dev.norm16 ? WGPUTextureFormat_RG16Unorm : WGPUTextureFormat_RG16Float;
|
|
case P::R16G16_FLOAT: return WGPUTextureFormat_RG16Float;
|
|
case P::R16G16_UINT: return WGPUTextureFormat_RG16Uint;
|
|
case P::R16G16_SINT: return WGPUTextureFormat_RG16Sint;
|
|
case P::R16G16_SNORM: return dev.norm16 ? WGPUTextureFormat_RG16Snorm : WGPUTextureFormat_RG16Float;
|
|
case P::BC1_RGBA_UNORM: return dev.bc ? WGPUTextureFormat_BC1RGBAUnorm : WGPUTextureFormat_RGBA8Unorm;
|
|
case P::BC1_RGBA_SRGB: return dev.bc ? WGPUTextureFormat_BC1RGBAUnormSrgb : WGPUTextureFormat_RGBA8UnormSrgb;
|
|
case P::BC2_UNORM: return dev.bc ? WGPUTextureFormat_BC2RGBAUnorm : WGPUTextureFormat_RGBA8Unorm;
|
|
case P::BC2_SRGB: return dev.bc ? WGPUTextureFormat_BC2RGBAUnormSrgb : WGPUTextureFormat_RGBA8UnormSrgb;
|
|
case P::BC3_UNORM: return dev.bc ? WGPUTextureFormat_BC3RGBAUnorm : WGPUTextureFormat_RGBA8Unorm;
|
|
case P::BC3_SRGB: return dev.bc ? WGPUTextureFormat_BC3RGBAUnormSrgb : WGPUTextureFormat_RGBA8UnormSrgb;
|
|
case P::BC4_UNORM: return dev.bc ? WGPUTextureFormat_BC4RUnorm : WGPUTextureFormat_R8Unorm;
|
|
case P::BC4_SNORM: return dev.bc ? WGPUTextureFormat_BC4RSnorm : WGPUTextureFormat_R8Snorm;
|
|
case P::BC5_UNORM: return dev.bc ? WGPUTextureFormat_BC5RGUnorm : WGPUTextureFormat_RG8Unorm;
|
|
case P::BC5_SNORM: return dev.bc ? WGPUTextureFormat_BC5RGSnorm : WGPUTextureFormat_RG8Snorm;
|
|
case P::BC7_UNORM: return dev.bc ? WGPUTextureFormat_BC7RGBAUnorm : WGPUTextureFormat_RGBA8Unorm;
|
|
case P::BC7_SRGB: return dev.bc ? WGPUTextureFormat_BC7RGBAUnormSrgb : WGPUTextureFormat_RGBA8UnormSrgb;
|
|
case P::BC6H_UFLOAT: return dev.bc ? WGPUTextureFormat_BC6HRGBUfloat : WGPUTextureFormat_RGBA16Float;
|
|
case P::BC6H_SFLOAT: return dev.bc ? WGPUTextureFormat_BC6HRGBFloat : WGPUTextureFormat_RGBA16Float;
|
|
case P::D32_FLOAT: return WGPUTextureFormat_Depth32Float;
|
|
case P::D16_UNORM: return WGPUTextureFormat_Depth16Unorm;
|
|
case P::X8_D24_UNORM: return WGPUTextureFormat_Depth24Plus;
|
|
case P::S8_UINT: return WGPUTextureFormat_Stencil8;
|
|
case P::D24_UNORM_S8_UINT:
|
|
case P::S8_UINT_D24_UNORM:
|
|
return WGPUTextureFormat_Depth24PlusStencil8;
|
|
case P::D32_FLOAT_S8_UINT:
|
|
return dev.depth32_stencil8 ? WGPUTextureFormat_Depth32FloatStencil8 : WGPUTextureFormat_Depth24PlusStencil8;
|
|
default:
|
|
if (VideoCore::Surface::IsPixelFormatASTC(pf)) // decoded on upload
|
|
return VideoCore::Surface::IsPixelFormatSRGB(pf) ? WGPUTextureFormat_RGBA8UnormSrgb
|
|
: WGPUTextureFormat_RGBA8Unorm;
|
|
return WGPUTextureFormat_Undefined;
|
|
}
|
|
}
|
|
|
|
// ---- clears ----
|
|
namespace {
|
|
|
|
/// A clear of part of a target (the scissor), of some components only, or of some stencil bits: a triangle over the
|
|
/// target with the clear value, drawn with that scissor and write mask (suyu: BlitImageHelper::ClearColor /
|
|
/// ClearDepthStencil). One pipeline per format, mask and value (TOTK uses a handful).
|
|
WGPURenderPipeline partial_clear_pipeline(WGPUTextureFormat color_format, uint32_t write_mask, const float* color,
|
|
WGPUTextureFormat depth_format, bool depth, bool stencil, float depth_value,
|
|
uint32_t stencil_write_mask) {
|
|
uint32_t cb[4]{}, db = 0;
|
|
if (color) std::memcpy(cb, color, sizeof cb);
|
|
std::memcpy(&db, &depth_value, 4);
|
|
char key[160];
|
|
std::snprintf(key, sizeof key, "%u/%u/%08x%08x%08x%08x/%u/%d%d/%08x/%u", (unsigned)color_format, write_mask, cb[0],
|
|
cb[1], cb[2], cb[3], (unsigned)depth_format, (int)depth, (int)stencil, db, stencil_write_mask);
|
|
static std::unordered_map<std::string, WGPURenderPipeline> cache;
|
|
if (auto it = cache.find(key); it != cache.end()) return it->second;
|
|
std::string wgsl = "@vertex fn vs(@builtin(vertex_index) i: u32) -> @builtin(position) vec4<f32> {\n"
|
|
" let p = vec2<f32>(f32((i << 1u) & 2u), f32(i & 2u));\n"
|
|
" return vec4<f32>(p * 2.0 - 1.0, 0.0, 1.0);\n}\n";
|
|
char b[320];
|
|
if (color_format != WGPUTextureFormat_Undefined)
|
|
std::snprintf(b, sizeof b,
|
|
"@fragment fn fs() -> @location(0) vec4<f32> { return vec4<f32>(bitcast<f32>(%uu), "
|
|
"bitcast<f32>(%uu), bitcast<f32>(%uu), bitcast<f32>(%uu)); }\n", cb[0], cb[1], cb[2], cb[3]);
|
|
else
|
|
std::snprintf(b, sizeof b, "@fragment fn fs() -> @builtin(frag_depth) f32 { return bitcast<f32>(%uu); }\n",
|
|
db);
|
|
wgsl += b;
|
|
WGPUShaderSourceWGSL src{};
|
|
src.chain.sType = WGPUSType_ShaderSourceWGSL;
|
|
src.code = WGPUStringView{wgsl.data(), wgsl.size()};
|
|
WGPUShaderModuleDescriptor md{};
|
|
md.nextInChain = &src.chain;
|
|
scope_push();
|
|
WGPUShaderModule m = wgpuDeviceCreateShaderModule(dev.device, &md);
|
|
WGPUColorTargetState t{};
|
|
t.format = color_format;
|
|
t.writeMask = write_mask;
|
|
WGPUFragmentState fs{};
|
|
fs.module = m;
|
|
fs.entryPoint = sv("fs");
|
|
fs.targetCount = color_format != WGPUTextureFormat_Undefined ? 1 : 0;
|
|
fs.targets = &t;
|
|
WGPUDepthStencilState ds{};
|
|
WGPURenderPipelineDescriptor d{};
|
|
d.vertex.module = m;
|
|
d.vertex.entryPoint = sv("vs");
|
|
d.primitive.topology = WGPUPrimitiveTopology_TriangleList;
|
|
d.multisample.count = 1;
|
|
d.multisample.mask = 0xFFFFFFFF;
|
|
d.fragment = &fs;
|
|
if (depth_format != WGPUTextureFormat_Undefined) {
|
|
const bool st = stencil && has_stencil(depth_format);
|
|
ds.format = depth_format;
|
|
ds.depthWriteEnabled = depth ? WGPUOptionalBool_True : WGPUOptionalBool_False;
|
|
ds.depthCompare = WGPUCompareFunction_Always;
|
|
ds.stencilFront = ds.stencilBack = {WGPUCompareFunction_Always, WGPUStencilOperation_Keep,
|
|
WGPUStencilOperation_Keep,
|
|
st ? WGPUStencilOperation_Replace : WGPUStencilOperation_Keep};
|
|
ds.stencilReadMask = st ? 0xFF : 0;
|
|
ds.stencilWriteMask = st ? stencil_write_mask : 0;
|
|
d.depthStencil = &ds;
|
|
}
|
|
WGPURenderPipeline p = wgpuDeviceCreateRenderPipeline(dev.device, &d);
|
|
const std::string error = scope_pop();
|
|
wgpuShaderModuleRelease(m);
|
|
if (!error.empty()) {
|
|
log_limited("partial_clear", 8, "rx: partial clear pipeline not created: %s", error.c_str());
|
|
if (p) wgpuRenderPipelineRelease(p);
|
|
p = nullptr;
|
|
}
|
|
cache[key] = p;
|
|
return p;
|
|
}
|
|
|
|
/// The rectangle a clear covers in a w x h target: the scissor when the clear uses it (suyu's GetScissorState).
|
|
/// False when it is empty.
|
|
bool clear_rect(const Regs& r, uint32_t w, uint32_t h, uint32_t rect[4]) {
|
|
uint32_t x0 = 0, y0 = 0, x1 = w, y1 = h;
|
|
if (r.clear_control.use_scissor && r.scissor_test[0].enable) {
|
|
const auto& s = r.scissor_test[0];
|
|
x0 = std::min<uint32_t>(s.min_x, w);
|
|
y0 = std::min<uint32_t>(s.min_y, h);
|
|
x1 = std::min<uint32_t>(s.max_x, w);
|
|
y1 = std::min<uint32_t>(s.max_y, h);
|
|
}
|
|
rect[0] = x0;
|
|
rect[1] = y0;
|
|
rect[2] = x1 > x0 ? x1 - x0 : 0;
|
|
rect[3] = y1 > y0 ? y1 - y0 : 0;
|
|
return rect[2] && rect[3];
|
|
}
|
|
|
|
void draw_partial_clear(WGPURenderPipeline p, const uint32_t rect[4], uint32_t stencil_ref) {
|
|
rp_set_pipeline(pass, p);
|
|
rp_set_scissor(pass, rect[0], rect[1], rect[2], rect[3]);
|
|
rp_set_stencil_reference(pass, stencil_ref);
|
|
rp_draw(pass, 3, 1, 0, 0);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
void clear(Channel& ch, const Regs& r) {
|
|
const auto& cs = r.clear_surface;
|
|
// A target past rt_control.count is not bound, and clearing it does nothing (suyu: FindColorBuffer gives no view
|
|
// for index >= rt_control.count). TOTK clears targets 2 and 3 of its half-resolution lighting pass with only
|
|
// target 0 enabled, right before the full-resolution composite reads them: clearing them took the cave's ambient
|
|
// light off Link's skin (left orange by the subsurface term). RX_CLEAR_ALL_RTS=1: the old behavior.
|
|
static const bool all_rts = env_flag("RX_CLEAR_ALL_RTS");
|
|
const bool any_color = cs.R || cs.G || cs.B || cs.A;
|
|
const bool color = any_color && (all_rts || cs.RT < r.rt_control.count);
|
|
if (any_color && !color) count("clear_unbound_rt");
|
|
const bool depth = cs.Z, stencil = cs.S;
|
|
bool forced = false;
|
|
if (!ch.render_on) { // RX_COND_FORCE=<guest hex | WxH>,...: clears of those targets run anyway (debugging)
|
|
static std::vector<std::pair<uint64_t, uint64_t>> force; // (guest, 0) or (0, w << 32 | h)
|
|
static bool parsed;
|
|
if (!parsed) {
|
|
parsed = true;
|
|
if (const char* s = getenv("RX_COND_FORCE"))
|
|
for (const char* p = s; *p;) {
|
|
char* end;
|
|
const uint64_t v = strtoull(p, &end, 16);
|
|
if (end == p) break;
|
|
if (*end == 'x') { // a size: decimal
|
|
char* e2;
|
|
const uint64_t w = strtoull(p, nullptr, 10), h = strtoull(end + 1, &e2, 10);
|
|
force.push_back({0, w << 32 | h});
|
|
end = e2;
|
|
} else {
|
|
force.push_back({v, 0});
|
|
}
|
|
p = *end == ',' ? end + 1 : end;
|
|
}
|
|
}
|
|
const Image* rt = cs.RT < 8 && (cs.R || cs.G || cs.B || cs.A) ? rt_image(r, cs.RT) : nullptr;
|
|
const Image* z = cs.Z || cs.S ? zeta_image(r) : nullptr;
|
|
for (const auto& [g, wh] : force)
|
|
for (const Image* im : {rt, z})
|
|
if (im && (g ? im->guest == g : ((uint64_t)im->width << 32 | im->height) == wh)) forced = true;
|
|
}
|
|
if (!ch.render_on && !forced) {
|
|
count("clear_conditional_off");
|
|
if (static const uint64_t watch = getenv("RX_WATCH_RT") ? strtoull(getenv("RX_WATCH_RT"), nullptr, 16) : 0;
|
|
watch && cs.RT < 8 && (uint32_t)r.rt[cs.RT].Address() == (uint32_t)watch) {
|
|
static int logged;
|
|
uint64_t v[4]{};
|
|
gpu_read(r.render_enable.Address(), v, sizeof v);
|
|
if (logged++ < 200)
|
|
hos_log("rx: watch %08llx: clear skipped (render enable %u at %llx: %llx / %llx) frame %lld",
|
|
(unsigned long long)watch, (unsigned)r.render_enable.mode,
|
|
(unsigned long long)r.render_enable.Address(), (unsigned long long)v[0],
|
|
(unsigned long long)v[2], (long long)frame_number);
|
|
}
|
|
if (frame_dump_active()) {
|
|
uint64_t v[4]{};
|
|
gpu_read(r.render_enable.Address(), v, sizeof v);
|
|
char b[320];
|
|
const Image* rt = cs.RT < 8 ? rt_image(r, cs.RT) : nullptr;
|
|
const Image* z = zeta_image(r);
|
|
std::snprintf(b, sizeof b, "clear skipped (render enable %u at %llx: %llx %llx / %llx %llx): raw %#x rt %u %08llx "
|
|
"%ux%u zeta %08llx", (unsigned)r.render_enable.mode, (unsigned long long)r.render_enable.Address(),
|
|
(unsigned long long)v[0], (unsigned long long)v[1], (unsigned long long)v[2],
|
|
(unsigned long long)v[3], cs.raw, (unsigned)cs.RT,
|
|
(unsigned long long)(rt ? rt->guest : 0), rt ? rt->width : 0, rt ? rt->height : 0,
|
|
(unsigned long long)(z ? z->guest : 0));
|
|
dump_draw(b);
|
|
}
|
|
return;
|
|
}
|
|
st_clears++;
|
|
static const bool whole = env_flag("RX_WHOLE_CLEARS"); // (debugging: the old behavior, every clear whole)
|
|
if (color && cs.RT < 8) {
|
|
if (Image* im = rt_image(r, cs.RT); im && !whole) {
|
|
uint32_t rect[4];
|
|
if (!clear_rect(r, im->width, im->height, rect)) {
|
|
count("clear_empty");
|
|
goto depth_part;
|
|
}
|
|
const bool part = rect[0] || rect[1] || rect[2] < im->width || rect[3] < im->height;
|
|
const bool all = cs.R && cs.G && cs.B && cs.A;
|
|
if ((part || !all) && !VideoCore::Surface::IsPixelFormatInteger(im->pixel_format)) {
|
|
const uint32_t mask = (cs.R ? WGPUColorWriteMask_Red : 0) | (cs.G ? WGPUColorWriteMask_Green : 0) |
|
|
(cs.B ? WGPUColorWriteMask_Blue : 0) | (cs.A ? WGPUColorWriteMask_Alpha : 0);
|
|
if (WGPURenderPipeline p = partial_clear_pipeline(im->format, mask, r.clear_color.data(),
|
|
WGPUTextureFormat_Undefined, false, false, 0, 0)) {
|
|
Attachments at;
|
|
at.color[0] = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
std::array<WGPURenderPassColorAttachment, 8> c{};
|
|
c[0] = color_attachment(im, WGPULoadOp_Load, {});
|
|
dump_pass(at, "clear");
|
|
if (frame_dump_active()) {
|
|
char b[200];
|
|
std::snprintf(b, sizeof b, "clear color %g %g %g %g (mask %#x) rect %u,%u %ux%u",
|
|
r.clear_color[0], r.clear_color[1], r.clear_color[2], r.clear_color[3], cs.raw,
|
|
rect[0], rect[1], rect[2], rect[3]);
|
|
dump_draw(b);
|
|
}
|
|
begin_pass(at, c.data(), nullptr);
|
|
draw_partial_clear(p, rect, 0);
|
|
end_render_pass("pe_clear");
|
|
count("clear_partial");
|
|
goto depth_part;
|
|
}
|
|
} else if (part || !all) {
|
|
log_limited("clear_int_part", 4, "rx: partial clear of an integer target (%#x): cleared whole",
|
|
cs.raw);
|
|
}
|
|
}
|
|
}
|
|
if (color && cs.RT < 8) {
|
|
if (Image* im = rt_image(r, cs.RT)) {
|
|
Attachments at;
|
|
at.color[0] = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
if (clears_deferred()) {
|
|
PendingClear p{};
|
|
p.im = im;
|
|
p.color = true;
|
|
p.value = {r.clear_color[0], r.clear_color[1], r.clear_color[2], r.clear_color[3]};
|
|
add_pending_clear(p);
|
|
goto depth_part;
|
|
}
|
|
std::array<WGPURenderPassColorAttachment, 8> c{};
|
|
c[0] = color_attachment(im, WGPULoadOp_Clear,
|
|
{r.clear_color[0], r.clear_color[1], r.clear_color[2], r.clear_color[3]});
|
|
dump_pass(at, "clear");
|
|
if (frame_dump_active()) {
|
|
char b[256];
|
|
uint64_t v[3]{};
|
|
if (r.render_enable.mode != Regs::RenderEnable::Mode::True) gpu_read(r.render_enable.Address(), v, sizeof v);
|
|
std::snprintf(b, sizeof b, "clear color %g %g %g %g (mask %#x) render %u at %llx: %llx / %llx",
|
|
r.clear_color[0], r.clear_color[1], r.clear_color[2], r.clear_color[3], cs.raw,
|
|
(unsigned)r.render_enable.mode, (unsigned long long)r.render_enable.Address(),
|
|
(unsigned long long)v[0], (unsigned long long)v[2]);
|
|
dump_draw(b);
|
|
}
|
|
begin_pass(at, c.data(), nullptr);
|
|
end_render_pass("pe_clear");
|
|
}
|
|
}
|
|
depth_part:
|
|
if ((depth || stencil) && !whole) {
|
|
if (Image* im = zeta_image(r)) {
|
|
uint32_t rect[4];
|
|
if (!clear_rect(r, im->width, im->height, rect)) {
|
|
count("clear_empty");
|
|
return;
|
|
}
|
|
const bool part = rect[0] || rect[1] || rect[2] < im->width || rect[3] < im->height;
|
|
// (suyu: a stencil write mask other than 0xFF, or 0, clears only those bits)
|
|
const uint32_t smask = r.stencil_front_mask & 0xFF;
|
|
const bool stencil_part = stencil && has_stencil(im->format) && smask != 0xFF && smask != 0;
|
|
if (part || stencil_part) {
|
|
const bool has_d = im->format != WGPUTextureFormat_Stencil8;
|
|
if (WGPURenderPipeline p = partial_clear_pipeline(WGPUTextureFormat_Undefined, 0, nullptr, im->format,
|
|
depth && has_d, stencil,
|
|
depth_clear_value(im, r.clear_depth),
|
|
stencil_part ? smask : 0xFF)) {
|
|
Attachments at;
|
|
at.depth = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
WGPURenderPassDepthStencilAttachment d =
|
|
depth_attachment(im, WGPULoadOp_Load, WGPULoadOp_Load, 0.0f, 0);
|
|
dump_pass(at, "clear");
|
|
if (frame_dump_active()) {
|
|
char b[200];
|
|
std::snprintf(b, sizeof b, "clear depth %g (%d) stencil %u (%d) mask %#x rect %u,%u %ux%u",
|
|
r.clear_depth, (int)depth, r.clear_stencil & 0xFF, (int)stencil, smask, rect[0],
|
|
rect[1], rect[2], rect[3]);
|
|
dump_draw(b);
|
|
}
|
|
begin_pass(at, nullptr, &d);
|
|
draw_partial_clear(p, rect, r.clear_stencil & 0xFF);
|
|
end_render_pass("pe_clear");
|
|
count("clear_partial");
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (depth || stencil) {
|
|
if (Image* im = zeta_image(r)) {
|
|
Attachments at;
|
|
at.depth = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
if (clears_deferred()) {
|
|
PendingClear p{};
|
|
p.im = im;
|
|
p.depth = depth && im->format != WGPUTextureFormat_Stencil8;
|
|
p.stencil = stencil && has_stencil(im->format);
|
|
p.depth_value = depth_clear_value(im, r.clear_depth);
|
|
p.stencil_value = r.clear_stencil & 0xFF;
|
|
if (p.depth || p.stencil) add_pending_clear(p);
|
|
return;
|
|
}
|
|
WGPURenderPassDepthStencilAttachment d =
|
|
depth_attachment(im, depth ? WGPULoadOp_Clear : WGPULoadOp_Load,
|
|
stencil ? WGPULoadOp_Clear : WGPULoadOp_Load, depth_clear_value(im, r.clear_depth),
|
|
r.clear_stencil & 0xFF);
|
|
dump_pass(at, "clear");
|
|
if (frame_dump_active()) {
|
|
char b[256];
|
|
std::snprintf(b, sizeof b, "clear depth %g (%d) stencil %u (%d) raw %#x zeta %llx %ux%u fmt %#x "
|
|
"control %#x zwrite %u ro %#x scissor %u %u..%u %u..%u render %u/%u at %llx", r.clear_depth, (int)depth,
|
|
r.clear_stencil & 0xFF, (int)stencil, cs.raw, (unsigned long long)r.zeta.Address(),
|
|
r.zeta_size.width, r.zeta_size.height, (unsigned)r.zeta.format,
|
|
*(const uint32_t*)&r.clear_control, (unsigned)r.depth_write_enabled,
|
|
*(const uint32_t*)&r.zeta_read_only, (unsigned)r.scissor_test[0].enable,
|
|
(unsigned)r.scissor_test[0].min_x, (unsigned)r.scissor_test[0].max_x,
|
|
(unsigned)r.scissor_test[0].min_y, (unsigned)r.scissor_test[0].max_y,
|
|
(unsigned)r.render_enable.mode, (unsigned)r.render_enable_override,
|
|
(unsigned long long)r.render_enable.Address());
|
|
dump_draw(b);
|
|
}
|
|
begin_pass(at, nullptr, &d);
|
|
end_render_pass("pe_clear");
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- the native NVN layer: render targets and clears ----
|
|
Image* nvn_target(const nvn_tex* t, const nvn_view* v) {
|
|
if (!t || !t->gpu) return nullptr;
|
|
const nvn_texdesc& d = t->d;
|
|
const unsigned level = v && v->set_levels ? v->base_level : 0;
|
|
const unsigned layer = v && v->set_layers ? v->min_layer : 0;
|
|
const int format = v && v->set_format ? v->format : d.format;
|
|
const PixelFormat pf = nvn_pixel_format(format);
|
|
if (pf == PixelFormat::Invalid) {
|
|
log_limited("nvn_rt_format", 16, "rx: NVN render target format %#x has no pixel format", format);
|
|
return nullptr;
|
|
}
|
|
const bool one_row = d.target == NVN_TEXTURE_TARGET_1D || d.target == NVN_TEXTURE_TARGET_1D_ARRAY;
|
|
uint32_t w = std::max(1u, d.width >> level), h = one_row ? 1u : std::max(1u, d.height >> level);
|
|
// a compressed texture drawn through an uncompressed view of its block's size (compression on the GPU: TOTK
|
|
// renders its gloom noise volumes, 64x64x64 BC4, as RGBA16UI, a texel a 4x4 block): the target is the texture's
|
|
// size in blocks, as the GPU draws it, and is written back where the texture is decoded from (writeback_targets)
|
|
const PixelFormat tex_pf = format != d.format ? nvn_pixel_format(d.format) : pf;
|
|
const uint32_t bw = VideoCore::Surface::DefaultBlockWidth(tex_pf), bh = VideoCore::Surface::DefaultBlockHeight(tex_pf);
|
|
const bool block_view = tex_pf != PixelFormat::Invalid && (bw > 1 || bh > 1) &&
|
|
VideoCore::Surface::DefaultBlockWidth(pf) == 1 &&
|
|
VideoCore::Surface::BytesPerBlock(tex_pf) == VideoCore::Surface::BytesPerBlock(pf);
|
|
if (block_view) w = (w + bw - 1) / bw, h = one_row ? 1u : (h + bh - 1) / bh;
|
|
// (each level and layer is an image at its own address, as the targets of a layered or mipmapped texture are
|
|
// drawn one by one: the texture sampled whole is assembled from them, rx_texture.cpp)
|
|
uint64_t gpu = t->gpu + nvn_tex_level_offset(&d, level);
|
|
// a 3D texture's slices interleave in its memory (block linear): each slice drawn is an image of its own, filed
|
|
// under nvn_slice_key, and the 3D texture is assembled from them where it is sampled (assemble_3d_slices). They
|
|
// were all drawn as slice 0: TOTK renders its colour-grading LUT slice by slice (the loading map lost its blues).
|
|
uint64_t key = 0;
|
|
if (d.target == NVN_TEXTURE_TARGET_3D) {
|
|
const uint64_t guest = nvn_gpu_guest(gpu);
|
|
if (!guest) return nullptr;
|
|
key = nvn_slice_key(guest, layer);
|
|
} else if (layer) {
|
|
gpu += (uint64_t)layer * t->L.layer_size;
|
|
}
|
|
Image* im = find_render_target(gpu, w, h, pf, key);
|
|
if (im) {
|
|
im->block_view = block_view;
|
|
if (key) im->slice = layer, im->volume_depth = std::max(1u, d.depth >> level);
|
|
im->pitch_linear = nvn_tex_is_linear(&d);
|
|
unsigned th = 0, td = 0;
|
|
nvn_tex_level_tiles(&d, level, &th, &td);
|
|
im->bl_height = th;
|
|
im->bl_depth = td;
|
|
im->reg_width = im->pitch_linear ? nvn_tex_linear_stride(&d) : w;
|
|
// (what a write-back may cover: this level of this layer)
|
|
const uint64_t lo = nvn_tex_level_offset(&d, level);
|
|
const uint64_t hi = level + 1 < std::max(1u, d.levels) ? nvn_tex_level_offset(&d, level + 1)
|
|
: (t->L.layer_size ? t->L.layer_size : t->L.size);
|
|
im->storage_limit = hi > lo ? hi - lo : 0;
|
|
}
|
|
return im;
|
|
}
|
|
|
|
/// (mkw-recomp) The whole texture as one GPU image (all its layers, or its 3D volume) for compute programs that write
|
|
/// it as a layered or 3D image: filed at the texture's storage like a render target, so draws that sample the texture
|
|
/// afterwards take it as drawn by the GPU. Null when it cannot be a storage texture.
|
|
Image* nvn_storage_image(const nvn_tex* t, const nvn_view* v, bool volume) {
|
|
if (!t || !t->gpu) return nullptr;
|
|
const nvn_texdesc& d = t->d;
|
|
const unsigned level = v && v->set_levels ? v->base_level : 0;
|
|
const int format = v && v->set_format ? v->format : d.format;
|
|
const PixelFormat pf = nvn_pixel_format(format);
|
|
if (pf == PixelFormat::Invalid) return nullptr;
|
|
WGPUTextureFormat wf = wgpu_format(pf);
|
|
// (as the render targets: 8-bit SNORM and 16-bit norm formats as half floats, which storage textures can be)
|
|
switch (wf) {
|
|
case WGPUTextureFormat_RGBA8Snorm: case WGPUTextureFormat_RGBA16Unorm: case WGPUTextureFormat_RGBA16Snorm:
|
|
wf = WGPUTextureFormat_RGBA16Float; break;
|
|
case WGPUTextureFormat_RG16Unorm: case WGPUTextureFormat_RG16Snorm: wf = WGPUTextureFormat_RG16Float; break;
|
|
case WGPUTextureFormat_R16Unorm: case WGPUTextureFormat_R16Snorm: wf = WGPUTextureFormat_R16Float; break;
|
|
default: break;
|
|
}
|
|
if (!wgsl_storage_format(wf)) {
|
|
log_limited("storage_image_format", 8, "rx: storage image format %#x (NVN %#x) is not a storage format",
|
|
(unsigned)wf, format);
|
|
return nullptr;
|
|
}
|
|
const uint32_t w = std::max(1u, d.width >> level), h = std::max(1u, d.height >> level);
|
|
const uint32_t layers = volume ? 1u : std::max(1u, v && v->set_layers && v->nlayers ? (uint32_t)v->nlayers : d.depth);
|
|
const uint32_t depth = volume ? std::max(1u, d.depth >> level) : 1u;
|
|
if (w > dev.limits.maxTextureDimension2D || h > dev.limits.maxTextureDimension2D || layers > dev.limits.maxTextureArrayLayers)
|
|
return nullptr;
|
|
const uint64_t gpu = t->gpu + nvn_tex_level_offset(&d, level) + (v && v->set_layers && !volume ? v->min_layer * t->L.layer_size : 0);
|
|
const uint64_t guest = nvn_gpu_guest(gpu);
|
|
if (!guest) return nullptr;
|
|
const WGPUTextureDimension dim = volume ? WGPUTextureDimension_3D : WGPUTextureDimension_2D;
|
|
auto it = images.find(guest);
|
|
if (it != images.end() && it->second && it->second->width == w && it->second->height == h &&
|
|
it->second->layers == layers && it->second->dimension == dim && it->second->format == wf &&
|
|
(!volume || it->second->volume_depth == depth)) {
|
|
it->second->last_use = frame_number;
|
|
return it->second.get();
|
|
}
|
|
auto im = std::make_unique<Image>();
|
|
im->gpu_va = gpu, im->guest = guest, im->width = w, im->height = h, im->layers = layers;
|
|
im->pixel_format = pf, im->format = wf, im->dimension = dim, im->rendered = true, im->last_use = frame_number;
|
|
if (volume) im->volume_depth = depth;
|
|
WGPUTextureDescriptor td{};
|
|
// (render passes may target it too: it is filed as the render target at its address)
|
|
td.usage = WGPUTextureUsage_TextureBinding | WGPUTextureUsage_StorageBinding | WGPUTextureUsage_CopySrc |
|
|
WGPUTextureUsage_CopyDst | WGPUTextureUsage_RenderAttachment;
|
|
td.dimension = dim;
|
|
td.size = {w, h, volume ? depth : layers};
|
|
td.format = wf;
|
|
td.mipLevelCount = 1;
|
|
td.sampleCount = 1;
|
|
scope_push();
|
|
im->tex = wgpuDeviceCreateTexture(dev.device, &td);
|
|
WGPUTextureViewDescriptor vd{};
|
|
vd.format = wf;
|
|
vd.dimension = volume ? WGPUTextureViewDimension_3D : WGPUTextureViewDimension_2D;
|
|
vd.mipLevelCount = 1;
|
|
vd.arrayLayerCount = 1;
|
|
vd.aspect = WGPUTextureAspect_All;
|
|
im->attachment = wgpuTextureCreateView(im->tex, &vd);
|
|
if (const std::string error = scope_pop(); !error.empty()) {
|
|
log_limited("storage_image", 8, "rx: storage image %ux%ux%u not created: %s", w, h, volume ? depth : layers,
|
|
error.c_str());
|
|
return nullptr;
|
|
}
|
|
Image* raw = im.get();
|
|
if (it != images.end() && it->second) spare_images.emplace(guest, std::move(it->second));
|
|
images[guest] = std::move(im);
|
|
note_image_change(guest);
|
|
log_limited("storage_image_made", 16, "rx: storage image %ux%u x%u%s format %#x at guest %#llx", w, h,
|
|
volume ? depth : layers, volume ? " (3D)" : "", (unsigned)wf, (unsigned long long)guest);
|
|
return raw;
|
|
}
|
|
|
|
Attachments nvn_attachments(NvnState& s) {
|
|
// (images can be replaced or made by any lookup: the answer holds while none was, in this frame)
|
|
if (!s.at_dirty && s.at_images == images_changed && s.at_frame == frame_number) return s.at;
|
|
Attachments at;
|
|
for (int i = 0; i < std::min(s.rt.n, 8); i++)
|
|
at.color[i] = nvn_target(s.rt.color[i], s.rt.has_cview[i] ? &s.rt.cview[i] : nullptr);
|
|
at.depth = nvn_target(s.rt.depth, s.rt.has_dview ? &s.rt.dview : nullptr);
|
|
for (int i = 0; i < 8; i++) // (one image attached once: two targets can name one storage)
|
|
for (int j = 0; j < i && at.color[i]; j++)
|
|
if (at.color[j] == at.color[i]) at.color[i] = nullptr;
|
|
if (at.depth)
|
|
for (Image*& c : at.color)
|
|
if (c == at.depth) c = nullptr;
|
|
for (Image* im : at.color)
|
|
if (im && !at.width) at.width = im->width, at.height = im->height;
|
|
if (!at.width && at.depth) at.width = at.depth->width, at.height = at.depth->height;
|
|
for (auto& im : at.color)
|
|
if (im && (im->width != at.width || im->height != at.height)) {
|
|
log_limited("nvn_rt_size", 8, "rx: NVN color target %ux%u dropped from a %ux%u pass", im->width, im->height,
|
|
at.width, at.height);
|
|
im = nullptr;
|
|
}
|
|
if (at.depth && (at.depth->width != at.width || at.depth->height != at.height)) {
|
|
log_limited("nvn_zt_size", 8, "rx: NVN depth target %ux%u dropped from a %ux%u pass", at.depth->width,
|
|
at.depth->height, at.width, at.height);
|
|
at.depth = nullptr;
|
|
}
|
|
s.at = at;
|
|
s.at_dirty = false;
|
|
s.at_images = images_changed; // (after the lookups: they can make images)
|
|
s.at_frame = frame_number;
|
|
return at;
|
|
}
|
|
|
|
namespace {
|
|
/// The rectangle of a w x h target an NVN clear covers: the scissor, when one was set (upper-left origin)
|
|
bool nvn_clear_rect(uint32_t w, uint32_t h, const int32_t* sc, uint32_t rect[4]) {
|
|
int64_t x0 = 0, y0 = 0, x1 = w, y1 = h;
|
|
if (sc && sc[2] > 0 && sc[3] > 0) {
|
|
x0 = std::max<int64_t>(x0, sc[0]);
|
|
y0 = std::max<int64_t>(y0, sc[1]);
|
|
x1 = std::min<int64_t>(x1, (int64_t)sc[0] + sc[2]);
|
|
y1 = std::min<int64_t>(y1, (int64_t)sc[1] + sc[3]);
|
|
}
|
|
if (x1 <= x0 || y1 <= y0) return false;
|
|
rect[0] = (uint32_t)x0, rect[1] = (uint32_t)y0, rect[2] = (uint32_t)(x1 - x0), rect[3] = (uint32_t)(y1 - y0);
|
|
return true;
|
|
}
|
|
} // namespace
|
|
|
|
void nvn_clear_color(Image* im, const float rgba[4], uint32_t mask, const int32_t* scissor) {
|
|
if (!im || !(mask & 15)) return;
|
|
st_clears++;
|
|
uint32_t rect[4];
|
|
if (!nvn_clear_rect(im->width, im->height, scissor, rect)) {
|
|
count("clear_empty");
|
|
return;
|
|
}
|
|
const bool part = rect[0] || rect[1] || rect[2] < im->width || rect[3] < im->height;
|
|
const bool all = (mask & 15) == 15;
|
|
const bool integer = VideoCore::Surface::IsPixelFormatInteger(im->pixel_format);
|
|
if ((part || !all) && !integer) {
|
|
const uint32_t wmask = (mask & 1 ? WGPUColorWriteMask_Red : 0) | (mask & 2 ? WGPUColorWriteMask_Green : 0) |
|
|
(mask & 4 ? WGPUColorWriteMask_Blue : 0) | (mask & 8 ? WGPUColorWriteMask_Alpha : 0);
|
|
if (WGPURenderPipeline p = partial_clear_pipeline(im->format, wmask, rgba, WGPUTextureFormat_Undefined, false,
|
|
false, 0, 0)) {
|
|
Attachments at;
|
|
at.color[0] = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
std::array<WGPURenderPassColorAttachment, 8> c{};
|
|
c[0] = color_attachment(im, WGPULoadOp_Load, {});
|
|
begin_pass(at, c.data(), nullptr);
|
|
draw_partial_clear(p, rect, 0);
|
|
end_render_pass("pe_clear");
|
|
count("clear_partial");
|
|
return;
|
|
}
|
|
} else if (part || !all) {
|
|
log_limited("nvn_clear_int_part", 4, "rx: partial clear of an NVN integer target: cleared whole");
|
|
}
|
|
// (integer targets: the values are the integers' bits)
|
|
WGPUColor value{rgba[0], rgba[1], rgba[2], rgba[3]};
|
|
if (integer) {
|
|
int32_t v[4];
|
|
std::memcpy(v, rgba, 16);
|
|
const bool sign = VideoCore::Surface::IsPixelFormatSignedInteger(im->pixel_format);
|
|
value = sign ? WGPUColor{(double)v[0], (double)v[1], (double)v[2], (double)v[3]}
|
|
: WGPUColor{(double)(uint32_t)v[0], (double)(uint32_t)v[1], (double)(uint32_t)v[2],
|
|
(double)(uint32_t)v[3]};
|
|
}
|
|
if (clears_deferred()) {
|
|
PendingClear p{};
|
|
p.im = im;
|
|
p.color = true;
|
|
p.value = value;
|
|
add_pending_clear(p);
|
|
return;
|
|
}
|
|
Attachments at;
|
|
at.color[0] = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
std::array<WGPURenderPassColorAttachment, 8> c{};
|
|
c[0] = color_attachment(im, WGPULoadOp_Clear, value);
|
|
begin_pass(at, c.data(), nullptr);
|
|
end_render_pass("pe_clear");
|
|
}
|
|
|
|
void nvn_clear_depth_stencil(Image* im, bool depth, float dv, bool stencil, uint32_t sv, uint32_t smask,
|
|
const int32_t* scissor) {
|
|
if (!im) return;
|
|
depth = depth && im->format != WGPUTextureFormat_Stencil8;
|
|
stencil = stencil && (smask & 0xFF) && has_stencil(im->format);
|
|
if (!depth && !stencil) return;
|
|
st_clears++;
|
|
uint32_t rect[4];
|
|
if (!nvn_clear_rect(im->width, im->height, scissor, rect)) {
|
|
count("clear_empty");
|
|
return;
|
|
}
|
|
const bool part = rect[0] || rect[1] || rect[2] < im->width || rect[3] < im->height;
|
|
const bool stencil_part = stencil && (smask & 0xFF) != 0xFF;
|
|
if (part || stencil_part) {
|
|
if (WGPURenderPipeline p = partial_clear_pipeline(WGPUTextureFormat_Undefined, 0, nullptr, im->format, depth,
|
|
stencil, depth_clear_value(im, dv),
|
|
stencil_part ? smask & 0xFF : 0xFF)) {
|
|
Attachments at;
|
|
at.depth = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
WGPURenderPassDepthStencilAttachment d = depth_attachment(im, WGPULoadOp_Load, WGPULoadOp_Load, 0.0f, 0);
|
|
begin_pass(at, nullptr, &d);
|
|
draw_partial_clear(p, rect, sv & 0xFF);
|
|
end_render_pass("pe_clear");
|
|
count("clear_partial");
|
|
return;
|
|
}
|
|
}
|
|
if (clears_deferred()) {
|
|
PendingClear p{};
|
|
p.im = im;
|
|
p.depth = depth;
|
|
p.stencil = stencil;
|
|
p.depth_value = depth_clear_value(im, dv);
|
|
p.stencil_value = sv & 0xFF;
|
|
add_pending_clear(p);
|
|
return;
|
|
}
|
|
Attachments at;
|
|
at.depth = im;
|
|
at.width = im->width;
|
|
at.height = im->height;
|
|
WGPURenderPassDepthStencilAttachment d = depth_attachment(im, depth ? WGPULoadOp_Clear : WGPULoadOp_Load,
|
|
stencil ? WGPULoadOp_Clear : WGPULoadOp_Load,
|
|
depth_clear_value(im, dv), sv & 0xFF);
|
|
begin_pass(at, nullptr, &d);
|
|
end_render_pass("pe_clear");
|
|
}
|
|
|
|
} // namespace rx
|
|
|
|
using namespace rx;
|
|
|
|
extern "C" {
|
|
|
|
int rx_start(void* hwnd) {
|
|
stats_on = getenv("RX_STATS") && std::atoi(getenv("RX_STATS")) != 0;
|
|
frame_dump_enabled = getenv("RX_DUMP_FRAME") || getenv("RX_DUMP_AT");
|
|
#ifdef __EMSCRIPTEN__
|
|
on_render_thread = true; // (the web: rx_start runs on the render thread)
|
|
#endif
|
|
std::lock_guard lk(mu);
|
|
if (started) return 1;
|
|
if (getenv("RX_OFF")) return 0;
|
|
#if RX_WEB_MT
|
|
bool made = false; // (the device belongs to the WebGPU thread: made there)
|
|
on_gpu("init_device", [&] { made = init_device(static_cast<HWND>(hwnd)); });
|
|
if (!made) {
|
|
#else
|
|
if (!init_device(static_cast<HWND>(hwnd))) {
|
|
#endif
|
|
hos_log("rx: no WebGPU device: the window shows the window buffers' memory");
|
|
return 0;
|
|
}
|
|
started = true;
|
|
return 1;
|
|
}
|
|
|
|
int rx_active(void) {
|
|
return started;
|
|
}
|
|
|
|
void rx_resize(int width, int height) {
|
|
InterlockedExchange(&want_w, width);
|
|
InterlockedExchange(&want_h, height);
|
|
present_window_size(width, height);
|
|
}
|
|
|
|
uint64_t rx_channel_zpass(void* channel) {
|
|
return channel ? static_cast<Channel*>(channel)->zpass : 0;
|
|
}
|
|
|
|
} // extern "C"
|
|
namespace rx {
|
|
namespace {
|
|
struct PendingWriteback { uint64_t guest, id; };
|
|
std::vector<PendingWriteback> pending_writebacks;
|
|
} // namespace
|
|
|
|
void queue_target_writeback(Image* im) {
|
|
static const bool off = env_flag("RX_NO_TARGET_WRITEBACK");
|
|
// (a 3D slice, filed under nvn_slice_key: only a block view's, written into its volume)
|
|
if (off || !im || im->writeback_queued || ((im->guest >> 32) && !im->block_view)) return;
|
|
im->writeback_queued = true;
|
|
pending_writebacks.push_back({im->guest, im->id});
|
|
}
|
|
|
|
/// What a render target writeback needs once its pixels are read back (a copy: the image may be gone by then)
|
|
struct TargetWriteback {
|
|
uint64_t guest = 0, gpu_va = 0, storage_limit = 0;
|
|
uint32_t w = 0, h = 0, bpp = 0, row = 0, reg_width = 0, bl_height = 0, bl_depth = 0;
|
|
uint32_t slice = 0, depth = 1; // a 3D texture's slice: which, of how many
|
|
bool pitch_linear = false, block_view = false;
|
|
int pf = 0;
|
|
uint8_t norm16 = 0; // the texture is the float16 stand-in of a 16-bit unorm (1) / snorm (2) format (3 / 4: float32)
|
|
uint32_t src_row = 0; // bytes per row read back (the texture's texels; row: the guest's)
|
|
};
|
|
|
|
static float half_value(uint16_t h) {
|
|
const uint32_t e = h >> 10 & 31, m = h & 1023;
|
|
const float f = e == 0 ? std::ldexp((float)m, -24) : e == 31 ? (m ? NAN : INFINITY) : std::ldexp(1.0f + m / 1024.0f, (int)e - 15);
|
|
return h >> 15 ? -f : f;
|
|
}
|
|
|
|
/// 1 / 2 when <texture> is the float16 stand-in rx draws a 16-bit unorm / snorm target into (no 16-bit norm
|
|
/// render targets without RX_NORM16_RT): same size per texel, values to convert; 3 / 4 for a float32 stand-in (RGBA
|
|
/// 16: twice the size per texel). 0 otherwise.
|
|
static uint8_t norm16_stand_in(PixelFormat pf, WGPUTextureFormat texture) {
|
|
using P = PixelFormat;
|
|
switch (pf) {
|
|
case P::R16_UNORM: return texture == WGPUTextureFormat_R16Float ? 1 : 0;
|
|
case P::R16G16_UNORM: return texture == WGPUTextureFormat_RG16Float ? 1 : 0;
|
|
case P::R16G16B16A16_UNORM: return texture == WGPUTextureFormat_RGBA16Float ? 1 : texture == WGPUTextureFormat_RGBA32Float ? 3 : 0;
|
|
case P::R16_SNORM: return texture == WGPUTextureFormat_R16Float ? 2 : 0;
|
|
case P::R16G16_SNORM: return texture == WGPUTextureFormat_RG16Float ? 2 : 0;
|
|
case P::R16G16B16A16_SNORM: return texture == WGPUTextureFormat_RGBA16Float ? 2 : texture == WGPUTextureFormat_RGBA32Float ? 4 : 0;
|
|
default: return 0;
|
|
}
|
|
}
|
|
|
|
/// The read-back rows q (row bytes apart) into guest memory, in the target's layout (pitch or block linear).
|
|
/// counted: false off the renderer's thread (count() is that thread's)
|
|
void apply_target_writeback(const TargetWriteback& t, const uint8_t* q, bool counted) {
|
|
const uint32_t w = t.w, h = t.h, bpp = t.bpp, row = t.row;
|
|
std::vector<uint8_t> conv;
|
|
if (t.norm16 >= 3) { // (float32 back to the guest's 16-bit norm values: rows src_row apart, 16 bytes a texel)
|
|
conv.assign((size_t)row * h, 0);
|
|
for (uint32_t y = 0; y < h; y++)
|
|
for (uint32_t i = 0; i < w * 4; i++) {
|
|
float f;
|
|
std::memcpy(&f, q + (size_t)y * t.src_row + 4 * i, 4);
|
|
if (f != f) f = 0;
|
|
const uint16_t v = t.norm16 == 3 ? (uint16_t)std::lround(std::clamp(f, 0.0f, 1.0f) * 65535.0f)
|
|
: (uint16_t)(int16_t)std::lround(std::clamp(f, -1.0f, 1.0f) * 32767.0f);
|
|
std::memcpy(conv.data() + (size_t)y * row + 2 * i, &v, 2);
|
|
}
|
|
q = conv.data();
|
|
if (counted) count("rt_writeback_norm16");
|
|
} else if (t.norm16) { // (float16 back to the guest's 16-bit norm values)
|
|
conv.assign(q, q + (size_t)row * h);
|
|
for (uint32_t y = 0; y < h; y++) {
|
|
uint8_t* p = conv.data() + (size_t)y * row;
|
|
for (uint32_t i = 0; i < w * bpp / 2; i++) {
|
|
uint16_t v;
|
|
std::memcpy(&v, p + 2 * i, 2);
|
|
float f = half_value(v);
|
|
if (f != f) f = 0;
|
|
v = t.norm16 == 1 ? (uint16_t)std::lround(std::clamp(f, 0.0f, 1.0f) * 65535.0f)
|
|
: (uint16_t)(int16_t)std::lround(std::clamp(f, -1.0f, 1.0f) * 32767.0f);
|
|
std::memcpy(p + 2 * i, &v, 2);
|
|
}
|
|
}
|
|
q = conv.data();
|
|
if (counted) count("rt_writeback_norm16");
|
|
}
|
|
uint64_t avail = 0;
|
|
uint8_t* out = const_cast<uint8_t*>(hos_gpu_ptr(t.gpu_va, &avail));
|
|
if (t.pitch_linear) {
|
|
const uint32_t pitch = std::max(t.reg_width, w * bpp);
|
|
const uint64_t need = (uint64_t)pitch * (h - 1) + (uint64_t)w * bpp;
|
|
if (out && avail >= need && (!t.storage_limit || need <= t.storage_limit))
|
|
for (uint32_t y = 0; y < h; y++) std::memcpy(out + (uint64_t)y * pitch, q + (uint64_t)y * row, (size_t)w * bpp);
|
|
} else {
|
|
const uint32_t sw = std::max(t.reg_width, w); // (the surface's width: its GOB row stride)
|
|
const uint32_t depth = std::max(1u, t.depth);
|
|
const size_t size = Tegra::Texture::CalculateSize(true, bpp, sw, h, depth, t.bl_height, t.bl_depth);
|
|
if (t.storage_limit && size > t.storage_limit) { // (a native NVN target: never past its storage)
|
|
log_limited("wb_limit", 16, "rx: writeback of %ux%u (%zu bytes, block height %u) is larger than its "
|
|
"storage (%llu bytes): skipped", w, h, size, t.bl_height, (unsigned long long)t.storage_limit);
|
|
if (counted) count("rt_writeback_too_big");
|
|
} else if (out && avail >= size && t.slice < depth) {
|
|
// (a 3D texture's slice: its volume as memory holds it, the slice replaced; slices interleave in memory)
|
|
std::vector<uint8_t> lin((size_t)sw * h * depth * bpp);
|
|
if (depth > 1)
|
|
Tegra::Texture::UnswizzleTexture(lin, std::span<const uint8_t>(out, size), bpp, sw, h, depth,
|
|
t.bl_height, t.bl_depth);
|
|
uint8_t* at = lin.data() + (size_t)t.slice * sw * h * bpp;
|
|
for (uint32_t y = 0; y < h; y++) std::memcpy(at + (size_t)y * sw * bpp, q + (uint64_t)y * row, (size_t)w * bpp);
|
|
// (the surface's own bytes first: the GOBs' unused corners keep what memory held)
|
|
std::vector<uint8_t> tiled(out, out + size);
|
|
Tegra::Texture::SwizzleTexture(tiled, lin, bpp, sw, h, depth, t.bl_height, t.bl_depth);
|
|
std::memcpy(out, tiled.data(), size);
|
|
if (counted) count("rt_writeback_bytes", (int64_t)size);
|
|
// (a block view: the compressed texture is decoded again from what it drew)
|
|
if (t.block_view && counted) textures_written(t.guest & ((1ull << 40) - 1), size);
|
|
}
|
|
}
|
|
if (counted) count("rt_writebacks");
|
|
// (RX_LOG_WRITEBACKS=<first frame>)
|
|
static const int64_t log_wb = getenv("RX_LOG_WRITEBACKS") ? atoll(getenv("RX_LOG_WRITEBACKS")) : -1;
|
|
static int logged;
|
|
if (log_wb >= 0 && frame_number >= log_wb && logged++ < 2000)
|
|
hos_log("rx: writeback %08llx %ux%u format %d frame %lld", (unsigned long long)t.guest, w, h, t.pf,
|
|
(long long)frame_number);
|
|
}
|
|
|
|
/// (the threaded web build: writebacks recorded into a batch and not in guest memory yet; the native NVN layer's
|
|
/// drains wait for them, rx_target_writebacks_pending, and the game's memcpy for the ones it reads,
|
|
/// rx_wait_writebacks_reading)
|
|
std::atomic<int> target_writebacks_pending{0};
|
|
std::mutex wb_spans_mu;
|
|
std::vector<std::pair<uintptr_t, uintptr_t>> wb_spans; // (the host memory each pending one writes)
|
|
void drop_wb_span(std::pair<uintptr_t, uintptr_t> s) {
|
|
std::lock_guard lk(wb_spans_mu);
|
|
if (auto it = std::find(wb_spans.begin(), wb_spans.end(), s); it != wb_spans.end()) wb_spans.erase(it);
|
|
}
|
|
|
|
/// The queued render targets' contents to guest memory, laid out as the target registers said (block linear or
|
|
/// pitch), when the texture holds the guest format's bytes as they are (no 16-bit norm substitute, color only).
|
|
/// The threaded web build: the copy is recorded into the batch and read back after its submit, on the WebGPU thread,
|
|
/// instead of waiting here for every batch before it and for the GPU (~35 a second at the castle, each one stopping
|
|
/// the replay); the native NVN layer's fences and drains wait for it instead.
|
|
void writeback_targets() {
|
|
if (pending_writebacks.empty()) return;
|
|
std::vector<PendingWriteback> list;
|
|
list.swap(pending_writebacks);
|
|
if (!RX_WEB_MT) wait_submitted();
|
|
for (const PendingWriteback& p : list) {
|
|
Image* im = image_at(p.guest);
|
|
if (!im || im->id != p.id) continue;
|
|
im->writeback_queued = false;
|
|
const PixelFormat pf = im->pixel_format;
|
|
// (an sRGB target's texture may be its linear twin, viewed as sRGB: the same bytes)
|
|
const bool same_bytes = wgpu_format(pf) == im->format || srgb_pair(wgpu_format(pf)) == im->format;
|
|
const uint8_t norm16 = same_bytes ? 0 : norm16_stand_in(pf, im->format);
|
|
if (im->layers != 1 || im->dimension != WGPUTextureDimension_2D || (!same_bytes && !norm16) ||
|
|
VideoCore::Surface::GetFormatType(pf) != VideoCore::Surface::SurfaceType::ColorTexture ||
|
|
VideoCore::Surface::IsPixelFormatASTC(pf) || VideoCore::Surface::DefaultBlockWidth(pf) != 1) {
|
|
count("rt_writeback_skipped");
|
|
log_limited("rt_writeback_skipped", 16, "rx: writeback of %08llx %ux%u skipped: layers %u dim %d pixel format "
|
|
"%d texture format %#x (guest's %#x)", (unsigned long long)im->guest, im->width, im->height,
|
|
im->layers, (int)im->dimension, (int)pf, (unsigned)im->format, (unsigned)wgpu_format(pf));
|
|
continue;
|
|
}
|
|
TargetWriteback t;
|
|
t.guest = im->guest, t.gpu_va = im->gpu_va, t.storage_limit = im->storage_limit;
|
|
t.bpp = VideoCore::Surface::BytesPerBlock(pf);
|
|
t.w = im->width, t.h = im->height;
|
|
t.row = (t.w * t.bpp + 255) & ~255u;
|
|
t.reg_width = im->reg_width, t.bl_height = im->bl_height, t.bl_depth = im->bl_depth;
|
|
t.pitch_linear = im->pitch_linear;
|
|
t.block_view = im->block_view;
|
|
if (im->guest >> 32) t.slice = im->slice, t.depth = std::max(1u, im->volume_depth);
|
|
t.pf = (int)pf;
|
|
t.norm16 = norm16;
|
|
t.src_row = norm16 >= 3 ? (t.w * 16 + 255) & ~255u : t.row;
|
|
WGPUBufferDescriptor bd{};
|
|
bd.label = sv("rx target writeback");
|
|
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
|
|
bd.size = (uint64_t)t.src_row * t.h;
|
|
WGPUBuffer staging = wgpuDeviceCreateBuffer(dev.device, &bd);
|
|
WGPUTexelCopyTextureInfo src{};
|
|
src.texture = im->tex;
|
|
src.aspect = WGPUTextureAspect_All;
|
|
WGPUTexelCopyBufferInfo dst{};
|
|
dst.buffer = staging;
|
|
dst.layout.bytesPerRow = t.src_row;
|
|
dst.layout.rowsPerImage = t.h;
|
|
const WGPUExtent3D extent{t.w, t.h, 1};
|
|
if (RX_WEB_MT) {
|
|
end_render_pass("pe_writeback");
|
|
ce_copy_texture_to_buffer(encoder(), &src, &dst, &extent);
|
|
// (the memory it will write: as apply_target_writeback lays it out, at most)
|
|
uint64_t avail = 0;
|
|
const uintptr_t at = (uintptr_t)hos_gpu_ptr(t.gpu_va, &avail);
|
|
const uint64_t span = t.pitch_linear ? (uint64_t)std::max(t.reg_width, t.w * t.bpp) * t.h
|
|
: Tegra::Texture::CalculateSize(true, t.bpp, std::max(t.reg_width, t.w), t.h,
|
|
1, t.bl_height, t.bl_depth);
|
|
const std::pair<uintptr_t, uintptr_t> s{at, at + span};
|
|
{
|
|
std::lock_guard lk(wb_spans_mu);
|
|
wb_spans.push_back(s);
|
|
}
|
|
target_writebacks_pending++;
|
|
struct Pending {
|
|
TargetWriteback t;
|
|
WGPUBuffer staging;
|
|
uint64_t size;
|
|
std::pair<uintptr_t, uintptr_t> span;
|
|
};
|
|
auto* pw = new Pending{t, staging, bd.size, s};
|
|
open_afters.push_back([pw](bool submitted) { // (the WebGPU thread, after the batch's submit)
|
|
if (!submitted) {
|
|
wgpuBufferRelease(pw->staging);
|
|
drop_wb_span(pw->span);
|
|
delete pw;
|
|
target_writebacks_pending--;
|
|
return;
|
|
}
|
|
WGPUBufferMapCallbackInfo mc{};
|
|
mc.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
|
|
auto* pw = static_cast<Pending*>(u1);
|
|
if (status == WGPUMapAsyncStatus_Success) {
|
|
apply_target_writeback(
|
|
pw->t, static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(pw->staging, 0, pw->size)),
|
|
false);
|
|
wgpuBufferUnmap(pw->staging);
|
|
}
|
|
wgpuBufferRelease(pw->staging);
|
|
drop_wb_span(pw->span);
|
|
delete pw;
|
|
target_writebacks_pending--;
|
|
};
|
|
mc.userdata1 = pw;
|
|
wgpuBufferMapAsync(pw->staging, WGPUMapMode_Read, 0, pw->size, mc);
|
|
});
|
|
continue;
|
|
}
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
wgpuCommandEncoderCopyTextureToBuffer(e, &src, &dst, &extent);
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(e);
|
|
struct Done { bool complete = false; WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error; } done;
|
|
WGPUBufferMapCallbackInfo mc{};
|
|
mc.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
|
|
auto* dn = static_cast<Done*>(u1);
|
|
dn->status = status;
|
|
dn->complete = true;
|
|
};
|
|
mc.userdata1 = &done;
|
|
wgpuBufferMapAsync(staging, WGPUMapMode_Read, 0, bd.size, mc);
|
|
while (!done.complete) wgpuDevicePoll(dev.device, true, nullptr);
|
|
if (done.status == WGPUMapAsyncStatus_Success) {
|
|
apply_target_writeback(t, static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(staging, 0, bd.size)), true);
|
|
wgpuBufferUnmap(staging);
|
|
}
|
|
wgpuBufferRelease(staging);
|
|
}
|
|
if (RX_WEB_MT) flush_batch(); // (the copies go now)
|
|
}
|
|
int rx_target_writebacks_pending() { return target_writebacks_pending.load(); }
|
|
|
|
extern "C" void rx_wait_writebacks_reading(const void* p, uint64_t n) {
|
|
if (target_writebacks_pending.load(std::memory_order_relaxed) <= 0) return;
|
|
const uintptr_t a = (uintptr_t)p, b = a + n;
|
|
const int64_t t0 = hos_now_ns();
|
|
bool waited = false;
|
|
for (;;) {
|
|
bool hit = false;
|
|
{
|
|
std::lock_guard lk(wb_spans_mu);
|
|
for (const auto& [lo, hi] : wb_spans) hit |= a < hi && lo < b;
|
|
}
|
|
if (!hit) break;
|
|
waited = true;
|
|
std::this_thread::sleep_for(std::chrono::microseconds(100));
|
|
}
|
|
#if RX_WEB_MT
|
|
if (waited) mt_note_wait("(memcpy: writebacks)", hos_now_ns() - t0);
|
|
#endif
|
|
}
|
|
uint64_t last_batch_serial() {
|
|
return batch_serial.load();
|
|
}
|
|
|
|
bool batch_done(uint64_t serial) {
|
|
if (!serial) return true;
|
|
if (batch_serial_run.load() < serial) return false;
|
|
wgpuDevicePoll(dev.device, false, nullptr);
|
|
if (gpu_done_serial.load() < serial) return false;
|
|
std::lock_guard lk(inflight_mu);
|
|
return readback_serials.empty() || *readback_serials.begin() > serial;
|
|
}
|
|
|
|
void wait_batch(uint64_t serial) {
|
|
if (!started || !serial) return;
|
|
{
|
|
std::unique_lock lk(submit_mu);
|
|
submitted_cv.wait(lk, [&] { return batch_serial_run.load() >= serial; });
|
|
}
|
|
#if RX_WEB_MT
|
|
while (!batch_done(serial)) rx_web_yield(); // (the WebGPU thread's event loop runs the done callbacks)
|
|
#elif RX_WEB
|
|
while (!batch_done(serial)) wgpuInstanceProcessEvents(dev.instance);
|
|
#else
|
|
uint64_t index = 0;
|
|
{
|
|
std::lock_guard lk(index_mu);
|
|
// (this batch's last submission, or a later batch's: submissions finish in order)
|
|
for (uint64_t s = serial; s < serial + 1024 && s <= batch_serial_run.load(); s++)
|
|
if (serial_index[s % 1024].serial == s && serial_index[s % 1024].index) {
|
|
index = serial_index[s % 1024].index;
|
|
break;
|
|
}
|
|
}
|
|
const int64_t t0 = hos_now_ns();
|
|
if (index) {
|
|
const WGPUSubmissionIndex i = index;
|
|
wgpuDevicePoll(dev.device, true, &i);
|
|
}
|
|
while (!batch_done(serial)) wgpuDevicePoll(dev.device, true, nullptr);
|
|
count("t_fence_wait", hos_now_ns() - t0);
|
|
#endif
|
|
}
|
|
} // namespace rx
|
|
extern "C" {
|
|
|
|
void* rx_channel_new(void) {
|
|
return new Channel();
|
|
}
|
|
|
|
int64_t rx_stat_draws(void) { // (hos_prof.c's HOS_AB: is the game in the world yet)
|
|
return st_draws;
|
|
}
|
|
|
|
const uint8_t* rx_3d_methods(void) {
|
|
static const std::array<uint8_t, 0x1000> table = method_table();
|
|
return table.data();
|
|
}
|
|
|
|
void rx_3d_method(void* channel, const uint32_t* words, uint32_t method, uint32_t arg) {
|
|
std::lock_guard lk(mu);
|
|
Channel& ch = *static_cast<Channel*>(channel);
|
|
const Regs& r = *reinterpret_cast<const Regs*>(words);
|
|
const uint32_t bg = RX_REG(bind_groups);
|
|
if (method >= bg && method < bg + 5 * 8 && (method - bg) % 8 == 4) {
|
|
const uint32_t stage = (method - bg) / 8;
|
|
const auto& b = r.bind_groups[stage];
|
|
ConstBuffer& cb = ch.cbufs[stage][b.shader_slot % 18];
|
|
cb.enabled = b.valid != 0;
|
|
cb.address = r.const_buffer.Address();
|
|
cb.size = r.const_buffer.size;
|
|
static int binding_trace_count;
|
|
static const bool trace_bindings = env_flag("RX_TRACE_BINDINGS");
|
|
if (trace_bindings && binding_trace_count++ < 128)
|
|
hos_log("rx: cbuf bind stage %u slot %u valid %u address %#llx size %#x before device %u",
|
|
stage, (uint32_t)b.shader_slot, cb.enabled, (unsigned long long)cb.address, cb.size, !started);
|
|
return;
|
|
}
|
|
if (!started) return;
|
|
if (method == 0xFFFF) { // a macro finished: its instanced draws run
|
|
draw_deferred(ch, r);
|
|
return;
|
|
}
|
|
switch (method) {
|
|
case RX_REG(invalidate_texture_data_cache):
|
|
case RX_REG(tiled_cache_barrier):
|
|
case RX_REG(invalidate_texture_cache_no_wfi):
|
|
case RX_REG(invalidate_texture_data_cache_lines):
|
|
texture_barrier(method, arg);
|
|
return;
|
|
case RX_REG(clear_report_value):
|
|
if (arg == (uint32_t)Regs::ClearReport::ZPassPixelCount) ch.zpass = 0;
|
|
return;
|
|
case RX_REG(render_enable.mode):
|
|
case RX_REG(render_enable_override):
|
|
ch.render_on = render_enabled(r);
|
|
if (frame_dump_active()) {
|
|
uint64_t v[4]{};
|
|
if (r.render_enable.mode >= Regs::RenderEnable::Mode::Conditional) gpu_read(r.render_enable.Address(), v, sizeof v);
|
|
char b[200];
|
|
std::snprintf(b, sizeof b, "render enable %u override %u at %llx (%llx %llx / %llx %llx): %s",
|
|
(unsigned)r.render_enable.mode, (unsigned)r.render_enable_override,
|
|
(unsigned long long)r.render_enable.Address(), (unsigned long long)v[0],
|
|
(unsigned long long)v[1], (unsigned long long)v[2], (unsigned long long)v[3],
|
|
ch.render_on ? "on" : "off");
|
|
dump_draw(b);
|
|
}
|
|
return;
|
|
case RX_REG(clear_surface):
|
|
if (rt_history_on()) rt_history_note(r, ch.render_on);
|
|
clear(ch, r);
|
|
return;
|
|
case RX_REG(draw.begin):
|
|
if (rt_history_on()) rt_history_note(r, ch.render_on);
|
|
if (static const uint64_t on_rt = getenv("RX_DUMP_ON_RT") ? strtoull(getenv("RX_DUMP_ON_RT"), nullptr, 16) : 0;
|
|
on_rt)
|
|
for (uint32_t i = 0; i < 8; i++)
|
|
if (r.rt[i].format != Tegra::RenderTargetFormat::NONE &&
|
|
(r.rt[i].Address() == on_rt || // (or a size: RX_DUMP_ON_RT=<width>0000<height> in hex, e.g. 120000104)
|
|
(on_rt < 0x100000000ull && on_rt >> 16 == r.rt[i].width && (on_rt & 0xFFFF) == r.rt[i].height)))
|
|
dump_start_now();
|
|
draw_begin(ch, r);
|
|
return;
|
|
case RX_REG(draw.end):
|
|
draw_end(ch, r, 1, false);
|
|
return;
|
|
case RX_REG(index_buffer.count):
|
|
ch.draw_indexed = true;
|
|
return;
|
|
case RX_REG(index_buffer32_subsequent):
|
|
case RX_REG(index_buffer16_subsequent):
|
|
case RX_REG(index_buffer8_subsequent):
|
|
ch.instance_count++;
|
|
[[fallthrough]];
|
|
case RX_REG(index_buffer32_first):
|
|
case RX_REG(index_buffer16_first):
|
|
case RX_REG(index_buffer8_first):
|
|
draw_index_small(ch, r, arg);
|
|
return;
|
|
case RX_REG(draw_inline_index):
|
|
push_inline_index(ch, arg);
|
|
return;
|
|
case RX_REG(inline_index_2x16.even):
|
|
push_inline_index(ch, arg & 0xFFFF);
|
|
push_inline_index(ch, arg >> 16);
|
|
return;
|
|
case RX_REG(inline_index_4x8.index0):
|
|
for (int i = 0; i < 4; i++) push_inline_index(ch, (arg >> (8 * i)) & 0xFF);
|
|
return;
|
|
case RX_REG(vertex_array_instance_first):
|
|
draw_array_instanced(ch, r, r.vertex_array_instance_first, false);
|
|
return;
|
|
case RX_REG(vertex_array_instance_subsequent):
|
|
draw_array_instanced(ch, r, r.vertex_array_instance_subsequent, true);
|
|
return;
|
|
case RX_REG(draw_texture.src_y0):
|
|
log_limited("draw_texture", 4, "rx: draw_texture not drawn yet");
|
|
return;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
void rx_flush(void) {
|
|
if (!started) return;
|
|
std::lock_guard lk(mu);
|
|
flush_batch();
|
|
writeback_targets();
|
|
}
|
|
|
|
/// Every batch so far submitted and run, and the readbacks after them done (the native NVN layer's fences).
|
|
void rx_wait_idle(void) {
|
|
if (!started) return;
|
|
{
|
|
std::lock_guard lk(mu);
|
|
flush_batch();
|
|
}
|
|
wait_submitted();
|
|
for (;;) {
|
|
{
|
|
std::lock_guard lk(inflight_mu);
|
|
if (inflight.empty()) break;
|
|
}
|
|
if (readbacks_pending <= 0) break;
|
|
wgpuDevicePoll(dev.device, true, nullptr);
|
|
}
|
|
}
|
|
|
|
void rx_begin_submission(void) {
|
|
if (!started) return;
|
|
std::lock_guard lk(mu);
|
|
flush_batch(); // (none should be open)
|
|
buffer_cache_scan();
|
|
count("submissions");
|
|
}
|
|
|
|
/// (mkw-recomp) Whether what shaders wrote of this memory is in the buffer cache (or guest memory) already: no batch's
|
|
/// copy of it waits to be written back. Indirect arguments are then read in place on the GPU, without a sync.
|
|
int rx_gpu_writes_in_cache(uint64_t gpu_va, uint64_t size) {
|
|
if (!started) return 1;
|
|
auto overlaps = [&](const Writeback& w) { return w.gpu_va < gpu_va + size && gpu_va < w.gpu_va + w.size; };
|
|
{
|
|
std::lock_guard lk(mu);
|
|
for (const Writeback& w : open_writebacks)
|
|
if (overlaps(w)) return 0;
|
|
}
|
|
std::lock_guard lk(inflight_mu);
|
|
for (const Writeback& w : inflight)
|
|
if (overlaps(w)) return 0;
|
|
return 1;
|
|
}
|
|
|
|
int rx_gpu_writes_pending(uint64_t gpu_va, uint64_t size) {
|
|
if (!started) return 0;
|
|
auto overlaps = [&](const Writeback& w) { return w.gpu_va < gpu_va + size && gpu_va < w.gpu_va + w.size; };
|
|
{
|
|
std::lock_guard lk(mu);
|
|
for (const Writeback& w : open_writebacks)
|
|
if (overlaps(w)) return 1;
|
|
if (gpu_written(gpu_va, size)) return 1; // a dispatch wrote it in the buffer cache
|
|
}
|
|
std::lock_guard lk(inflight_mu);
|
|
for (const Writeback& w : inflight)
|
|
if (overlaps(w)) return 1;
|
|
return 0;
|
|
}
|
|
|
|
#ifdef __EMSCRIPTEN__
|
|
// The web: WebGPU objects belong to the render thread, so a sync asked for by another thread (the parsing thread,
|
|
// before the CPU reads what a shader wrote) is done there, between its records (rx_web_idle), and waited for.
|
|
namespace {
|
|
struct SyncRequest {
|
|
uint64_t gpu_va, size;
|
|
bool done = false;
|
|
};
|
|
std::mutex sync_mu;
|
|
std::condition_variable sync_cv;
|
|
std::deque<SyncRequest*> sync_requests;
|
|
} // namespace
|
|
#endif
|
|
|
|
#ifdef __EMSCRIPTEN__
|
|
std::atomic<volatile uint32_t*> idle_word{nullptr}; // (rx_web_idle's wake word: sync requests bump it)
|
|
#endif
|
|
|
|
void rx_sync_gpu_writes(uint64_t gpu_va, uint64_t size) {
|
|
if (!started) return;
|
|
#ifdef __EMSCRIPTEN__
|
|
if (!on_render_thread) {
|
|
SyncRequest r{gpu_va, size};
|
|
std::unique_lock lk(sync_mu);
|
|
sync_requests.push_back(&r);
|
|
if (volatile uint32_t* w = idle_word.load()) { // (the render thread may be waiting for work)
|
|
__atomic_add_fetch(w, 1, __ATOMIC_SEQ_CST);
|
|
emscripten_futex_wake((void*)w, 1);
|
|
}
|
|
sync_cv.wait(lk, [&] { return r.done; });
|
|
return;
|
|
}
|
|
#endif
|
|
const int64_t t0 = hos_now_ns();
|
|
{
|
|
std::lock_guard lk(mu);
|
|
flush_batch();
|
|
}
|
|
wait_submitted();
|
|
for (;;) {
|
|
{
|
|
std::lock_guard lk(inflight_mu);
|
|
if (inflight.empty()) break;
|
|
}
|
|
if (readbacks_pending <= 0) break;
|
|
wgpuDevicePoll(dev.device, true, nullptr);
|
|
}
|
|
{
|
|
std::lock_guard lk(mu);
|
|
if (gpu_written(gpu_va, size)) buffer_cache_sync_to_guest(gpu_va, size);
|
|
}
|
|
count("gpu_write_syncs");
|
|
count("t_gpu_write_sync", hos_now_ns() - t0);
|
|
}
|
|
|
|
#ifdef __EMSCRIPTEN__
|
|
/// The render thread with no records to run (web): serves sync requests, then waits in its event loop (WebGPU's
|
|
/// callbacks: readbacks run there) until the parse thread bumps *wake (its condition variable's sequence: new
|
|
/// records), a sync request comes, or 2 ms pass.
|
|
extern "C" void rx_web_idle(volatile uint32_t* wake, uint32_t seen) {
|
|
idle_word = wake;
|
|
on_render_thread = true;
|
|
for (;;) {
|
|
SyncRequest* r = nullptr;
|
|
{
|
|
std::lock_guard lk(sync_mu);
|
|
if (!sync_requests.empty()) {
|
|
r = sync_requests.front();
|
|
sync_requests.pop_front();
|
|
}
|
|
}
|
|
if (!r) break;
|
|
rx_sync_gpu_writes(r->gpu_va, r->size);
|
|
{
|
|
std::lock_guard lk(sync_mu);
|
|
r->done = true;
|
|
}
|
|
sync_cv.notify_all();
|
|
}
|
|
{
|
|
std::lock_guard lk(sync_mu);
|
|
if (!sync_requests.empty()) return;
|
|
}
|
|
rx_web_wait_word((uint32_t)(uintptr_t)wake, seen, 2.0);
|
|
}
|
|
#endif
|
|
|
|
uint32_t* rx_collect_written(int64_t* count) {
|
|
*count = -1;
|
|
if (!started) return nullptr;
|
|
std::vector<uint32_t> pages;
|
|
if (!collect_written_pages(pages)) return nullptr;
|
|
auto* out = static_cast<uint32_t*>(malloc((pages.size() + 1) * sizeof(uint32_t)));
|
|
if (!pages.empty()) std::memcpy(out, pages.data(), pages.size() * sizeof(uint32_t));
|
|
*count = (int64_t)pages.size();
|
|
return out;
|
|
}
|
|
|
|
void rx_begin_submission_written(uint32_t* pages, int64_t n) {
|
|
if (!started) {
|
|
free(pages);
|
|
return;
|
|
}
|
|
std::lock_guard lk(mu);
|
|
flush_batch(); // (none should be open)
|
|
if (n >= 0) buffer_cache_apply_written(pages, (size_t)n);
|
|
else buffer_cache_scan();
|
|
count("submissions");
|
|
free(pages);
|
|
}
|
|
|
|
void rx_memory_written(uint64_t gpu_va, uint64_t size) {
|
|
if (!started) return;
|
|
std::lock_guard lk(mu);
|
|
buffer_cache_written(gpu_va, size);
|
|
}
|
|
|
|
/// HOS_FRAME_HIST: called after each present with the render thread's busy time for the frame.
|
|
void rx_frame_spike(int64_t busy_ns, int64_t threshold_ns) {
|
|
static int64_t logged_frame = -1000;
|
|
if (busy_ns > threshold_ns && (threshold_ns <= 18000000 || frame_number - logged_frame >= 120) &&
|
|
(logged_frame = frame_number, true))
|
|
hos_log("rx: slow frame %lld: %.1f ms busy: %lld draws, decode %.1f ms, waits %.1f ms (trims %.1f), %lld KB "
|
|
"texture uploads, %lld bind groups made, %lld translations queued; in place: translations %.1f ms, "
|
|
"pipelines %.1f ms",
|
|
(long long)frame_number, busy_ns / 1e6, (long long)spike.draws, spike.decode_ns / 1e6,
|
|
spike.wait_ns / 1e6, spike.trim_ns / 1e6, (long long)(spike.upload_bytes >> 10),
|
|
(long long)spike.bind_groups, (long long)spike.translations, spike.translate_ns / 1e6,
|
|
spike.pipeline_ns / 1e6);
|
|
spike = {};
|
|
}
|
|
|
|
/// (mkw-recomp) the window's crop for the next present: the frame is the top-left crop_w x crop_h of the window
|
|
/// texture (MKW renders its scene at a dynamic size into a 3840x2160 window texture and crops it); 0: all of it
|
|
static uint32_t present_crop_w, present_crop_h;
|
|
void rx_present_crop(uint32_t w, uint32_t h) { present_crop_w = w, present_crop_h = h; }
|
|
|
|
int rx_present(uint64_t guest, uint32_t width, uint32_t height) {
|
|
if (!started) return 0;
|
|
std::lock_guard lk(mu);
|
|
flush_batch();
|
|
#if !RX_WEB
|
|
rdoc_present();
|
|
#endif
|
|
if (static const bool watch_buf = getenv("RX_WATCH_BUF") != nullptr; watch_buf) watch_buffer_frame();
|
|
if (frame_dump_active()) {
|
|
wait_submitted();
|
|
const char* dir = getenv("RX_DUMP_DIR");
|
|
dump_frame_images(dir ? dir : "rx_frame");
|
|
}
|
|
const int64_t present_start = hos_now_ns();
|
|
struct PresentTimer {
|
|
int64_t t0;
|
|
~PresentTimer() { count("t_present", hos_now_ns() - t0); }
|
|
} present_timer{present_start};
|
|
sync_dispatch_writes();
|
|
install_decoded_textures();
|
|
#if RX_WEB
|
|
wgpuInstanceProcessEvents(dev.instance); // (pipelines made asynchronously: their callbacks)
|
|
pipecache_step();
|
|
#endif
|
|
native_cache_step();
|
|
report_period();
|
|
#if !RX_WEB
|
|
if (static const bool ab = getenv("HOS_AB") != nullptr; ab) { // (the patched wgpu flips its switch with HOS_AB)
|
|
using SetPhase = void (*)(uint32_t);
|
|
static const auto set_phase =
|
|
reinterpret_cast<SetPhase>(GetProcAddress(GetModuleHandleA("wgpu_native.dll"), "wgpuTotkSetAbPhase"));
|
|
if (set_phase) set_phase(ab_flipped() ? 1u : 0u);
|
|
}
|
|
#endif
|
|
// RX_DUMP_ON_DRAWS=<n> (with RX_DUMP_FRAME=99999999): the dump starts RX_DUMP_ON_DELAY frames after the first
|
|
// frame with more than n draws (gameplay: thousands; title and loading screens: ~100)
|
|
if (static const int64_t on_draws = getenv("RX_DUMP_ON_DRAWS") ? atoll(getenv("RX_DUMP_ON_DRAWS")) : 0; on_draws) {
|
|
static int64_t last_draws = 0, fired = -1;
|
|
static const int64_t delay = getenv("RX_DUMP_ON_DELAY") ? atoll(getenv("RX_DUMP_ON_DELAY")) : 0;
|
|
static const int64_t arm_ns = getenv("RX_DUMP_ON_ARM") ? (int64_t)(atof(getenv("RX_DUMP_ON_ARM")) * 1e9) : 0;
|
|
static const int64_t armed_from = hos_now_ns() + arm_ns; // (and RX_DUMP_ON_ARM seconds in)
|
|
if (fired < 0 && st_draws - last_draws > on_draws && hos_now_ns() >= armed_from) fired = frame_number;
|
|
last_draws = st_draws;
|
|
frame_number++;
|
|
if (fired >= 0 && frame_number == fired + 1 + delay) dump_start_now(); // (the frame starting now)
|
|
} else {
|
|
frame_number++;
|
|
}
|
|
{ // the first gameplay-sized frame (over 2500 draws; the title ~400-1500, loading screens ~100): a load is done
|
|
static int64_t first_present_ns = hos_now_ns(), prev_draws = 0;
|
|
static bool logged = false;
|
|
if (!logged && st_draws - prev_draws > 2500) {
|
|
logged = true;
|
|
hos_log("rx: first gameplay frame (%lld draws) %.2f s after the first present",
|
|
(long long)(st_draws - prev_draws), (hos_now_ns() - first_present_ns) / 1e9);
|
|
}
|
|
prev_draws = st_draws;
|
|
}
|
|
trim_decoded_textures();
|
|
if (frame_number % 600 == 300) retire_unused_segments();
|
|
if (frame_number % 600 == 450) reset_idle_stream_memory();
|
|
if (frame_number % 600 == 150) trim_pipelines();
|
|
// (RX_MEM_EVERY=<frames>: the memory report that often; default once a minute at 60 fps)
|
|
static const int64_t mem_every = getenv("RX_MEM_EVERY") ? std::max(1LL, atoll(getenv("RX_MEM_EVERY"))) : 3600;
|
|
if (frame_number % mem_every == std::min<int64_t>(900, mem_every / 2)) memory_report();
|
|
if (frame_number % 60 == 0) // spare render targets not drawn for 120 frames go (mkw-recomp: was 600 / 600)
|
|
for (auto it = spare_images.begin(); it != spare_images.end();)
|
|
it = it->second->last_use < frame_number - 120 ? spare_images.erase(it) : std::next(it);
|
|
Image* im = image_at(guest);
|
|
const bool debug_present = env_flag("RX_DEBUG_PRESENT");
|
|
if (debug_present) im = nullptr;
|
|
static int present_trace;
|
|
if (env_flag("RX_TRACE") && present_trace++ < 12)
|
|
hos_log("rx: present #%d guest %#llx target %p %ux%u format %d rendered %d", present_trace,
|
|
(unsigned long long)guest, (void*)im, width, height, im ? (int)im->format : -1,
|
|
im ? im->rendered : 0);
|
|
if (!im) {
|
|
log_limited("present_none", 8, "rx: present of %ux%u at guest %#llx: nothing drawn there", width, height,
|
|
(unsigned long long)guest);
|
|
} else if (is_depth_format(im->format)) {
|
|
im = nullptr;
|
|
}
|
|
if (env_flag("RX_READBACK_TARGET") || env_flag("RX_DEBUG_TARGET")) wait_submitted();
|
|
static int64_t next_readback_frame = 120;
|
|
if (im && frame_number >= next_readback_frame && env_flag("RX_READBACK_TARGET")) {
|
|
next_readback_frame += 240;
|
|
const uint32_t channels = im->format == WGPUTextureFormat_R8Unorm ? 1 :
|
|
im->format == WGPUTextureFormat_RG8Unorm ? 2 :
|
|
im->format == WGPUTextureFormat_RGBA8Unorm ||
|
|
im->format == WGPUTextureFormat_RGBA8UnormSrgb
|
|
? 4 : 0;
|
|
if (!channels) {
|
|
log_limited("readback_format", 4, "rx: target readback unsupported format %#x", (unsigned)im->format);
|
|
}
|
|
const size_t bytes_per_pixel = channels ? channels : 4;
|
|
const size_t bytes_per_row = ((size_t)im->width * bytes_per_pixel + 255) & ~size_t(255);
|
|
const size_t read_size = bytes_per_row * im->height;
|
|
WGPUBufferDescriptor buffer_desc{};
|
|
buffer_desc.size = read_size;
|
|
buffer_desc.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
|
|
WGPUBuffer buffer = wgpuDeviceCreateBuffer(dev.device, &buffer_desc);
|
|
WGPUCommandEncoder read_encoder = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
WGPUTexelCopyTextureInfo source{};
|
|
source.texture = im->tex;
|
|
WGPUTexelCopyBufferInfo destination{};
|
|
destination.buffer = buffer;
|
|
destination.layout.bytesPerRow = (uint32_t)bytes_per_row;
|
|
destination.layout.rowsPerImage = im->height;
|
|
const WGPUExtent3D extent{im->width, im->height, 1};
|
|
wgpuCommandEncoderCopyTextureToBuffer(read_encoder, &source, &destination, &extent);
|
|
WGPUCommandBuffer read_cb = wgpuCommandEncoderFinish(read_encoder, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &read_cb);
|
|
wgpuCommandBufferRelease(read_cb);
|
|
wgpuCommandEncoderRelease(read_encoder);
|
|
struct ReadResult {
|
|
std::mutex mutex;
|
|
std::condition_variable ready;
|
|
bool complete = false;
|
|
WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error;
|
|
} result;
|
|
WGPUBufferMapCallbackInfo callback{};
|
|
callback.mode = WGPUCallbackMode_AllowSpontaneous;
|
|
callback.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* userdata, void*) {
|
|
auto& result = *static_cast<ReadResult*>(userdata);
|
|
{
|
|
std::lock_guard lock(result.mutex);
|
|
result.status = status;
|
|
result.complete = true;
|
|
}
|
|
result.ready.notify_one();
|
|
};
|
|
callback.userdata1 = &result;
|
|
wgpuBufferMapAsync(buffer, WGPUMapMode_Read, 0, read_size, callback);
|
|
wgpuDevicePoll(dev.device, true, nullptr);
|
|
{
|
|
std::unique_lock lock(result.mutex);
|
|
result.ready.wait(lock, [&] { return result.complete; });
|
|
}
|
|
if (result.status == WGPUMapAsyncStatus_Success) {
|
|
const uint8_t* pixels = (const uint8_t*)wgpuBufferGetConstMappedRange(buffer, 0, read_size);
|
|
if (pixels) {
|
|
uint64_t visible = 0;
|
|
uint8_t max_channel = 0;
|
|
for (uint32_t y = 0; y < im->height; y++) {
|
|
for (uint32_t x = 0; x < im->width; x++) {
|
|
const uint8_t* pixel = pixels + (size_t)y * bytes_per_row + (size_t)x * bytes_per_pixel;
|
|
const uint8_t g = channels >= 2 ? pixel[1] : pixel[0];
|
|
const uint8_t b = channels >= 3 ? pixel[2] : channels == 1 ? pixel[0] : 0;
|
|
visible += pixel[0] > 2 || g > 2 || b > 2;
|
|
max_channel = std::max({max_channel, pixel[0], g, b});
|
|
}
|
|
}
|
|
hos_log("rx: target readback guest %#llx %ux%u format %#x: %llu nonblack pixels, max RGB %u",
|
|
(unsigned long long)guest, im->width, im->height, (unsigned)im->format,
|
|
(unsigned long long)visible, max_channel);
|
|
}
|
|
wgpuBufferUnmap(buffer);
|
|
} else {
|
|
hos_log("rx: target readback map failed (%d)", (int)result.status);
|
|
}
|
|
wgpuBufferRelease(buffer);
|
|
}
|
|
if (im && env_flag("RX_DEBUG_TARGET")) {
|
|
WGPUCommandEncoder debug_encoder = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
WGPURenderPassColorAttachment debug_color{};
|
|
debug_color.view = im->attachment;
|
|
debug_color.depthSlice = WGPU_DEPTH_SLICE_UNDEFINED;
|
|
debug_color.loadOp = WGPULoadOp_Clear;
|
|
debug_color.storeOp = WGPUStoreOp_Store;
|
|
debug_color.clearValue = {0.0, 1.0, 0.0, 1.0};
|
|
WGPURenderPassDescriptor debug_pass_desc{};
|
|
debug_pass_desc.colorAttachmentCount = 1;
|
|
debug_pass_desc.colorAttachments = &debug_color;
|
|
WGPURenderPassEncoder debug_pass = wgpuCommandEncoderBeginRenderPass(debug_encoder, &debug_pass_desc);
|
|
wgpuRenderPassEncoderEnd(debug_pass);
|
|
wgpuRenderPassEncoderRelease(debug_pass);
|
|
WGPUCommandBuffer debug_cb = wgpuCommandEncoderFinish(debug_encoder, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &debug_cb);
|
|
wgpuCommandBufferRelease(debug_cb);
|
|
wgpuCommandEncoderRelease(debug_encoder);
|
|
}
|
|
screenshot(im);
|
|
flash_probe(im);
|
|
if (!surface) { // headless: the frame's end still queued (nothing presented), so the frames-ahead caps hold
|
|
queue_present([] {});
|
|
return im != nullptr;
|
|
}
|
|
// the blit into the window and the present run on the submit thread, after the frame's batches
|
|
WGPUTextureView view = nullptr;
|
|
if (im) {
|
|
// the bytes as they are: an sRGB window buffer holds encoded values, and the surface is not sRGB
|
|
const WGPUTextureFormat raw = srgb_pair(im->format);
|
|
const bool srgb = im->format == WGPUTextureFormat_RGBA8UnormSrgb || im->format == WGPUTextureFormat_BGRA8UnormSrgb;
|
|
view = srgb ? im->view(WGPUTextureViewDimension_2D, false, 0, 0, 1, raw) : im->attachment;
|
|
wgpuTextureViewAddRef(view);
|
|
}
|
|
uint32_t sw = im ? im->width : 0, sh = im ? im->height : 0;
|
|
// a crop: copied into a texture of its size, which is presented instead (a sampler cannot take a part of a view)
|
|
if (im && view && present_crop_w && present_crop_h && (present_crop_w < sw || present_crop_h < sh) &&
|
|
!is_depth_format(im->format)) {
|
|
static WGPUTexture crop_tex;
|
|
static WGPUTextureView crop_view;
|
|
static uint32_t crop_w, crop_h;
|
|
static WGPUTextureFormat crop_format;
|
|
const uint32_t cw = std::min(present_crop_w, sw), ch = std::min(present_crop_h, sh);
|
|
if (!crop_tex || crop_w != cw || crop_h != ch || crop_format != im->format) {
|
|
if (crop_view) wgpuTextureViewRelease(crop_view);
|
|
if (crop_tex) wgpuTextureRelease(crop_tex);
|
|
WGPUTextureDescriptor td{};
|
|
td.usage = WGPUTextureUsage_TextureBinding | WGPUTextureUsage_CopyDst | WGPUTextureUsage_CopySrc;
|
|
td.dimension = WGPUTextureDimension_2D;
|
|
td.size = {cw, ch, 1};
|
|
td.format = im->format;
|
|
td.mipLevelCount = 1;
|
|
td.sampleCount = 1;
|
|
const WGPUTextureFormat raw = srgb_pair(im->format);
|
|
td.viewFormatCount = raw != im->format ? 1 : 0;
|
|
td.viewFormats = &raw;
|
|
crop_tex = wgpuDeviceCreateTexture(dev.device, &td);
|
|
WGPUTextureViewDescriptor vd{};
|
|
vd.format = raw;
|
|
vd.dimension = WGPUTextureViewDimension_2D;
|
|
vd.mipLevelCount = 1;
|
|
vd.arrayLayerCount = 1;
|
|
vd.aspect = WGPUTextureAspect_All;
|
|
crop_view = wgpuTextureCreateView(crop_tex, &vd);
|
|
crop_w = cw, crop_h = ch, crop_format = im->format;
|
|
}
|
|
WGPUCommandEncoder ce = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
WGPUTexelCopyTextureInfo from{}, to{};
|
|
from.texture = im->tex, to.texture = crop_tex;
|
|
from.aspect = to.aspect = WGPUTextureAspect_All;
|
|
const WGPUExtent3D size{cw, ch, 1};
|
|
wgpuCommandEncoderCopyTextureToTexture(ce, &from, &to, &size);
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(ce, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(ce);
|
|
wgpuTextureViewRelease(view);
|
|
view = crop_view;
|
|
wgpuTextureViewAddRef(view);
|
|
sw = cw, sh = ch;
|
|
}
|
|
const LONG ww = want_w, wh = want_h;
|
|
queue_present([view, sw, sh, ww, wh, debug_present] { present_on_submit_thread(view, sw, sh, ww, wh, debug_present); });
|
|
return im != nullptr;
|
|
}
|
|
|
|
const char* rx_describe(void) {
|
|
return started ? dev.description.c_str() : "off";
|
|
}
|
|
|
|
void rx_stats(int64_t* draws, int64_t* skipped, int64_t* pipelines) {
|
|
*draws = st_draws;
|
|
*skipped = st_skipped;
|
|
*pipelines = st_pipelines;
|
|
}
|
|
|
|
} // extern "C"
|
|
|
|
namespace rx {
|
|
std::string rt_history_find(uint64_t lo, uint64_t hi) {
|
|
std::string out;
|
|
std::vector<std::pair<uint64_t, RtSeen>> sorted;
|
|
for (const auto& [va, h] : rt_history)
|
|
if (va >= lo && va < hi) sorted.push_back({va, h});
|
|
std::sort(sorted.begin(), sorted.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
|
|
for (const auto& [va, h] : sorted) {
|
|
char b[120];
|
|
std::snprintf(b, sizeof b, " named %#llx %ux%u f%#x frame %lld%s", (unsigned long long)va, h.width, h.height,
|
|
h.format, (long long)h.frame, h.on ? "" : " (render off)");
|
|
out += b;
|
|
}
|
|
return out;
|
|
}
|
|
} // namespace rx
|
|
|
|
/// (debugging, any thread, no lock: a guess) the render target images over guest [lo, lo + len)
|
|
extern "C" void rx_debug_images_at(uint64_t lo, uint64_t len, char* out, size_t cap) {
|
|
size_t n = 0;
|
|
out[0] = 0;
|
|
for (const auto& [g, im] : rx::all_images())
|
|
if (im && im->guest < lo + len && lo < im->guest + (uint64_t)im->width * im->height * 4 && n + 80 < cap)
|
|
n += std::snprintf(out + n, cap - n, " rt %08llx %ux%u fmt %d", (unsigned long long)im->guest, im->width,
|
|
im->height, (int)im->format);
|
|
}
|
|
|
|
namespace rx {
|
|
bool guest_watch_hit(uint64_t va, uint64_t size) {
|
|
static uint64_t lo = 1, hi = 0;
|
|
if (lo == 1) {
|
|
lo = 0;
|
|
if (const char* v = getenv("RX_GUEST_WATCH")) {
|
|
char* e;
|
|
lo = strtoull(v, &e, 16);
|
|
hi = *e == '-' ? strtoull(e + 1, nullptr, 16) : 0;
|
|
}
|
|
}
|
|
if (!hi) return false;
|
|
const uint64_t g = hos_gpu_translate(va);
|
|
return g && g < hi && g + size > lo;
|
|
}
|
|
} // namespace rx
|
|
|
|
/// the live input file's "!shots <ms>" (hos/nn_hid.c): another screenshot interval from now on
|
|
extern "C" void rx_set_screenshot_ms(int64_t ms) { rx::screenshot_interval = ms * 1000000LL; }
|
|
|
|
extern "C" void rx_flash_dark_arm(void) { rx::flash_dark_armed = true; }
|