mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
1604 lines
76 KiB
C++
1604 lines
76 KiB
C++
// The native NVN layer's replay (HOS_NVN_NATIVE=1): the command records hos/nvn_native.c made, run at the submit with
|
|
// the renderer. See rx_nvn.h. Each record becomes renderer work built from NVN state directly: render targets from
|
|
// NVN texture objects, clears, draws and dispatches from the bound programs and state objects. Effects the CPU can see
|
|
// (events, counters, uniform updates, buffer copies) happen in the order of the recordings.
|
|
#include <atomic>
|
|
#include <vector>
|
|
#include <thread>
|
|
#include <deque>
|
|
#include <condition_variable>
|
|
#include <cstring>
|
|
#include <mutex>
|
|
#include <unordered_set>
|
|
#include <unordered_map>
|
|
|
|
#include "rx_internal.h"
|
|
#include "rx_nvn.h"
|
|
#if RX_WEB
|
|
#include <emscripten/threading.h>
|
|
#include <pthread.h>
|
|
#endif
|
|
|
|
extern "C" {
|
|
uint32_t* rx_collect_written(int64_t* count);
|
|
void rx_begin_submission_written(uint32_t* pages, int64_t n);
|
|
void rx_flush(void);
|
|
void rx_memory_written(uint64_t gpu_va, uint64_t size);
|
|
int rx_gpu_writes_pending(uint64_t gpu_va, uint64_t size);
|
|
int rx_gpu_writes_in_cache(uint64_t gpu_va, uint64_t size);
|
|
void rx_sync_gpu_writes(uint64_t gpu_va, uint64_t size);
|
|
int rx_present(uint64_t guest, uint32_t width, uint32_t height);
|
|
void rx_present_crop(uint32_t w, uint32_t h); // (rx_core.cpp: the window crop for the next present)
|
|
void rx_wait_idle(void);
|
|
void hos_host_thread_register_self(const char* name);
|
|
void hos_thread_boost(int level);
|
|
#if RX_WEB
|
|
void rx_web_idle(volatile uint32_t* wake, uint32_t seen);
|
|
#endif
|
|
#if defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
|
|
void hos_mark_written(const void* p, size_t n);
|
|
#endif
|
|
}
|
|
|
|
namespace rx {
|
|
/// Guest memory this code wrote with the CPU: the soft write watch's mark (Windows' write watch sees it by itself)
|
|
inline void rx_mark_written(const void* p, size_t n) {
|
|
#if defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
|
|
hos_mark_written(p, n);
|
|
#else
|
|
(void)p, (void)n;
|
|
#endif
|
|
}
|
|
|
|
// ---- formats ----
|
|
PixelFormat nvn_pixel_format(int f) {
|
|
using P = PixelFormat;
|
|
switch (f) {
|
|
case NVN_FORMAT_R8: return P::R8_UNORM;
|
|
case NVN_FORMAT_R8SN: return P::R8_SNORM;
|
|
case NVN_FORMAT_R8UI: return P::R8_UINT;
|
|
case NVN_FORMAT_R8I: return P::R8_SINT;
|
|
case NVN_FORMAT_R16F: return P::R16_FLOAT;
|
|
case NVN_FORMAT_R16: return P::R16_UNORM;
|
|
case NVN_FORMAT_R16SN: return P::R16_SNORM;
|
|
case NVN_FORMAT_R16UI: return P::R16_UINT;
|
|
case NVN_FORMAT_R16I: return P::R16_SINT;
|
|
case NVN_FORMAT_R32F: return P::R32_FLOAT;
|
|
case NVN_FORMAT_R32UI: return P::R32_UINT;
|
|
case NVN_FORMAT_R32I: return P::R32_SINT;
|
|
case NVN_FORMAT_RG8: return P::R8G8_UNORM;
|
|
case NVN_FORMAT_RG8SN: return P::R8G8_SNORM;
|
|
case NVN_FORMAT_RG8UI: return P::R8G8_UINT;
|
|
case NVN_FORMAT_RG8I: return P::R8G8_SINT;
|
|
case NVN_FORMAT_RG16F: return P::R16G16_FLOAT;
|
|
case NVN_FORMAT_RG16: return P::R16G16_UNORM;
|
|
case NVN_FORMAT_RG16SN: return P::R16G16_SNORM;
|
|
case NVN_FORMAT_RG16UI: return P::R16G16_UINT;
|
|
case NVN_FORMAT_RG16I: return P::R16G16_SINT;
|
|
case NVN_FORMAT_RG32F: return P::R32G32_FLOAT;
|
|
case NVN_FORMAT_RG32UI: return P::R32G32_UINT;
|
|
case NVN_FORMAT_RG32I: return P::R32G32_SINT;
|
|
case NVN_FORMAT_RGB32F: return P::R32G32B32_FLOAT;
|
|
case NVN_FORMAT_RGBA8: case NVN_FORMAT_RGBX8: return P::A8B8G8R8_UNORM;
|
|
case NVN_FORMAT_RGBA8SN: case NVN_FORMAT_RGBX8SN: return P::A8B8G8R8_SNORM;
|
|
case NVN_FORMAT_RGBA8UI: case NVN_FORMAT_RGBX8UI: return P::A8B8G8R8_UINT;
|
|
case NVN_FORMAT_RGBA8I: case NVN_FORMAT_RGBX8I: return P::A8B8G8R8_SINT;
|
|
case NVN_FORMAT_RGBA16F: return P::R16G16B16A16_FLOAT;
|
|
case NVN_FORMAT_RGBX16F: return P::R16G16B16X16_FLOAT;
|
|
case NVN_FORMAT_RGBA16: case NVN_FORMAT_RGBX16: return P::R16G16B16A16_UNORM;
|
|
case NVN_FORMAT_RGBA16SN: case NVN_FORMAT_RGBX16SN: return P::R16G16B16A16_SNORM;
|
|
case NVN_FORMAT_RGBA16UI: case NVN_FORMAT_RGBX16UI: return P::R16G16B16A16_UINT;
|
|
case NVN_FORMAT_RGBA16I: case NVN_FORMAT_RGBX16I: return P::R16G16B16A16_SINT;
|
|
case NVN_FORMAT_RGBA32F: case NVN_FORMAT_RGBX32F: return P::R32G32B32A32_FLOAT;
|
|
case NVN_FORMAT_RGBA32UI: case NVN_FORMAT_RGBX32UI: return P::R32G32B32A32_UINT;
|
|
case NVN_FORMAT_RGBA32I: case NVN_FORMAT_RGBX32I: return P::R32G32B32A32_SINT;
|
|
case NVN_FORMAT_STENCIL8: return P::S8_UINT;
|
|
case NVN_FORMAT_DEPTH16: return P::D16_UNORM;
|
|
case NVN_FORMAT_DEPTH24: return P::X8_D24_UNORM;
|
|
case NVN_FORMAT_DEPTH32F: return P::D32_FLOAT;
|
|
case NVN_FORMAT_DEPTH24_STENCIL8: return P::S8_UINT_D24_UNORM;
|
|
case NVN_FORMAT_DEPTH32F_STENCIL8: return P::D32_FLOAT_S8_UINT;
|
|
case NVN_FORMAT_RGBX8_SRGB: case NVN_FORMAT_RGBA8_SRGB: return P::A8B8G8R8_SRGB;
|
|
case NVN_FORMAT_RGBA4: return P::A4B4G4R4_UNORM;
|
|
case NVN_FORMAT_RGB5A1: return P::A1B5G5R5_UNORM;
|
|
case NVN_FORMAT_RGB565: return P::B5G6R5_UNORM;
|
|
case NVN_FORMAT_BGR565: return P::R5G6B5_UNORM;
|
|
case NVN_FORMAT_RGB10A2: return P::A2B10G10R10_UNORM;
|
|
case NVN_FORMAT_RGB10A2UI: return P::A2B10G10R10_UINT;
|
|
case NVN_FORMAT_R11G11B10F: return P::B10G11R11_FLOAT;
|
|
case NVN_FORMAT_RGB9E5F: return P::E5B9G9R9_FLOAT;
|
|
case NVN_FORMAT_RGB_DXT1: case NVN_FORMAT_RGBA_DXT1: return P::BC1_RGBA_UNORM;
|
|
case NVN_FORMAT_RGB_DXT1_SRGB: case NVN_FORMAT_RGBA_DXT1_SRGB: return P::BC1_RGBA_SRGB;
|
|
case NVN_FORMAT_RGBA_DXT3: return P::BC2_UNORM;
|
|
case NVN_FORMAT_RGBA_DXT3_SRGB: return P::BC2_SRGB;
|
|
case NVN_FORMAT_RGBA_DXT5: return P::BC3_UNORM;
|
|
case NVN_FORMAT_RGBA_DXT5_SRGB: return P::BC3_SRGB;
|
|
case NVN_FORMAT_RGTC1_UNORM: return P::BC4_UNORM;
|
|
case NVN_FORMAT_RGTC1_SNORM: return P::BC4_SNORM;
|
|
case NVN_FORMAT_RGTC2_UNORM: return P::BC5_UNORM;
|
|
case NVN_FORMAT_RGTC2_SNORM: return P::BC5_SNORM;
|
|
case NVN_FORMAT_BPTC_UNORM: return P::BC7_UNORM;
|
|
case NVN_FORMAT_BPTC_UNORM_SRGB: return P::BC7_SRGB;
|
|
case NVN_FORMAT_BPTC_SFLOAT: return P::BC6H_SFLOAT;
|
|
case NVN_FORMAT_BPTC_UFLOAT: return P::BC6H_UFLOAT;
|
|
case NVN_FORMAT_RGBA_ASTC_4x4: return P::ASTC_2D_4X4_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_5x4: return P::ASTC_2D_5X4_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_5x5: return P::ASTC_2D_5X5_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_6x5: return P::ASTC_2D_6X5_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_6x6: return P::ASTC_2D_6X6_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_8x5: return P::ASTC_2D_8X5_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_8x6: return P::ASTC_2D_8X6_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_8x8: return P::ASTC_2D_8X8_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_10x5: return P::ASTC_2D_10X5_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_10x6: return P::ASTC_2D_10X6_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_10x8: return P::ASTC_2D_10X8_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_10x10: return P::ASTC_2D_10X10_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_12x10: return P::ASTC_2D_12X10_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_12x12: return P::ASTC_2D_12X12_UNORM;
|
|
case NVN_FORMAT_RGBA_ASTC_4x4_SRGB: return P::ASTC_2D_4X4_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_5x4_SRGB: return P::ASTC_2D_5X4_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_5x5_SRGB: return P::ASTC_2D_5X5_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_6x5_SRGB: return P::ASTC_2D_6X5_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_6x6_SRGB: return P::ASTC_2D_6X6_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_8x5_SRGB: return P::ASTC_2D_8X5_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_8x6_SRGB: return P::ASTC_2D_8X6_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_8x8_SRGB: return P::ASTC_2D_8X8_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_10x5_SRGB: return P::ASTC_2D_10X5_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_10x6_SRGB: return P::ASTC_2D_10X6_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_10x8_SRGB: return P::ASTC_2D_10X8_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_10x10_SRGB: return P::ASTC_2D_10X10_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_12x10_SRGB: return P::ASTC_2D_12X10_SRGB;
|
|
case NVN_FORMAT_RGBA_ASTC_12x12_SRGB: return P::ASTC_2D_12X12_SRGB;
|
|
case NVN_FORMAT_BGRA8: case NVN_FORMAT_BGRX8: return P::B8G8R8A8_UNORM;
|
|
case NVN_FORMAT_BGRA8_SRGB: case NVN_FORMAT_BGRX8_SRGB: return P::B8G8R8A8_SRGB;
|
|
default: return P::Invalid;
|
|
}
|
|
}
|
|
|
|
// ---- texture and sampler descriptor registrations ----
|
|
namespace {
|
|
struct TexReg {
|
|
nvn_tex t{};
|
|
nvn_view v{};
|
|
bool has_view = false, ok = false;
|
|
bool released = false; // (its texture was finalized: the registration names the one living at its storage)
|
|
};
|
|
struct SampReg {
|
|
nvn_samp_par p{};
|
|
bool ok = false;
|
|
};
|
|
std::mutex reg_mu; // (registrations come from any game thread)
|
|
std::vector<TexReg> tex_regs;
|
|
std::vector<SampReg> samp_regs;
|
|
// registrations by their texture's serial: a finalized texture marks the ones naming it (a set of every finalized
|
|
// serial grew for the whole session: TOTK finalizes transient textures every frame)
|
|
std::unordered_map<uint64_t, std::vector<int>> regs_by_serial;
|
|
// per handle, bumped whenever what the handle resolves to may change (registered again, its texture finalized): the
|
|
// texture binding memo (rx_texture.cpp) holds across frames while it stays (read without the lock: fixed arrays)
|
|
std::atomic<uint32_t> tex_version[1 << 20], samp_version[1 << 12];
|
|
} // namespace
|
|
|
|
uint32_t nvn_texture_version(int id) { return id >= 0 && id < (1 << 20) ? tex_version[id].load(std::memory_order_relaxed) : 0; }
|
|
uint32_t nvn_sampler_version(int id) { return id >= 0 && id < (1 << 12) ? samp_version[id].load(std::memory_order_relaxed) : 0; }
|
|
bool nvn_texture_follows_storage(int id) {
|
|
std::lock_guard lk(reg_mu);
|
|
return id >= 0 && (size_t)id < tex_regs.size() && tex_regs[id].ok && tex_regs[id].released;
|
|
}
|
|
|
|
void nvn_register_texture(int id, const nvn_tex& t, const nvn_view* v) {
|
|
if (id < 0 || id >= (1 << 20)) return;
|
|
std::lock_guard lk(reg_mu);
|
|
if ((size_t)id >= tex_regs.size()) tex_regs.resize((size_t)id + 1024);
|
|
TexReg& r = tex_regs[id];
|
|
if (r.ok) { // (no longer named by this handle)
|
|
auto it = regs_by_serial.find(r.t.serial);
|
|
if (it != regs_by_serial.end()) {
|
|
std::erase(it->second, id);
|
|
if (it->second.empty()) regs_by_serial.erase(it);
|
|
}
|
|
}
|
|
r.t = t;
|
|
r.has_view = v != nullptr;
|
|
r.v = v ? *v : nvn_view{};
|
|
r.ok = true;
|
|
r.released = false;
|
|
regs_by_serial[t.serial].push_back(id);
|
|
tex_version[id].fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
void nvn_register_sampler(int id, const nvn_samp_par& p) {
|
|
if (id < 0 || id >= (1 << 12)) return;
|
|
std::lock_guard lk(reg_mu);
|
|
if ((size_t)id >= samp_regs.size()) samp_regs.resize((size_t)id + 256);
|
|
samp_regs[id].p = p;
|
|
samp_regs[id].ok = true;
|
|
samp_version[id].fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
void nvn_texture_released(uint64_t serial) {
|
|
std::lock_guard lk(reg_mu);
|
|
auto it = regs_by_serial.find(serial);
|
|
if (it == regs_by_serial.end()) return;
|
|
for (int id : it->second) {
|
|
tex_regs[id].released = true;
|
|
tex_version[id].fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
}
|
|
bool nvn_texture_of(int id, nvn_tex* t, nvn_view* v, bool* has_view) {
|
|
std::lock_guard lk(reg_mu);
|
|
if (id < 0 || (size_t)id >= tex_regs.size() || !tex_regs[id].ok) return false;
|
|
const TexReg& r = tex_regs[id];
|
|
*t = r.t;
|
|
*v = r.v;
|
|
*has_view = r.has_view;
|
|
// the game re-initialises transient targets every frame at the same storage without registering them again (a
|
|
// descriptor names memory on the console): a finalized texture's registration names the one living there now
|
|
if (r.released) {
|
|
nvn_tex live;
|
|
if (nvn_tex_live_at(r.t.gpu, &live)) *t = live;
|
|
}
|
|
return true;
|
|
}
|
|
bool nvn_sampler_of(int id, nvn_samp_par* p) {
|
|
std::lock_guard lk(reg_mu);
|
|
if (id < 0 || (size_t)id >= samp_regs.size() || !samp_regs[id].ok) return false;
|
|
*p = samp_regs[id].p;
|
|
return true;
|
|
}
|
|
|
|
// ---- the driver's constant banks (what NVN-compiled programs read for their bindings) ----
|
|
// c0: the storage buffer table (16 bytes a slot: address, size) at 0x110 (vertex), 0x510 (fragment), 0x310 (compute);
|
|
// compute also has its grid at 0x200 and its uniform buffers' table at 0x210; vertex programs clamp the point size to
|
|
// 0x610. c2: texture handles at 0x168 + 8 slot and 0x20 + 8 slot (BindTexture's combined handles, 32 slots), sampler
|
|
// handles at 0x568 + 8 slot, image handles at 0x120 + 8 slot. (Measured on TOTK by the MIT-licensed reference layer.)
|
|
// (mkw-recomp) NVN2 compute programs read a c0 packed for the program: the workgroup counts at 0, then from 0x10 the
|
|
// uniform buffer table (16 bytes a slot: address, size), the storage buffer table (the same), texture handles and
|
|
// image handles (8 bytes a slot), as many slots of each as the program's control section says (+0xb1 uniform
|
|
// buffers, +0xb2 storage buffers, +0xb3 textures, +0xb6 images). Found from 173 programs' reads, all of which fit.
|
|
// Graphics programs keep the fixed layout below (c2 texture handles at 0x20 + 8 slot).
|
|
struct Nvn2CsLayout {
|
|
uint32_t ubo = 0, ssbo = 0, tex = 0, img = 0; // slot counts
|
|
uint32_t ssbo_at() const { return 0x10 + 16 * ubo; }
|
|
uint32_t tex_at() const { return ssbo_at() + 16 * ssbo; }
|
|
uint32_t img_at() const { return tex_at() + 8 * tex; }
|
|
uint32_t end() const { return img_at() + 8 * img; }
|
|
};
|
|
Nvn2CsLayout nvn2_cs_layout(const NvnState& s) {
|
|
static std::mutex mu;
|
|
static std::unordered_map<uint64_t, Nvn2CsLayout> by_serial;
|
|
std::lock_guard lk(mu);
|
|
if (auto it = by_serial.find(s.prog.serial); it != by_serial.end()) return it->second;
|
|
Nvn2CsLayout l;
|
|
for (uint32_t i = 0; i < s.prog.nstage && i < 6; i++) {
|
|
uint8_t c[0x60];
|
|
if (!s.prog.sd[i].control || !gpu_read(s.prog.sd[i].control + 0x58, c, sizeof c)) continue;
|
|
uint32_t magic = 0;
|
|
uint16_t st = 0;
|
|
gpu_read(s.prog.sd[i].control, &magic, 4);
|
|
gpu_read(s.prog.sd[i].control + 0x56, &st, 2);
|
|
if (magic != 0x98761234 || st != 5) continue;
|
|
l.ubo = c[0xb1 - 0x58], l.ssbo = c[0xb2 - 0x58], l.tex = c[0xb3 - 0x58], l.img = c[0xb6 - 0x58];
|
|
break;
|
|
}
|
|
if (by_serial.size() > 4096) by_serial.clear();
|
|
by_serial[s.prog.serial] = l;
|
|
return l;
|
|
}
|
|
|
|
void nvn_driver_bank(const NvnState& s, int bank, int stage, uint8_t d[0xA00]) {
|
|
std::memset(d, 0, 0xA00);
|
|
if (stage < 0 || stage > 5) return;
|
|
if (bank == 0 && stage == 5) { // (NVN2 compute: see Nvn2CsLayout)
|
|
const Nvn2CsLayout l = nvn2_cs_layout(s);
|
|
std::memcpy(d, s.grid, sizeof s.grid);
|
|
auto entry = [&](uint32_t at, const nvn_p_bind_buf& e) {
|
|
if (at + 16 > 0xA00) return;
|
|
const uint32_t n = (uint32_t)e.size;
|
|
std::memcpy(d + at, &e.addr, 8);
|
|
std::memcpy(d + at + 8, &n, 4);
|
|
};
|
|
for (uint32_t r = 0; r < l.ubo && r < 16; r++) entry(0x10 + 16 * r, s.ubo[5][r]);
|
|
for (uint32_t r = 0; r < l.ssbo && r < 16; r++) entry(l.ssbo_at() + 16 * r, s.ssbo[5][r]);
|
|
for (uint32_t r = 0; r < l.tex && r < 128 && l.tex_at() + 8 * r + 4 <= 0xA00; r++) {
|
|
const uint32_t th = (uint32_t)s.tex[5][r];
|
|
std::memcpy(d + l.tex_at() + 8 * r, &th, 4);
|
|
}
|
|
for (uint32_t r = 0; r < l.img && r < 32 && l.img_at() + 8 * r + 4 <= 0xA00; r++) {
|
|
const uint32_t ih = (uint32_t)s.img[5][r];
|
|
std::memcpy(d + l.img_at() + 8 * r, &ih, 4);
|
|
}
|
|
return;
|
|
}
|
|
if (bank == 0) {
|
|
const int base = stage == 0 ? 0x110 : stage == 5 ? 0x310 : stage == 1 ? 0x510 : -1;
|
|
if (base >= 0)
|
|
for (int r = 0; r < 16; r++) {
|
|
const uint64_t a = s.ssbo[stage][r].addr;
|
|
const uint32_t n = (uint32_t)s.ssbo[stage][r].size;
|
|
std::memcpy(d + base + 16 * r, &a, 8);
|
|
std::memcpy(d + base + 16 * r + 8, &n, 4);
|
|
}
|
|
if (stage == 5) {
|
|
std::memcpy(d + 0x200, s.grid, sizeof s.grid);
|
|
for (int r = 0; r < 16; r++) {
|
|
const uint64_t a = s.ubo[5][r].addr;
|
|
const uint32_t n = (uint32_t)s.ubo[5][r].size;
|
|
std::memcpy(d + 0x210 + 16 * r, &a, 8);
|
|
std::memcpy(d + 0x210 + 16 * r + 8, &n, 4);
|
|
}
|
|
} else if (stage == 0) {
|
|
const float point_max = 2047.9375f;
|
|
std::memcpy(d + 0x610, &point_max, 4);
|
|
}
|
|
} else if (bank == 2) {
|
|
for (int r = 0; r < 128; r++) {
|
|
const uint32_t th = (uint32_t)s.tex[stage][r], sh = (uint32_t)s.samp[stage][r];
|
|
std::memcpy(d + 0x168 + 8 * r, &th, 4);
|
|
std::memcpy(d + 0x568 + 8 * r, &sh, 4);
|
|
}
|
|
for (int r = 0; r < 32; r++) {
|
|
const uint32_t th = (uint32_t)s.tex[stage][r];
|
|
std::memcpy(d + 0x20 + 8 * r, &th, 4);
|
|
}
|
|
for (int r = 0; r < 9; r++) {
|
|
const uint32_t ih = (uint32_t)s.img[stage][r];
|
|
std::memcpy(d + 0x120 + 8 * r, &ih, 4);
|
|
}
|
|
}
|
|
}
|
|
/// One word of a driver bank, as nvn_driver_bank lays it out (without building the 2.5 KB bank: the programs' handle
|
|
/// and buffer reads, several per draw)
|
|
uint32_t nvn_driver_bank_word(const NvnState& s, int bank, int stage, uint32_t offset) {
|
|
if (stage < 0 || stage > 5 || offset + 4 > 0xA00 || (offset & 3)) {
|
|
uint8_t d[0xA00];
|
|
nvn_driver_bank(s, bank, stage, d);
|
|
uint32_t w = 0;
|
|
if (offset + 4 <= sizeof d) std::memcpy(&w, d + offset, 4);
|
|
return w;
|
|
}
|
|
auto lo = [](uint64_t v) { return (uint32_t)v; };
|
|
auto hi = [](uint64_t v) { return (uint32_t)(v >> 32); };
|
|
if (bank == 2) {
|
|
if (offset & 7) return 0; // (the upper words of the 8-byte slots)
|
|
if (offset >= 0x568 && offset < 0x568 + 8 * 128) return lo(s.samp[stage][(offset - 0x568) / 8]);
|
|
if (offset >= 0x168 && offset < 0x168 + 8 * 128) return lo(s.tex[stage][(offset - 0x168) / 8]);
|
|
if (offset >= 0x120 && offset < 0x120 + 8 * 9) return lo(s.img[stage][(offset - 0x120) / 8]);
|
|
if (offset >= 0x20 && offset < 0x20 + 8 * 32) return lo(s.tex[stage][(offset - 0x20) / 8]);
|
|
return 0;
|
|
}
|
|
if (bank != 0) return 0;
|
|
if (stage == 5) { // (NVN2 compute: see Nvn2CsLayout)
|
|
const Nvn2CsLayout l = nvn2_cs_layout(s);
|
|
if (offset < 0x10) return offset < sizeof s.grid ? s.grid[offset / 4] : 0;
|
|
auto entry = [&](const nvn_p_bind_buf& e, uint32_t in) -> uint32_t {
|
|
return in == 0 ? lo(e.addr) : in == 4 ? hi(e.addr) : in == 8 ? (uint32_t)e.size : 0;
|
|
};
|
|
if (offset < l.ssbo_at()) return (offset - 0x10) / 16 < 16 ? entry(s.ubo[5][(offset - 0x10) / 16], (offset - 0x10) % 16) : 0;
|
|
if (offset < l.tex_at())
|
|
return (offset - l.ssbo_at()) / 16 < 16 ? entry(s.ssbo[5][(offset - l.ssbo_at()) / 16], (offset - l.ssbo_at()) % 16) : 0;
|
|
if (offset < l.img_at()) return (offset & 7) ? 0 : lo(s.tex[5][std::min<uint32_t>((offset - l.tex_at()) / 8, 127)]);
|
|
if (offset < l.end()) return (offset & 7) ? 0 : lo(s.img[5][std::min<uint32_t>((offset - l.img_at()) / 8, 31)]);
|
|
return 0;
|
|
}
|
|
const uint32_t base = stage == 0 ? 0x110 : stage == 5 ? 0x310 : stage == 1 ? 0x510 : 0;
|
|
auto table = [&](const nvn_p_bind_buf* t, uint32_t at) -> uint32_t { // 16-byte entries: address, size, 0
|
|
const nvn_p_bind_buf& e = t[(offset - at) / 16];
|
|
switch ((offset - at) % 16) {
|
|
case 0: return lo(e.addr);
|
|
case 4: return hi(e.addr);
|
|
case 8: return (uint32_t)e.size;
|
|
default: return 0;
|
|
}
|
|
};
|
|
if (stage == 5) {
|
|
if (offset >= 0x200 && offset < 0x200 + sizeof s.grid) return s.grid[(offset - 0x200) / 4];
|
|
if (offset >= 0x210 && offset < 0x210 + 16 * 16) return table(s.ubo[5], 0x210);
|
|
}
|
|
if (base && offset >= base && offset < base + 16 * 16) return table(s.ssbo[stage], base);
|
|
if (stage == 0 && offset == 0x610) return std::bit_cast<uint32_t>(2047.9375f);
|
|
return 0;
|
|
}
|
|
|
|
|
|
namespace {
|
|
|
|
std::mutex nvn_mu; // one replay at a time (submits come from several game threads)
|
|
std::atomic<uint64_t> submitted{0};
|
|
NvnState state; // the queue's state
|
|
|
|
/* Tegra GPU ticks (19.2 MHz) of now: what counters' timestamps hold */
|
|
/* Counter timestamps: within a frame the time runs at RX_NVN_TS_SCALE (default 0.1), from the real time at its start.
|
|
TOTK's dynamic resolution reads the frame's GPU time from them, and the replay's CPU time is not the GPU's (several
|
|
times the GPU's work): after every load it dropped to 1280x720 for ~1.5 s, and in heavy scenes for good. */
|
|
bool ts_new_frame = true;
|
|
uint64_t gpu_ticks() {
|
|
// ((mkw-recomp) default 1: MKW's frame pacer compares the timestamps with its CPU ticks to pick 30 or 60 fps; at
|
|
// 0.1 it saw a GPU ten times as fast, chose 60, missed it and switched back, over and over: 16-21 fps)
|
|
static const double scale = getenv("RX_NVN_TS_SCALE") ? atof(getenv("RX_NVN_TS_SCALE")) : 1.0;
|
|
// ((mkw-recomp) on the CPU's tick clock, as the console's GPU and CPU share one: see nvn_native.c)
|
|
const uint64_t real = hos_ticks() * (1000000000ull / 31250000ull);
|
|
static uint64_t origin_real, origin_out, last_out;
|
|
if (ts_new_frame) {
|
|
ts_new_frame = false;
|
|
origin_real = real;
|
|
origin_out = std::max(real, last_out);
|
|
}
|
|
const uint64_t out = origin_out + (uint64_t)((double)(real - origin_real) * scale);
|
|
last_out = std::max(last_out, out);
|
|
return last_out * 12 / 625;
|
|
}
|
|
|
|
/* a record's payload: variable-length records (bindings of n items) are shorter than their type (the rest: zeros) */
|
|
template <class T>
|
|
T payload(const uint8_t* a, uint32_t bytes) {
|
|
T v{};
|
|
std::memcpy(&v, a, std::min<size_t>(sizeof v, bytes));
|
|
return v;
|
|
}
|
|
|
|
// ---- copies: on guest memory, as the console's copy engine makes them (textures in their own layout) ----
|
|
// A copy reads what guest memory holds: a source a shader wrote on the GPU is synced first; one a render pass drew
|
|
// (a render target) is counted (nvn_copy_from_rt) until copies of those run on the GPU.
|
|
|
|
/* one level of a texture as the copy engine addresses it: sizes in blocks (a compressed format's 4x4 texels) */
|
|
struct CopySurface {
|
|
uint64_t base = 0, layer_size = 0, level_size = 0; // GPU address of the level in the first layer; bytes
|
|
uint32_t bpb = 0, bw = 1, bh = 1; // bytes per block, block size in texels
|
|
uint32_t width = 0, height = 0, depth = 1; // the level in blocks (depth: a 3D texture's slices)
|
|
uint32_t base_layer = 0;
|
|
int target = 0;
|
|
bool linear = false;
|
|
uint32_t stride = 0; // linear: bytes per row
|
|
unsigned tile_h = 0, tile_d = 0; // block linear: block height / depth (log2 GOBs)
|
|
};
|
|
|
|
/* a GOB is 64 bytes x 8 rows (512 bytes) swizzled inside; a block is 1 GOB wide, 2^bh GOBs tall and 2^bd deep;
|
|
blocks run left to right, then top to bottom, then front to back (x in bytes, y in rows) */
|
|
uint64_t bl_offset(uint32_t x, uint32_t y, uint32_t z, uint32_t width_bytes, uint32_t height, uint32_t bh, uint32_t bd) {
|
|
const uint64_t gobs_x = (width_bytes + 63) >> 6;
|
|
const uint64_t blocks_y = (height + (8u << bh) - 1) >> (3 + bh);
|
|
const uint64_t block = 512ull << (bh + bd);
|
|
// (the Switch 2's GOB swizzle: the Tegra X1's with offset bits 5 and 6 swapped)
|
|
const uint64_t gob = ((x & 63) >> 5) << 8 | ((y & 7) >> 2) << 7 | ((x & 31) >> 4) << 6 | ((y >> 1) & 1) << 5 |
|
|
(y & 1) << 4 | (x & 15);
|
|
return (uint64_t)(z >> bd) * gobs_x * blocks_y * block + (uint64_t)(y >> (3 + bh)) * gobs_x * block +
|
|
(uint64_t)(x >> 6) * block + ((uint64_t)(z & ((1u << bd) - 1)) << (9 + bh)) +
|
|
((uint64_t)((y >> 3) & ((1u << bh) - 1)) << 9) + gob;
|
|
}
|
|
|
|
bool copy_surface(const nvn_tex& t, const nvn_view* v, CopySurface& o) {
|
|
int bw = 1, bh = 1;
|
|
const int bpb = nvn_fmt_bpb(t.d.format, &bw, &bh);
|
|
if (bpb <= 0 || !t.gpu) return false;
|
|
const uint32_t level = v && v->set_levels ? v->base_level : 0;
|
|
const bool one_d = t.d.target == NVN_TEXTURE_TARGET_1D || t.d.target == NVN_TEXTURE_TARGET_1D_ARRAY;
|
|
o.target = t.d.target;
|
|
o.bpb = (uint32_t)bpb, o.bw = (uint32_t)bw, o.bh = (uint32_t)bh;
|
|
o.width = (std::max(1u, t.d.width >> level) + o.bw - 1) / o.bw;
|
|
o.height = one_d ? 1 : (std::max(1u, t.d.height >> level) + o.bh - 1) / o.bh;
|
|
o.depth = t.d.target == NVN_TEXTURE_TARGET_3D ? std::max(1u, t.d.depth >> level) : 1;
|
|
o.base = t.gpu + nvn_tex_level_offset(&t.d, level);
|
|
o.layer_size = t.L.layer_size;
|
|
o.base_layer = v && v->set_layers ? v->min_layer : 0;
|
|
o.linear = nvn_tex_is_linear(&t.d) != 0;
|
|
if (o.linear) {
|
|
o.stride = t.d.stride ? t.d.stride : nvn_tex_linear_stride(&t.d);
|
|
o.level_size = (uint64_t)o.stride * o.height;
|
|
} else {
|
|
nvn_tex_level_tiles(&t.d, level, &o.tile_h, &o.tile_d);
|
|
const uint64_t gobs_x = (o.width * o.bpb + 63) >> 6;
|
|
const uint64_t blocks_y = (o.height + (8u << o.tile_h) - 1) >> (3 + o.tile_h);
|
|
const uint64_t blocks_z = (o.depth + (1u << o.tile_d) - 1) >> o.tile_d;
|
|
o.level_size = gobs_x * blocks_y * blocks_z * (512ull << (o.tile_h + o.tile_d));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/* a copy region in blocks: x, y, w, h; z, d (3D slices, or array layers when `layers`) */
|
|
struct CopyBox {
|
|
uint32_t x = 0, y = 0, w = 0, h = 0, z = 0, d = 0;
|
|
bool layers = false;
|
|
};
|
|
CopyBox copy_box(const CopySurface& s, const NVNcopyRegion& r) {
|
|
CopyBox b;
|
|
const auto clampi = [](int v) { return (uint32_t)std::max(v, 0); };
|
|
b.x = clampi(r.xoffset) / s.bw;
|
|
b.w = (clampi(r.width) + s.bw - 1) / s.bw;
|
|
b.y = clampi(r.yoffset) / s.bh;
|
|
b.h = (clampi(r.height) + s.bh - 1) / s.bh;
|
|
b.z = clampi(r.zoffset);
|
|
b.d = std::max(1u, clampi(r.depth));
|
|
switch (s.target) {
|
|
case NVN_TEXTURE_TARGET_1D: b.y = 0, b.h = 1, b.z = 0, b.d = 1; break;
|
|
case NVN_TEXTURE_TARGET_1D_ARRAY:
|
|
b.z = clampi(r.yoffset), b.d = std::max(1u, clampi(r.height)), b.layers = true;
|
|
b.y = 0, b.h = 1;
|
|
break;
|
|
case NVN_TEXTURE_TARGET_3D: break;
|
|
case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: case NVN_TEXTURE_TARGET_CUBEMAP:
|
|
case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY:
|
|
b.layers = true;
|
|
break;
|
|
default: b.z = 0, b.d = 1; break;
|
|
}
|
|
// (clipped to the level)
|
|
b.w = b.x < s.width ? std::min(b.w, s.width - b.x) : 0;
|
|
b.h = b.y < s.height ? std::min(b.h, s.height - b.y) : 0;
|
|
if (!b.layers) b.d = b.z < s.depth ? std::min(b.d, s.depth - b.z) : 0;
|
|
return b;
|
|
}
|
|
|
|
/* host memory of GPU addresses for a run of copies: a 64 KiB page at a time (virtual pools map pages that size) */
|
|
struct HostPages {
|
|
uint64_t page = ~0ull;
|
|
uint8_t* host = nullptr;
|
|
uint8_t* at(uint64_t gpu) {
|
|
if (gpu >> 16 != page) {
|
|
page = gpu >> 16;
|
|
host = nvn_gpu_host(page << 16);
|
|
if (host && nvn_gpu_contiguous(page << 16, 0x10000) < 0x10000) host = nullptr;
|
|
}
|
|
return host ? host + (gpu & 0xFFFF) : nullptr;
|
|
}
|
|
};
|
|
|
|
/* one row of blocks of the box (row `row` of slice / layer k): fn(GPU address, byte offset in the row, bytes) for
|
|
each run contiguous in the texture's layout (16 bytes inside a GOB; a linear texture's whole row) */
|
|
template <class Fn>
|
|
void surface_row(const CopySurface& s, const CopyBox& b, uint32_t row, uint32_t k, Fn&& fn) {
|
|
const uint64_t layer = b.layers ? (uint64_t)(s.base_layer + b.z + k) : s.base_layer;
|
|
const uint32_t z = b.layers ? 0 : b.z + k;
|
|
const uint64_t base = s.base + layer * s.layer_size;
|
|
const uint32_t x0 = b.x * s.bpb, n = b.w * s.bpb, y = b.y + row;
|
|
if (s.linear) {
|
|
fn(base + (uint64_t)y * s.stride + x0, 0u, n);
|
|
return;
|
|
}
|
|
for (uint32_t x = 0; x < n;) {
|
|
const uint32_t bx = x0 + x, run = std::min(16 - (bx & 15), n - x);
|
|
fn(base + bl_offset(bx, y, z, s.width * s.bpb, s.height, s.tile_h, s.tile_d), x, run);
|
|
x += run;
|
|
}
|
|
}
|
|
|
|
/* (mkw-recomp) RX_INDIRECT_SYNC=1: indirect draws and dispatches read their arguments back, as before */
|
|
bool indirect_draws_on_gpu() {
|
|
static const bool sync = env_flag("RX_INDIRECT_SYNC");
|
|
return !sync && buffer_cache_active();
|
|
}
|
|
|
|
/* (mkw-recomp) An indirect draw command as one GPU multi-draw: its arguments (and count) copied on the GPU from where
|
|
they are in the buffer cache into a ring of their own (packed; a buffer bound for writing in the pass could hold
|
|
them), the draws drawn from there. False: not so (the caller reads the arguments back). Reading them back stopped
|
|
the GPU 2-3 times a race frame. */
|
|
bool nvn_indirect_on_gpu(NvnState& s, const nvn_p_draw_indirect& d, uint32_t words, uint32_t stride, uint32_t n) {
|
|
if (!indirect_draws_on_gpu() || (d.count_addr && !dev.multi_draw_count)) return false;
|
|
const uint64_t span = (uint64_t)(n - 1) * stride + 4 * words, packed = (uint64_t)n * 4 * words;
|
|
if (span > (8u << 20) || (stride != 4 * words && n > 1024)) return false;
|
|
if (!rx_gpu_writes_in_cache(d.indirect, span) || (d.count_addr && !rx_gpu_writes_in_cache(d.count_addr, 4))) return false;
|
|
const BufferRef args = guest_buffer(d.indirect, span);
|
|
if (!args.buffer || args.offset % 4) return false;
|
|
BufferRef cnt;
|
|
if (d.count_addr) {
|
|
cnt = guest_buffer(d.count_addr, 4);
|
|
if (!cnt.buffer || cnt.offset % 4) return false;
|
|
}
|
|
static WGPUBuffer ring;
|
|
static uint64_t at;
|
|
constexpr uint64_t kRing = 8u << 20;
|
|
if (!ring) {
|
|
WGPUBufferDescriptor bd{};
|
|
bd.label = sv("rx indirect draws");
|
|
bd.size = kRing;
|
|
bd.usage = WGPUBufferUsage_Indirect | WGPUBufferUsage_CopyDst;
|
|
ring = wgpuDeviceCreateBuffer(dev.device, &bd);
|
|
}
|
|
NvnDraw w;
|
|
w.prim = d.prim, w.indexed = d.indexed != 0, w.index_type = d.itype, w.index_gpu = d.ibuf;
|
|
w.count = 1, w.instances = 1, w.indirect_max = n;
|
|
// packed arguments are read where they are: in a render pass the buffer cache's segments are only read (vertices,
|
|
// indices, read-only storage; written storage is copied), which indirect use may share. A copy ended the pass
|
|
// each time (63 times a race frame: the replay's draw time 10 -> 17 ms). RX_INDIRECT_COPY=1: always copied
|
|
static const bool copy_always = env_flag("RX_INDIRECT_COPY");
|
|
if (stride == 4 * words && !copy_always) {
|
|
w.indirect = args.buffer, w.indirect_offset = args.offset;
|
|
if (d.count_addr) w.count_buffer = cnt.buffer, w.count_offset = cnt.offset;
|
|
count("indirect_in_place");
|
|
} else {
|
|
const uint64_t need = packed + 256;
|
|
if (at + need > kRing) at = 0; // (in turn: each copy runs before its draws, in order)
|
|
const uint64_t off = at, count_off = (off + packed + 15) & ~15ull;
|
|
at = (count_off + 16 + 255) & ~255ull;
|
|
end_render_pass("pe_indirect_args");
|
|
if (stride == 4 * words) {
|
|
ce_copy_buffer_to_buffer(encoder(), args.buffer, args.offset, ring, off, packed);
|
|
} else {
|
|
for (uint32_t i = 0; i < n; i++)
|
|
ce_copy_buffer_to_buffer(encoder(), args.buffer, args.offset + (uint64_t)i * stride, ring,
|
|
off + (uint64_t)i * 4 * words, 4 * words);
|
|
}
|
|
if (d.count_addr) ce_copy_buffer_to_buffer(encoder(), cnt.buffer, cnt.offset, ring, count_off, 4);
|
|
w.indirect = ring, w.indirect_offset = off;
|
|
if (d.count_addr) w.count_buffer = ring, w.count_offset = count_off;
|
|
}
|
|
nvn_draw_indirect_refused = false;
|
|
nvn_draw(s, w);
|
|
if (nvn_draw_indirect_refused) return false;
|
|
count("nvn_indirect_gpu", n);
|
|
return true;
|
|
}
|
|
|
|
/* the source's memory holds what the GPU wrote (a shader's stores reach guest memory after their batch ran) */
|
|
void copy_source_synced(uint64_t gpu, uint64_t size, const char* why = "sync_copy") {
|
|
if (size && rx_gpu_writes_pending(gpu, size)) count(why), rx_sync_gpu_writes(gpu, size);
|
|
}
|
|
/* a copy out of a render target the GPU drew: its pixels into guest memory first, where the copy reads them (they
|
|
are only in its texture). TOTK renders each fused weapon's inventory icon into a 256x144 target, copies it to a
|
|
buffer, crops it on the CPU and uploads it with WriteTexels: read from guest memory, the icons were black. (The
|
|
threaded web build writes targets back after the batch: not in time for this copy.) */
|
|
void sync_rendered_source(const CopySurface& s) {
|
|
if (RX_WEB_MT) return;
|
|
const uint64_t guest = nvn_gpu_guest(s.base);
|
|
Image* im = guest ? image_at(guest) : nullptr;
|
|
if (!im || !im->rendered) return;
|
|
queue_target_writeback(im);
|
|
flush_batch();
|
|
writeback_targets();
|
|
count("nvn_copy_rt_synced");
|
|
}
|
|
void copy_note_target(const CopySurface& s, bool to) {
|
|
const uint64_t guest = nvn_gpu_guest(s.base);
|
|
if (Image* im = guest ? image_at(guest) : nullptr; im && im->rendered) {
|
|
const char* what = to ? "nvn_copy_to_rt" : "nvn_copy_from_rt";
|
|
count(what);
|
|
log_limited(what, 4, "rx: NVN copy %s render target %ux%u at %#llx: through guest memory", to ? "to" : "from",
|
|
im->width, im->height, (unsigned long long)s.base);
|
|
}
|
|
}
|
|
uint64_t copy_from(const CopySurface& s, const CopyBox& b) {
|
|
return s.base + (uint64_t)(s.base_layer + (b.layers ? b.z : 0)) * s.layer_size;
|
|
}
|
|
uint64_t copy_span(const CopySurface& s, const CopyBox& b) {
|
|
return b.layers ? (uint64_t)(b.d - 1) * s.layer_size + s.level_size : s.level_size;
|
|
}
|
|
|
|
/* buffer <-> texture: the buffer's rows `row_stride` apart (0: packed), its slices / layers row_stride * rows apart */
|
|
void nvn_copy_buffer_texture(const nvn_p_copy_bt& c, int64_t row_stride, bool to_texture) {
|
|
CopySurface s;
|
|
if (!copy_surface(*c.tex, c.has_view ? &c.view : nullptr, s)) {
|
|
log_limited("nvn_copy_fmt", 8, "rx: NVN copy of texture format %d: no layout", c.tex->d.format);
|
|
return;
|
|
}
|
|
const CopyBox b = copy_box(s, c.r);
|
|
if (!b.w || !b.h || !b.d) return;
|
|
const uint64_t pitch = row_stride > 0 ? (uint64_t)row_stride : (uint64_t)b.w * s.bpb;
|
|
const uint64_t image = pitch * b.h, total = image * b.d;
|
|
if (to_texture) copy_source_synced(c.buf, total);
|
|
else copy_source_synced(copy_from(s, b), copy_span(s, b)), sync_rendered_source(s);
|
|
copy_note_target(s, to_texture);
|
|
static std::vector<uint8_t> row;
|
|
row.resize((size_t)b.w * s.bpb);
|
|
HostPages tex;
|
|
for (uint32_t k = 0; k < b.d; k++)
|
|
for (uint32_t y = 0; y < b.h; y++) {
|
|
const uint64_t at = c.buf + k * image + y * pitch;
|
|
if (to_texture && !nvn_gpu_read(at, row.data(), row.size())) continue;
|
|
surface_row(s, b, y, k, [&](uint64_t gpu, uint32_t off, uint32_t n) {
|
|
uint8_t* h = tex.at(gpu);
|
|
if (h && (gpu & 0xFFFF) + n <= 0x10000) {
|
|
if (to_texture) std::memcpy(h, row.data() + off, n);
|
|
else std::memcpy(row.data() + off, h, n);
|
|
} else if (to_texture) {
|
|
nvn_gpu_write(gpu, row.data() + off, n);
|
|
} else {
|
|
nvn_gpu_read(gpu, row.data() + off, n);
|
|
}
|
|
});
|
|
if (!to_texture) nvn_gpu_write(at, row.data(), row.size());
|
|
}
|
|
if (to_texture) rx_memory_written(copy_from(s, b), copy_span(s, b));
|
|
else rx_memory_written(c.buf, total);
|
|
count(to_texture ? "nvn_copies_b2t" : "nvn_copies_t2b");
|
|
if (static const bool trace = getenv("RX_NVN_TEXELS_LOG") != nullptr; trace) {
|
|
const uint64_t guest = nvn_gpu_guest(s.base);
|
|
const Image* im = guest ? image_at(guest) : nullptr;
|
|
uint64_t nz = 0;
|
|
std::vector<uint8_t> all((size_t)std::min<uint64_t>(total, 64u << 20));
|
|
if (nvn_gpu_read(c.buf, all.data(), all.size()))
|
|
for (uint8_t v : all) nz += v != 0;
|
|
log_limited("nvn_texels", 100000, "rx: NVN copy %s %08llx %ux%u format %d (%s %ux%u) %s buffer %llx at %d,%d %dx%d "
|
|
"(%llu nonzero)", to_texture ? "to texture" : "from texture", (unsigned long long)guest,
|
|
c.tex->d.width, c.tex->d.height, c.tex->d.format, im && im->rendered ? "a render target" : "memory",
|
|
im ? im->width : 0, im ? im->height : 0, to_texture ? "<-" : "->", (unsigned long long)c.buf,
|
|
c.r.xoffset, c.r.yoffset, c.r.width, c.r.height, (unsigned long long)nz);
|
|
}
|
|
}
|
|
|
|
/* texture -> texture: block for block; a scaled or mirrored copy takes the nearest source block */
|
|
void nvn_copy_texture_texture(const nvn_p_copy_tt& c) {
|
|
CopySurface ss, ds;
|
|
if (!copy_surface(*c.src, c.has_sv ? &c.sv : nullptr, ss) || !copy_surface(*c.dst, c.has_dv ? &c.dv : nullptr, ds))
|
|
return;
|
|
if (ss.bpb != ds.bpb || ss.bw != ds.bw || ss.bh != ds.bh) {
|
|
log_limited("nvn_copy_tt_fmt", 8, "rx: NVN texture copy from format %d to %d: blocks differ, not copied",
|
|
c.src->d.format, c.dst->d.format);
|
|
count("nvn_copies_skipped");
|
|
return;
|
|
}
|
|
const CopyBox sb = copy_box(ss, c.sr), db = copy_box(ds, c.dr);
|
|
if (!sb.w || !sb.h || !sb.d || !db.w || !db.h || !db.d) return;
|
|
copy_source_synced(copy_from(ss, sb), copy_span(ss, sb));
|
|
sync_rendered_source(ss);
|
|
copy_note_target(ss, false);
|
|
copy_note_target(ds, true);
|
|
const bool mirror_x = c.flags & NVN_COPY_FLAGS_MIRROR_X_BIT;
|
|
const bool scaled = sb.w != db.w || sb.h != db.h || sb.d != db.d;
|
|
if (scaled || (c.flags & (NVN_COPY_FLAGS_MIRROR_X_BIT | NVN_COPY_FLAGS_MIRROR_Y_BIT | NVN_COPY_FLAGS_MIRROR_Z_BIT)))
|
|
count("nvn_copies_scaled");
|
|
static std::vector<uint8_t> src_row, row;
|
|
src_row.resize((size_t)sb.w * ss.bpb);
|
|
row.resize((size_t)db.w * ds.bpb);
|
|
HostPages sp, dp;
|
|
for (uint32_t k = 0; k < db.d; k++)
|
|
for (uint32_t y = 0; y < db.h; y++) {
|
|
uint32_t sk = (uint32_t)((uint64_t)k * sb.d / db.d), sy = (uint32_t)((uint64_t)y * sb.h / db.h);
|
|
if (c.flags & NVN_COPY_FLAGS_MIRROR_Z_BIT) sk = sb.d - 1 - sk;
|
|
if (c.flags & NVN_COPY_FLAGS_MIRROR_Y_BIT) sy = sb.h - 1 - sy;
|
|
surface_row(ss, sb, sy, sk, [&](uint64_t gpu, uint32_t off, uint32_t n) {
|
|
uint8_t* h = sp.at(gpu);
|
|
if (h && (gpu & 0xFFFF) + n <= 0x10000) std::memcpy(src_row.data() + off, h, n);
|
|
else nvn_gpu_read(gpu, src_row.data() + off, n);
|
|
});
|
|
if (!scaled && !mirror_x) {
|
|
row = src_row;
|
|
} else {
|
|
for (uint32_t x = 0; x < db.w; x++) {
|
|
uint32_t sx = (uint32_t)((uint64_t)x * sb.w / db.w);
|
|
if (mirror_x) sx = sb.w - 1 - sx;
|
|
std::memcpy(row.data() + (size_t)x * ds.bpb, src_row.data() + (size_t)sx * ss.bpb, ds.bpb);
|
|
}
|
|
}
|
|
surface_row(ds, db, y, k, [&](uint64_t gpu, uint32_t off, uint32_t n) {
|
|
uint8_t* h = dp.at(gpu);
|
|
if (h && (gpu & 0xFFFF) + n <= 0x10000) std::memcpy(h, row.data() + off, n);
|
|
else nvn_gpu_write(gpu, row.data() + off, n);
|
|
});
|
|
}
|
|
rx_memory_written(copy_from(ds, db), copy_span(ds, db));
|
|
count("nvn_copies_t2t");
|
|
}
|
|
|
|
// RX_NVN_COND_RECHECK=1 (debugging): the counters render conditions read are read again at the next present; the
|
|
// ones that changed in between are logged (the CPU wrote them after the replay read them)
|
|
struct CondRead { uint64_t addr, x, y; int64_t frame; };
|
|
std::vector<CondRead> cond_reads;
|
|
std::mutex cond_mu;
|
|
void cond_recheck_note(uint64_t addr, uint64_t x, uint64_t y) {
|
|
std::lock_guard lk(cond_mu);
|
|
if (cond_reads.size() < 4096) cond_reads.push_back({addr, x, y, frame_number});
|
|
}
|
|
void cond_recheck_present() {
|
|
std::vector<CondRead> v;
|
|
{
|
|
std::lock_guard lk(cond_mu);
|
|
v.swap(cond_reads);
|
|
}
|
|
static int logged = 0;
|
|
for (const CondRead& c : v) {
|
|
uint64_t x = 0, y = 0;
|
|
nvn_gpu_read(c.addr, &x, 8);
|
|
nvn_gpu_read(c.addr + 16, &y, 8);
|
|
if (c.x == c.y && x != y && logged++ < 300) // (equal at the replay, different later: written after it)
|
|
hos_log("rx: render condition at %llx read %llx / %llx at the replay, %llx / %llx at the present (frame %lld)",
|
|
(unsigned long long)c.addr, (unsigned long long)c.x, (unsigned long long)c.y, (unsigned long long)x,
|
|
(unsigned long long)y, (long long)c.frame);
|
|
}
|
|
}
|
|
|
|
[[gnu::noinline]] void replay(NvnState& s, const uint8_t* p) { // (its own frame in crash reports)
|
|
for (;;) {
|
|
nvn_op_hdr h;
|
|
std::memcpy(&h, p, sizeof h);
|
|
const uint8_t* a = p + 8;
|
|
const uint32_t avail = h.size - 8;
|
|
switch (h.op) {
|
|
case NOP_END: return;
|
|
case NOP_BEGIN: case NOP_JUMP: case NOP_CALL: break;
|
|
// ---- what the CPU sees ----
|
|
case NOP_SIGNAL_EVENT: {
|
|
const auto e = payload<nvn_p_signal>(a, avail);
|
|
uint32_t v = 0;
|
|
if (e.mode == NVN_EVENT_SIGNAL_MODE_ADD) nvn_gpu_read(e.gpu, &v, 4);
|
|
v = e.mode == NVN_EVENT_SIGNAL_MODE_ADD ? v + e.value : e.value;
|
|
nvn_gpu_write(e.gpu, &v, 4);
|
|
if (frame_dump_active()) {
|
|
char b[96];
|
|
std::snprintf(b, sizeof b, "nvn event %llx = %u", (unsigned long long)e.gpu, v);
|
|
dump_draw(b);
|
|
}
|
|
break;
|
|
}
|
|
case NOP_REPORT_COUNTER: {
|
|
const auto c = payload<nvn_p_counter>(a, avail);
|
|
// occlusion counts: what was drawn since the last reset (NvnState::samples_passed). Games subtract two
|
|
// reports around a draw: a constant made every object hidden (Ultrahand never found its target).
|
|
// RX_OCCLUSION_CONSTANT=1: the old constant
|
|
// Counted with occlusion queries (occlusion_report), the counter word is written once the GPU is done.
|
|
static const bool constant = getenv("RX_OCCLUSION_CONSTANT") != nullptr;
|
|
const bool samples = c.type == NVN_COUNTER_TYPE_SAMPLES_PASSED;
|
|
const bool queried = samples && !constant && occlusion_report(c.addr, s.samples_unqueried);
|
|
const uint64_t rec[2] = {samples ? constant ? 0x10000ull : s.samples_passed : 0, gpu_ticks()};
|
|
if (queried) nvn_gpu_write(c.addr + 8, &rec[1], 8);
|
|
else nvn_gpu_write(c.addr, rec, sizeof rec);
|
|
if (frame_dump_active()) {
|
|
char b[96];
|
|
if (queried)
|
|
std::snprintf(b, sizeof b, "nvn counter %llu at %llx = (queried)", (unsigned long long)c.type,
|
|
(unsigned long long)c.addr);
|
|
else
|
|
std::snprintf(b, sizeof b, "nvn counter %llu at %llx = %llu", (unsigned long long)c.type,
|
|
(unsigned long long)c.addr, (unsigned long long)rec[0]);
|
|
dump_draw(b);
|
|
}
|
|
break;
|
|
}
|
|
case NOP_RESET_COUNTER: {
|
|
const auto c = payload<nvn_p_counter>(a, avail);
|
|
if (c.type == NVN_COUNTER_TYPE_SAMPLES_PASSED) {
|
|
s.samples_passed = s.samples_unqueried = 0;
|
|
occlusion_reset();
|
|
}
|
|
if (frame_dump_active()) dump_draw("nvn counter reset");
|
|
break;
|
|
}
|
|
case NOP_COPY_B2B: {
|
|
const auto c = payload<nvn_p_copy_bb>(a, avail);
|
|
if (frame_dump_active()) {
|
|
char b[160];
|
|
std::snprintf(b, sizeof b, "nvn copy buffer %llx -> %llx, %llu bytes", (unsigned long long)c.src,
|
|
(unsigned long long)c.dst, (unsigned long long)c.size);
|
|
dump_draw(b);
|
|
}
|
|
for (uint64_t done = 0; done < c.size;) {
|
|
uint8_t buf[65536];
|
|
const uint64_t k = std::min<uint64_t>(sizeof buf, c.size - done);
|
|
if (!nvn_gpu_read(c.src + done, buf, k) || !nvn_gpu_write(c.dst + done, buf, k)) break;
|
|
done += k;
|
|
}
|
|
rx_memory_written(c.dst, c.size);
|
|
break;
|
|
}
|
|
case NOP_UPDATE_UB: {
|
|
const auto u = payload<nvn_p_update_ub>(a, avail);
|
|
nvn_gpu_write(u.dst, a + sizeof u, u.size);
|
|
rx_memory_written(u.dst, u.size);
|
|
break;
|
|
}
|
|
// ---- state ----
|
|
case NOP_SET_RT: s.rt = payload<nvn_p_set_rt>(a, avail); s.at_dirty = true; break;
|
|
case NOP_VIEWPORTS: {
|
|
const auto v = payload<nvn_p_viewports>(a, avail);
|
|
for (int i = 0; i < v.n && v.first + i < 16; i++) std::memcpy(s.viewport[v.first + i], v.v[i], 16);
|
|
break;
|
|
}
|
|
case NOP_SCISSOR: {
|
|
const auto c = payload<nvn_p_scissor>(a, avail);
|
|
s.scissor[0] = c.x, s.scissor[1] = c.y, s.scissor[2] = c.w, s.scissor[3] = c.h;
|
|
break;
|
|
}
|
|
case NOP_DEPTH_RANGE: {
|
|
const auto d = payload<nvn_p_depth_range>(a, avail);
|
|
s.depth_near = d.n, s.depth_far = d.f;
|
|
break;
|
|
}
|
|
case NOP_BIND_PROGRAM: {
|
|
const auto b = payload<nvn_p_bind_program>(a, avail);
|
|
if (b.prog) s.prog = *b.prog;
|
|
s.has_prog = b.prog != nullptr && b.prog->nstage > 0;
|
|
s.prog_epoch++;
|
|
break;
|
|
}
|
|
case NOP_BIND_STATE: {
|
|
const auto b = payload<nvn_p_bind_state>(a, avail);
|
|
switch (b.kind) {
|
|
case 0: {
|
|
nvn_blend_st st;
|
|
std::memcpy(&st, b.bytes, sizeof st);
|
|
s.blend[st.target & 7] = st;
|
|
break;
|
|
}
|
|
case 1: std::memcpy(&s.color, b.bytes, sizeof s.color); break;
|
|
case 2: std::memcpy(&s.chmask, b.bytes, sizeof s.chmask); break;
|
|
case 3: std::memcpy(&s.ms, b.bytes, sizeof s.ms); break;
|
|
case 4: std::memcpy(&s.poly, b.bytes, sizeof s.poly); break;
|
|
case 5: std::memcpy(&s.ds, b.bytes, sizeof s.ds); break;
|
|
default: break;
|
|
}
|
|
break;
|
|
}
|
|
case NOP_BIND_VATTR: {
|
|
const auto v = payload<nvn_p_bind_vattr>(a, avail);
|
|
decltype(s.vattr) old;
|
|
std::memcpy(old, s.vattr, sizeof old);
|
|
std::memset(s.vattr, 0, sizeof s.vattr);
|
|
for (int i = 0; i < v.n && i < 16; i++) s.vattr[i] = v.st[i];
|
|
const int n = std::min(v.n, 16);
|
|
if (n != s.nvattr || std::memcmp(old, s.vattr, sizeof old)) {
|
|
s.nvattr = n;
|
|
s.prog_epoch++;
|
|
s.vtx_hash = nvn_vertex_hash(s);
|
|
}
|
|
break;
|
|
}
|
|
case NOP_BIND_VSTREAM: {
|
|
const auto v = payload<nvn_p_bind_vstream>(a, avail);
|
|
bool changed = false;
|
|
for (int i = 0; i < v.n && i < 16; i++) {
|
|
changed |= std::memcmp(&s.vstream[i], &v.st[i], sizeof v.st[i]) != 0;
|
|
s.vstream[i] = v.st[i];
|
|
}
|
|
if (changed) s.prog_epoch++, s.vtx_hash = nvn_vertex_hash(s);
|
|
break;
|
|
}
|
|
case NOP_BIND_VBUF: {
|
|
const auto v = payload<nvn_p_bind_vbuf>(a, avail);
|
|
for (int i = 0; i < v.n && v.first + i < 16; i++) s.vbuf[v.first + i] = v.r[i];
|
|
break;
|
|
}
|
|
case NOP_BIND_BUFS: {
|
|
const auto b = payload<nvn_p_bind_bufs>(a, avail);
|
|
for (int i = 0; i < b.n && i < 16; i++) {
|
|
const int index = b.first + i;
|
|
if (b.stage < 0 || b.stage >= 6 || index < 0 || index >= 16) continue;
|
|
nvn_p_bind_buf& u = (b.ssbo ? s.ssbo : s.ubo)[b.stage][index];
|
|
u.stage = b.stage, u.index = index, u.addr = b.r[i].address, u.size = b.r[i].size;
|
|
}
|
|
break;
|
|
}
|
|
case NOP_BIND_UBO: case NOP_BIND_SSBO: {
|
|
const auto b = payload<nvn_p_bind_buf>(a, avail);
|
|
if (b.stage >= 0 && b.stage < 6 && b.index >= 0 && b.index < 16)
|
|
(h.op == NOP_BIND_UBO ? s.ubo : s.ssbo)[b.stage][b.index] = b;
|
|
break;
|
|
}
|
|
case NOP_BIND_TEX: case NOP_BIND_SAMP: case NOP_BIND_IMG: {
|
|
nvn_p_bind_handles b;
|
|
std::memset(&b, 0, sizeof b);
|
|
std::memcpy(&b, a, std::min<size_t>(sizeof b, h.size - 8));
|
|
if (b.stage < 0 || b.stage > 5 || b.first < 0) break;
|
|
uint64_t* to = h.op == NOP_BIND_TEX ? s.tex[b.stage] : h.op == NOP_BIND_SAMP ? s.samp[b.stage] : s.img[b.stage];
|
|
const int lim = h.op == NOP_BIND_IMG ? 32 : 128;
|
|
bool changed = false;
|
|
for (int k = 0; k < b.n && b.first + k < lim; k++) {
|
|
changed |= to[b.first + k] != b.h[k];
|
|
to[b.first + k] = b.h[k];
|
|
}
|
|
if (changed) s.prog_epoch++;
|
|
break;
|
|
}
|
|
case NOP_STENCIL: {
|
|
const auto st = payload<nvn_p_stencil>(a, avail);
|
|
for (int f = 0; f < 2; f++) {
|
|
if (!(st.face & (1 << f))) continue;
|
|
if (st.which == 0) s.stencil_value_mask[f] = (uint32_t)st.value_mask;
|
|
else if (st.which == 1) s.stencil_mask[f] = (uint32_t)st.mask;
|
|
else s.stencil_ref[f] = (uint32_t)st.ref;
|
|
}
|
|
break;
|
|
}
|
|
case NOP_POLY_OFFSET: {
|
|
const auto o = payload<nvn_p_poly_offset>(a, avail);
|
|
s.poly_offset[0] = o.factor, s.poly_offset[1] = o.units, s.poly_offset[2] = o.clamp;
|
|
break;
|
|
}
|
|
case NOP_BLEND_COLOR: std::memcpy(s.blend_color, payload<nvn_p_blend_color>(a, avail).rgba, 16); break;
|
|
case NOP_PRIMITIVE_RESTART: {
|
|
const auto r = payload<nvn_p_restart>(a, avail);
|
|
s.restart = r.enable != 0, s.restart_index = (uint32_t)r.index;
|
|
break;
|
|
}
|
|
case NOP_DEPTH_CLAMP: s.depth_clamp = payload<nvn_p_flag>(a, avail).value != 0; break;
|
|
case NOP_RASTER_DISCARD: s.raster_discard = payload<nvn_p_flag>(a, avail).value != 0; break;
|
|
case NOP_RENDER_ENABLE:
|
|
s.render_enabled = payload<nvn_p_flag>(a, avail).value != 0;
|
|
if (frame_dump_active()) dump_draw(s.render_enabled ? "nvn render on" : "nvn render off");
|
|
break;
|
|
case NOP_RENDER_COND: {
|
|
const auto r = payload<nvn_p_render_cond>(a, avail);
|
|
uint64_t x = 0, y = 0;
|
|
nvn_gpu_read(r.addr, &x, 8);
|
|
nvn_gpu_read(r.addr + 16, &y, 8);
|
|
s.render_enabled = r.mode == 0 ? x == y : x != y;
|
|
if (static const bool recheck = getenv("RX_NVN_COND_RECHECK") != nullptr; recheck) cond_recheck_note(r.addr, x, y);
|
|
if (frame_dump_active()) {
|
|
char b[128];
|
|
std::snprintf(b, sizeof b, "nvn render if %s at %llx: %llx / %llx: %s", r.mode ? "not equal" : "equal",
|
|
(unsigned long long)r.addr, (unsigned long long)x, (unsigned long long)y,
|
|
s.render_enabled ? "on" : "off");
|
|
dump_draw(b);
|
|
}
|
|
break;
|
|
}
|
|
case NOP_SET_TEX_POOL: case NOP_SET_SAMP_POOL: break;
|
|
case NOP_COPY_STRIDE: s.copy_stride = payload<nvn_p_copy_stride>(a, avail).stride; break;
|
|
case NOP_BARRIER: { // (what was drawn is visible to texture reads after it: feedback snapshots are taken again)
|
|
int32_t flags = 0;
|
|
std::memcpy(&flags, a, std::min<uint32_t>(4, avail));
|
|
// The pass goes on: what a barrier makes visible is only read through a feedback snapshot (taken again
|
|
// after it, which ends the pass then: snapshot_for_feedback) or storage the pass wrote (draw_storage ends
|
|
// it first). Ending it at every barrier was ~97 pass breaks a frame at the castle, each one a render pass
|
|
// begun again in wgpu and the driver and its attachments stored and loaded. RX_BARRIER_ENDS_PASS=1: old.
|
|
if (static const bool ends = env_flag("RX_BARRIER_ENDS_PASS"); ends) end_render_pass("pe_nvn_barrier");
|
|
texture_barrier(0xB000, (uint32_t)flags);
|
|
break;
|
|
}
|
|
// ---- work ----
|
|
case NOP_CLEAR_COLOR: {
|
|
if (!s.render_enabled) {
|
|
if (frame_dump_active()) dump_draw("nvn clear skipped (render off)");
|
|
break;
|
|
}
|
|
const auto c = payload<nvn_p_clear_color>(a, avail);
|
|
const Attachments at = nvn_attachments(s);
|
|
if (frame_dump_active() && c.index >= 0 && c.index < 8 && at.color[c.index]) {
|
|
char b[128];
|
|
std::snprintf(b, sizeof b, "nvn clear c%d %llx to %g %g %g %g mask %x", c.index,
|
|
(unsigned long long)at.color[c.index]->guest, c.rgba[0], c.rgba[1], c.rgba[2], c.rgba[3], c.mask);
|
|
dump_draw(b);
|
|
}
|
|
if (c.index >= 0 && c.index < 8) nvn_clear_color(at.color[c.index], c.rgba, (uint32_t)c.mask, s.scissor);
|
|
break;
|
|
}
|
|
case NOP_CLEAR_DS: {
|
|
if (!s.render_enabled) break;
|
|
const auto c = payload<nvn_p_clear_ds>(a, avail);
|
|
const Attachments at = nvn_attachments(s);
|
|
nvn_clear_depth_stencil(at.depth, c.depth_mask != 0, c.depth, c.stencil_mask != 0, (uint32_t)c.stencil,
|
|
(uint32_t)c.stencil_mask, s.scissor);
|
|
break;
|
|
}
|
|
case NOP_CLEAR_TEX: {
|
|
if (!s.render_enabled) break;
|
|
const auto c = payload<nvn_p_clear_tex>(a, avail);
|
|
Image* im = nvn_target(c.tex, c.has_view ? &c.view : nullptr);
|
|
if (frame_dump_active()) {
|
|
char b[200];
|
|
std::snprintf(b, sizeof b, "nvn clear texture %08llx %ux%u at %d,%d %dx%d to %g %g %g %g mask %x%s",
|
|
c.tex ? (unsigned long long)nvn_gpu_guest(c.tex->gpu) : 0ull, c.tex ? c.tex->d.width : 0,
|
|
c.tex ? c.tex->d.height : 0, c.r.xoffset, c.r.yoffset, c.r.width, c.r.height, c.c.f[0],
|
|
c.c.f[1], c.c.f[2], c.c.f[3], (unsigned)c.mask, im ? "" : " (no target)");
|
|
dump_draw(b);
|
|
}
|
|
if (!im) break;
|
|
const int32_t rect[4] = {c.r.xoffset, c.r.yoffset, c.r.width, c.r.height};
|
|
if (is_depth_format(im->format)) nvn_clear_depth_stencil(im, true, c.c.f[0], false, 0, 0, rect);
|
|
else nvn_clear_color(im, c.c.f, (uint32_t)c.mask, rect);
|
|
break;
|
|
}
|
|
case NOP_DRAW: {
|
|
if (!s.render_enabled || s.raster_discard) {
|
|
if (frame_dump_active()) dump_draw(s.raster_discard ? "nvn draw skipped (raster discard)" : "nvn draw skipped (render off)");
|
|
break;
|
|
}
|
|
const auto d = payload<nvn_p_draw>(a, avail);
|
|
NvnDraw w;
|
|
w.prim = d.prim, w.first = (uint32_t)d.first, w.count = (uint32_t)d.count;
|
|
w.base_instance = (uint32_t)d.base_instance, w.instances = (uint32_t)std::max(1, d.instances);
|
|
s.samples_passed += (uint64_t)std::max(1u, w.count) * w.instances * 16; // (an estimate: some passed)
|
|
const uint64_t queried = occlusion_queried();
|
|
nvn_draw(s, w);
|
|
if (occlusion_queried() == queried) s.samples_unqueried += (uint64_t)std::max(1u, w.count) * w.instances * 16;
|
|
break;
|
|
}
|
|
case NOP_DRAW_IDX: {
|
|
if (!s.render_enabled || s.raster_discard) {
|
|
if (frame_dump_active()) dump_draw(s.raster_discard ? "nvn draw skipped (raster discard)" : "nvn draw skipped (render off)");
|
|
break;
|
|
}
|
|
const auto d = payload<nvn_p_draw_idx>(a, avail);
|
|
NvnDraw w;
|
|
w.indexed = true;
|
|
w.prim = d.prim, w.index_type = d.itype, w.count = (uint32_t)d.count, w.index_gpu = d.ibuf;
|
|
w.base_vertex = d.base_vertex, w.base_instance = (uint32_t)d.base_instance;
|
|
w.instances = (uint32_t)std::max(1, d.instances);
|
|
s.samples_passed += (uint64_t)std::max(1u, w.count) * w.instances * 16;
|
|
const uint64_t queried = occlusion_queried();
|
|
nvn_draw(s, w);
|
|
if (occlusion_queried() == queried) s.samples_unqueried += (uint64_t)std::max(1u, w.count) * w.instances * 16;
|
|
break;
|
|
}
|
|
case NOP_DRAW_INDIRECT: { // (mkw-recomp) arguments from memory, after the GPU's writes there are in
|
|
if (!s.render_enabled || s.raster_discard) break;
|
|
const auto d = payload<nvn_p_draw_indirect>(a, avail);
|
|
uint32_t n = (uint32_t)std::max(0, d.max);
|
|
{ // (mkw-recomp) drawn by the GPU from its arguments where it can be
|
|
const uint32_t words = d.indexed ? 5 : 4, stride = d.stride >= (int)(4 * words) ? (uint32_t)d.stride : 4 * words;
|
|
if (n && d.indirect && n <= 65536 && nvn_indirect_on_gpu(s, d, words, stride, n)) break;
|
|
}
|
|
if (d.count_addr) {
|
|
count("indirect_with_count");
|
|
copy_source_synced(d.count_addr, 4, "sync_indirect_count");
|
|
uint32_t drawn = 0;
|
|
if (!nvn_gpu_read(d.count_addr, &drawn, 4)) break;
|
|
n = std::min(n, drawn);
|
|
}
|
|
const uint32_t words = d.indexed ? 5 : 4, stride = d.stride >= (int)(4 * words) ? (uint32_t)d.stride : 4 * words;
|
|
if (!n || !d.indirect || n > 65536) break;
|
|
std::vector<uint32_t> args((size_t)n * stride / 4 + words);
|
|
copy_source_synced(d.indirect, (uint64_t)(n - 1) * stride + 4 * words, "sync_indirect");
|
|
if (!nvn_gpu_read(d.indirect, args.data(), (uint64_t)(n - 1) * stride + 4 * words)) break;
|
|
const uint32_t unit = d.itype == 0 ? 1 : d.itype == 1 ? 2 : 4; // (NVNindexType: byte, short, int)
|
|
for (uint32_t i = 0; i < n; i++) {
|
|
const uint32_t* w = args.data() + (size_t)i * stride / 4;
|
|
if (!w[0] || !w[1]) continue;
|
|
NvnDraw draw;
|
|
draw.prim = d.prim, draw.count = w[0], draw.instances = w[1];
|
|
if (d.indexed) {
|
|
draw.indexed = true, draw.index_type = d.itype;
|
|
draw.index_gpu = d.ibuf + (uint64_t)w[2] * unit;
|
|
draw.base_vertex = (int32_t)w[3], draw.base_instance = w[4];
|
|
} else {
|
|
draw.first = w[2], draw.base_instance = w[3];
|
|
}
|
|
if (draw.base_instance) count("indirect_base_instance");
|
|
s.samples_passed += (uint64_t)draw.count * draw.instances * 16;
|
|
nvn_draw(s, draw);
|
|
}
|
|
count("nvn_indirect_draws", n);
|
|
break;
|
|
}
|
|
case NOP_DISPATCH_INDIRECT: {
|
|
const auto d = payload<nvn_p_dispatch_indirect>(a, avail);
|
|
uint32_t g[3] = {0, 0, 0};
|
|
// (mkw-recomp) read on the GPU where the arguments are in the buffer cache
|
|
if (d.addr && indirect_draws_on_gpu() && rx_gpu_writes_in_cache(d.addr, 12)) {
|
|
nvn_dispatch(s, 0, 0, 0, d.addr);
|
|
break;
|
|
}
|
|
copy_source_synced(d.addr, 12, "sync_dispatch_indirect");
|
|
if (d.addr && nvn_gpu_read(d.addr, g, 12) && g[0] && g[1] && g[2]) nvn_dispatch(s, g[0], g[1], g[2]);
|
|
break;
|
|
}
|
|
case NOP_DISPATCH: {
|
|
const auto d = payload<nvn_p_dispatch>(a, avail);
|
|
if (frame_dump_active()) {
|
|
char b[96];
|
|
std::snprintf(b, sizeof b, "nvn dispatch %d x %d x %d", d.x, d.y, d.z);
|
|
dump_draw(b);
|
|
}
|
|
nvn_dispatch(s, (uint32_t)d.x, (uint32_t)d.y, (uint32_t)d.z);
|
|
break;
|
|
}
|
|
case NOP_COPY_B2T: case NOP_COPY_T2B: {
|
|
const auto c = payload<nvn_p_copy_bt>(a, avail);
|
|
const bool to_texture = h.op == NOP_COPY_B2T;
|
|
if (frame_dump_active()) {
|
|
char b[200];
|
|
std::snprintf(b, sizeof b, "nvn copy %s buffer %llx %s texture %08llx %ux%u at %d,%d,%d %dx%dx%d",
|
|
to_texture ? "from" : "to", (unsigned long long)c.buf, to_texture ? "->" : "<-",
|
|
c.tex ? (unsigned long long)nvn_gpu_guest(c.tex->gpu) : 0ull, c.tex ? c.tex->d.width : 0,
|
|
c.tex ? c.tex->d.height : 0, c.r.xoffset, c.r.yoffset, c.r.zoffset, c.r.width, c.r.height,
|
|
c.r.depth);
|
|
dump_draw(b);
|
|
}
|
|
nvn_copy_buffer_texture(c, s.copy_stride, to_texture);
|
|
break;
|
|
}
|
|
case NOP_COPY_T2T: {
|
|
const auto c = payload<nvn_p_copy_tt>(a, avail);
|
|
if (frame_dump_active()) {
|
|
char b[220];
|
|
std::snprintf(b, sizeof b, "nvn copy texture %08llx %ux%u (%d,%d %dx%d) -> %08llx %ux%u (%d,%d %dx%d) flags %x",
|
|
c.src ? (unsigned long long)nvn_gpu_guest(c.src->gpu) : 0ull, c.src ? c.src->d.width : 0,
|
|
c.src ? c.src->d.height : 0, c.sr.xoffset, c.sr.yoffset, c.sr.width, c.sr.height,
|
|
c.dst ? (unsigned long long)nvn_gpu_guest(c.dst->gpu) : 0ull, c.dst ? c.dst->d.width : 0,
|
|
c.dst ? c.dst->d.height : 0, c.dr.xoffset, c.dr.yoffset, c.dr.width, c.dr.height,
|
|
(unsigned)c.flags);
|
|
dump_draw(b);
|
|
}
|
|
nvn_copy_texture_texture(c);
|
|
break;
|
|
}
|
|
default:
|
|
log_limited("nvn_op", 16, "rx: NVN record %u not replayed", h.op);
|
|
break;
|
|
}
|
|
p += h.size;
|
|
}
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace rx
|
|
|
|
using namespace rx;
|
|
|
|
extern "C" {
|
|
|
|
// ---- submits and presents: replayed in order on the replay thread ----
|
|
// A submit used to replay on the game thread that called it (TOTK's frame thread: ~14 ms a frame in the open world,
|
|
// the game waiting for its own draws). The submits and presents now queue for one "NVN replay" thread, as the
|
|
// console's GPU front end takes them, and a fence is reached when the replay passed it: the game runs up to a frame
|
|
// ahead of the replay. The game's command recordings stay alive until the replay passed every submit made before
|
|
// they were released (hos/nvn_native.c). RX_NVN_SYNC=1: replayed at the call, as before. On the web the replay thread
|
|
// is also the render thread (WebGPU objects belong to the worker that made them): it takes the page's canvas, starts
|
|
// the renderer and waits for jobs in its event loop (rx_web_idle: WebGPU's callbacks run there).
|
|
} // extern "C"
|
|
namespace rx {
|
|
namespace {
|
|
struct NvnJob {
|
|
std::vector<uint64_t> handles;
|
|
uint64_t scan = 0; // (a submit: the write scan its push started; 0: scanned when replayed)
|
|
bool present = false;
|
|
uint64_t present_guest = 0;
|
|
uint32_t present_w = 0, present_h = 0, present_crop_w = 0, present_crop_h = 0;
|
|
};
|
|
std::mutex job_mu;
|
|
std::condition_variable job_cv, job_done_cv;
|
|
std::deque<NvnJob> jobs;
|
|
uint64_t jobs_pushed = 0; // (under job_mu)
|
|
int presents_queued = 0; // (under job_mu)
|
|
std::atomic<uint64_t> jobs_replayed{0};
|
|
|
|
/* RX_NVN_FRAMES_AHEAD=<n>: presents the game may queue ahead of the replay (default (mkw-recomp) 2: with 1 the game and the
|
|
replay took turns, ~24 fps in a race; with 2 they overlap and the race holds its 30) */
|
|
int max_presents_ahead() {
|
|
static const int n = getenv("RX_NVN_FRAMES_AHEAD") ? std::max(1, atoi(getenv("RX_NVN_FRAMES_AHEAD"))) : 2;
|
|
return n;
|
|
}
|
|
|
|
bool replay_sync() {
|
|
static const bool sync = !RX_WEB && getenv("RX_NVN_SYNC") != nullptr;
|
|
return sync;
|
|
}
|
|
|
|
/* The CPU's writes before a submit are collected on the "NVN write scan" thread, started when the game submits:
|
|
GetWriteWatch over the ~2 GB the GPU can see costs ~2 ms, which the replay thread spent at the start of every
|
|
submit (11% of its time while running through the castle). A submit's replay waits for the scan its push started;
|
|
one scan may answer several pushes (its pages go to the first replay that takes them: invalidating early is
|
|
harmless). RX_NVN_SCAN_INLINE=1: scanned on the replay thread, as before. */
|
|
std::mutex scan_mu;
|
|
std::condition_variable scan_cv, scan_done_cv;
|
|
uint64_t scan_requested = 0, scan_done = 0; // (under scan_mu)
|
|
std::vector<uint32_t> scan_pages; // collected, not taken yet (under scan_mu)
|
|
bool scan_failed = false; // (under scan_mu: no write watch; every replay scans for itself)
|
|
|
|
void scan_main() {
|
|
hos_host_thread_register_self("NVN write scan");
|
|
hos_thread_boost(1);
|
|
for (;;) {
|
|
uint64_t target;
|
|
{
|
|
std::unique_lock lk(scan_mu);
|
|
scan_cv.wait(lk, [] { return scan_requested > scan_done; });
|
|
target = scan_requested;
|
|
}
|
|
int64_t n = -1;
|
|
uint32_t* pages = rx_collect_written(&n);
|
|
{
|
|
std::lock_guard lk(scan_mu);
|
|
if (n < 0) scan_failed = true;
|
|
else scan_pages.insert(scan_pages.end(), pages, pages + n);
|
|
scan_done = target;
|
|
}
|
|
free(pages);
|
|
scan_done_cv.notify_all();
|
|
}
|
|
}
|
|
|
|
uint64_t request_scan() {
|
|
static const bool inline_scan = getenv("RX_NVN_SCAN_INLINE") != nullptr;
|
|
if (inline_scan) return 0;
|
|
static std::once_flag started;
|
|
std::call_once(started, [] { std::thread(scan_main).detach(); });
|
|
uint64_t serial;
|
|
{
|
|
std::lock_guard lk(scan_mu);
|
|
serial = ++scan_requested;
|
|
}
|
|
scan_cv.notify_one();
|
|
return serial;
|
|
}
|
|
|
|
void run_submit(const uint64_t* handles, size_t n, uint64_t scan = 0) {
|
|
std::lock_guard lk(nvn_mu);
|
|
{ // what the CPU wrote since the last submit: the buffer cache and decoded textures learn of it
|
|
int64_t nw = -1;
|
|
uint32_t* pages = nullptr;
|
|
bool scanned = false;
|
|
if (scan) {
|
|
const int64_t t0 = hos_now_ns();
|
|
std::unique_lock sl(scan_mu);
|
|
scan_done_cv.wait(sl, [&] { return scan_done >= scan; });
|
|
if (!scan_failed) {
|
|
nw = (int64_t)scan_pages.size();
|
|
pages = static_cast<uint32_t*>(malloc((scan_pages.size() + 1) * sizeof(uint32_t)));
|
|
if (nw) std::memcpy(pages, scan_pages.data(), scan_pages.size() * sizeof(uint32_t));
|
|
scan_pages.clear();
|
|
scanned = true;
|
|
}
|
|
sl.unlock();
|
|
count("t_nvn_scan_wait", hos_now_ns() - t0);
|
|
}
|
|
if (!scanned) pages = rx_collect_written(&nw);
|
|
rx_begin_submission_written(pages, nw);
|
|
}
|
|
{
|
|
std::lock_guard rl(mu);
|
|
for (size_t i = 0; i < n; i++) replay(state, reinterpret_cast<const uint8_t*>(static_cast<uintptr_t>(handles[i])));
|
|
}
|
|
rx_flush();
|
|
submitted++;
|
|
}
|
|
|
|
void run_present(uint64_t guest, uint32_t w, uint32_t h) {
|
|
ts_new_frame = true;
|
|
if (static const bool recheck = getenv("RX_NVN_COND_RECHECK") != nullptr; recheck) cond_recheck_present();
|
|
std::lock_guard lk(nvn_mu);
|
|
rx_present(guest, w, h);
|
|
}
|
|
|
|
#if RX_WEB
|
|
uint32_t job_seq; // (bumped at each push: the replay thread's rx_web_idle wakes on it)
|
|
#endif
|
|
std::mutex ready_mu;
|
|
std::condition_variable ready_cv;
|
|
bool ready = false; // (the replay thread has started the renderer: under ready_mu)
|
|
|
|
void replay_main() {
|
|
hos_host_thread_register_self("NVN replay");
|
|
hos_thread_boost(1);
|
|
#if RX_WEB
|
|
rx_start(nullptr); // (the device is made on the thread that makes every WebGPU call)
|
|
#endif
|
|
{
|
|
std::lock_guard lk(ready_mu);
|
|
ready = true;
|
|
}
|
|
ready_cv.notify_all();
|
|
for (;;) {
|
|
NvnJob j;
|
|
{
|
|
std::unique_lock lk(job_mu);
|
|
#if RX_WEB
|
|
while (jobs.empty()) {
|
|
const uint32_t seen = __atomic_load_n(&job_seq, __ATOMIC_SEQ_CST);
|
|
lk.unlock();
|
|
rx_web_idle(&job_seq, seen);
|
|
lk.lock();
|
|
}
|
|
#else
|
|
job_cv.wait(lk, [] { return !jobs.empty(); });
|
|
#endif
|
|
j = std::move(jobs.front());
|
|
jobs.pop_front();
|
|
}
|
|
if (j.present) rx_present_crop(j.present_crop_w, j.present_crop_h), run_present(j.present_guest, j.present_w, j.present_h);
|
|
else run_submit(j.handles.data(), j.handles.size(), j.scan);
|
|
{
|
|
std::lock_guard lk(job_mu);
|
|
if (j.present) presents_queued--;
|
|
jobs_replayed++;
|
|
}
|
|
job_done_cv.notify_all();
|
|
}
|
|
}
|
|
|
|
void* replay_entry(void*) {
|
|
replay_main();
|
|
return nullptr;
|
|
}
|
|
|
|
/* The replay thread, started once (and on the web before the first submit or present: until it has started the
|
|
renderer, rx_active() is false there and the work would be dropped) */
|
|
void start_replay() {
|
|
static std::once_flag once;
|
|
std::call_once(once, [] {
|
|
#if RX_WEB
|
|
pthread_attr_t a;
|
|
pthread_attr_init(&a);
|
|
pthread_attr_setstacksize(&a, 1 << 20);
|
|
if (!RX_WEB_MT) // (the canvas the page shows; the threaded build: the WebGPU thread takes it, rx_web_mt.cpp)
|
|
emscripten_pthread_attr_settransferredcanvases(&a, "#screen");
|
|
pthread_t t;
|
|
pthread_create(&t, &a, replay_entry, nullptr);
|
|
pthread_attr_destroy(&a);
|
|
pthread_detach(t);
|
|
#else
|
|
std::thread(replay_main).detach();
|
|
#endif
|
|
std::unique_lock lk(ready_mu);
|
|
ready_cv.wait(lk, [] { return ready; });
|
|
});
|
|
}
|
|
|
|
void push_job(NvnJob&& j) {
|
|
start_replay();
|
|
std::unique_lock lk(job_mu);
|
|
// (the game at most one frame and 64 submits ahead of the replay)
|
|
const auto room = [&] { return jobs.size() < 64 && (!j.present || presents_queued < max_presents_ahead()); };
|
|
if (!room()) {
|
|
const int64_t t0 = hos_now_ns();
|
|
job_done_cv.wait(lk, room);
|
|
count(j.present ? "t_nvn_present_wait" : "t_nvn_submit_wait", hos_now_ns() - t0);
|
|
}
|
|
if (j.present) presents_queued++;
|
|
jobs.push_back(std::move(j));
|
|
jobs_pushed++;
|
|
lk.unlock();
|
|
job_cv.notify_one();
|
|
#if RX_WEB
|
|
__atomic_add_fetch(&job_seq, 1, __ATOMIC_SEQ_CST);
|
|
emscripten_futex_wake(&job_seq, 1);
|
|
#endif
|
|
}
|
|
|
|
/* The threaded web build reads render targets back to guest memory after the replay (rx_core.cpp writeback_targets):
|
|
a fence or a drain also waits for those */
|
|
void wait_writebacks() {
|
|
if (!RX_WEB_MT || rx_target_writebacks_pending() <= 0) return;
|
|
const int64_t t0 = hos_now_ns();
|
|
while (rx_target_writebacks_pending() > 0) std::this_thread::sleep_for(std::chrono::microseconds(100));
|
|
#if RX_WEB_MT
|
|
mt_note_wait("(fences and drains: writebacks)", hos_now_ns() - t0);
|
|
#endif
|
|
}
|
|
|
|
/* every job pushed so far replayed (the game is about to look at what they wrote, or to replace what they read) */
|
|
void drain() {
|
|
if (replay_sync()) return;
|
|
{
|
|
std::unique_lock lk(job_mu);
|
|
job_done_cv.wait(lk, [] { return jobs_replayed.load() >= jobs_pushed; });
|
|
}
|
|
wait_writebacks();
|
|
}
|
|
} // namespace
|
|
} // namespace rx
|
|
extern "C" {
|
|
|
|
void rx_nvn_submit(const uint64_t* handles, int n) {
|
|
if (RX_WEB) rx::start_replay();
|
|
if (!rx_active() || n <= 0) return;
|
|
if (replay_sync()) {
|
|
run_submit(handles, (size_t)n);
|
|
std::lock_guard lk(job_mu);
|
|
jobs_pushed++;
|
|
jobs_replayed++;
|
|
return;
|
|
}
|
|
NvnJob j;
|
|
j.handles.assign(handles, handles + n);
|
|
j.scan = request_scan();
|
|
push_job(std::move(j));
|
|
}
|
|
uint64_t rx_nvn_pushed(void) {
|
|
std::lock_guard lk(job_mu);
|
|
return jobs_pushed;
|
|
}
|
|
uint64_t rx_nvn_replayed(void) { return jobs_replayed.load(); }
|
|
/* A fence: the submits made so far. It is reached when they were replayed, as the emulated GPU's syncpoints are when
|
|
its command processor parsed the work: what the CPU changes after that does not reach work already recorded
|
|
(uniforms are copied at the draw, memory uploads go to the queue in order), and what shaders write comes back to
|
|
guest memory by itself. RX_NVN_GPU_FENCES=1: also the GPU ran them (the replay waits for it). */
|
|
uint64_t rx_nvn_fence(void) { return rx_nvn_pushed(); }
|
|
int rx_nvn_wait(uint64_t fence, uint64_t timeout_ns) {
|
|
// RX_NVN_EARLY_FENCES=1 (experiment): reached at the submit, before the replay
|
|
static const bool early = getenv("RX_NVN_EARLY_FENCES") != nullptr;
|
|
if (early) return 1;
|
|
if (jobs_replayed.load() < fence) {
|
|
if (!timeout_ns) return 0;
|
|
const int64_t t0 = hos_now_ns();
|
|
std::unique_lock lk(job_mu);
|
|
job_done_cv.wait(lk, [&] { return jobs_replayed.load() >= fence; });
|
|
count("nvn_fence_waits");
|
|
count("t_nvn_fence_wait", hos_now_ns() - t0);
|
|
#if RX_WEB_MT
|
|
mt_note_wait("(fences: the replay)", hos_now_ns() - t0);
|
|
#endif
|
|
}
|
|
// (writebacks: the game's memcpy waits for the ones it reads, rx_wait_writebacks_reading; a fence waited for all
|
|
// of them, ~20 ms each, about the GPU's round trip in the browser. RX_FENCE_WRITEBACKS=1: as before)
|
|
static const bool fence_writebacks = env_flag("RX_FENCE_WRITEBACKS");
|
|
if (RX_WEB_MT && fence_writebacks && rx_target_writebacks_pending() > 0) {
|
|
if (!timeout_ns) return 0;
|
|
wait_writebacks();
|
|
}
|
|
static const bool gpu_fences = getenv("RX_NVN_GPU_FENCES") != nullptr;
|
|
if (gpu_fences) wait_batch(last_batch_serial());
|
|
return 1;
|
|
}
|
|
void rx_nvn_finish(void) {
|
|
drain();
|
|
rx_wait_idle();
|
|
}
|
|
void rx_nvn_present(const nvn_tex* t, const int crop[4]) {
|
|
// (mkw-recomp) the window's crop (x, y ignored: MKW crops from the top left)
|
|
const uint32_t cw = crop && crop[2] > 0 ? (uint32_t)crop[2] : 0, ch = crop && crop[3] > 0 ? (uint32_t)crop[3] : 0;
|
|
if (RX_WEB) rx::start_replay();
|
|
if (!t || !rx_active()) return;
|
|
const uint64_t guest = nvn_gpu_guest(t->gpu);
|
|
if (replay_sync()) {
|
|
rx_present_crop(cw, ch);
|
|
run_present(guest, t->d.width, t->d.height);
|
|
return;
|
|
}
|
|
NvnJob j;
|
|
j.present = true;
|
|
j.present_crop_w = cw, j.present_crop_h = ch;
|
|
j.present_guest = guest;
|
|
j.present_w = t->d.width;
|
|
j.present_h = t->d.height;
|
|
push_job(std::move(j));
|
|
}
|
|
void rx_nvn_register_texture(int id, const nvn_tex* t, const nvn_view* v) {
|
|
if (t) nvn_register_texture(id, *t, v);
|
|
}
|
|
void rx_nvn_register_sampler(int id, const nvn_samp_par* p) {
|
|
if (p) nvn_register_sampler(id, *p);
|
|
}
|
|
void rx_nvn_set_texture_pool_size(int n) { (void)n; }
|
|
void rx_nvn_texture_released(const nvn_tex* t) {
|
|
if (t && t->serial) nvn_texture_released(t->serial);
|
|
}
|
|
void rx_nvn_program_released(const nvn_prog* p) { (void)p; }
|
|
/* ReadTexels / WriteTexels: the texture's memory in its own layout, through the copy engine's addressing (a texture
|
|
the GPU rendered is read as guest memory holds it: nvn_copy_from_rt counts those) */
|
|
static void texel_copy(const nvn_tex* t, unsigned level, unsigned layer0, const NVNcopyRegion* r, uint8_t* data,
|
|
size_t row, size_t img, bool write) {
|
|
nvn_view v{};
|
|
v.set_levels = 1, v.base_level = (uint8_t)level, v.nlevels = 1;
|
|
v.set_layers = 1, v.min_layer = (uint16_t)layer0, v.nlayers = 1;
|
|
CopySurface s;
|
|
if (!copy_surface(*t, &v, s)) {
|
|
log_limited("nvn_texels_fmt", 8, "rx: NVN %sTexels of texture format %d: no layout", write ? "Write" : "Read",
|
|
t->d.format);
|
|
return;
|
|
}
|
|
const CopyBox b = copy_box(s, *r);
|
|
if (!b.w || !b.h || !b.d) return;
|
|
const size_t pitch = row ? row : (size_t)b.w * s.bpb, image = img ? img : pitch * b.h;
|
|
if (!write) {
|
|
copy_source_synced(copy_from(s, b), copy_span(s, b));
|
|
copy_note_target(s, false);
|
|
}
|
|
HostPages tex;
|
|
for (uint32_t k = 0; k < b.d; k++)
|
|
for (uint32_t y = 0; y < b.h; y++) {
|
|
uint8_t* line = data + k * image + y * pitch;
|
|
surface_row(s, b, y, k, [&](uint64_t gpu, uint32_t off, uint32_t n) {
|
|
uint8_t* h = tex.at(gpu);
|
|
if (h && (gpu & 0xFFFF) + n <= 0x10000) {
|
|
if (write) std::memcpy(h, line + off, n), rx_mark_written(h, n);
|
|
else std::memcpy(line + off, h, n);
|
|
} else if (write) {
|
|
nvn_gpu_write(gpu, line + off, n);
|
|
} else {
|
|
nvn_gpu_read(gpu, line + off, n);
|
|
}
|
|
});
|
|
}
|
|
// (a CPU write: the write watch reports it at the next submit)
|
|
count(write ? "nvn_write_texels" : "nvn_read_texels");
|
|
if (write) note_texels_written(nvn_gpu_guest(t->gpu)); // (RX_DUMP_TEX=texels: the last ones are dumped)
|
|
if (static const bool trace = getenv("RX_NVN_TEXELS_LOG") != nullptr; trace)
|
|
log_limited("nvn_texels", 100000, "rx: NVN %sTexels %08llx %ux%u format %d level %u layer %u at %d,%d,%d %dx%dx%d",
|
|
write ? "Write" : "Read", (unsigned long long)nvn_gpu_guest(t->gpu), t->d.width, t->d.height,
|
|
t->d.format, level, layer0, r->xoffset, r->yoffset, r->zoffset, r->width, r->height, r->depth);
|
|
}
|
|
void rx_nvn_read_texels(const nvn_tex* t, unsigned level, unsigned layer0, const NVNcopyRegion* r, uint8_t* dst,
|
|
size_t row, size_t img) {
|
|
drain(); // (what the submits before it wrote)
|
|
std::lock_guard lk(nvn_mu);
|
|
texel_copy(t, level, layer0, r, dst, row, img, false);
|
|
}
|
|
void rx_nvn_write_texels(const nvn_tex* t, unsigned level, unsigned layer0, const NVNcopyRegion* r, const uint8_t* src,
|
|
size_t row, size_t img) {
|
|
drain(); // (after the submits before it read the texture)
|
|
std::lock_guard lk(nvn_mu);
|
|
texel_copy(t, level, layer0, r, const_cast<uint8_t*>(src), row, img, true);
|
|
}
|
|
/* the CPU is about to read memory a shader may have written on the GPU (InvalidateMappedRange): nothing to do here.
|
|
TOTK invalidates ranges every frame; a GPU sync at each (as rx_sync_gpu_writes makes) halved the frame rate, and the
|
|
emulated path never synced there either: shader writes come back to guest memory by themselves (readbacks). */
|
|
void rx_nvn_invalidate(uint64_t gpu, uint64_t size) { (void)gpu, (void)size; }
|
|
|
|
} // extern "C"
|