mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
2407 lines
126 KiB
C++
2407 lines
126 KiB
C++
// Textures and samplers: a shader's texture handle -> the texture header (TIC) and sampler header (TSC) in the bound
|
|
// pools -> a WebGPU texture view and sampler. A texture is the render target that memory holds when the GPU drew
|
|
// it, or else is decoded from guest memory (block-linear layout, ASTC, packed 16-bit formats) and kept while that
|
|
// memory does not change. Layout arithmetic is suyu's (video_core/texture_cache/util.cpp, image_info.cpp).
|
|
#include <algorithm>
|
|
#include <unordered_set>
|
|
#include <atomic>
|
|
#include <condition_variable>
|
|
#include <deque>
|
|
#include <mutex>
|
|
#include <thread>
|
|
#include <cstdint>
|
|
#include <map>
|
|
#include <numeric>
|
|
#include <span>
|
|
|
|
#include "common/alignment.h"
|
|
#include "common/div_ceil.h"
|
|
#include "video_core/texture_cache/format_lookup_table.h"
|
|
#include "video_core/textures/astc.h"
|
|
#include "video_core/textures/decoders.h"
|
|
#include "video_core/textures/workers.h"
|
|
|
|
#include "rx_bcn.h"
|
|
#include "rx_internal.h"
|
|
|
|
namespace rx {
|
|
namespace {
|
|
|
|
using Tegra::Texture::TICEntry;
|
|
using Tegra::Texture::TSCEntry;
|
|
using VideoCore::Surface::BytesPerBlock;
|
|
using VideoCore::Surface::DefaultBlockHeight;
|
|
using VideoCore::Surface::DefaultBlockWidth;
|
|
using Tegra::Texture::GOB_SIZE_SHIFT;
|
|
using Tegra::Texture::GOB_SIZE_X;
|
|
using Tegra::Texture::GOB_SIZE_X_SHIFT;
|
|
using Tegra::Texture::GOB_SIZE_Y;
|
|
using Tegra::Texture::GOB_SIZE_Y_SHIFT;
|
|
using Tegra::Texture::GOB_SIZE_Z;
|
|
|
|
// ---- suyu's block-linear layout arithmetic (texture_cache/util.cpp) ----
|
|
struct Extent2D {
|
|
u32 width, height;
|
|
};
|
|
struct Extent3D {
|
|
u32 width, height, depth;
|
|
};
|
|
struct LevelInfo {
|
|
Extent3D size;
|
|
Extent3D block;
|
|
Extent2D tile_size;
|
|
u32 bpp_log2;
|
|
u32 tile_width_spacing;
|
|
u32 num_levels;
|
|
};
|
|
|
|
constexpr u32 AdjustTileSize(u32 shift, u32 unit_factor, u32 dimension) {
|
|
if (shift == 0) return 0;
|
|
u32 x = unit_factor << (shift - 1);
|
|
if (x >= dimension) {
|
|
while (--shift) {
|
|
x >>= 1;
|
|
if (x < dimension) break;
|
|
}
|
|
}
|
|
return shift;
|
|
}
|
|
constexpr u32 AdjustMipSize(u32 size, u32 level) {
|
|
return std::max<u32>(size >> level, 1);
|
|
}
|
|
constexpr Extent3D AdjustMipSize(Extent3D size, u32 level) {
|
|
return {AdjustMipSize(size.width, level), AdjustMipSize(size.height, level), AdjustMipSize(size.depth, level)};
|
|
}
|
|
template <u32 GOB_EXTENT>
|
|
constexpr u32 AdjustMipBlockSize(u32 num_tiles, u32 block_size, u32 level) {
|
|
do {
|
|
while (block_size > 0 && num_tiles <= (1U << (block_size - 1)) * GOB_EXTENT) --block_size;
|
|
} while (level--);
|
|
return block_size;
|
|
}
|
|
constexpr Extent3D AdjustMipBlockSize(Extent3D num_tiles, Extent3D block_size, u32 level, u32 num_levels) {
|
|
return {AdjustMipBlockSize<GOB_SIZE_X>(num_tiles.width, block_size.width, level),
|
|
AdjustMipBlockSize<GOB_SIZE_Y>(num_tiles.height, block_size.height, level),
|
|
level == 0 && num_levels == 1 ? block_size.depth
|
|
: AdjustMipBlockSize<GOB_SIZE_Z>(num_tiles.depth, block_size.depth, level)};
|
|
}
|
|
constexpr Extent3D AdjustTileSize(Extent3D size, Extent2D tile) {
|
|
return {Common::DivCeil(size.width, tile.width), Common::DivCeil(size.height, tile.height), size.depth};
|
|
}
|
|
constexpr u32 BytesPerBlockLog2(u32 bytes) {
|
|
return std::countl_zero(bytes) ^ 0x1F;
|
|
}
|
|
constexpr u32 AdjustSize(u32 size, u32 level, u32 block_size) {
|
|
return Common::DivCeil(AdjustMipSize(size, level), block_size);
|
|
}
|
|
constexpr Extent3D NumLevelBlocks(const LevelInfo& info, u32 level) {
|
|
return {AdjustSize(info.size.width, level, info.tile_size.width) << info.bpp_log2,
|
|
AdjustSize(info.size.height, level, info.tile_size.height), AdjustMipSize(info.size.depth, level)};
|
|
}
|
|
constexpr Extent3D TileShift(const LevelInfo& info, u32 level) {
|
|
if (level == 0 && info.num_levels == 1) return info.block;
|
|
const Extent3D blocks = NumLevelBlocks(info, level);
|
|
return {AdjustTileSize(info.block.width, GOB_SIZE_X, blocks.width),
|
|
AdjustTileSize(info.block.height, GOB_SIZE_Y, blocks.height),
|
|
AdjustTileSize(info.block.depth, GOB_SIZE_Z, blocks.depth)};
|
|
}
|
|
constexpr Extent2D GobSize(u32 bpp_log2, u32 block_height, u32 tile_width_spacing) {
|
|
return {GOB_SIZE_X_SHIFT - bpp_log2 + tile_width_spacing, GOB_SIZE_Y_SHIFT + block_height};
|
|
}
|
|
constexpr bool IsSmallerThanGobSize(Extent3D num_tiles, Extent2D gob, u32 block_depth) {
|
|
return num_tiles.width <= (1U << gob.width) || num_tiles.height <= (1U << gob.height) ||
|
|
num_tiles.depth < (1U << block_depth);
|
|
}
|
|
constexpr u32 StrideAlignment(Extent3D num_tiles, Extent3D block, Extent2D gob, u32 bpp_log2) {
|
|
return IsSmallerThanGobSize(num_tiles, gob, block.depth) ? GOB_SIZE_X_SHIFT - bpp_log2 : gob.width;
|
|
}
|
|
constexpr Extent2D NumGobs(const LevelInfo& info, u32 level) {
|
|
const Extent3D blocks = NumLevelBlocks(info, level);
|
|
const Extent2D gobs{Common::DivCeilLog2(blocks.width, GOB_SIZE_X_SHIFT),
|
|
Common::DivCeilLog2(blocks.height, GOB_SIZE_Y_SHIFT)};
|
|
const Extent2D gob = GobSize(info.bpp_log2, info.block.height, info.tile_width_spacing);
|
|
const bool is_small = IsSmallerThanGobSize(blocks, gob, info.block.depth);
|
|
const u32 alignment = is_small ? 0 : info.tile_width_spacing;
|
|
return {Common::AlignUpLog2(gobs.width, alignment), gobs.height};
|
|
}
|
|
constexpr Extent3D LevelTiles(const LevelInfo& info, u32 level) {
|
|
const Extent3D blocks = NumLevelBlocks(info, level);
|
|
const Extent3D tile_shift = TileShift(info, level);
|
|
const Extent2D gobs = NumGobs(info, level);
|
|
return {Common::DivCeilLog2(gobs.width, tile_shift.width), Common::DivCeilLog2(gobs.height, tile_shift.height),
|
|
Common::DivCeilLog2(blocks.depth, tile_shift.depth)};
|
|
}
|
|
constexpr u32 CalculateLevelSize(const LevelInfo& info, u32 level) {
|
|
const Extent3D tile_shift = TileShift(info, level);
|
|
const Extent3D tiles = LevelTiles(info, level);
|
|
const u32 num_tiles = tiles.width * tiles.height * tiles.depth;
|
|
return num_tiles << (GOB_SIZE_SHIFT + tile_shift.width + tile_shift.height + tile_shift.depth);
|
|
}
|
|
constexpr u32 AlignLayerSize(u32 size_bytes, Extent3D size, Extent3D block, u32 tile_size_y,
|
|
u32 tile_width_spacing) {
|
|
if (tile_width_spacing > 0) {
|
|
const u32 alignment_log2 = GOB_SIZE_SHIFT + tile_width_spacing + block.height + block.depth;
|
|
return Common::AlignUpLog2(size_bytes, alignment_log2);
|
|
}
|
|
const u32 aligned_height = Common::AlignUp(size.height, tile_size_y);
|
|
while (block.height != 0 && aligned_height <= (1U << (block.height - 1)) * GOB_SIZE_Y) --block.height;
|
|
while (block.depth != 0 && size.depth <= (1U << (block.depth - 1))) --block.depth;
|
|
const u32 block_shift = GOB_SIZE_SHIFT + block.height + block.depth;
|
|
const u32 num_blocks = size_bytes >> block_shift;
|
|
return size_bytes != num_blocks << block_shift ? (num_blocks + 1) << block_shift : size_bytes;
|
|
}
|
|
|
|
// ---- texture descriptions ----
|
|
struct TexInfo {
|
|
PixelFormat format{};
|
|
WGPUTextureDimension dimension = WGPUTextureDimension_2D;
|
|
Extent3D size{1, 1, 1}; // depth: 3D textures only
|
|
u32 layers = 1, levels = 1;
|
|
Extent3D block{0, 0, 0};
|
|
u32 tile_width_spacing = 0;
|
|
bool linear = false;
|
|
u32 pitch = 0;
|
|
bool valid = false;
|
|
};
|
|
|
|
TexInfo tex_info(const TICEntry& t) {
|
|
TexInfo i;
|
|
i.format = VideoCommon::PixelFormatFromTextureInfo(t.format, t.r_type, t.g_type, t.b_type, t.a_type,
|
|
t.srgb_conversion);
|
|
i.levels = std::min<u32>(t.max_mip_level + 1, 16);
|
|
i.linear = t.IsPitchLinear();
|
|
if (i.linear) i.pitch = t.Pitch();
|
|
else i.block = {t.block_width, t.block_height, t.block_depth};
|
|
i.tile_width_spacing = t.tile_width_spacing;
|
|
using T = Tegra::Texture::TextureType;
|
|
T type = t.texture_type;
|
|
if (t.Depth() > 1 || t.BaseLayer() != 0) {
|
|
if (type == T::Texture1D) type = T::Texture1DArray;
|
|
else if (type == T::Texture2D) type = T::Texture2DArray;
|
|
else if (type == T::TextureCubemap) type = T::TextureCubeArray;
|
|
}
|
|
i.size = {t.Width(), t.Height(), 1};
|
|
switch (type) {
|
|
case T::Texture1D:
|
|
i.dimension = WGPUTextureDimension_1D;
|
|
i.size.height = 1;
|
|
break;
|
|
case T::Texture1DArray:
|
|
i.dimension = WGPUTextureDimension_1D;
|
|
i.size.height = 1;
|
|
i.layers = t.BaseLayer() + t.Depth();
|
|
break;
|
|
case T::Texture2D:
|
|
case T::Texture2DNoMipmap:
|
|
i.layers = t.BaseLayer() + 1;
|
|
break;
|
|
case T::Texture2DArray:
|
|
i.layers = t.BaseLayer() + t.Depth();
|
|
break;
|
|
case T::TextureCubemap:
|
|
i.layers = t.BaseLayer() + 6;
|
|
break;
|
|
case T::TextureCubeArray:
|
|
i.layers = t.BaseLayer() + t.Depth() * 6;
|
|
break;
|
|
case T::Texture3D:
|
|
i.dimension = WGPUTextureDimension_3D;
|
|
i.size.depth = t.Depth();
|
|
break;
|
|
default:
|
|
return i; // buffers: not textures
|
|
}
|
|
if (i.linear) i.levels = 1;
|
|
if (i.dimension == WGPUTextureDimension_1D) i.levels = 1; // WebGPU: 1D textures have one level
|
|
i.valid = i.size.width && i.size.height;
|
|
return i;
|
|
}
|
|
|
|
/// What a decoded texture is stored as when its format has no WebGPU equivalent.
|
|
enum class Convert { None, Astc, Packed16, Rgb32f, Bc1, Bc2, Bc3, Bc4, Bc4Snorm, Bc5, Bc5Snorm, Unsupported };
|
|
Convert conversion(PixelFormat f) {
|
|
using P = PixelFormat;
|
|
if (VideoCore::Surface::IsPixelFormatASTC(f)) return Convert::Astc;
|
|
switch (f) {
|
|
case P::R5G6B5_UNORM:
|
|
case P::B5G6R5_UNORM:
|
|
case P::A1R5G5B5_UNORM:
|
|
case P::A1B5G5R5_UNORM:
|
|
case P::A5B5G5R1_UNORM:
|
|
case P::A4B4G4R4_UNORM:
|
|
case P::G4R4_UNORM:
|
|
return Convert::Packed16;
|
|
case P::R32G32B32_FLOAT:
|
|
return Convert::Rgb32f;
|
|
case P::R16G16B16A16_UNORM: case P::R16G16B16A16_SNORM: case P::R16_UNORM: case P::R16_SNORM:
|
|
case P::R16G16_UNORM: case P::R16G16_SNORM:
|
|
return dev.norm16 ? Convert::None : Convert::Unsupported;
|
|
case P::BC1_RGBA_UNORM: case P::BC1_RGBA_SRGB:
|
|
return dev.bc ? Convert::None : Convert::Bc1;
|
|
case P::BC2_UNORM: case P::BC2_SRGB:
|
|
return dev.bc ? Convert::None : Convert::Bc2;
|
|
case P::BC3_UNORM: case P::BC3_SRGB:
|
|
return dev.bc ? Convert::None : Convert::Bc3;
|
|
case P::BC4_UNORM:
|
|
return dev.bc ? Convert::None : Convert::Bc4;
|
|
case P::BC4_SNORM:
|
|
return dev.bc ? Convert::None : Convert::Bc4Snorm;
|
|
case P::BC5_UNORM:
|
|
return dev.bc ? Convert::None : Convert::Bc5;
|
|
case P::BC5_SNORM:
|
|
return dev.bc ? Convert::None : Convert::Bc5Snorm;
|
|
case P::BC7_UNORM: case P::BC7_SRGB: case P::BC6H_UFLOAT: case P::BC6H_SFLOAT:
|
|
return dev.bc ? Convert::None : Convert::Unsupported;
|
|
default:
|
|
if (VideoCore::Surface::GetFormatType(f) != VideoCore::Surface::SurfaceType::ColorTexture &&
|
|
!is_depth_format(wgpu_format(f)))
|
|
return Convert::Unsupported;
|
|
return wgpu_format(f) == WGPUTextureFormat_Undefined ? Convert::Unsupported : Convert::None;
|
|
}
|
|
}
|
|
|
|
uint8_t expand(uint32_t v, int bits) {
|
|
return bits == 0 ? 0 : (uint8_t)((v * 255u + ((1u << bits) - 1) / 2) / ((1u << bits) - 1));
|
|
}
|
|
|
|
void convert_packed16(PixelFormat f, const uint8_t* src, uint8_t* dst, size_t pixels) {
|
|
using P = PixelFormat;
|
|
for (size_t i = 0; i < pixels; i++) {
|
|
uint32_t v = f == P::G4R4_UNORM ? src[i] : (uint32_t)(src[2 * i] | src[2 * i + 1] << 8);
|
|
uint8_t r = 0, g = 0, b = 0, a = 255;
|
|
switch (f) {
|
|
case P::R5G6B5_UNORM: r = expand(v >> 11, 5); g = expand(v >> 5 & 63, 6); b = expand(v & 31, 5); break;
|
|
case P::B5G6R5_UNORM: b = expand(v >> 11, 5); g = expand(v >> 5 & 63, 6); r = expand(v & 31, 5); break;
|
|
case P::A1R5G5B5_UNORM:
|
|
a = v >> 15 ? 255 : 0; r = expand(v >> 10 & 31, 5); g = expand(v >> 5 & 31, 5); b = expand(v & 31, 5);
|
|
break;
|
|
case P::A1B5G5R5_UNORM:
|
|
a = v >> 15 ? 255 : 0; b = expand(v >> 10 & 31, 5); g = expand(v >> 5 & 31, 5); r = expand(v & 31, 5);
|
|
break;
|
|
case P::A5B5G5R1_UNORM:
|
|
r = v & 1 ? 255 : 0; g = expand(v >> 1 & 31, 5); b = expand(v >> 6 & 31, 5); a = expand(v >> 11, 5);
|
|
break;
|
|
case P::A4B4G4R4_UNORM:
|
|
r = expand(v & 15, 4); g = expand(v >> 4 & 15, 4); b = expand(v >> 8 & 15, 4); a = expand(v >> 12, 4);
|
|
break;
|
|
case P::G4R4_UNORM: r = expand(v & 15, 4); g = expand(v >> 4, 4); break;
|
|
default: break;
|
|
}
|
|
dst[4 * i] = r;
|
|
dst[4 * i + 1] = g;
|
|
dst[4 * i + 2] = b;
|
|
dst[4 * i + 3] = a;
|
|
}
|
|
}
|
|
|
|
/// Applies the header's component swizzle to RGBA8 texels.
|
|
void swizzle_rgba8(const TICEntry& t, uint8_t* p, size_t pixels) {
|
|
using S = Tegra::Texture::SwizzleSource;
|
|
const S src[4] = {t.x_source, t.y_source, t.z_source, t.w_source};
|
|
if (src[0] == S::R && src[1] == S::G && src[2] == S::B && src[3] == S::A) return;
|
|
for (size_t i = 0; i < pixels; i++) {
|
|
uint8_t in[4] = {p[4 * i], p[4 * i + 1], p[4 * i + 2], p[4 * i + 3]};
|
|
for (int c = 0; c < 4; c++) {
|
|
switch (src[c]) {
|
|
case S::Zero: p[4 * i + c] = 0; break;
|
|
case S::R: p[4 * i + c] = in[0]; break;
|
|
case S::G: p[4 * i + c] = in[1]; break;
|
|
case S::B: p[4 * i + c] = in[2]; break;
|
|
case S::A: p[4 * i + c] = in[3]; break;
|
|
default: p[4 * i + c] = 255; break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
std::unordered_map<uint64_t, std::unique_ptr<Image>> decoded; // by TIC contents
|
|
uint64_t decoded_generation; // bumped whenever the decoded-texture map changes
|
|
// The same, by the 1 MB pieces of guest memory each covers (textures_written runs per written page: a walk back
|
|
// over every texture within the largest one's size was ~4% of the GPU thread)
|
|
constexpr uint64_t kBucketShift = 20;
|
|
std::unordered_map<uint64_t, std::vector<Image*>> decoded_by_bucket;
|
|
|
|
void remember(Image* im) {
|
|
const uint64_t last = (im->guest + std::max<uint64_t>(im->guest_bytes, 1) - 1) >> kBucketShift;
|
|
for (uint64_t b = im->guest >> kBucketShift; b <= last; b++) decoded_by_bucket[b].push_back(im);
|
|
}
|
|
|
|
void forget(Image* im) {
|
|
const uint64_t last = (im->guest + std::max<uint64_t>(im->guest_bytes, 1) - 1) >> kBucketShift;
|
|
for (uint64_t b = im->guest >> kBucketShift; b <= last; b++) {
|
|
auto it = decoded_by_bucket.find(b);
|
|
if (it == decoded_by_bucket.end()) continue;
|
|
auto& v = it->second;
|
|
v.erase(std::remove(v.begin(), v.end(), im), v.end());
|
|
if (v.empty()) decoded_by_bucket.erase(it);
|
|
}
|
|
}
|
|
std::unordered_map<uint64_t, WGPUSampler> samplers;
|
|
|
|
} // namespace
|
|
|
|
namespace { uint64_t sample_hash(uint64_t va, uint64_t size); }
|
|
|
|
// RX_DUMP_TEX=texels: the textures the game last wrote with WriteTexels (rx_nvn.cpp) instead of a list of addresses
|
|
static std::mutex texels_mu;
|
|
static std::vector<uint64_t> texels_written;
|
|
void note_texels_written(uint64_t guest) {
|
|
static const bool on = getenv("RX_DUMP_TEX") && !strcmp(getenv("RX_DUMP_TEX"), "texels");
|
|
if (!on || !guest) return;
|
|
std::lock_guard lk(texels_mu);
|
|
if (std::find(texels_written.begin(), texels_written.end(), guest) != texels_written.end()) return;
|
|
texels_written.push_back(guest);
|
|
if (texels_written.size() > 16) texels_written.erase(texels_written.begin());
|
|
}
|
|
|
|
void dump_tex_list(const char* dir) {
|
|
const char* s = getenv("RX_DUMP_TEX");
|
|
if (!s) return;
|
|
std::string from_texels;
|
|
if (!strcmp(s, "texels")) {
|
|
std::lock_guard lk(texels_mu);
|
|
for (uint64_t g : texels_written) {
|
|
char b[24];
|
|
std::snprintf(b, sizeof b, "%llx,", (unsigned long long)g);
|
|
from_texels += b;
|
|
}
|
|
s = from_texels.c_str();
|
|
}
|
|
int order = 800;
|
|
for (const char* p = s; *p;) {
|
|
char* end;
|
|
const uint64_t va = strtoull(p, &end, 16);
|
|
if (end == p) break;
|
|
p = *end == ',' ? end + 1 : end;
|
|
for (auto& [key, im] : decoded)
|
|
if (im && (im->gpu_va == va || im->guest == va) && im->tex) {
|
|
dump_texture(im->tex, va, im->width, im->height, im->format, dir, order++);
|
|
// what guest memory holds now, against the decoded copy
|
|
std::vector<uint8_t> mem(im->guest_bytes);
|
|
gpu_read(im->gpu_va, mem.data(), mem.size());
|
|
size_t nonzero = 0;
|
|
for (uint8_t b : mem) nonzero += b != 0;
|
|
char path[512];
|
|
std::snprintf(path, sizeof path, "%s/tex_%llx_guest.bin", dir, (unsigned long long)va);
|
|
if (FILE* f = std::fopen(path, "wb")) { std::fwrite(mem.data(), 1, mem.size(), f); std::fclose(f); }
|
|
hos_log("rx-frame: texture %#llx (gpu %#llx) %ux%u format %d saved; guest %llu bytes, %zu nonzero; stale %d "
|
|
"decoded_seq %llu now %llu; hash %s; rendered %d; layers %u dim %d levels %u", (unsigned long long)va, (unsigned long long)im->gpu_va, im->width,
|
|
im->height, (int)im->format, (unsigned long long)im->guest_bytes, nonzero, (int)im->stale,
|
|
(unsigned long long)im->decoded_seq, (unsigned long long)memory_seq(),
|
|
sample_hash(im->gpu_va, im->guest_bytes) == im->content_hash ? "same" : "changed",
|
|
(int)im->rendered, im->layers, (int)im->dimension, im->levels);
|
|
}
|
|
// a render target there: what its guest memory holds (the GPU's image is what draws see)
|
|
for (const auto& [g, im] : all_images())
|
|
if (im && (im->gpu_va == va || im->guest == va)) {
|
|
const uint64_t bytes = (uint64_t)im->width * im->height * BytesPerBlock(im->pixel_format);
|
|
std::vector<uint8_t> mem(bytes);
|
|
gpu_read(im->gpu_va, mem.data(), mem.size());
|
|
size_t nonzero = 0;
|
|
for (uint8_t b : mem) nonzero += b != 0;
|
|
char path[512];
|
|
std::snprintf(path, sizeof path, "%s/rt_%llx_guest.bin", dir, (unsigned long long)va);
|
|
if (FILE* f = std::fopen(path, "wb")) { std::fwrite(mem.data(), 1, mem.size(), f); std::fclose(f); }
|
|
hos_log("rx-frame: render target %#llx (gpu %#llx) %ux%u pixel format %d: guest %llu bytes, %zu nonzero; "
|
|
"rendered %d version %llu", (unsigned long long)va, (unsigned long long)im->gpu_va, im->width,
|
|
im->height, (int)im->pixel_format, (unsigned long long)bytes, nonzero, (int)im->rendered,
|
|
(unsigned long long)im->version);
|
|
}
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
|
|
/// A few samples of a texture's memory: what tells that the game wrote new contents.
|
|
uint64_t sample_hash(uint64_t va, uint64_t size) {
|
|
uint64_t h = 0xcbf29ce484222325ull;
|
|
uint8_t buf[64];
|
|
const uint64_t n = std::min<uint64_t>(size, 64);
|
|
for (int i = 0; i < 16; i++) {
|
|
const uint64_t off = size > n ? (size - n) * i / 15 & ~7ull : 0;
|
|
gpu_read(va + off, buf, n);
|
|
h = hash_bytes(buf, n, h);
|
|
}
|
|
return h;
|
|
}
|
|
|
|
/// Where a (layer, level) of a block-linear texture starts, from its address.
|
|
uint64_t subresource_offset(const TexInfo& info, uint32_t layer, uint32_t level) {
|
|
const u32 bpp = BytesPerBlock(info.format);
|
|
const Extent2D tile{DefaultBlockWidth(info.format), DefaultBlockHeight(info.format)};
|
|
const LevelInfo li{info.size, info.block, tile, BytesPerBlockLog2(bpp), info.tile_width_spacing, info.levels};
|
|
u32 layer_size = 0, offset = 0;
|
|
for (u32 l = 0; l < info.levels; l++) {
|
|
if (l == level) offset = layer_size;
|
|
layer_size += CalculateLevelSize(li, l);
|
|
}
|
|
const u32 stride = AlignLayerSize(layer_size, info.size, li.block, tile.height, info.tile_width_spacing);
|
|
return (uint64_t)layer * stride + offset;
|
|
}
|
|
|
|
uint64_t guest_size(const TexInfo& info) {
|
|
const u32 bpp = BytesPerBlock(info.format);
|
|
const Extent2D tile{DefaultBlockWidth(info.format), DefaultBlockHeight(info.format)};
|
|
if (info.linear) return (uint64_t)info.pitch * Common::DivCeil(info.size.height, tile.height);
|
|
const LevelInfo li{info.size, info.block, tile, BytesPerBlockLog2(bpp), info.tile_width_spacing, info.levels};
|
|
u32 layer = 0;
|
|
for (u32 l = 0; l < info.levels; l++) layer += CalculateLevelSize(li, l);
|
|
const u32 stride = AlignLayerSize(layer, info.size, li.block, tile.height, info.tile_width_spacing);
|
|
return (uint64_t)stride * (info.layers - 1) + layer;
|
|
}
|
|
|
|
/// The browser: WebGPU objects belong to the render thread, so a decode worker describes the texture
|
|
/// (thread_local deferring) and streams its decoded layers to the render thread, which makes the texture at the first
|
|
/// one and writes them at each present. At most kDeferredBudget bytes wait (the host heap above the guest window is
|
|
/// ~800 MB: a 350 MB ASTC array decoded whole ran it out), so a big texture arrives over a few frames.
|
|
struct DeferredUpload {
|
|
uint32_t level = 0, origin_y = 0, origin_z = 0;
|
|
WGPUTexelCopyBufferLayout layout{};
|
|
WGPUExtent3D extent{};
|
|
std::vector<uint8_t> data;
|
|
};
|
|
struct DeferredTexture {
|
|
WGPUTextureDescriptor desc{};
|
|
WGPUTextureFormat pair = WGPUTextureFormat_Undefined;
|
|
WGPUTexture tex = nullptr; // (render thread)
|
|
};
|
|
struct StreamedUpload {
|
|
DeferredTexture* texture;
|
|
DeferredUpload upload;
|
|
};
|
|
thread_local DeferredTexture* deferring;
|
|
thread_local bool gpu_force_cpu; // (RX_GPU_VERIFY: the CPU decode to compare with)
|
|
std::unordered_set<WGPUSampler> nonfiltering_samplers; // (debugging the browser's sampler checks)
|
|
std::unordered_set<uint64_t> rewritten_textures; // guest addresses whose background decode was rewritten meanwhile
|
|
thread_local char decoding_what[160]; // (debugging: HOS_BIG_ALLOC_MB names the texture being decoded)
|
|
extern "C" const char* rx_decoding_what() { return decoding_what; }
|
|
#ifdef __EMSCRIPTEN__
|
|
extern "C" void* sbrk(intptr_t);
|
|
uintptr_t heap_top() { return (uintptr_t)sbrk(0); }
|
|
#else
|
|
uintptr_t heap_top() { return 0; }
|
|
#endif
|
|
std::mutex decode_mu;
|
|
std::condition_variable decode_cv;
|
|
std::deque<StreamedUpload> upload_stream; // (decode_mu)
|
|
std::atomic<int64_t> deferred_bytes{0};
|
|
constexpr int64_t kDeferredBudget = 32ll << 20;
|
|
|
|
void stream_upload(DeferredUpload&& u) {
|
|
const int64_t n = (int64_t)u.data.size();
|
|
std::unique_lock lk(decode_mu);
|
|
decode_cv.wait(lk, [n] {
|
|
const int64_t b = deferred_bytes.load();
|
|
return b == 0 || b + n <= kDeferredBudget;
|
|
});
|
|
deferred_bytes += n;
|
|
upload_stream.push_back({deferring, std::move(u)});
|
|
}
|
|
|
|
WGPUTexture deferred_texture(DeferredTexture& t) { // (render thread)
|
|
if (!t.tex) {
|
|
if (t.pair != WGPUTextureFormat_Undefined) t.desc.viewFormats = &t.pair;
|
|
t.tex = wgpuDeviceCreateTexture(dev.device, &t.desc);
|
|
}
|
|
return t.tex;
|
|
}
|
|
|
|
/// BC textures whose size is not a multiple of 4 stay BC: the texture is made a whole number of blocks (3686 ->
|
|
/// 3688: 0.05% of stretch) and each level's copy fits its padded size. Decoded to RGBA8 on the CPU they were 8x the
|
|
/// memory and a 108 MB buffer for TOTK's 7373x3686 maps (more than the browser build's heap had free).
|
|
/// RX_BC_DECODE_ODD=1: decoded as before.
|
|
bool odd_bc_decoded() {
|
|
static const bool on = env_flag("RX_BC_DECODE_ODD");
|
|
return on;
|
|
}
|
|
|
|
/// Decodes the texture into a new WebGPU texture.
|
|
std::unique_ptr<Image> decode(const TICEntry& tic, const TexInfo& info_in, uint64_t va) {
|
|
TexInfo info = info_in;
|
|
// (mkw-recomp) Large 2D textures and arrays leave out their top levels down to RX_TEXTURE_MAX_DIM (default 2048;
|
|
// 0: all): MKW's scenes sample 1200+ textures, 4.5 GB decoded, which with the rest ran the GPU out of memory, and
|
|
// its terrain samples three sparse 8192x8192 x68 arrays (1.4-5.8 GB whole). A texture keeps at least two levels;
|
|
// the levels kept have their layout, so the data starts at the first one kept, and an array's layers keep their
|
|
// stride (layer_stride_kept, and the bytes it spans: total_kept).
|
|
u32 layer_stride_kept = 0;
|
|
uint64_t total_kept = 0;
|
|
if (static const u32 max_dim = getenv("RX_TEXTURE_MAX_DIM") ? (u32)atoi(getenv("RX_TEXTURE_MAX_DIM")) : 2048;
|
|
max_dim && info.dimension == WGPUTextureDimension_2D && info.size.depth == 1 && !info.linear &&
|
|
std::max(info.size.width, info.size.height) > max_dim && info.levels > 2) {
|
|
const Extent2D tile{DefaultBlockWidth(info.format), DefaultBlockHeight(info.format)};
|
|
const u32 bpp_log2 = BytesPerBlockLog2(BytesPerBlock(info.format));
|
|
if (info.layers > 1) {
|
|
const LevelInfo full{info.size, info.block, tile, bpp_log2, info.tile_width_spacing, info.levels};
|
|
u32 size = 0;
|
|
for (u32 l = 0; l < info.levels; l++) size += CalculateLevelSize(full, l);
|
|
layer_stride_kept = AlignLayerSize(size, info.size, full.block, tile.height, info.tile_width_spacing);
|
|
}
|
|
while (std::max(info.size.width, info.size.height) > max_dim && info.levels > 2) {
|
|
const LevelInfo li{info.size, info.block, tile, bpp_log2, info.tile_width_spacing, info.levels};
|
|
va += CalculateLevelSize(li, 0);
|
|
info.size.width = std::max(1u, info.size.width >> 1), info.size.height = std::max(1u, info.size.height >> 1);
|
|
info.levels--;
|
|
count("tex_levels_dropped");
|
|
}
|
|
if (layer_stride_kept) {
|
|
const LevelInfo kept{info.size, info.block, tile, bpp_log2, info.tile_width_spacing, info.levels};
|
|
u32 size = 0;
|
|
for (u32 l = 0; l < info.levels; l++) size += CalculateLevelSize(kept, l);
|
|
total_kept = (uint64_t)(info.layers - 1) * layer_stride_kept + size;
|
|
}
|
|
}
|
|
const Convert cv = conversion(info.format);
|
|
if (cv == Convert::Unsupported) {
|
|
log_limited("tex_format", 24, "rx: texture format %d not decoded", (int)info.format);
|
|
return nullptr;
|
|
}
|
|
const WGPUTextureFormat guest_format = wgpu_format(info.format);
|
|
const u32 bpp = BytesPerBlock(info.format);
|
|
const Extent2D tile{DefaultBlockWidth(info.format), DefaultBlockHeight(info.format)};
|
|
if (info.size.width > dev.limits.maxTextureDimension2D || info.size.height > dev.limits.maxTextureDimension2D ||
|
|
info.layers > dev.limits.maxTextureArrayLayers)
|
|
return nullptr;
|
|
// (mkw-recomp) MKW's sparse textures (8192x8192 x68 and the like: gigabytes whole, a few tiles resident) are not
|
|
// made: the GPU runs out of memory and the failed texture invalidates the draw's bind group.
|
|
// RX_TEXTURE_MAX_MB (default 768): the largest texture made, in guest bytes; larger ones sample the placeholder.
|
|
{
|
|
static const uint64_t budget = (getenv("RX_TEXTURE_MAX_MB") ? atoll(getenv("RX_TEXTURE_MAX_MB")) : 768) << 20;
|
|
if (guest_size(info) > budget) {
|
|
log_limited("tex_budget", 16, "rx: texture %ux%u x%u format %d (%llu MB) over the budget: not made",
|
|
info.size.width, info.size.height, info.layers, (int)info.format,
|
|
(unsigned long long)(guest_size(info) >> 20));
|
|
return nullptr;
|
|
}
|
|
}
|
|
const bool guest_compressed = VideoCore::Surface::IsPixelFormatBCn(info.format);
|
|
const u32 physical_width = guest_compressed ? Common::AlignUp(info.size.width, 4) : info.size.width;
|
|
const u32 physical_height = guest_compressed ? Common::AlignUp(info.size.height, 4) : info.size.height;
|
|
const bool has_cpu_bc_decoder =
|
|
info.format == PixelFormat::BC1_RGBA_UNORM || info.format == PixelFormat::BC1_RGBA_SRGB ||
|
|
info.format == PixelFormat::BC2_UNORM || info.format == PixelFormat::BC2_SRGB ||
|
|
info.format == PixelFormat::BC3_UNORM || info.format == PixelFormat::BC3_SRGB ||
|
|
info.format == PixelFormat::BC4_UNORM || info.format == PixelFormat::BC4_SNORM ||
|
|
info.format == PixelFormat::BC5_UNORM || info.format == PixelFormat::BC5_SNORM;
|
|
const bool decode_bc = cv == Convert::Bc1 || cv == Convert::Bc2 || cv == Convert::Bc3 || cv == Convert::Bc4 ||
|
|
cv == Convert::Bc4Snorm || cv == Convert::Bc5 || cv == Convert::Bc5Snorm ||
|
|
(guest_compressed && has_cpu_bc_decoder &&
|
|
((odd_bc_decoded() && ((info.size.width & 3) || (info.size.height & 3))) ||
|
|
info.dimension == WGPUTextureDimension_3D));
|
|
const bool decode_any_compressed = decode_bc;
|
|
static const bool astc_bc7 = env_flag("RX_ASTC_BC7");
|
|
const bool to_bc7 = astc_bc7 && dev.bc && cv == Convert::Astc && info.dimension == WGPUTextureDimension_2D &&
|
|
!(info.size.width & 3) && !(info.size.height & 3);
|
|
const WGPUTextureFormat format = decode_any_compressed
|
|
? (VideoCore::Surface::IsPixelFormatSRGB(info.format)
|
|
? WGPUTextureFormat_RGBA8UnormSrgb
|
|
: WGPUTextureFormat_RGBA8Unorm)
|
|
: to_bc7 ? (VideoCore::Surface::IsPixelFormatSRGB(info.format)
|
|
? WGPUTextureFormat_BC7RGBAUnormSrgb
|
|
: WGPUTextureFormat_BC7RGBAUnorm)
|
|
: guest_format;
|
|
|
|
const uint64_t total = total_kept ? total_kept : guest_size(info);
|
|
// RX_LOG_BIG_DECODES=<MB> (debugging): each texture decoded whose GPU copy is at least that large (the first 60)
|
|
if (static const uint64_t big_mb = getenv("RX_LOG_BIG_DECODES") ? atoll(getenv("RX_LOG_BIG_DECODES")) : 0; big_mb) {
|
|
const uint64_t gpu_bytes = (uint64_t)info.size.width * info.size.height * info.size.depth * info.layers *
|
|
(decode_any_compressed || cv == Convert::Astc ? 4 : std::max<u32>(bpp, 1)) * 4 / 3;
|
|
static int n;
|
|
if (gpu_bytes >= (big_mb << 20) && n++ < 60)
|
|
hos_log("rx: decode %ux%u x%u depth %u levels %u format %d (convert %d): %llu MB guest, ~%llu MB on the GPU",
|
|
info.size.width, info.size.height, info.layers, info.size.depth, info.levels, (int)info.format, (int)cv,
|
|
(unsigned long long)(total >> 20), (unsigned long long)(gpu_bytes >> 20));
|
|
}
|
|
// made by the GPU from the guest's bytes as they are (rx_texture_gpu.cpp), 2D (arrays) in block-linear layout:
|
|
// - ASTC decoded there: the texture is RGBA8Unorm, as storage textures cannot be sRGB, with the sRGB encoding as
|
|
// its view format (its Image keeps the guest's encoding: views and copies are made in it)
|
|
// - other textures uploaded as they are (BC, 4-16 byte texels), from 256 KB: unswizzled there
|
|
const bool gpu_layout_ok = !gpu_force_cpu && !deferring && !info.linear && info.dimension == WGPUTextureDimension_2D &&
|
|
info.size.depth == 1 && total <= dev.limits.maxStorageBufferBindingSize;
|
|
const bool astc_gpu = gpu_layout_ok && cv == Convert::Astc && !to_bc7 && astc_gpu_on();
|
|
const bool swizzle_gpu = gpu_layout_ok && cv == Convert::None && !decode_any_compressed && !to_bc7 &&
|
|
(bpp == 4 || bpp == 8 || bpp == 16) && !is_depth_format(format) && total >= (256u << 10) &&
|
|
unswizzle_gpu_on();
|
|
std::vector<uint8_t> scratch;
|
|
std::snprintf(decoding_what, sizeof decoding_what, "%ux%u x%u depth %u levels %u format %d linear %d total %llu",
|
|
info.size.width, info.size.height, info.layers, info.size.depth, info.levels, (int)info.format,
|
|
(int)info.linear, (unsigned long long)total);
|
|
const uint8_t* src = gpu_span(va, total);
|
|
if (total > (32u << 20)) {
|
|
static int logged;
|
|
if (logged++ < 20)
|
|
hos_log("rx: big texture %ux%u x%u (depth %u) levels %u format %d: %llu MB in guest memory%s", info.size.width,
|
|
info.size.height, info.layers, info.size.depth, info.levels, (int)info.format,
|
|
(unsigned long long)(total >> 20), src ? "" : " (copied: not one mapping)");
|
|
}
|
|
if (!src) {
|
|
scratch.resize(total);
|
|
gpu_read(va, scratch.data(), total);
|
|
src = scratch.data();
|
|
}
|
|
|
|
auto im = std::make_unique<Image>();
|
|
im->gpu_va = va;
|
|
im->guest = hos_gpu_translate(va);
|
|
im->width = info.size.width;
|
|
im->height = info.size.height;
|
|
im->layers = info.dimension == WGPUTextureDimension_3D ? info.size.depth : info.layers;
|
|
im->levels = info.levels;
|
|
im->pixel_format = info.format;
|
|
im->format = format;
|
|
im->storage = astc_gpu;
|
|
im->dimension = info.dimension;
|
|
WGPUTextureDescriptor d{};
|
|
d.usage = WGPUTextureUsage_TextureBinding | WGPUTextureUsage_CopyDst | WGPUTextureUsage_CopySrc |
|
|
(astc_gpu ? WGPUTextureUsage_StorageBinding : 0);
|
|
d.dimension = info.dimension;
|
|
d.size = {decode_any_compressed ? info.size.width : physical_width,
|
|
decode_any_compressed ? info.size.height : physical_height, im->layers};
|
|
d.format = astc_gpu ? WGPUTextureFormat_RGBA8Unorm : format;
|
|
d.mipLevelCount = info.levels;
|
|
d.sampleCount = 1;
|
|
const WGPUTextureFormat pair = srgb_pair(d.format);
|
|
if (pair != WGPUTextureFormat_Undefined) {
|
|
d.viewFormatCount = 1;
|
|
d.viewFormats = &pair;
|
|
}
|
|
if (deferring) {
|
|
deferring->desc = d;
|
|
deferring->pair = pair;
|
|
} else {
|
|
scope_push();
|
|
im->tex = wgpuDeviceCreateTexture(dev.device, &d);
|
|
}
|
|
if (const std::string error = deferring ? std::string() : scope_pop(); !error.empty()) {
|
|
log_limited("tex_create", 16, "rx: texture %ux%u x%u format %d not created: %s", info.size.width,
|
|
info.size.height, info.layers, (int)info.format, error.c_str());
|
|
return nullptr;
|
|
}
|
|
|
|
const LevelInfo li{info.size, info.block, tile, BytesPerBlockLog2(bpp), info.tile_width_spacing, info.levels};
|
|
u32 layer_size = 0;
|
|
for (u32 l = 0; l < info.levels; l++) layer_size += CalculateLevelSize(li, l);
|
|
const u32 layer_stride = layer_stride_kept ? layer_stride_kept
|
|
: AlignLayerSize(layer_size, info.size, li.block, tile.height, info.tile_width_spacing);
|
|
const Extent2D gob = GobSize(li.bpp_log2, info.block.height, info.tile_width_spacing);
|
|
std::vector<uint8_t> level_data, converted;
|
|
size_t level_offset = 0;
|
|
const u32 copy_layers = info.dimension == WGPUTextureDimension_3D ? 1 : info.layers;
|
|
bool gpu_decoded = false;
|
|
if (astc_gpu || swizzle_gpu) {
|
|
std::vector<BlockLinearLevel> lv;
|
|
size_t off = 0;
|
|
for (u32 level = 0; level < info.levels; level++) {
|
|
const Extent3D tiles = AdjustTileSize(AdjustMipSize(info.size, level), tile);
|
|
const Extent3D block = AdjustMipBlockSize(tiles, li.block, level, li.num_levels);
|
|
const u32 stride = Common::AlignUpLog2(tiles.width, StrideAlignment(tiles, info.block, gob, li.bpp_log2)) * bpp;
|
|
const u32 x_shift = 9 + block.height + block.depth; // (a GOB is 512 bytes, 64 wide)
|
|
lv.push_back({(uint32_t)off, tiles.width, tiles.height, ((stride + 63) >> 6) << x_shift, x_shift, block.height});
|
|
off += CalculateLevelSize(li, level);
|
|
}
|
|
const int64_t t0 = hos_now_ns();
|
|
gpu_decoded = astc_gpu ? astc_gpu_decode(im->tex, src, total, lv.data(), (int)lv.size(), copy_layers,
|
|
layer_stride, tile.width, tile.height)
|
|
: unswizzle_gpu(im->tex, src, total, lv.data(), (int)lv.size(), copy_layers, layer_stride,
|
|
li.bpp_log2, tile.width, tile.height);
|
|
if (static const bool log_inplace = env_flag("RX_LOG_INPLACE"); log_inplace && total > (1u << 20))
|
|
hos_log("rx: %s %ux%u x%u levels %u format %d on the GPU: %.2f ms, %llu KB", astc_gpu ? "ASTC" : "unswizzle",
|
|
info.size.width, info.size.height, copy_layers, info.levels, (int)info.format,
|
|
(hos_now_ns() - t0) / 1e6, (unsigned long long)(total >> 10));
|
|
// RX_GPU_VERIFY=1 (or RX_ASTC_VERIFY=1): each texture also made by the CPU, and the two compared (level by
|
|
// level, first and last layer; ASTC within 1 of 255: the GPU rounds where the CPU truncates)
|
|
static const bool verify = env_flag("RX_GPU_VERIFY") || env_flag("RX_ASTC_VERIFY");
|
|
if (verify && gpu_decoded) {
|
|
gpu_force_cpu = true;
|
|
const auto cpu = decode(tic, info, va);
|
|
gpu_force_cpu = false;
|
|
for (u32 level = 0; cpu && level < info.levels; level++) {
|
|
const Extent3D ls = AdjustMipSize(info.size, level);
|
|
const Extent3D tiles = AdjustTileSize(ls, tile);
|
|
// (RGBA8 texels for ASTC; whole blocks as they are otherwise)
|
|
const uint32_t w = astc_gpu ? ls.width : tiles.width * tile.width;
|
|
const uint32_t h = astc_gpu ? ls.height : tiles.height * tile.height;
|
|
const uint32_t row = astc_gpu ? ls.width * 4 : tiles.width * bpp, rows = astc_gpu ? ls.height : tiles.height;
|
|
const int tolerance = astc_gpu ? 1 : 0;
|
|
for (u32 layer : {0u, copy_layers - 1}) {
|
|
const auto a = read_level(im->tex, level, layer, w, h, row, rows);
|
|
const auto b = read_level(cpu->tex, level, layer, w, h, row, rows);
|
|
size_t bad = 0, first = SIZE_MAX;
|
|
int worst = 0;
|
|
for (size_t i = 0; i < a.size() && i < b.size(); i++) {
|
|
const int diff = std::abs((int)a[i] - (int)b[i]);
|
|
if (diff > tolerance) {
|
|
bad++;
|
|
if (first == SIZE_MAX) first = i;
|
|
}
|
|
worst = std::max(worst, diff);
|
|
}
|
|
static int logged;
|
|
if ((bad || a.size() != b.size()) && logged++ < 60)
|
|
hos_log("rx: GPU verify (%s) %ux%u x%u format %d level %u layer %u: %zu of %zu bytes differ "
|
|
"(worst %d), first at byte %zu of row %zu", astc_gpu ? "ASTC" : "unswizzle",
|
|
info.size.width, info.size.height, copy_layers, (int)info.format, level, layer, bad,
|
|
a.size(), worst, first == SIZE_MAX ? 0 : first % row,
|
|
first == SIZE_MAX ? 0 : first / row);
|
|
static std::atomic<int64_t> compared, differ;
|
|
const int64_t n = ++compared, nd = differ += bad || a.size() != b.size() || a.empty();
|
|
if (n % 200 == 0 || n == 20)
|
|
hos_log("rx: GPU verify: %lld level/layer pairs compared, %lld differ", (long long)n,
|
|
(long long)nd);
|
|
if (copy_layers == 1) break;
|
|
}
|
|
}
|
|
}
|
|
if (gpu_decoded && !rx_side_thread) spike.upload_bytes += (int64_t)total;
|
|
}
|
|
for (u32 level = 0; !gpu_decoded && level < info.levels; level++) {
|
|
const Extent3D level_size = AdjustMipSize(info.size, level);
|
|
const Extent3D tiles = AdjustTileSize(level_size, tile);
|
|
const size_t layer_bytes = (size_t)tiles.width * tiles.height * tiles.depth * bpp;
|
|
// ASTC arrays: the layers of a level decoded at once, each by a thread of its own (unswizzle, decode,
|
|
// upload). One after another, with the ASTC pool waiting between small slices, the terrain's 1024x1024 x121
|
|
// material array took 1.2 s of the title screen's first frame (RX_SERIAL_LAYERS=1: the old way). Not on the
|
|
// web: there only the render thread may call WebGPU, and each worker uploads its own layer.
|
|
static const bool serial_layers = RX_WEB || env_flag("RX_SERIAL_LAYERS");
|
|
if (!serial_layers && cv == Convert::Astc && !deferring && !to_bc7 && !decode_any_compressed && !info.linear &&
|
|
copy_layers >= 4 && level_size.depth == 1 && (size_t)level_size.width * level_size.height * 4 <= (8u << 20)) {
|
|
const Extent3D block = AdjustMipBlockSize(tiles, li.block, level, li.num_levels);
|
|
const u32 stride_alignment = StrideAlignment(tiles, info.block, gob, li.bpp_log2);
|
|
const unsigned nthreads = std::max(1u, std::min<unsigned>(copy_layers, std::thread::hardware_concurrency() - 1));
|
|
std::atomic<u32> next_layer{0};
|
|
static std::atomic<int64_t> t_swz, t_dec, t_up;
|
|
auto work = [&] {
|
|
std::vector<uint8_t> swz, rgba((size_t)level_size.width * level_size.height * 4);
|
|
for (u32 layer; (layer = next_layer++) < copy_layers;) {
|
|
const uint64_t off = level_offset + (uint64_t)layer * layer_stride;
|
|
if (off >= total) continue;
|
|
const int64_t a0 = hos_now_ns();
|
|
swz.assign(layer_bytes, 0);
|
|
Tegra::Texture::UnswizzleTexture(swz, std::span<const u8>(src + off, total - off), bpp, tiles.width,
|
|
tiles.height, tiles.depth, block.height, block.depth,
|
|
stride_alignment);
|
|
const int64_t a1 = hos_now_ns();
|
|
Tegra::Texture::ASTC::DecompressSerial(swz, level_size.width, level_size.height, 1, tile.width,
|
|
tile.height, rgba);
|
|
const int64_t a2 = hos_now_ns();
|
|
t_swz += a1 - a0;
|
|
t_dec += a2 - a1;
|
|
WGPUTexelCopyTextureInfo d{};
|
|
d.texture = im->tex;
|
|
d.mipLevel = level;
|
|
d.origin = {0, 0, layer};
|
|
d.aspect = WGPUTextureAspect_All;
|
|
WGPUTexelCopyBufferLayout lay{};
|
|
lay.bytesPerRow = level_size.width * 4;
|
|
lay.rowsPerImage = level_size.height;
|
|
const WGPUExtent3D ext{level_size.width, level_size.height, 1};
|
|
const int64_t a3 = hos_now_ns();
|
|
wgpuQueueWriteTexture(dev.queue, &d, rgba.data(), rgba.size(), &lay, &ext);
|
|
t_up += hos_now_ns() - a3;
|
|
}
|
|
};
|
|
std::vector<std::thread> threads;
|
|
for (unsigned t = 1; t < nthreads; t++) threads.emplace_back(work);
|
|
work();
|
|
for (std::thread& t : threads) t.join();
|
|
if (static const bool log_inplace = env_flag("RX_LOG_INPLACE"); log_inplace && level == info.levels - 1)
|
|
hos_log("rx: parallel layers %ux%u x%u: thread time unswizzle %.0f ms, ASTC %.0f ms, upload %.0f ms "
|
|
"(%u threads)", info.size.width, info.size.height, copy_layers, t_swz.exchange(0) / 1e6,
|
|
t_dec.exchange(0) / 1e6, t_up.exchange(0) / 1e6, nthreads);
|
|
if (!rx_side_thread) spike.upload_bytes += (int64_t)level_size.width * level_size.height * 4 * copy_layers;
|
|
level_offset += CalculateLevelSize(li, level);
|
|
continue;
|
|
}
|
|
for (u32 layer = 0; layer < copy_layers; layer++) {
|
|
level_data.assign(layer_bytes, 0);
|
|
if (info.linear) {
|
|
for (u32 y = 0; y < tiles.height; y++) {
|
|
const uint64_t off = (uint64_t)y * info.pitch;
|
|
if (off + (uint64_t)tiles.width * bpp <= total)
|
|
std::memcpy(level_data.data() + (size_t)y * tiles.width * bpp, src + off,
|
|
(size_t)tiles.width * bpp);
|
|
}
|
|
} else {
|
|
const Extent3D block = AdjustMipBlockSize(tiles, li.block, level, li.num_levels);
|
|
const u32 stride_alignment = StrideAlignment(tiles, info.block, gob, li.bpp_log2);
|
|
const uint64_t off = level_offset + (uint64_t)layer * layer_stride;
|
|
if (off >= total) break;
|
|
Tegra::Texture::UnswizzleTexture(level_data, std::span<const u8>(src + off, total - off), bpp,
|
|
tiles.width, tiles.height, tiles.depth, block.height, block.depth,
|
|
stride_alignment);
|
|
}
|
|
const uint8_t* upload = level_data.data();
|
|
uint32_t row_bytes = tiles.width * bpp, rows = tiles.height;
|
|
if (decode_any_compressed) {
|
|
const size_t pixels = (size_t)level_size.width * level_size.height * level_size.depth;
|
|
converted.assign(pixels * 4, 0);
|
|
bcn::Format bc_format;
|
|
switch (info.format) {
|
|
case PixelFormat::BC1_RGBA_UNORM: case PixelFormat::BC1_RGBA_SRGB: bc_format = bcn::Format::BC1; break;
|
|
case PixelFormat::BC2_UNORM: case PixelFormat::BC2_SRGB: bc_format = bcn::Format::BC2; break;
|
|
case PixelFormat::BC3_UNORM: case PixelFormat::BC3_SRGB: bc_format = bcn::Format::BC3; break;
|
|
case PixelFormat::BC4_SNORM: bc_format = bcn::Format::BC4Snorm; break;
|
|
case PixelFormat::BC5_UNORM: bc_format = bcn::Format::BC5; break;
|
|
case PixelFormat::BC5_SNORM: bc_format = bcn::Format::BC5Snorm; break;
|
|
default: bc_format = bcn::Format::BC4; break;
|
|
}
|
|
bcn::decode(level_data.data(), level_size.width, level_size.height, level_size.depth, bc_format,
|
|
converted.data());
|
|
upload = converted.data();
|
|
row_bytes = level_size.width * 4;
|
|
rows = level_size.height;
|
|
}
|
|
// a band of rows (y0, nrows) of RGBA8 for this level and layer, to the texture (or the render thread)
|
|
auto write_band = [&](std::vector<uint8_t>& data, uint32_t y0, uint32_t nrows) {
|
|
WGPUTexelCopyBufferLayout lay{};
|
|
lay.bytesPerRow = level_size.width * 4;
|
|
lay.rowsPerImage = nrows;
|
|
const WGPUExtent3D ext{level_size.width, nrows, 1};
|
|
const uint32_t z = info.dimension == WGPUTextureDimension_3D ? 0 : layer;
|
|
if (deferring) {
|
|
DeferredUpload u;
|
|
u.level = level;
|
|
u.origin_y = y0;
|
|
u.origin_z = z;
|
|
u.layout = lay;
|
|
u.extent = ext;
|
|
u.data = std::move(data);
|
|
stream_upload(std::move(u));
|
|
} else {
|
|
WGPUTexelCopyTextureInfo d{};
|
|
d.texture = im->tex;
|
|
d.mipLevel = level;
|
|
d.origin = {0, y0, z};
|
|
d.aspect = WGPUTextureAspect_All;
|
|
wgpuQueueWriteTexture(dev.queue, &d, data.data(), data.size(), &lay, &ext);
|
|
}
|
|
};
|
|
auto write_bc7_band = [&](const std::vector<uint8_t>& rgba, uint32_t y0, uint32_t nrows) {
|
|
const uint32_t bw = (level_size.width + 3) / 4, bh = (nrows + 3) / 4;
|
|
std::vector<uint8_t> bc((size_t)bw * bh * 16);
|
|
bc7_encode(rgba.data(), level_size.width, nrows, level_size.width * 4, bc.data());
|
|
WGPUTexelCopyBufferLayout lay{};
|
|
lay.bytesPerRow = bw * 16;
|
|
lay.rowsPerImage = bh;
|
|
const WGPUExtent3D ext{bw * 4, bh * 4, 1};
|
|
if (deferring) {
|
|
DeferredUpload u;
|
|
u.level = level;
|
|
u.origin_y = y0;
|
|
u.origin_z = layer;
|
|
u.layout = lay;
|
|
u.extent = ext;
|
|
u.data = std::move(bc);
|
|
stream_upload(std::move(u));
|
|
} else {
|
|
WGPUTexelCopyTextureInfo d{};
|
|
d.texture = im->tex;
|
|
d.mipLevel = level;
|
|
d.origin = {0, y0, layer};
|
|
d.aspect = WGPUTextureAspect_All;
|
|
wgpuQueueWriteTexture(dev.queue, &d, bc.data(), bc.size(), &lay, &ext);
|
|
}
|
|
};
|
|
bool banded = false;
|
|
switch (decode_any_compressed ? Convert::None : cv) {
|
|
case Convert::Astc:
|
|
if (level_size.depth == 1 && (size_t)level_size.width * level_size.height * 4 > (8u << 20)) {
|
|
// (in bands of ~8 MB: a 7373x3686 level is 108 MB of RGBA8, more than the browser build's host
|
|
// heap had free at once)
|
|
uint32_t brows = std::max<uint32_t>(1, (8u << 20) / (level_size.width * 4 * tile.height));
|
|
if (to_bc7) {
|
|
const uint32_t step = 4 / std::gcd(tile.height, 4u);
|
|
brows = std::max(step, brows / step * step);
|
|
}
|
|
for (uint32_t by = 0; by < tiles.height; by += brows) {
|
|
const uint32_t nb = std::min(brows, tiles.height - by);
|
|
const uint32_t y0 = by * tile.height;
|
|
const uint32_t h = std::min(nb * tile.height, level_size.height - y0);
|
|
const size_t off = (size_t)by * tiles.width * 16, len = (size_t)nb * tiles.width * 16;
|
|
if (off + len > level_data.size()) break;
|
|
std::vector<uint8_t> band((size_t)level_size.width * h * 4);
|
|
Tegra::Texture::ASTC::Decompress(std::span<const uint8_t>(level_data.data() + off, len),
|
|
level_size.width, h, 1, tile.width, tile.height, band);
|
|
if (to_bc7) write_bc7_band(band, y0, h);
|
|
else write_band(band, y0, h);
|
|
}
|
|
banded = true;
|
|
break;
|
|
}
|
|
converted.assign((size_t)level_size.width * level_size.height * level_size.depth * 4, 0);
|
|
Tegra::Texture::ASTC::Decompress(level_data, level_size.width, level_size.height, level_size.depth,
|
|
tile.width, tile.height, converted);
|
|
upload = converted.data();
|
|
row_bytes = level_size.width * 4;
|
|
rows = level_size.height;
|
|
if (to_bc7) {
|
|
const uint32_t bw = (level_size.width + 3) / 4, bh = (level_size.height + 3) / 4;
|
|
std::vector<uint8_t> bc((size_t)bw * bh * 16);
|
|
bc7_encode(converted.data(), level_size.width, level_size.height, level_size.width * 4, bc.data());
|
|
converted.swap(bc);
|
|
upload = converted.data();
|
|
row_bytes = bw * 16;
|
|
rows = bh;
|
|
}
|
|
break;
|
|
case Convert::Packed16:
|
|
converted.assign((size_t)tiles.width * tiles.height * tiles.depth * 4, 0);
|
|
convert_packed16(info.format, level_data.data(), converted.data(),
|
|
(size_t)tiles.width * tiles.height * tiles.depth);
|
|
upload = converted.data();
|
|
row_bytes = tiles.width * 4;
|
|
break;
|
|
case Convert::Rgb32f: {
|
|
const size_t n = (size_t)tiles.width * tiles.height * tiles.depth;
|
|
converted.assign(n * 16, 0);
|
|
for (size_t i = 0; i < n; i++) {
|
|
std::memcpy(converted.data() + i * 16, level_data.data() + i * 12, 12);
|
|
const float one = 1.0f;
|
|
std::memcpy(converted.data() + i * 16 + 12, &one, 4);
|
|
}
|
|
upload = converted.data();
|
|
row_bytes = tiles.width * 16;
|
|
break;
|
|
}
|
|
case Convert::Bc1:
|
|
case Convert::Bc2:
|
|
case Convert::Bc3:
|
|
case Convert::Bc4:
|
|
case Convert::Bc4Snorm:
|
|
case Convert::Bc5:
|
|
case Convert::Bc5Snorm: {
|
|
upload = converted.data();
|
|
row_bytes = level_size.width * 4;
|
|
rows = level_size.height;
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
if (banded) continue;
|
|
// Component mapping is applied by the shader for every format (including render targets).
|
|
WGPUTexelCopyTextureInfo dst{};
|
|
dst.texture = im->tex;
|
|
dst.mipLevel = level;
|
|
dst.origin = {0, 0, info.dimension == WGPUTextureDimension_3D ? 0 : layer};
|
|
dst.aspect = WGPUTextureAspect_All;
|
|
WGPUTexelCopyBufferLayout layout{};
|
|
layout.bytesPerRow = row_bytes;
|
|
layout.rowsPerImage = rows;
|
|
// compressed formats are copied in whole blocks: the extent is the block-aligned size
|
|
const bool compressed = !decode_any_compressed && cv == Convert::None && tile.width > 1;
|
|
WGPUExtent3D extent{compressed ? tiles.width * tile.width : to_bc7 ? (level_size.width + 3) / 4 * 4 : level_size.width,
|
|
compressed ? tiles.height * tile.height : to_bc7 ? (level_size.height + 3) / 4 * 4 : level_size.height,
|
|
info.dimension == WGPUTextureDimension_3D ? level_size.depth : 1};
|
|
const size_t upload_bytes = (size_t)row_bytes * rows * extent.depthOrArrayLayers;
|
|
if (deferring) {
|
|
DeferredUpload u;
|
|
u.level = level;
|
|
u.origin_z = dst.origin.z;
|
|
u.layout = layout;
|
|
u.extent = extent;
|
|
// (the decoded level itself, not a copy: one big texture's copy ran the host heap out)
|
|
std::vector<uint8_t>& from = upload == converted.data() ? converted : level_data;
|
|
if (upload == from.data() && from.size() >= upload_bytes) {
|
|
u.data = std::move(from);
|
|
from = {};
|
|
} else {
|
|
u.data.assign(upload, upload + upload_bytes);
|
|
}
|
|
stream_upload(std::move(u));
|
|
} else {
|
|
wgpuQueueWriteTexture(dev.queue, &dst, upload, upload_bytes, &layout, &extent);
|
|
}
|
|
if (!rx_side_thread) spike.upload_bytes += (int64_t)row_bytes * rows * extent.depthOrArrayLayers;
|
|
}
|
|
level_offset += CalculateLevelSize(li, level);
|
|
}
|
|
im->content_hash = sample_hash(va, total);
|
|
im->decoded_frame = frame_number;
|
|
im->guest_bytes = total;
|
|
if (!rx_side_thread) { // (a worker's texture gets these when it is installed)
|
|
im->checked_frame = frame_number;
|
|
im->decoded_seq = memory_seq();
|
|
}
|
|
return im;
|
|
}
|
|
|
|
// ---- textures decoded on a worker thread: a texture first drawn is decoded there (ASTC above all: the GPU has no
|
|
// ASTC, and new areas brought 40-60 ms frames of decoding) while its draws get the placeholder, as for programs and
|
|
// pipelines still being made; it is installed at the next present. A texture whose memory was written meanwhile is
|
|
// dropped (and decoded again when drawn). Used on the web only by default (decode_async). ----
|
|
struct DecodeJob {
|
|
uint64_t key = 0, va = 0, seq = 0;
|
|
TICEntry tic{};
|
|
TexInfo info{};
|
|
std::unique_ptr<Image> im;
|
|
std::unique_ptr<DeferredTexture> deferred; // (web)
|
|
};
|
|
std::deque<DecodeJob*> decode_queue;
|
|
std::vector<DecodeJob*> decode_done;
|
|
std::atomic<int> decodes_done{0};
|
|
std::unordered_map<uint64_t, DecodeJob*> decode_pending; // by key (render thread)
|
|
bool decode_pending_result; // the last texture_for_handle_uncached gave the placeholder for a texture on its way
|
|
|
|
const bool astc_workers_in_background = (Common::worker_thread_start = &background_thread, true);
|
|
|
|
void decode_worker() {
|
|
rx_side_thread = true;
|
|
background_thread();
|
|
for (;;) {
|
|
DecodeJob* j;
|
|
{
|
|
std::unique_lock lk(decode_mu);
|
|
decode_cv.wait(lk, [] { return !decode_queue.empty() && deferred_bytes.load() < kDeferredBudget; });
|
|
j = decode_queue.front();
|
|
decode_queue.pop_front();
|
|
}
|
|
if (RX_WEB) {
|
|
j->deferred = std::make_unique<DeferredTexture>();
|
|
deferring = j->deferred.get();
|
|
}
|
|
j->im = decode(j->tic, j->info, j->va);
|
|
deferring = nullptr;
|
|
static const bool trace = env_flag("RX_TRACE_DECODE");
|
|
if (trace)
|
|
hos_log("rx: decoded %ux%u x%u levels %u format %d (streamed %lld MB, heap top %u MB)", j->info.size.width,
|
|
j->info.size.height, j->info.layers, j->info.levels, (int)j->info.format,
|
|
(long long)(deferred_bytes.load() >> 20), (unsigned)(heap_top() >> 20));
|
|
std::lock_guard lk(decode_mu);
|
|
decode_done.push_back(j);
|
|
decodes_done++;
|
|
}
|
|
}
|
|
|
|
// Natively textures decode in place by default: ASTC and big block-linear textures are made by the GPU now (a few ms
|
|
// at most), and a placeholder drawn into anything the game keeps (the loading screen's map, terrain tiles baked once:
|
|
// light boxes on the horizon for seconds after a load) stays wrong. RX_ASYNC_DECODE=1: the worker, as before (the web
|
|
// keeps it: its CPU decodes and heap are the constraint).
|
|
bool decode_async() {
|
|
static const bool sync = RX_WEB ? env_flag("RX_SYNC_DECODE") : !env_flag("RX_ASYNC_DECODE");
|
|
return !sync && !frame_dump_active();
|
|
}
|
|
|
|
void queue_decode(uint64_t key, const TICEntry& tic, const TexInfo& info, uint64_t va) {
|
|
if (decode_pending.count(key)) return;
|
|
auto* j = new DecodeJob();
|
|
j->key = key;
|
|
j->va = va;
|
|
j->seq = memory_seq();
|
|
j->tic = tic;
|
|
j->info = info;
|
|
decode_pending[key] = j;
|
|
static bool started;
|
|
std::lock_guard lk(decode_mu);
|
|
if (!started) {
|
|
started = true;
|
|
// (one: ASTC fans out over its own thread pool; in the browser four workers each holding a whole decoded
|
|
// texture array ran the host heap out)
|
|
std::thread(decode_worker).detach();
|
|
}
|
|
decode_queue.push_back(j);
|
|
decode_cv.notify_one();
|
|
}
|
|
} // namespace
|
|
|
|
/// (the browser's memory log: decoded texture bytes waiting for the render thread, textures queued and decoded)
|
|
extern "C" void rx_texture_memory(int64_t* waiting, int64_t* queued, int64_t* pending) {
|
|
*waiting = deferred_bytes.load();
|
|
std::lock_guard lk(decode_mu);
|
|
*queued = (int64_t)decode_queue.size();
|
|
*pending = (int64_t)decode_done.size();
|
|
}
|
|
|
|
bool sampler_nonfiltering(WGPUSampler s) { return nonfiltering_samplers.count(s) != 0; }
|
|
|
|
void install_decoded_textures() {
|
|
std::vector<DecodeJob*> done;
|
|
std::deque<StreamedUpload> uploads;
|
|
if (RX_WEB) { // (one lock for both: a finished job's uploads all come with it)
|
|
std::lock_guard lk(decode_mu);
|
|
uploads.swap(upload_stream);
|
|
done.swap(decode_done);
|
|
decodes_done = 0;
|
|
} else {
|
|
if (!decodes_done.load(std::memory_order_relaxed)) return;
|
|
std::lock_guard lk(decode_mu);
|
|
done.swap(decode_done);
|
|
decodes_done = 0;
|
|
}
|
|
if (!uploads.empty()) {
|
|
int64_t n = 0;
|
|
for (StreamedUpload& su : uploads) {
|
|
const DeferredUpload& u = su.upload;
|
|
WGPUTexelCopyTextureInfo dst{};
|
|
dst.texture = deferred_texture(*su.texture);
|
|
dst.mipLevel = u.level;
|
|
dst.origin = {0, u.origin_y, u.origin_z};
|
|
dst.aspect = WGPUTextureAspect_All;
|
|
wgpuQueueWriteTexture(dev.queue, &dst, u.data.data(), u.data.size(), &u.layout, &u.extent);
|
|
n += (int64_t)u.data.size();
|
|
}
|
|
uploads.clear();
|
|
deferred_bytes -= n;
|
|
decode_cv.notify_all(); // (a worker waiting for room)
|
|
}
|
|
for (DecodeJob* j : done) {
|
|
decode_pending.erase(j->key);
|
|
Image* im = j->im.get();
|
|
const bool written = im && j->seq && memory_written_since(im->guest, std::max<uint64_t>(im->guest_bytes, 1), j->seq);
|
|
if (written || decoded.count(j->key)) {
|
|
count("tex_decode_dropped");
|
|
// (rewritten while it decoded: a texture the CPU or a copy rewrites every frame or two, such as a
|
|
// cutscene video's frames, would be dropped again and again and the draw would sample the gray
|
|
// placeholder for the whole movie; from now on it decodes where it is drawn)
|
|
if (written && rewritten_textures.insert(im->guest).second) {
|
|
static int logged;
|
|
if (logged++ < 20)
|
|
hos_log("rx: texture %ux%u format %d at %08llx was rewritten while it decoded: decoded where drawn "
|
|
"from now on", j->info.size.width, j->info.size.height, (int)j->info.format,
|
|
(unsigned long long)im->guest);
|
|
}
|
|
} else {
|
|
if (im && j->deferred) { // (web: the texture its layers were streamed into)
|
|
im->tex = deferred_texture(*j->deferred);
|
|
j->deferred->tex = nullptr;
|
|
}
|
|
if (im) {
|
|
im->checked_frame = frame_number;
|
|
im->decoded_seq = j->seq;
|
|
remember(im);
|
|
static int logged;
|
|
if (logged++ < 40)
|
|
hos_log("rx: texture %ux%u x%u levels %u format %d at GPU %#llx (decoded on the worker)",
|
|
j->info.size.width, j->info.size.height, j->info.layers, j->info.levels,
|
|
(int)j->info.format, (unsigned long long)j->va);
|
|
}
|
|
decoded[j->key] = std::move(j->im); // (null: known undecodable)
|
|
decoded_generation++;
|
|
count("tex_decoded_async");
|
|
}
|
|
if (j->deferred && j->deferred->tex) wgpuTextureRelease(j->deferred->tex); // (dropped)
|
|
delete j; // (a dropped texture goes with it: no draw used it)
|
|
}
|
|
}
|
|
|
|
namespace {
|
|
|
|
TexKind kind_of(WGPUTextureFormat f, PixelFormat pf) {
|
|
if (is_depth_format(f)) return TexKind::Depth;
|
|
if (VideoCore::Surface::IsPixelFormatInteger(pf))
|
|
return VideoCore::Surface::IsPixelFormatSignedInteger(pf) ? TexKind::Sint : TexKind::Uint;
|
|
const bool f32 = f == WGPUTextureFormat_R32Float || f == WGPUTextureFormat_RG32Float ||
|
|
f == WGPUTextureFormat_RGBA32Float;
|
|
// (the browser: 16-bit normalized formats (texture-formats-tier1) are not filterable there; bound as filterable,
|
|
// the bind group was invalid and its whole batch was dropped: half of Lookout Landing missing)
|
|
const bool norm16 = RX_WEB && (f == WGPUTextureFormat_R16Unorm || f == WGPUTextureFormat_RG16Unorm ||
|
|
f == WGPUTextureFormat_RGBA16Unorm || f == WGPUTextureFormat_R16Snorm ||
|
|
f == WGPUTextureFormat_RG16Snorm || f == WGPUTextureFormat_RGBA16Snorm);
|
|
return (f32 && !dev.float32_filterable) || norm16 ? TexKind::UnfilterableFloat : TexKind::Float;
|
|
}
|
|
|
|
WGPUAddressMode address_mode(Tegra::Texture::WrapMode m) {
|
|
using W = Tegra::Texture::WrapMode;
|
|
switch (m) {
|
|
case W::Wrap: return WGPUAddressMode_Repeat;
|
|
case W::Mirror: return WGPUAddressMode_MirrorRepeat;
|
|
default: return WGPUAddressMode_ClampToEdge;
|
|
}
|
|
}
|
|
|
|
WGPUCompareFunction compare_function(Tegra::Texture::DepthCompareFunc f) {
|
|
using D = Tegra::Texture::DepthCompareFunc;
|
|
switch (f) {
|
|
case D::Never: return WGPUCompareFunction_Never;
|
|
case D::Less: return WGPUCompareFunction_Less;
|
|
case D::Equal: return WGPUCompareFunction_Equal;
|
|
case D::LessEqual: return WGPUCompareFunction_LessEqual;
|
|
case D::Greater: return WGPUCompareFunction_Greater;
|
|
case D::NotEqual: return WGPUCompareFunction_NotEqual;
|
|
case D::GreaterEqual: return WGPUCompareFunction_GreaterEqual;
|
|
default: return WGPUCompareFunction_Always;
|
|
}
|
|
}
|
|
|
|
/// mode: 0 filtering as the header says, 1 never filtering (unfilterable textures), 2 comparison
|
|
WGPUSampler sampler_for(const TSCEntry& tsc, int mode) {
|
|
const uint64_t key = hash_bytes(&tsc, sizeof tsc, (uint64_t)mode);
|
|
auto it = samplers.find(key);
|
|
if (it != samplers.end()) {
|
|
if (mode == 1 && !nonfiltering_samplers.count(it->second))
|
|
log_limited("sampler_key", 6, "rx: sampler key %016llx (mode 1) found %p, not non-filtering (map %zu)",
|
|
(unsigned long long)key, (void*)it->second, samplers.size());
|
|
return it->second;
|
|
}
|
|
using F = Tegra::Texture::TextureFilter;
|
|
using M = Tegra::Texture::TextureMipmapFilter;
|
|
WGPUSamplerDescriptor d{};
|
|
d.addressModeU = address_mode(tsc.wrap_u);
|
|
d.addressModeV = address_mode(tsc.wrap_v);
|
|
d.addressModeW = address_mode(tsc.wrap_p);
|
|
const bool filter = mode != 1;
|
|
d.magFilter = filter && tsc.mag_filter == F::Linear ? WGPUFilterMode_Linear : WGPUFilterMode_Nearest;
|
|
d.minFilter = filter && tsc.min_filter == F::Linear ? WGPUFilterMode_Linear : WGPUFilterMode_Nearest;
|
|
d.mipmapFilter = filter && tsc.mipmap_filter == M::Linear ? WGPUMipmapFilterMode_Linear
|
|
: WGPUMipmapFilterMode_Nearest;
|
|
d.lodMinClamp = std::max(0.0f, tsc.MinLod());
|
|
d.lodMaxClamp = tsc.mipmap_filter == M::None ? d.lodMinClamp : std::max(d.lodMinClamp, tsc.MaxLod());
|
|
d.compare = mode == 2 ? (tsc.depth_compare_enabled ? compare_function(tsc.depth_compare_func)
|
|
: WGPUCompareFunction_LessEqual)
|
|
: WGPUCompareFunction_Undefined;
|
|
static const uint16_t aniso[8] = {1, 2, 4, 6, 8, 10, 12, 16};
|
|
const bool all_linear = d.magFilter == WGPUFilterMode_Linear && d.minFilter == WGPUFilterMode_Linear &&
|
|
d.mipmapFilter == WGPUMipmapFilterMode_Linear;
|
|
d.maxAnisotropy = all_linear ? aniso[tsc.max_anisotropy & 7] : 1;
|
|
WGPUSampler s = wgpuDeviceCreateSampler(dev.device, &d);
|
|
samplers[key] = s;
|
|
if (mode == 1) nonfiltering_samplers.insert(s);
|
|
return s;
|
|
}
|
|
|
|
struct Dummy {
|
|
Image d2, d1, d3, depth, uint2, sint2;
|
|
};
|
|
Dummy* dummies;
|
|
|
|
void make_dummy(Image& im, WGPUTextureDimension dim, WGPUTextureFormat f, uint32_t layers, const void* texel,
|
|
uint32_t texel_bytes) {
|
|
im.format = f;
|
|
im.dimension = dim;
|
|
im.width = im.height = 1;
|
|
im.layers = layers;
|
|
WGPUTextureDescriptor d{};
|
|
d.usage = WGPUTextureUsage_TextureBinding | WGPUTextureUsage_CopyDst |
|
|
(is_depth_format(f) ? WGPUTextureUsage_RenderAttachment : 0);
|
|
d.dimension = dim;
|
|
d.size = {1, 1, layers};
|
|
d.format = f;
|
|
d.mipLevelCount = 1;
|
|
d.sampleCount = 1;
|
|
im.tex = wgpuDeviceCreateTexture(dev.device, &d);
|
|
if (is_depth_format(f)) { // depth textures cannot be written: cleared by a render pass
|
|
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
|
|
for (uint32_t l = 0; l < layers; l++) {
|
|
WGPUTextureViewDescriptor vd{};
|
|
vd.format = f;
|
|
vd.dimension = WGPUTextureViewDimension_2D;
|
|
vd.mipLevelCount = 1;
|
|
vd.baseArrayLayer = l;
|
|
vd.arrayLayerCount = 1;
|
|
vd.aspect = WGPUTextureAspect_All;
|
|
WGPUTextureView v = wgpuTextureCreateView(im.tex, &vd);
|
|
WGPURenderPassDepthStencilAttachment da{};
|
|
da.view = v;
|
|
da.depthLoadOp = WGPULoadOp_Clear;
|
|
da.depthStoreOp = WGPUStoreOp_Store;
|
|
da.depthClearValue = 1.0f;
|
|
WGPURenderPassDescriptor pd{};
|
|
pd.depthStencilAttachment = &da;
|
|
WGPURenderPassEncoder p = wgpuCommandEncoderBeginRenderPass(e, &pd);
|
|
wgpuRenderPassEncoderEnd(p);
|
|
wgpuRenderPassEncoderRelease(p);
|
|
wgpuTextureViewRelease(v);
|
|
}
|
|
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
|
|
wgpuQueueSubmit(dev.queue, 1, &cb);
|
|
wgpuCommandBufferRelease(cb);
|
|
wgpuCommandEncoderRelease(e);
|
|
return;
|
|
}
|
|
for (uint32_t l = 0; l < layers; l++) {
|
|
WGPUTexelCopyTextureInfo dst{};
|
|
dst.texture = im.tex;
|
|
dst.origin = {0, 0, l};
|
|
dst.aspect = WGPUTextureAspect_All;
|
|
WGPUTexelCopyBufferLayout layout{};
|
|
layout.bytesPerRow = texel_bytes;
|
|
layout.rowsPerImage = 1;
|
|
WGPUExtent3D ext{1, 1, 1};
|
|
wgpuQueueWriteTexture(dev.queue, &dst, texel, texel_bytes, &layout, &ext);
|
|
}
|
|
}
|
|
|
|
Dummy& dummy() {
|
|
if (!dummies) {
|
|
dummies = new Dummy();
|
|
const uint8_t gray[4] = {128, 128, 128, 255};
|
|
const uint32_t zero[4] = {0, 0, 0, 0};
|
|
make_dummy(dummies->d2, WGPUTextureDimension_2D, WGPUTextureFormat_RGBA8Unorm, 6, gray, 4);
|
|
make_dummy(dummies->d1, WGPUTextureDimension_1D, WGPUTextureFormat_RGBA8Unorm, 1, gray, 4);
|
|
make_dummy(dummies->d3, WGPUTextureDimension_3D, WGPUTextureFormat_RGBA8Unorm, 1, gray, 4);
|
|
make_dummy(dummies->depth, WGPUTextureDimension_2D, WGPUTextureFormat_Depth32Float, 6, nullptr, 0);
|
|
make_dummy(dummies->uint2, WGPUTextureDimension_2D, WGPUTextureFormat_RGBA32Uint, 6, zero, 16);
|
|
make_dummy(dummies->sint2, WGPUTextureDimension_2D, WGPUTextureFormat_RGBA32Sint, 6, zero, 16);
|
|
}
|
|
return *dummies;
|
|
}
|
|
|
|
TSCEntry read_tsc(const Regs& r, uint32_t index) {
|
|
TSCEntry e{};
|
|
if (index <= r.tex_sampler.limit) gpu_read(r.tex_sampler.Address() + (uint64_t)index * sizeof e, &e, sizeof e);
|
|
return e;
|
|
}
|
|
|
|
bool layers_fit(WGPUTextureViewDimension dim, uint32_t layers) {
|
|
switch (dim) {
|
|
case WGPUTextureViewDimension_Cube: return layers >= 6;
|
|
case WGPUTextureViewDimension_CubeArray: return layers >= 6 && layers % 6 == 0;
|
|
default: return true;
|
|
}
|
|
}
|
|
|
|
/// A layered or mipmapped texture whose layers/levels the GPU rendered one at a time (array layers, cube faces,
|
|
/// mip chains drawn as separate render targets at their places in the texture's memory): those subresources come
|
|
/// from the render targets (copied when they were drawn again since), the rest from guest memory.
|
|
/// A 3D texture decoded from memory takes the slices drawn as render targets (nvn_target files them under
|
|
/// nvn_slice_key): each slice whose image was drawn since the last copy is copied into its depth slice (level 0).
|
|
void assemble_3d_slices(Image& im, const TexInfo& info, uint64_t guest) {
|
|
if (!guest || im.dimension != WGPUTextureDimension_3D || im.assembled_passes == pass_count()) return;
|
|
im.assembled_passes = pass_count();
|
|
const uint32_t depth = std::min<uint32_t>(info.size.depth, im.layers);
|
|
if (im.sub_versions.size() != depth) im.sub_versions.assign(depth, 0);
|
|
bool ended = false;
|
|
for (uint32_t z = 0; z < depth; z++) {
|
|
Image* rt = image_at(nvn_slice_key(guest, z));
|
|
if (!rt || !rt->rendered || rt->width != im.width || rt->height != im.height || rt->format != im.format) continue;
|
|
if (im.sub_versions[z] == rt->version) continue;
|
|
im.sub_versions[z] = rt->version;
|
|
if (!ended) end_render_pass("pe_assemble"), ended = true;
|
|
WGPUTexelCopyTextureInfo src{}, dst{};
|
|
src.texture = rt->tex;
|
|
dst.texture = im.tex;
|
|
dst.origin = {0, 0, z};
|
|
const WGPUExtent3D one{im.width, im.height, 1};
|
|
ce_copy_texture_to_texture(encoder(), &src, &dst, &one);
|
|
count("assembled_3d_slices");
|
|
}
|
|
}
|
|
|
|
void assemble_from_render_targets(Image& im, const TexInfo& info, uint64_t guest) {
|
|
static const bool always = env_flag("RX_ASSEMBLE_ALWAYS"); // (debugging: every look copies every level again)
|
|
if (!guest || (im.assembled_passes == pass_count() && !always)) return; // no render target drawn since the last look
|
|
im.assembled_passes = pass_count();
|
|
const uint32_t layers = std::min(info.layers, im.layers), levels = std::min(info.levels, im.levels);
|
|
if (im.sub_versions.size() != (size_t)layers * levels) {
|
|
im.sub_versions.assign((size_t)layers * levels, 0);
|
|
im.sub_addresses.resize((size_t)layers * levels);
|
|
im.sub_end = 0;
|
|
for (uint32_t layer = 0; layer < layers; layer++)
|
|
for (uint32_t level = 0; level < levels; level++) {
|
|
const uint64_t a = guest + subresource_offset(info, layer, level);
|
|
im.sub_addresses[(size_t)layer * levels + level] = a;
|
|
im.sub_end = std::max(im.sub_end, a + 1);
|
|
}
|
|
im.sub_images.clear();
|
|
}
|
|
// image_at per subresource, again only when an image appeared or went away inside this texture's memory (render
|
|
// targets change tens of times a frame elsewhere; a 121-layer, 11-level array is 1331 lookups)
|
|
static const bool no_subcache = env_flag("RX_NO_SUBCACHE");
|
|
if (im.sub_images_generation != images_generation() || no_subcache || im.sub_images.empty()) {
|
|
if (im.sub_images.empty() || no_subcache || images_changed_in(im.sub_images_generation, guest, im.sub_end)) {
|
|
im.sub_images.resize(im.sub_addresses.size());
|
|
im.sub_present.clear();
|
|
for (size_t i = 0; i < im.sub_addresses.size(); i++)
|
|
if ((im.sub_images[i] = image_at(im.sub_addresses[i]))) im.sub_present.push_back((uint32_t)i);
|
|
}
|
|
im.sub_images_generation = images_generation();
|
|
}
|
|
bool ended = false;
|
|
// (RX_DUMP_ASSEMBLE=1: frame dumps log the missing ones too, every time: a 121-layer array is 1331 lines)
|
|
static const bool dump_missing = env_flag("RX_DUMP_ASSEMBLE");
|
|
const bool all = dump_missing && frame_dump_active();
|
|
const size_t n = all ? im.sub_images.size() : im.sub_present.size();
|
|
for (size_t k = 0; k < n; k++) {
|
|
const uint32_t i = all ? (uint32_t)k : im.sub_present[k];
|
|
const uint32_t layer = i / levels, level = i % levels;
|
|
Image* rt = im.sub_images[i];
|
|
const uint32_t w = std::max(1u, im.width >> level), h = std::max(1u, im.height >> level);
|
|
if (all && (!rt || !rt->rendered || rt->width != w || rt->height != h || rt->format != im.format)) {
|
|
char b[256];
|
|
std::snprintf(b, sizeof b, "assemble %08llx layer %u level %u at %08llx: %s %ux%u fmt %x (want %ux%u fmt %x)",
|
|
(unsigned long long)im.guest, layer, level,
|
|
(unsigned long long)im.sub_addresses[(size_t)layer * levels + level],
|
|
rt ? (rt->rendered ? "rt" : "unrendered image") : "no image", rt ? rt->width : 0,
|
|
rt ? rt->height : 0, rt ? (unsigned)rt->format : 0, w, h, (unsigned)im.format);
|
|
dump_draw(b);
|
|
}
|
|
if (!rt) continue;
|
|
if (!rt || !rt->rendered || rt->width != w || rt->height != h || rt->format != im.format) continue;
|
|
uint64_t& seen = im.sub_versions[(size_t)layer * levels + level];
|
|
if (seen == rt->version && !always) continue;
|
|
static const uint64_t freeze = getenv("RX_ASSEMBLE_FREEZE") ? strtoull(getenv("RX_ASSEMBLE_FREEZE"), nullptr, 16) : 0;
|
|
if (freeze && im.guest == freeze && seen) continue; // (debugging: copied once, then never again)
|
|
seen = rt->version;
|
|
if (!ended) { // copies cannot be in a render pass
|
|
end_render_pass("pe_assemble");
|
|
ended = true;
|
|
}
|
|
WGPUTexelCopyTextureInfo src{}, dst{};
|
|
src.texture = rt->tex;
|
|
src.aspect = WGPUTextureAspect_All;
|
|
dst.texture = im.tex;
|
|
dst.mipLevel = level;
|
|
dst.origin = {0, 0, layer};
|
|
dst.aspect = WGPUTextureAspect_All;
|
|
const WGPUExtent3D extent{w, h, 1};
|
|
ce_copy_texture_to_texture(encoder(), &src, &dst, &extent);
|
|
count("assembled_copies");
|
|
if (frame_dump_active()) {
|
|
char b[256];
|
|
std::snprintf(b, sizeof b, "assemble %08llx %ux%u x%u levels %u fmt %x: layer %u level %u <- rt %08llx",
|
|
(unsigned long long)im.guest, im.width, im.height, im.layers, im.levels,
|
|
(unsigned)im.format, layer, level, (unsigned long long)rt->guest);
|
|
dump_draw(b);
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace
|
|
|
|
uint64_t image_bytes(const Image& im) {
|
|
double per_texel;
|
|
switch (im.format) {
|
|
case WGPUTextureFormat_BC1RGBAUnorm: case WGPUTextureFormat_BC1RGBAUnormSrgb:
|
|
case WGPUTextureFormat_BC4RUnorm: case WGPUTextureFormat_BC4RSnorm:
|
|
per_texel = 0.5;
|
|
break;
|
|
case WGPUTextureFormat_BC2RGBAUnorm: case WGPUTextureFormat_BC2RGBAUnormSrgb:
|
|
case WGPUTextureFormat_BC3RGBAUnorm: case WGPUTextureFormat_BC3RGBAUnormSrgb:
|
|
case WGPUTextureFormat_BC5RGUnorm: case WGPUTextureFormat_BC5RGSnorm:
|
|
case WGPUTextureFormat_BC6HRGBUfloat: case WGPUTextureFormat_BC6HRGBFloat:
|
|
case WGPUTextureFormat_BC7RGBAUnorm: case WGPUTextureFormat_BC7RGBAUnormSrgb:
|
|
case WGPUTextureFormat_R8Unorm: case WGPUTextureFormat_R8Snorm:
|
|
per_texel = 1;
|
|
break;
|
|
case WGPUTextureFormat_RG8Unorm: case WGPUTextureFormat_RG8Snorm: case WGPUTextureFormat_R16Float:
|
|
case WGPUTextureFormat_R16Unorm: case WGPUTextureFormat_Depth16Unorm:
|
|
per_texel = 2;
|
|
break;
|
|
case WGPUTextureFormat_RGBA16Float: case WGPUTextureFormat_RG32Float: case WGPUTextureFormat_RGBA16Unorm:
|
|
per_texel = 8;
|
|
break;
|
|
case WGPUTextureFormat_RGBA32Float:
|
|
per_texel = 16;
|
|
break;
|
|
default:
|
|
per_texel = 4;
|
|
break;
|
|
}
|
|
const double texels = (double)im.width * im.height * im.layers * (im.levels > 1 ? 4.0 / 3.0 : 1.0);
|
|
return (uint64_t)(texels * per_texel);
|
|
}
|
|
|
|
void decoded_texture_stats(size_t* n, uint64_t* bytes) {
|
|
*n = 0;
|
|
*bytes = 0;
|
|
for (const auto& [key, im] : decoded)
|
|
if (im) {
|
|
(*n)++;
|
|
*bytes += image_bytes(*im);
|
|
}
|
|
}
|
|
|
|
void trim_decoded_textures() {
|
|
static const uint64_t budget = [] {
|
|
const char* e = getenv("RX_TEXTURE_BUDGET_MB");
|
|
return (uint64_t)(e && atoi(e) > 0 ? atoi(e) : RX_WEB ? 640 : 2048) << 20;
|
|
}();
|
|
// (mkw-recomp: every 60 frames, unused for 180: MKW loads its title at a few frames a second, where TOTKPort's
|
|
// 600 / 900 never came round and decoded textures passed 4 GB until the GPU ran out of memory)
|
|
if (frame_number % 60 != 0) return;
|
|
uint64_t total = 0;
|
|
int dropped = 0;
|
|
std::vector<std::pair<int64_t, uint64_t>> old;
|
|
for (auto it = decoded.begin(); it != decoded.end();) {
|
|
Image* im = it->second.get();
|
|
if (!im) {
|
|
++it;
|
|
continue;
|
|
}
|
|
if (im->stale && im->decoded_seq && im->last_use < frame_number - 60) {
|
|
forget(im);
|
|
it = decoded.erase(it);
|
|
dropped++;
|
|
continue;
|
|
}
|
|
const uint64_t bytes = image_bytes(*im);
|
|
total += bytes;
|
|
if (im->last_use < frame_number - 60) old.emplace_back(im->last_use, it->first);
|
|
++it;
|
|
}
|
|
// RX_LOG_DECODED_TOP=1 (debugging): the largest decoded textures, when over budget (once)
|
|
if (static const bool top = env_flag("RX_LOG_DECODED_TOP"); top && total > budget) {
|
|
static bool done;
|
|
if (!done) {
|
|
done = true;
|
|
std::vector<std::pair<uint64_t, const Image*>> v;
|
|
for (const auto& [k, p] : decoded)
|
|
if (p) v.push_back({image_bytes(*p), p.get()});
|
|
std::sort(v.begin(), v.end(), [](const auto& a, const auto& b) { return a.first > b.first; });
|
|
for (size_t i = 0; i < v.size() && i < 24; i++)
|
|
hos_log("rx: decoded %3zu: %llu MB %ux%u x%u levels %u format %#x at %llx, last use %lld", i,
|
|
(unsigned long long)(v[i].first >> 20), v[i].second->width, v[i].second->height,
|
|
v[i].second->layers, v[i].second->levels, (unsigned)v[i].second->format,
|
|
(unsigned long long)v[i].second->guest, (long long)v[i].second->last_use);
|
|
}
|
|
}
|
|
uint64_t freed = 0;
|
|
if (total > budget && !old.empty()) {
|
|
std::sort(old.begin(), old.end());
|
|
// ((mkw-recomp) unused for 180 frames; when far over (by a quarter), for 60: in a race every texture had been
|
|
// used within 6 s, decoded textures passed 4 GB and the GPU ran out of memory)
|
|
const int64_t unused = total > budget + budget / 4 ? 60 : 180;
|
|
for (const auto& [use, key] : old) {
|
|
if (total - freed <= budget - budget / 8) break;
|
|
if (use >= frame_number - unused) break; // (sorted: the rest are newer)
|
|
auto it = decoded.find(key);
|
|
if (it == decoded.end() || !it->second) continue;
|
|
freed += image_bytes(*it->second);
|
|
forget(it->second.get());
|
|
decoded.erase(it);
|
|
dropped++;
|
|
}
|
|
}
|
|
if (dropped) {
|
|
decoded_generation++;
|
|
count("tex_trimmed", dropped);
|
|
static int64_t logged_at = -100000;
|
|
if (frame_number - logged_at > 3600) {
|
|
logged_at = frame_number;
|
|
hos_log("rx: decoded textures %llu MB (budget %llu MB): %d dropped (rewritten or unused), %llu MB freed",
|
|
(unsigned long long)((total - freed) >> 20), (unsigned long long)(budget >> 20), dropped,
|
|
(unsigned long long)(freed >> 20));
|
|
}
|
|
}
|
|
}
|
|
|
|
void textures_written(uint64_t guest, uint64_t size) {
|
|
if (decoded_by_bucket.empty() || !size) return;
|
|
for (uint64_t b = guest >> kBucketShift; b <= (guest + size - 1) >> kBucketShift; b++) {
|
|
auto it = decoded_by_bucket.find(b);
|
|
if (it == decoded_by_bucket.end()) continue;
|
|
for (Image* im : it->second)
|
|
if (im->guest < guest + size && im->guest + im->guest_bytes > guest) im->stale = true;
|
|
}
|
|
}
|
|
|
|
WGPUTextureViewDimension view_dimension(Shader::TextureType type) {
|
|
switch (type) {
|
|
case Shader::TextureType::Color1D: return WGPUTextureViewDimension_1D;
|
|
case Shader::TextureType::ColorArray2D: return WGPUTextureViewDimension_2DArray;
|
|
case Shader::TextureType::Color3D: return WGPUTextureViewDimension_3D;
|
|
case Shader::TextureType::ColorCube: return WGPUTextureViewDimension_Cube;
|
|
case Shader::TextureType::ColorArrayCube: return WGPUTextureViewDimension_CubeArray;
|
|
default: return WGPUTextureViewDimension_2D;
|
|
}
|
|
}
|
|
|
|
WGPUTextureView Image::view(WGPUTextureViewDimension dim, bool depth_aspect, uint32_t base_layer,
|
|
uint32_t base_level, uint32_t level_count, WGPUTextureFormat view_format) {
|
|
if (view_format != srgb_pair(format)) view_format = format; // (only the other encoding is a view format)
|
|
base_layer = std::min(base_layer, layers - 1);
|
|
base_level = std::min(base_level, levels - 1);
|
|
level_count = level_count ? std::min(level_count, levels - base_level) : levels - base_level;
|
|
const uint64_t key = (uint64_t)dim | (depth_aspect ? 0x100ull : 0) | (view_format != format ? 0x200ull : 0) |
|
|
((uint64_t)base_layer << 16) | ((uint64_t)base_level << 40) | ((uint64_t)level_count << 48);
|
|
for (auto& [k, v] : views)
|
|
if (k == key) return v;
|
|
WGPUTextureViewDescriptor d{};
|
|
d.format = view_format;
|
|
d.dimension = dim;
|
|
d.baseMipLevel = base_level;
|
|
d.mipLevelCount = level_count;
|
|
d.baseArrayLayer = base_layer;
|
|
switch (dim) {
|
|
case WGPUTextureViewDimension_1D:
|
|
case WGPUTextureViewDimension_2D:
|
|
case WGPUTextureViewDimension_3D: d.arrayLayerCount = 1; break;
|
|
case WGPUTextureViewDimension_Cube: d.arrayLayerCount = 6; break;
|
|
default: d.arrayLayerCount = dimension == WGPUTextureDimension_3D ? 1 : layers - base_layer; break;
|
|
}
|
|
d.aspect = depth_aspect ? WGPUTextureAspect_DepthOnly : WGPUTextureAspect_All;
|
|
// (mkw-recomp) a depth-only view of a depth-stencil texture takes its format from the aspect: wgpu 29 rejects the
|
|
// combined format there as a view format the texture does not list (MKW samples D24S8 depth; wgpu then panicked
|
|
// at the submit that used the invalid view)
|
|
if (depth_aspect && (format == WGPUTextureFormat_Depth24PlusStencil8 || format == WGPUTextureFormat_Depth32FloatStencil8))
|
|
d.format = WGPUTextureFormat_Undefined;
|
|
// (an sRGB view cannot have the storage usage: (mkw-recomp) nor can the other encoding's view of a render target
|
|
// compute programs write as an image; MKW's race samples one through an sRGB view, the draw left out)
|
|
if (storage || view_format != format) d.usage = WGPUTextureUsage_TextureBinding;
|
|
WGPUTextureView v = wgpuTextureCreateView(tex, &d);
|
|
views.emplace_back(key, v);
|
|
return v;
|
|
}
|
|
|
|
void texture_slots(uint8_t* dst, const TICEntry& tic, const TSCEntry& tsc) {
|
|
using W = Tegra::Texture::WrapMode;
|
|
auto border = [](W w) { return w == W::Border || w == W::MirrorOnceBorder; };
|
|
static const bool no_border = env_flag("RX_NO_BORDER"); // (debugging: border wrap as clamp)
|
|
const uint32_t axes = no_border ? 0u
|
|
: (border(tsc.wrap_u) ? 1u : 0u) | (border(tsc.wrap_v) ? 2u : 0u) |
|
|
(border(tsc.wrap_p) ? 4u : 0u);
|
|
const uint32_t compare = tsc.depth_compare_enabled ? (uint32_t)tsc.depth_compare_func.Value() & 7u : 3u;
|
|
const int32_t bias = (int32_t)std::lround(tsc.LodBias() * 256.0f);
|
|
const uint32_t slot[4] = {(uint32_t)tic.x_source.Value() | axes << 3, (uint32_t)tic.y_source.Value() | compare << 3,
|
|
(uint32_t)tic.z_source.Value() | (uint32_t)bias << 8, (uint32_t)tic.w_source.Value()};
|
|
std::memcpy(dst, slot, 16);
|
|
std::memcpy(dst + 16, tsc.border_color.data(), 16);
|
|
}
|
|
|
|
Tegra::Texture::TSCEntry read_tsc_for_dump(const Regs& r, uint32_t raw_handle) {
|
|
const auto [tic_index, tsc_index] =
|
|
Tegra::Texture::TexturePair(raw_handle, r.sampler_binding == Regs::SamplerBinding::ViaHeaderBinding);
|
|
(void)tic_index;
|
|
return read_tsc(r, tsc_index);
|
|
}
|
|
|
|
Tegra::Texture::TICEntry read_tic(const Regs& r, uint32_t index) {
|
|
TICEntry e{};
|
|
if (index <= r.tex_header.limit) gpu_read(r.tex_header.Address() + (uint64_t)index * sizeof e, &e, sizeof e);
|
|
return e;
|
|
}
|
|
|
|
BoundTexture dummy_texture(const Shader::TextureDescriptor& desc) {
|
|
Dummy& d = dummy();
|
|
const WGPUTextureViewDimension dim = view_dimension(desc.type);
|
|
BoundTexture b;
|
|
TSCEntry tsc{};
|
|
if (desc.is_depth) {
|
|
b.kind = TexKind::Depth;
|
|
b.view = d.depth.view(dim == WGPUTextureViewDimension_1D || dim == WGPUTextureViewDimension_3D
|
|
? WGPUTextureViewDimension_2D
|
|
: dim,
|
|
false);
|
|
b.sampler = sampler_for(tsc, 2);
|
|
return b;
|
|
}
|
|
Image& im = dim == WGPUTextureViewDimension_1D ? d.d1 : dim == WGPUTextureViewDimension_3D ? d.d3 : d.d2;
|
|
b.view = im.view(dim, false);
|
|
b.kind = TexKind::Float;
|
|
b.sampler = sampler_for(tsc, 0);
|
|
return b;
|
|
}
|
|
|
|
namespace {
|
|
|
|
Image* last_bound_image; // the image texture_for_handle_uncached bound (null: a placeholder)
|
|
bool last_assembled; // ...and it is assembled from render targets (assemble_from_render_targets)
|
|
BoundTexture texture_for_handle_uncached(const Regs& r, uint32_t raw, const Shader::TextureDescriptor& desc);
|
|
} // namespace
|
|
|
|
/// Draws bind the same textures again and again, frame after frame: the same header pair gives the same binding
|
|
/// while the image at the texture's address is the same one (and did not become rendered), no texture was decoded
|
|
/// or dropped, no GPU mapping changed and the texture's memory was not written (a decoded texture's stale flag).
|
|
/// A texture assembled from render targets still copies what was drawn since; a decoded texture without write
|
|
/// tracking still checks its memory once a frame (RX_NO_TEX_MEMO=1: always the full path; frame dumps always take
|
|
/// it).
|
|
BoundTexture texture_for_handle(const Regs& r, uint32_t raw, const Shader::TextureDescriptor& desc) {
|
|
static const bool no_memo = env_flag("RX_NO_TEX_MEMO");
|
|
if (no_memo || frame_dump_active()) {
|
|
const auto [tic_index, tsc_index] =
|
|
Tegra::Texture::TexturePair(raw, r.sampler_binding == Regs::SamplerBinding::ViaHeaderBinding);
|
|
BoundTexture b = texture_for_handle_uncached(r, raw, desc);
|
|
texture_slots(b.slots, read_tic(r, tic_index), read_tsc(r, tsc_index));
|
|
return b;
|
|
}
|
|
const bool via_header = r.sampler_binding == Regs::SamplerBinding::ViaHeaderBinding;
|
|
const auto [tic_index, tsc_index] = Tegra::Texture::TexturePair(raw, via_header);
|
|
TICEntry tic{};
|
|
TSCEntry tsc{};
|
|
bool have_headers = false;
|
|
auto load_headers = [&] {
|
|
if (have_headers) return;
|
|
tic = read_tic(r, tic_index);
|
|
tsc = read_tsc(r, tsc_index);
|
|
have_headers = true;
|
|
};
|
|
// The memo key is a hash of both headers: reading and hashing them for each of ~20k bindings a frame was itself
|
|
// ~2 ms. A front cache knows a handle's key while the headers' memory was not written since (page tracking) and
|
|
// the pools and mappings are the same.
|
|
struct Front {
|
|
uint64_t fkey = 0, key = 0, seq = 0, tic_guest = 0, tsc_guest = 0;
|
|
long maps = 0;
|
|
};
|
|
static Front front[8192];
|
|
const uint64_t pools[4] = {r.tex_header.Address() | (uint64_t)r.tex_header.limit << 44,
|
|
r.tex_sampler.Address() | (uint64_t)r.tex_sampler.limit << 44,
|
|
(uint64_t)raw << 32 | (uint64_t)desc.type << 8 | (desc.is_depth ? 2u : 0u) |
|
|
(via_header ? 1u : 0u),
|
|
0};
|
|
const uint64_t fkey = hash_bytes(pools, sizeof pools) | 1;
|
|
Front& f = front[fkey & 8191];
|
|
uint64_t key;
|
|
if (f.fkey == fkey && f.maps == maps_generation && !memory_written_since(f.tic_guest, sizeof(TICEntry), f.seq) &&
|
|
!memory_written_since(f.tsc_guest, sizeof(TSCEntry), f.seq)) {
|
|
key = f.key;
|
|
} else {
|
|
const uint64_t seq = memory_seq();
|
|
load_headers();
|
|
key = hash_bytes(&tic, sizeof tic, hash_bytes(&tsc, sizeof tsc, (uint64_t)desc.type << 8 | (desc.is_depth ? 1u : 0u)));
|
|
const uint64_t tic_guest = tic_index <= r.tex_header.limit
|
|
? hos_gpu_translate(r.tex_header.Address() + (uint64_t)tic_index * sizeof(TICEntry)) : 0;
|
|
const uint64_t tsc_guest = tsc_index <= r.tex_sampler.limit
|
|
? hos_gpu_translate(r.tex_sampler.Address() + (uint64_t)tsc_index * sizeof(TSCEntry)) : 0;
|
|
if (tic_guest && tsc_guest) f = {fkey, key, seq, tic_guest, tsc_guest, maps_generation};
|
|
else f.fkey = 0;
|
|
}
|
|
struct Memo {
|
|
uint64_t key = 0, frame = ~0ull, decoded = 0, guest = 0;
|
|
long maps = 0; // the GPU mappings' generation (guest = the texture's VA translated then)
|
|
Image* src = nullptr; // a decoded texture (its stale flag), or null
|
|
uint64_t src_id = 0, tic_key = 0; // ...its id, and its key in the decoded map
|
|
uint64_t at_id = 0; // image_at(guest) then (its id), and whether it was rendered
|
|
bool at_rendered = false;
|
|
Image* assembled = nullptr; // a texture assembled from render targets, and its header's info
|
|
TexInfo info;
|
|
BoundTexture b;
|
|
};
|
|
static Memo memo[8192];
|
|
Memo& m = memo[key & 8191];
|
|
if (m.key == key && m.decoded != decoded_generation && m.maps == maps_generation) {
|
|
// a texture was decoded or dropped since: that touches this binding only if it was its own (one texture the
|
|
// CPU rewrites every frame used to drop every memo entry every frame)
|
|
bool same = true;
|
|
if (m.src_id) {
|
|
auto it = decoded.find(m.tic_key);
|
|
same = it != decoded.end() && it->second && it->second->id == m.src_id;
|
|
if (same) m.src = it->second.get();
|
|
}
|
|
if (same) m.decoded = decoded_generation;
|
|
}
|
|
if (m.key == key && m.decoded == decoded_generation && m.maps == maps_generation) {
|
|
Image* at = m.guest ? image_at(m.guest) : nullptr;
|
|
// a later frame: what the uncached path does once a frame (a decoded texture without write tracking
|
|
// checks its memory's samples; images in use are stamped so they are not dropped)
|
|
const bool new_frame = m.frame != (uint64_t)frame_number;
|
|
if ((at ? at->id : 0) == m.at_id && (!at || at->rendered == m.at_rendered) && (!m.src || !m.src->stale) &&
|
|
!(new_frame && m.src && !m.src->decoded_seq)) {
|
|
if (new_frame) {
|
|
m.frame = (uint64_t)frame_number;
|
|
if (m.src) m.src->last_use = frame_number;
|
|
if (m.b.image) m.b.image->last_use = frame_number;
|
|
}
|
|
if (m.assembled) assemble_from_render_targets(*m.assembled, m.info, m.guest);
|
|
count("tex_memo_hits");
|
|
if (RX_WEB && m.b.kind == TexKind::UnfilterableFloat && !nonfiltering_samplers.count(m.b.sampler)) {
|
|
load_headers(); // (the browser validates the pair: see below)
|
|
m.b.sampler = sampler_for(tsc, 1);
|
|
log_limited("memo_sampler", 4, "rx: memo held a filtering sampler for an unfilterable texture");
|
|
}
|
|
return m.b; // (its slots too: the key holds both headers)
|
|
}
|
|
}
|
|
load_headers();
|
|
count(m.key != key ? "tex_memo_miss_key" : m.decoded != decoded_generation ? "tex_memo_miss_decoded"
|
|
: m.maps != maps_generation ? "tex_memo_miss_maps" : "tex_memo_miss_image");
|
|
last_bound_image = nullptr;
|
|
last_assembled = false;
|
|
decode_pending_result = false;
|
|
BoundTexture b = texture_for_handle_uncached(r, raw, desc);
|
|
texture_slots(b.slots, tic, tsc);
|
|
// (the browser rejects a filtering sampler with an unfilterable texture, and the batch with it)
|
|
if (RX_WEB && b.kind == TexKind::UnfilterableFloat && !nonfiltering_samplers.count(b.sampler)) {
|
|
const WGPUSampler before = b.sampler;
|
|
b.sampler = sampler_for(tsc, 1);
|
|
log_limited("uncached_sampler", 4, "rx: unfilterable texture %ux%u format %d came with a filtering sampler %p "
|
|
"-> %p (in set %d)", b.width, b.height, b.format, (void*)before, (void*)b.sampler,
|
|
(int)nonfiltering_samplers.count(b.sampler));
|
|
}
|
|
if (decode_pending_result) { // (the placeholder is not remembered: the texture comes in a frame or two)
|
|
m.key = 0;
|
|
return b;
|
|
}
|
|
const uint64_t va = (uint64_t)tic.address_high << 32 | tic.address_low;
|
|
m.key = key;
|
|
m.frame = (uint64_t)frame_number;
|
|
m.decoded = decoded_generation;
|
|
m.maps = maps_generation;
|
|
m.guest = va ? hos_gpu_translate(va) : 0;
|
|
Image* at = m.guest ? image_at(m.guest) : nullptr;
|
|
m.at_id = at ? at->id : 0;
|
|
m.at_rendered = at && at->rendered;
|
|
m.src = last_bound_image && !last_bound_image->rendered ? last_bound_image : nullptr;
|
|
m.src_id = m.src ? m.src->id : 0;
|
|
m.tic_key = hash_bytes(&tic, sizeof tic); // (as the decoded map keys it)
|
|
m.assembled = last_assembled ? last_bound_image : nullptr;
|
|
if (m.assembled) m.info = tex_info(tic);
|
|
m.b = b;
|
|
return b;
|
|
}
|
|
|
|
namespace {
|
|
BoundTexture texture_for_handle_uncached(const Regs& r, uint32_t raw, const Shader::TextureDescriptor& desc) {
|
|
const bool via_header = r.sampler_binding == Regs::SamplerBinding::ViaHeaderBinding;
|
|
const auto [tic_index, tsc_index] = Tegra::Texture::TexturePair(raw, via_header);
|
|
const TICEntry tic = read_tic(r, tic_index);
|
|
const TSCEntry tsc = read_tsc(r, tsc_index);
|
|
const uint64_t va = (uint64_t)tic.address_high << 32 | tic.address_low;
|
|
static int texture_trace_count;
|
|
static const bool trace_textures = env_flag("RX_TRACE_TEXTURES");
|
|
if (trace_textures && texture_trace_count++ < 24)
|
|
hos_log("rx: texture binding raw %#x TIC %u TSC %u VA %#llx TIC[%016llx %016llx %016llx %016llx]",
|
|
raw, tic_index, tsc_index, (unsigned long long)va, (unsigned long long)tic.raw[0],
|
|
(unsigned long long)tic.raw[1], (unsigned long long)tic.raw[2],
|
|
(unsigned long long)tic.raw[3]);
|
|
if (!va) return dummy_texture(desc);
|
|
const TexInfo info = tex_info(tic);
|
|
if (trace_textures && texture_trace_count <= 24)
|
|
hos_log("rx: texture info valid %u %ux%u x%u levels %u format %d", (unsigned)info.valid, info.size.width,
|
|
info.size.height, info.layers, info.levels, (int)info.format);
|
|
if (!info.valid) return dummy_texture(desc);
|
|
const WGPUTextureViewDimension dim = view_dimension(desc.type);
|
|
|
|
Image* im = nullptr;
|
|
// Padded attachments can have a larger backing extent than the TIC's logical sampling extent.
|
|
const uint64_t guest = hos_gpu_translate(va);
|
|
{ // RX_TIC_LOG=<guest hex>: the header of the texture there, once (debugging)
|
|
static const uint64_t tic_log = getenv("RX_TIC_LOG") ? strtoull(getenv("RX_TIC_LOG"), nullptr, 16) : 0;
|
|
static int logged;
|
|
if (tic_log && guest == tic_log && logged++ < 2)
|
|
hos_log("rx: TIC at %#llx: %016llx %016llx %016llx %016llx: %ux%ux%u layers %u levels %u block %u,%u,%u "
|
|
"dim %d linear %d format %d view %d", (unsigned long long)guest, (unsigned long long)tic.raw[0],
|
|
(unsigned long long)tic.raw[1], (unsigned long long)tic.raw[2], (unsigned long long)tic.raw[3],
|
|
info.size.width, info.size.height, info.size.depth, info.layers, info.levels, info.block.width,
|
|
info.block.height, info.block.depth, (int)info.dimension, (int)info.linear, (int)info.format,
|
|
(int)dim);
|
|
}
|
|
if (Image* rt = guest ? image_at(guest) : nullptr;
|
|
rt && rt->rendered && rt->width >= info.size.width && rt->height >= info.size.height &&
|
|
rt->dimension == info.dimension && rt->layers >= info.layers && rt->levels >= info.levels &&
|
|
layers_fit(dim, rt->layers) &&
|
|
((dim == WGPUTextureViewDimension_3D) == (rt->dimension == WGPUTextureDimension_3D)) &&
|
|
((dim == WGPUTextureViewDimension_1D) == (rt->dimension == WGPUTextureDimension_1D)))
|
|
im = rt; // (a mipmapped one whose levels were drawn one by one, a bloom pyramid: assembled below; sampled as
|
|
// its first level alone, its blurred levels were the unblurred one: particles as squares of glow)
|
|
if (!im && frame_dump_active())
|
|
if (Image* rt = guest ? image_at(guest) : nullptr) {
|
|
char b[256];
|
|
std::snprintf(b, sizeof b,
|
|
"render target %08llx %ux%u x%u dim %d rendered %d not taken for a TIC %ux%u x%u dim %d "
|
|
"(view %d, format %d)",
|
|
(unsigned long long)guest, rt->width, rt->height, rt->layers, (int)rt->dimension,
|
|
(int)rt->rendered, info.size.width, info.size.height, info.layers, (int)info.dimension,
|
|
(int)dim, (int)info.format);
|
|
dump_draw(b);
|
|
}
|
|
if (!im) {
|
|
const uint64_t key = hash_bytes(&tic, sizeof tic);
|
|
auto it = decoded.find(key);
|
|
if (it != decoded.end()) {
|
|
if (!it->second) return dummy_texture(desc); // known undecodable
|
|
im = it->second.get();
|
|
static const uint64_t redecode = getenv("RX_REDECODE") ? strtoull(getenv("RX_REDECODE"), nullptr, 16) : 0;
|
|
if ((redecode == va || redecode == im->guest) && im->checked_frame != frame_number) im->stale = true; // (debugging)
|
|
if (im->decoded_seq) { // its pages were written since (textures_written)
|
|
if (im->stale) {
|
|
forget(im);
|
|
decoded.erase(it);
|
|
decoded_generation++;
|
|
im = nullptr;
|
|
}
|
|
} else if (im->checked_frame != frame_number) { // first use this frame: did the samples change?
|
|
im->checked_frame = frame_number;
|
|
if (sample_hash(va, guest_size(info)) != im->content_hash) {
|
|
forget(im);
|
|
decoded.erase(it);
|
|
decoded_generation++;
|
|
im = nullptr;
|
|
}
|
|
}
|
|
}
|
|
// (a one-off draw decodes in place: a placeholder would stay in what it renders, a map made once; not big
|
|
// textures: whole in memory at once, where the background decode streams them, and the web's heap is small)
|
|
static const bool linear_async = env_flag("RX_ASYNC_LINEAR");
|
|
const bool cheap = !linear_async && (info.linear || rewritten_textures.count(guest)) &&
|
|
guest_size(info) <= (RX_WEB ? (4u << 20) : (16u << 20));
|
|
if (!im && decode_async() && !cheap &&
|
|
!(draw_must_run && guest_size(info) <= (RX_WEB ? (4u << 20) : (64u << 20)))) {
|
|
queue_decode(key, tic, info, va);
|
|
decode_pending_result = true;
|
|
count("tex_decoding");
|
|
return dummy_texture(desc);
|
|
}
|
|
if (!im) {
|
|
const int64_t t0 = hos_now_ns();
|
|
auto fresh = decode(tic, info, va);
|
|
count("t_decode", hos_now_ns() - t0);
|
|
spike.decode_ns += hos_now_ns() - t0;
|
|
if (static const bool log_inplace = env_flag("RX_LOG_INPLACE"); log_inplace && hos_now_ns() - t0 > 5000000)
|
|
hos_log("rx: decoded in place (%s) %ux%u x%u levels %u format %d linear %d: %.1f ms, %llu KB guest",
|
|
draw_must_run ? "one-off draw" : decode_async() ? "cheap" : "sync", info.size.width,
|
|
info.size.height, info.layers, info.levels, (int)info.format, (int)info.linear,
|
|
(hos_now_ns() - t0) / 1e6, (unsigned long long)(guest_size(info) >> 10));
|
|
im = fresh.get();
|
|
if (im) remember(im);
|
|
decoded[key] = std::move(fresh);
|
|
decoded_generation++;
|
|
if (!im) return dummy_texture(desc);
|
|
static int logged;
|
|
if (logged++ < 40)
|
|
hos_log("rx: texture %ux%u x%u levels %u format %d at GPU %#llx", info.size.width, info.size.height,
|
|
info.layers, info.levels, (int)info.format, (unsigned long long)va);
|
|
}
|
|
}
|
|
im->last_use = frame_number;
|
|
last_bound_image = im;
|
|
if (!im->rendered && !info.linear && info.dimension != WGPUTextureDimension_3D && (info.layers > 1 || info.levels > 1)) {
|
|
assemble_from_render_targets(*im, info, guest);
|
|
last_assembled = guest != 0;
|
|
}
|
|
const uint32_t base_layer = im->dimension == WGPUTextureDimension_3D ? 0 : tic.BaseLayer();
|
|
const uint32_t base_level = tic.res_min_mip_level;
|
|
const uint32_t level_count = (uint32_t)tic.res_max_mip_level >= base_level
|
|
? (uint32_t)tic.res_max_mip_level - base_level + 1 : 1;
|
|
// the encoding the header asks for (a render target drawn as sRGB can be read raw, and the other way round)
|
|
const WGPUTextureFormat wanted = wgpu_format(info.format);
|
|
if (im->rendered && wanted != im->format && wanted != srgb_pair(im->format) && !is_depth_format(im->format)) {
|
|
static std::unordered_map<uint64_t, bool> seen; // (each target and format pair once)
|
|
if (seen.size() < 200 && seen.emplace(im->guest ^ ((uint64_t)wanted << 40) ^ ((uint64_t)im->format << 48), true).second)
|
|
hos_log("rx: render target %08llx %ux%u format %#x sampled as format %#x (pixel format %d)",
|
|
(unsigned long long)im->guest, im->width, im->height, (unsigned)im->format, (unsigned)wanted,
|
|
(int)info.format);
|
|
}
|
|
auto texture_view = [&](bool depth) {
|
|
return im->view(dim, depth, base_layer, base_level, level_count, wanted);
|
|
};
|
|
BoundTexture b;
|
|
b.image = im->rendered ? im : nullptr;
|
|
b.source = im->rendered ? 'r' : 't';
|
|
if (im->rendered) dump_sampled(im);
|
|
b.guest = im->guest;
|
|
b.width = im->width;
|
|
b.height = im->height;
|
|
b.format = (int)im->format;
|
|
const TexKind kind = kind_of(im->format, im->pixel_format);
|
|
if (!layers_fit(dim, im->layers) || (dim == WGPUTextureViewDimension_3D) != (im->dimension == WGPUTextureDimension_3D) ||
|
|
(dim == WGPUTextureViewDimension_1D) != (im->dimension == WGPUTextureDimension_1D)) {
|
|
log_limited("tex_dim", 16, "rx: texture of %u layers (dimension %d) bound as view dimension %d: %ux%u format %d "
|
|
"at %#llx (TIC %ux%ux%u layers %u dimension %d, %s)", im->layers, (int)im->dimension, (int)dim,
|
|
im->width, im->height, (int)im->format, (unsigned long long)im->guest, info.size.width,
|
|
info.size.height, info.size.depth, info.layers, (int)info.dimension,
|
|
im->rendered ? "rendered" : "decoded");
|
|
count("tex_dim_dummy");
|
|
return dummy_texture(desc);
|
|
}
|
|
if (desc.is_depth) {
|
|
if (kind != TexKind::Depth) return dummy_texture(desc);
|
|
b.kind = TexKind::Depth;
|
|
b.view = texture_view(has_stencil(im->format));
|
|
b.sampler = sampler_for(tsc, 2);
|
|
return b;
|
|
}
|
|
switch (kind) {
|
|
case TexKind::Float:
|
|
b.kind = TexKind::Float;
|
|
b.view = texture_view(false);
|
|
b.sampler = sampler_for(tsc, 0);
|
|
return b;
|
|
case TexKind::UnfilterableFloat:
|
|
case TexKind::Depth: // a depth buffer read as a color texture
|
|
b.kind = TexKind::UnfilterableFloat;
|
|
b.view = texture_view(kind == TexKind::Depth && has_stencil(im->format));
|
|
b.sampler = sampler_for(tsc, 1);
|
|
return b;
|
|
default: // integer textures: the WGSL backend declares every color texture as f32 for now
|
|
log_limited("tex_int", 8, "rx: integer texture (format %d) bound to a float binding", (int)im->pixel_format);
|
|
return dummy_texture(desc);
|
|
}
|
|
}
|
|
} // namespace
|
|
|
|
// ---- the native NVN layer: textures and samplers from NVN objects ----
|
|
bool nvn_sampler_of(int id, nvn_samp_par* p); // rx_nvn.cpp
|
|
namespace {
|
|
|
|
WGPUAddressMode nvn_address_mode(uint8_t w) {
|
|
switch (w) {
|
|
case NVN_WRAP_MODE_REPEAT: return WGPUAddressMode_Repeat;
|
|
case NVN_WRAP_MODE_MIRRORED_REPEAT: return WGPUAddressMode_MirrorRepeat;
|
|
default: return WGPUAddressMode_ClampToEdge; // (border: the WGSL backend tests the coordinates, see the slots)
|
|
}
|
|
}
|
|
|
|
WGPUCompareFunction nvn_compare_function(uint8_t f) {
|
|
switch (f) {
|
|
case NVN_COMPARE_FUNC_NEVER: return WGPUCompareFunction_Never;
|
|
case NVN_COMPARE_FUNC_LESS: return WGPUCompareFunction_Less;
|
|
case NVN_COMPARE_FUNC_EQUAL: return WGPUCompareFunction_Equal;
|
|
case NVN_COMPARE_FUNC_LEQUAL: return WGPUCompareFunction_LessEqual;
|
|
case NVN_COMPARE_FUNC_GREATER: return WGPUCompareFunction_Greater;
|
|
case NVN_COMPARE_FUNC_NOTEQUAL: return WGPUCompareFunction_NotEqual;
|
|
case NVN_COMPARE_FUNC_GEQUAL: return WGPUCompareFunction_GreaterEqual;
|
|
default: return WGPUCompareFunction_Always;
|
|
}
|
|
}
|
|
|
|
/// mode: 0 filtering as the sampler says, 1 never filtering (unfilterable textures), 2 comparison
|
|
WGPUSampler nvn_sampler(const nvn_samp_par& p, int mode) {
|
|
static std::unordered_map<uint64_t, WGPUSampler> cache;
|
|
const uint64_t key = hash_bytes(&p, sizeof p, 0x5a1d0 + (uint64_t)mode);
|
|
if (auto it = cache.find(key); it != cache.end()) return it->second;
|
|
WGPUSamplerDescriptor d{};
|
|
d.addressModeU = nvn_address_mode(p.ws);
|
|
d.addressModeV = nvn_address_mode(p.wt);
|
|
d.addressModeW = nvn_address_mode(p.wr);
|
|
const bool filter = mode != 1;
|
|
const bool min_linear = p.minf == NVN_MIN_FILTER_LINEAR || p.minf == NVN_MIN_FILTER_LINEAR_MIPMAP_NEAREST ||
|
|
p.minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR;
|
|
const bool mips = p.minf >= NVN_MIN_FILTER_NEAREST_MIPMAP_NEAREST;
|
|
const bool mip_linear = p.minf == NVN_MIN_FILTER_NEAREST_MIPMAP_LINEAR || p.minf == NVN_MIN_FILTER_LINEAR_MIPMAP_LINEAR;
|
|
d.magFilter = filter && p.magf == NVN_MAG_FILTER_LINEAR ? WGPUFilterMode_Linear : WGPUFilterMode_Nearest;
|
|
d.minFilter = filter && min_linear ? WGPUFilterMode_Linear : WGPUFilterMode_Nearest;
|
|
d.mipmapFilter = filter && mip_linear ? WGPUMipmapFilterMode_Linear : WGPUMipmapFilterMode_Nearest;
|
|
d.lodMinClamp = std::clamp(p.lod_min, 0.0f, 32.0f);
|
|
d.lodMaxClamp = mips ? std::clamp(p.lod_max, d.lodMinClamp, 32.0f) : d.lodMinClamp;
|
|
d.compare = mode == 2 ? (p.cmp_mode ? nvn_compare_function(p.cmp_func) : WGPUCompareFunction_LessEqual)
|
|
: WGPUCompareFunction_Undefined;
|
|
const bool all_linear = d.magFilter == WGPUFilterMode_Linear && d.minFilter == WGPUFilterMode_Linear &&
|
|
d.mipmapFilter == WGPUMipmapFilterMode_Linear;
|
|
d.maxAnisotropy = all_linear ? (uint16_t)std::clamp(p.aniso, 1.0f, 16.0f) : 1;
|
|
if (p.reduction) log_limited("nvn_reduction", 4, "rx: NVN min/max sampler reduction: averaged");
|
|
WGPUSampler s = wgpuDeviceCreateSampler(dev.device, &d);
|
|
cache[key] = s;
|
|
if (mode == 1) nonfiltering_samplers.insert(s);
|
|
return s;
|
|
}
|
|
|
|
/// A texture's two uniform block slots (as texture_slots makes them from a header and sampler): the component
|
|
/// swizzle with the sampler's border axes, depth compare function and LOD bias, then the border color
|
|
void nvn_texture_slots(uint8_t* dst, const nvn_tex& t, const nvn_view* v, const nvn_samp_par& p) {
|
|
uint8_t sw[4] = {(uint8_t)t.d.swizzle_r, (uint8_t)t.d.swizzle_g, (uint8_t)t.d.swizzle_b, (uint8_t)t.d.swizzle_a};
|
|
if (v && v->set_swizzle) std::memcpy(sw, v->swz, 4);
|
|
uint32_t src[4];
|
|
for (int i = 0; i < 4; i++) // NVN ZERO 0, ONE 1, R..A 2..5 -> the header's sources (ONE: one as a float, 7)
|
|
src[i] = sw[i] == NVN_TEXTURE_SWIZZLE_ONE ? 7u : sw[i];
|
|
auto border = [](uint8_t w) { return w == NVN_WRAP_MODE_CLAMP_TO_BORDER || w == NVN_WRAP_MODE_MIRROR_CLAMP_TO_BORDER; };
|
|
const uint32_t axes = (border(p.ws) ? 1u : 0u) | (border(p.wt) ? 2u : 0u) | (border(p.wr) ? 4u : 0u);
|
|
const uint32_t compare = p.cmp_mode ? (uint32_t)(p.cmp_func - 1) & 7u : 3u;
|
|
const int32_t bias = (int32_t)std::lround(p.lod_bias * 256.0f);
|
|
const uint32_t slot[4] = {src[0] | axes << 3, src[1] | compare << 3, src[2] | (uint32_t)bias << 8, src[3]};
|
|
std::memcpy(dst, slot, 16);
|
|
std::memcpy(dst + 16, p.border, 16);
|
|
}
|
|
|
|
/// The texture description an NVN texture and view give (the texture's whole storage; the view picks levels and
|
|
/// layers). valid false: not a texture the renderer samples (buffer textures, unknown formats).
|
|
TexInfo nvn_tex_info(const nvn_tex& t, const nvn_view* v) {
|
|
TexInfo i;
|
|
const nvn_texdesc& d = t.d;
|
|
i.format = nvn_pixel_format(v && v->set_format ? v->format : d.format);
|
|
if (i.format == PixelFormat::Invalid) return i;
|
|
i.levels = std::clamp<u32>(d.levels, 1, 16);
|
|
i.linear = nvn_tex_is_linear(&d);
|
|
if (i.linear) i.pitch = nvn_tex_linear_stride(&d);
|
|
else i.block = {0, t.L.tileH, t.L.tileD};
|
|
i.size = {std::max(1u, d.width), std::max(1u, d.height), 1};
|
|
switch (d.target) {
|
|
case NVN_TEXTURE_TARGET_1D:
|
|
i.dimension = WGPUTextureDimension_1D;
|
|
i.size.height = 1;
|
|
break;
|
|
case NVN_TEXTURE_TARGET_1D_ARRAY:
|
|
i.dimension = WGPUTextureDimension_1D;
|
|
i.layers = std::max(1u, d.height);
|
|
i.size.height = 1;
|
|
break;
|
|
case NVN_TEXTURE_TARGET_2D: case NVN_TEXTURE_TARGET_RECTANGLE: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE:
|
|
break;
|
|
case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY:
|
|
i.layers = std::max(1u, d.depth);
|
|
break;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP: i.layers = 6; break;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: i.layers = std::max(6u, d.depth); break; // (depth counts layer-faces)
|
|
case NVN_TEXTURE_TARGET_3D:
|
|
i.dimension = WGPUTextureDimension_3D;
|
|
i.size.depth = std::max(1u, d.depth);
|
|
break;
|
|
default: return i; // buffer textures
|
|
}
|
|
if (i.linear) i.levels = 1;
|
|
if (i.dimension == WGPUTextureDimension_1D) i.levels = 1;
|
|
i.valid = true;
|
|
return i;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
/// The levels a view of the texture reaches (a view of its first level alone can sample the render target itself)
|
|
static uint32_t nvn_levels_used(const TexInfo& info, const nvn_view* v) {
|
|
if (v && v->set_levels && v->nlevels) return std::min<uint32_t>(info.levels, (uint32_t)v->base_level + v->nlevels);
|
|
return info.levels;
|
|
}
|
|
|
|
/// Decoded textures being decoded again (their memory was rewritten): the old image, drawn until the new one is in
|
|
static std::unordered_map<uint64_t, std::unique_ptr<Image>> nvn_stale_images;
|
|
|
|
/// What nvn_texture_resolve settled on, for the memo in nvn_texture: the texture's guest address (the image there is
|
|
/// checked again), the decoded image it took (its stale flag), the image assembled from render targets (assembled
|
|
/// again at each use) and its info.
|
|
struct NvnTexResolved {
|
|
uint64_t guest = 0;
|
|
Image* src = nullptr;
|
|
Image* assembled = nullptr;
|
|
Image* used = nullptr; // (the image bound: its last use is renewed on memo hits)
|
|
uint64_t dec_key = 0; // (src's key in decoded: a hit checks it is still the image there)
|
|
TexInfo info{};
|
|
bool memo_ok = false; // (a placeholder or a stale image drawn while decoding: not remembered)
|
|
};
|
|
|
|
BoundTexture nvn_texture_resolve(const NvnState& s, uint64_t tex_handle, uint64_t samp_handle,
|
|
const Shader::TextureDescriptor& desc, NvnTexResolved& res) {
|
|
(void)s;
|
|
const int tid = (int)(tex_handle & 0xFFFFF);
|
|
const int sid = (int)(((samp_handle ? samp_handle : tex_handle) >> 20) & 0xFFF);
|
|
nvn_tex t;
|
|
nvn_view v;
|
|
bool has_view = false;
|
|
if (!tid || !nvn_texture_of(tid, &t, &v, &has_view) || !t.gpu) return dummy_texture(desc);
|
|
nvn_samp_par sp{};
|
|
if (!nvn_sampler_of(sid, &sp)) {
|
|
sp.minf = NVN_MIN_FILTER_LINEAR, sp.magf = NVN_MAG_FILTER_LINEAR;
|
|
sp.ws = sp.wt = sp.wr = NVN_WRAP_MODE_CLAMP_TO_EDGE;
|
|
sp.lod_max = 1000.0f, sp.aniso = 1.0f;
|
|
}
|
|
const nvn_view* vp = has_view ? &v : nullptr;
|
|
const TexInfo info = nvn_tex_info(t, vp);
|
|
if (!info.valid) return dummy_texture(desc);
|
|
const WGPUTextureViewDimension dim = view_dimension(desc.type);
|
|
const uint64_t va = t.gpu, guest = nvn_gpu_guest(va);
|
|
res.guest = guest;
|
|
res.info = info;
|
|
bool from_stale = false;
|
|
Image* im = nullptr;
|
|
// drawn by the GPU: the render target at the texture's storage
|
|
if (Image* rt = guest ? image_at(guest) : nullptr;
|
|
rt && rt->rendered && rt->width >= info.size.width && rt->height >= info.size.height &&
|
|
rt->dimension == info.dimension && rt->layers >= info.layers && rt->levels >= nvn_levels_used(info, vp) &&
|
|
layers_fit(dim, rt->layers) &&
|
|
((dim == WGPUTextureViewDimension_3D) == (rt->dimension == WGPUTextureDimension_3D)) &&
|
|
((dim == WGPUTextureViewDimension_1D) == (rt->dimension == WGPUTextureDimension_1D)))
|
|
im = rt; // (a texture whose layers or levels were drawn one by one is assembled below, as in texture_for_handle)
|
|
if (!im) { // from memory: decoded (and cached by what it was decoded from)
|
|
// (field by field: TexInfo's padding bytes are not initialized; the texture object's own fields, such as its
|
|
// swizzle, do not change what is decoded: one decode per layout, as for a TIC)
|
|
const uint32_t fields[] = {(uint32_t)info.format, (uint32_t)info.dimension, info.size.width, info.size.height,
|
|
info.size.depth, info.layers, info.levels, info.block.width, info.block.height,
|
|
info.block.depth, info.tile_width_spacing, info.linear ? 1u : 0u, info.pitch};
|
|
const uint64_t key = hash_bytes(fields, sizeof fields, va ^ 0x4e564e00ull);
|
|
res.dec_key = key;
|
|
auto it = decoded.find(key);
|
|
if (it != decoded.end()) {
|
|
if (!it->second) return dummy_texture(desc);
|
|
im = it->second.get();
|
|
// RX_REDECODE=<guest hex> (debugging): that texture decoded again every frame
|
|
static const uint64_t redecode = getenv("RX_REDECODE") ? strtoull(getenv("RX_REDECODE"), nullptr, 16) : 0;
|
|
if (redecode == guest && im->decoded_seq && im->checked_frame != frame_number)
|
|
im->checked_frame = frame_number, im->stale = true;
|
|
// (mkw-recomp) a large texture is decoded again at most every 300 frames: MKW renders into a 67 MB depth
|
|
// array (101 layers) whose layers the GPU draws one by one each frame, and each frame's writeback made it
|
|
// stale; decoding it again (stale memory, as the drawn layers live on the GPU) took the frame's time
|
|
const bool recent_big = guest_size(info) >= (16u << 20) && frame_number - im->decoded_frame < 300;
|
|
if (recent_big) {
|
|
} else if (im->decoded_seq) {
|
|
if (im->stale) { // (rewritten: decoded again; the old one is drawn until the new one is in)
|
|
forget(im);
|
|
nvn_stale_images[key] = std::move(it->second);
|
|
decoded.erase(it);
|
|
decoded_generation++;
|
|
im = nullptr;
|
|
}
|
|
} else if (im->checked_frame != frame_number) {
|
|
im->checked_frame = frame_number;
|
|
if (sample_hash(va, guest_size(info)) != im->content_hash) {
|
|
forget(im);
|
|
decoded.erase(it);
|
|
decoded_generation++;
|
|
im = nullptr;
|
|
}
|
|
}
|
|
}
|
|
const TICEntry no_tic{};
|
|
const bool cheap = (info.linear || rewritten_textures.count(guest)) && guest_size(info) <= (16u << 20);
|
|
if (!im && decode_async() && !cheap && !(draw_must_run && guest_size(info) <= (64u << 20))) {
|
|
queue_decode(key, no_tic, info, va);
|
|
decode_pending_result = true;
|
|
count("tex_decoding");
|
|
// (a texture decoded again after the CPU rewrote it: its old image meanwhile, not a placeholder (a flash
|
|
// of black or gray for the frames the decode takes))
|
|
auto f = nvn_stale_images.find(key);
|
|
if (f == nvn_stale_images.end() || !f->second) return dummy_texture(desc);
|
|
im = f->second.get();
|
|
from_stale = true;
|
|
count("tex_stale_drawn");
|
|
} else if (im && !nvn_stale_images.empty()) {
|
|
nvn_stale_images.erase(key); // (its new image is in)
|
|
}
|
|
if (!im) {
|
|
// (a render target drew this memory on the GPU only and does not fit as the texture: the decode reads
|
|
// what guest memory held before, unless it was written back. The gloom noise volumes were such a case:
|
|
// block views, written back now. Logged once an address, to find others)
|
|
const Image* under = guest ? image_at(guest) : nullptr;
|
|
if (!under && guest && info.dimension == WGPUTextureDimension_3D) under = image_at(nvn_slice_key(guest, 0));
|
|
// (layers, levels and 3D slices of the texture's own size and kind are assembled from their targets)
|
|
const bool assembled = under && under->width == info.size.width && under->height == info.size.height &&
|
|
!VideoCore::Surface::IsPixelFormatBCn(info.format) &&
|
|
!VideoCore::Surface::IsPixelFormatASTC(info.format);
|
|
if (under && under->rendered && !under->block_view && !assembled) {
|
|
static std::unordered_set<uint64_t> told;
|
|
if (told.size() < 256 && told.insert(guest).second)
|
|
hos_log("rx: texture %08llx %ux%ux%u format %d decoded from memory a render target drew on the GPU "
|
|
"(%ux%u format %#x%s)", (unsigned long long)guest, info.size.width, info.size.height,
|
|
std::max(info.size.depth, info.layers), (int)info.format, under->width, under->height,
|
|
(unsigned)under->format, under->guest >> 32 ? ", 3D slices" : "");
|
|
}
|
|
const int64_t t0 = hos_now_ns();
|
|
auto fresh = decode(no_tic, info, va);
|
|
count("t_decode", hos_now_ns() - t0);
|
|
spike.decode_ns += hos_now_ns() - t0;
|
|
im = fresh.get();
|
|
if (im) remember(im);
|
|
decoded[key] = std::move(fresh);
|
|
decoded_generation++;
|
|
if (!im) return dummy_texture(desc);
|
|
}
|
|
}
|
|
im->last_use = frame_number;
|
|
res.used = im;
|
|
if (!im->rendered && !info.linear && info.dimension != WGPUTextureDimension_3D && (info.layers > 1 || info.levels > 1)) {
|
|
assemble_from_render_targets(*im, info, guest);
|
|
res.assembled = im;
|
|
} else if (!im->rendered && info.dimension == WGPUTextureDimension_3D) {
|
|
assemble_3d_slices(*im, info, guest);
|
|
res.assembled = im;
|
|
}
|
|
if (!im->rendered) res.src = im;
|
|
res.memo_ok = !from_stale;
|
|
const uint32_t base_layer = im->dimension == WGPUTextureDimension_3D ? 0 : (vp && vp->set_layers ? vp->min_layer : 0);
|
|
const uint32_t base_level = vp && vp->set_levels ? vp->base_level : 0;
|
|
const uint32_t level_count = vp && vp->set_levels && vp->nlevels ? vp->nlevels : 0;
|
|
const WGPUTextureFormat wanted = wgpu_format(info.format);
|
|
auto texture_view = [&](bool depth) {
|
|
return im->view(dim, depth, base_layer, base_level, level_count, wanted);
|
|
};
|
|
if (im->rendered && wanted != im->format && frame_dump_active()) { // (diagnostic: drawn as one format, read as another)
|
|
char b[160];
|
|
std::snprintf(b, sizeof b, "texture %llx %ux%u read as format %#x, drawn as %#x", (unsigned long long)guest,
|
|
info.size.width, info.size.height, (unsigned)wanted, (unsigned)im->format);
|
|
dump_draw(b);
|
|
}
|
|
BoundTexture b;
|
|
b.image = im->rendered ? im : nullptr;
|
|
b.source = im->rendered ? 'r' : 't';
|
|
if (im->rendered) dump_sampled(im);
|
|
b.guest = im->guest;
|
|
b.width = im->width;
|
|
b.height = im->height;
|
|
b.format = (int)im->format;
|
|
nvn_texture_slots(b.slots, t, vp, sp);
|
|
const TexKind kind = kind_of(im->format, im->pixel_format);
|
|
if (!layers_fit(dim, im->layers) || (dim == WGPUTextureViewDimension_3D) != (im->dimension == WGPUTextureDimension_3D) ||
|
|
(dim == WGPUTextureViewDimension_1D) != (im->dimension == WGPUTextureDimension_1D)) {
|
|
count("tex_dim_dummy");
|
|
return dummy_texture(desc);
|
|
}
|
|
if (desc.is_depth) {
|
|
if (kind != TexKind::Depth) return dummy_texture(desc);
|
|
b.kind = TexKind::Depth;
|
|
b.view = texture_view(has_stencil(im->format));
|
|
b.sampler = nvn_sampler(sp, 2);
|
|
return b;
|
|
}
|
|
switch (kind) {
|
|
case TexKind::Float:
|
|
b.kind = TexKind::Float;
|
|
b.view = texture_view(false);
|
|
b.sampler = nvn_sampler(sp, 0);
|
|
return b;
|
|
case TexKind::UnfilterableFloat:
|
|
case TexKind::Depth:
|
|
b.kind = TexKind::UnfilterableFloat;
|
|
b.view = texture_view(kind == TexKind::Depth && has_stencil(im->format));
|
|
b.sampler = nvn_sampler(sp, 1);
|
|
return b;
|
|
default:
|
|
log_limited("tex_int", 8, "rx: integer texture (format %d) bound to a float binding", (int)im->pixel_format);
|
|
return dummy_texture(desc);
|
|
}
|
|
}
|
|
|
|
|
|
/// The native path's texture bindings, remembered (rx_draw's per-draw lookups were 16% of the replay thread): the same
|
|
/// handles and view type give the same binding while no texture was decoded or dropped, no image changed address, the
|
|
/// image at the texture's memory is the same one (and as rendered or not), and the decoded image is not stale. Across
|
|
/// frames too while neither handle was registered again or had its texture finalized (per-handle versions,
|
|
/// rx_nvn.cpp; 914 of ~1500 misses a frame were the frame's first use), except for a handle that follows its storage
|
|
/// (a finalized transient target: what lives there can change any frame) and an image decoded without write tracking
|
|
/// (checked by its samples once a frame). RX_NO_NVN_TEX_MEMO=1: every lookup resolves again.
|
|
BoundTexture nvn_texture(const NvnState& s, uint64_t tex_handle, uint64_t samp_handle,
|
|
const Shader::TextureDescriptor& desc) {
|
|
static const bool no_memo = env_flag("RX_NO_NVN_TEX_MEMO");
|
|
NvnTexResolved res;
|
|
if (no_memo || frame_dump_active()) return nvn_texture_resolve(s, tex_handle, samp_handle, desc, res);
|
|
struct Memo {
|
|
uint64_t tex = 0, samp = 0, frame = ~0ull, decoded = 0, images = 0, guest = 0, at_id = 0;
|
|
uint64_t src_checked = ~0ull; // (decoded_generation when src was last found in the map)
|
|
uint32_t dkey = 0, tver = 0, sver = 0;
|
|
uint64_t dec_key = 0;
|
|
bool at_rendered = false, valid = false, per_frame = false;
|
|
Image* src = nullptr;
|
|
Image* assembled = nullptr;
|
|
Image* used = nullptr;
|
|
TexInfo info{};
|
|
BoundTexture b;
|
|
};
|
|
static Memo memo[4096];
|
|
const uint32_t dkey = (uint32_t)desc.type | (desc.is_depth ? 0x100u : 0u) | (desc.is_multisample ? 0x200u : 0u);
|
|
const uint64_t key = hash_bytes(&tex_handle, 8, samp_handle ^ (uint64_t)dkey << 40);
|
|
Memo& m = memo[key & 4095];
|
|
const int tid = (int)(tex_handle & 0xFFFFF), sid = (int)(((samp_handle ? samp_handle : tex_handle) >> 20) & 0xFFF);
|
|
static const bool frame_only = env_flag("RX_NVN_TEX_MEMO_FRAME"); // (the memo within a frame only, as before)
|
|
// an entry holds while: the same handles and view type, neither handle changed (versions); then a bound image
|
|
// (decoded or rendered) is checked on its own: a decoded one is still the decode of its key and not stale, the image
|
|
// at the texture's memory is the same one (both below). A binding without one (a dummy) holds within the frame
|
|
// while no texture was decoded or dropped and no image moved, as before.
|
|
bool ok = m.valid && m.tex == tex_handle && m.samp == samp_handle && m.dkey == dkey &&
|
|
m.tver == nvn_texture_version(tid) && m.sver == nvn_sampler_version(sid);
|
|
if (ok && (!m.used || m.per_frame || frame_only))
|
|
ok = m.frame == (uint64_t)frame_number && (m.used || (m.decoded == decoded_generation && m.images == images_generation()));
|
|
if (ok && m.src && m.src_checked != decoded_generation) { // (the map unchanged since: src is still its entry)
|
|
auto it = decoded.find(m.dec_key);
|
|
ok = it != decoded.end() && it->second.get() == m.src;
|
|
if (ok) m.src_checked = decoded_generation;
|
|
}
|
|
if (ok) {
|
|
Image* at = m.guest ? image_at(m.guest) : nullptr;
|
|
if ((at ? at->id : 0) == m.at_id && (!at || at->rendered == m.at_rendered) && (!m.src || !m.src->stale)) {
|
|
if (m.assembled) {
|
|
if (m.info.dimension == WGPUTextureDimension_3D) assemble_3d_slices(*m.assembled, m.info, m.guest);
|
|
else assemble_from_render_targets(*m.assembled, m.info, m.guest);
|
|
}
|
|
count("nvn_tex_memo_hits");
|
|
if (m.used) m.used->last_use = frame_number;
|
|
if (m.b.image) dump_sampled(m.b.image);
|
|
m.frame = (uint64_t)frame_number;
|
|
return m.b;
|
|
}
|
|
}
|
|
const uint32_t tver = nvn_texture_version(tid), sver = nvn_sampler_version(sid); // (before: a change meanwhile misses)
|
|
const BoundTexture b = nvn_texture_resolve(s, tex_handle, samp_handle, desc, res);
|
|
m.valid = res.memo_ok && b.view != nullptr;
|
|
if (!m.valid) return b;
|
|
m.tver = tver, m.sver = sver, m.used = res.used, m.dec_key = res.dec_key;
|
|
m.per_frame = nvn_texture_follows_storage(tid) || (res.src && !res.src->decoded_seq);
|
|
Image* at = res.guest ? image_at(res.guest) : nullptr;
|
|
m.tex = tex_handle, m.samp = samp_handle, m.dkey = dkey, m.frame = (uint64_t)frame_number;
|
|
m.decoded = decoded_generation, m.images = images_generation();
|
|
m.guest = res.guest, m.at_id = at ? at->id : 0, m.at_rendered = at && at->rendered;
|
|
m.src = res.src, m.assembled = res.assembled, m.info = res.info, m.b = b;
|
|
m.src_checked = ~0ull;
|
|
if (m.src) { // (checked as a hit checks it: a source not in the map is looked up each time, as before)
|
|
auto it = decoded.find(m.dec_key);
|
|
if (it != decoded.end() && it->second.get() == m.src) m.src_checked = decoded_generation;
|
|
}
|
|
return b;
|
|
}
|
|
|
|
} // namespace rx
|