mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
What was merged from NX2navi (DLSS frame generation, AMD FSR 4 in Setup builds, texture detail by video memory, sprint, volume, frame caps, safer saves, the above-60 fixes), the ultrawide HUD at the screen edges, camera speed, movies without the AV1 Video Extension, the 5120x2880 + RenoDX HDR start fix, the renderer work of 11-14 and the first float changes of the lift.
5108 lines
306 KiB
C++
5108 lines
306 KiB
C++
// Textures: one VkImage per texture storage (vk_texture.h). After totk-pc-recomp's runtime/vk_texture.c (MIT,
|
|
// Copyright (c) 2026 the totk-pc-recomp authors): the cache keyed by storage address plus descriptor, uploads on first
|
|
// use and when page write generations show the storage written, the GPU-owned rule (an image the GPU wrote is the
|
|
// content), every image in GENERAL layout, deferred destruction. The GPU ASTC decode is Eden's
|
|
// (video_core/host_shaders/astc_decoder.comp and renderer_vulkan/vk_compute_pass.cpp ASTCDecoderPass, GPL), the CPU
|
|
// unswizzle suyu's (video_core/textures/decoders.cpp, with the Switch 2 GOB swizzle).
|
|
//
|
|
// Formats share images by class: an image is made in the format first asked for (sRGB as its UNORM twin), with
|
|
// MUTABLE_FORMAT, so that a texture rendered as RGBA8 and sampled as RGBA8_SRGB, or written as R32UI and read as
|
|
// RGBA8, is one image. A class is the bytes per block for uncompressed colour, the format family for BC and ASTC,
|
|
// the format itself for depth / stencil.
|
|
#include <algorithm>
|
|
#define WIN32_LEAN_AND_MEAN
|
|
#ifndef NOMINMAX
|
|
#define NOMINMAX
|
|
#endif
|
|
#include <windows.h>
|
|
#include <psapi.h>
|
|
#if defined(__linux__)
|
|
#include <malloc.h>
|
|
#endif
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <cstring>
|
|
#include <map>
|
|
#include <deque>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <thread>
|
|
#include <tuple>
|
|
#include <unordered_set>
|
|
#include <vector>
|
|
|
|
#include "vk_texture.h"
|
|
#include "vk_menu.h"
|
|
#include "vk_draw.h"
|
|
#include "vk_gprof.h"
|
|
#include "vk_dump.h"
|
|
#include "vk_jobs.h"
|
|
|
|
#include <span>
|
|
|
|
#include "video_core/textures/astc.h"
|
|
#include "video_core/textures/decoders.h"
|
|
|
|
#include "host_shaders/astc_decoder_comp_spv.h"
|
|
#include "host_shaders/bc_encoder_comp_spv.h"
|
|
#include "bc_levels.h"
|
|
#include "astc_bc_policy.h"
|
|
#include "tex_detail.h"
|
|
#include "vram_compact.h"
|
|
#include "texture_gone.h"
|
|
#include "keep_rendered.h"
|
|
#include "target_overlap.h"
|
|
#include "host_shaders/block_linear_unswizzle_2d_comp_spv.h"
|
|
#include "shaders/swizzle_back_comp_spv.h"
|
|
#include "vk_work.h" // (after the others: counts the commands that do GPU work, for full_barrier)
|
|
|
|
namespace mkwvk {
|
|
namespace {
|
|
|
|
uint64_t mix(uint64_t h, uint64_t v) {
|
|
h ^= v + 0x9e3779b97f4a7c15ull + (h << 6) + (h >> 2);
|
|
return h * 0xff51afd7ed558ccdull;
|
|
}
|
|
|
|
// ---- formats ----
|
|
std::mutex fmt_mu;
|
|
std::unordered_map<int, VkFormatProperties> fmt_cache;
|
|
VkFormatProperties format_props(VkFormat f) {
|
|
std::lock_guard lk(fmt_mu);
|
|
auto it = fmt_cache.find((int)f);
|
|
if (it != fmt_cache.end()) return it->second;
|
|
VkFormatProperties p{};
|
|
vkGetPhysicalDeviceFormatProperties(dev.physical, f, &p);
|
|
fmt_cache[(int)f] = p;
|
|
return p;
|
|
}
|
|
bool is_bc(VkFormat f) {
|
|
return f >= VK_FORMAT_BC1_RGB_UNORM_BLOCK && f <= VK_FORMAT_BC7_SRGB_BLOCK;
|
|
}
|
|
bool is_astc(VkFormat f) {
|
|
return f >= VK_FORMAT_ASTC_4x4_UNORM_BLOCK && f <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK;
|
|
}
|
|
/// The UNORM twin of an sRGB format (the image's format: storage writes and every view format of the class)
|
|
VkFormat linear_twin(VkFormat f) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8G8B8A8_SRGB: return VK_FORMAT_R8G8B8A8_UNORM;
|
|
case VK_FORMAT_B8G8R8A8_SRGB: return VK_FORMAT_B8G8R8A8_UNORM;
|
|
case VK_FORMAT_BC1_RGB_SRGB_BLOCK: return VK_FORMAT_BC1_RGB_UNORM_BLOCK;
|
|
case VK_FORMAT_BC1_RGBA_SRGB_BLOCK: return VK_FORMAT_BC1_RGBA_UNORM_BLOCK;
|
|
case VK_FORMAT_BC2_SRGB_BLOCK: return VK_FORMAT_BC2_UNORM_BLOCK;
|
|
case VK_FORMAT_BC3_SRGB_BLOCK: return VK_FORMAT_BC3_UNORM_BLOCK;
|
|
case VK_FORMAT_BC7_SRGB_BLOCK: return VK_FORMAT_BC7_UNORM_BLOCK;
|
|
default:
|
|
if (is_astc(f) && ((f - VK_FORMAT_ASTC_4x4_UNORM_BLOCK) & 1)) return (VkFormat)(f - 1);
|
|
return f;
|
|
}
|
|
}
|
|
/// 3-component formats are not image formats on NVIDIA: the image holds 4 (alpha one), the upload widens them
|
|
VkFormat widened(VkFormat f, int* from_bpb) {
|
|
switch (f) {
|
|
case VK_FORMAT_R8G8B8_UNORM: *from_bpb = 3; return VK_FORMAT_R8G8B8A8_UNORM;
|
|
case VK_FORMAT_R8G8B8_SNORM: *from_bpb = 3; return VK_FORMAT_R8G8B8A8_SNORM;
|
|
case VK_FORMAT_R8G8B8_UINT: *from_bpb = 3; return VK_FORMAT_R8G8B8A8_UINT;
|
|
case VK_FORMAT_R8G8B8_SINT: *from_bpb = 3; return VK_FORMAT_R8G8B8A8_SINT;
|
|
case VK_FORMAT_R16G16B16_SFLOAT: *from_bpb = 6; return VK_FORMAT_R16G16B16A16_SFLOAT;
|
|
case VK_FORMAT_R16G16B16_UNORM: *from_bpb = 6; return VK_FORMAT_R16G16B16A16_UNORM;
|
|
case VK_FORMAT_R16G16B16_SNORM: *from_bpb = 6; return VK_FORMAT_R16G16B16A16_SNORM;
|
|
case VK_FORMAT_R16G16B16_UINT: *from_bpb = 6; return VK_FORMAT_R16G16B16A16_UINT;
|
|
case VK_FORMAT_R16G16B16_SINT: *from_bpb = 6; return VK_FORMAT_R16G16B16A16_SINT;
|
|
case VK_FORMAT_R32G32B32_SFLOAT: *from_bpb = 12; return VK_FORMAT_R32G32B32A32_SFLOAT;
|
|
case VK_FORMAT_R32G32B32_UINT: *from_bpb = 12; return VK_FORMAT_R32G32B32A32_UINT;
|
|
case VK_FORMAT_R32G32B32_SINT: *from_bpb = 12; return VK_FORMAT_R32G32B32A32_SINT;
|
|
default: *from_bpb = 0; return f;
|
|
}
|
|
}
|
|
/// A storage-capable view format of the image's class (sRGB views cannot be storage)
|
|
VkFormat storage_view_format(VkFormat f) {
|
|
f = linear_twin(f);
|
|
if (format_props(f).optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) return f;
|
|
switch (f) { // (same class, a format storage can be)
|
|
case VK_FORMAT_B8G8R8A8_UNORM: case VK_FORMAT_A2B10G10R10_UNORM_PACK32: case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
|
|
case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32: case VK_FORMAT_R8G8B8A8_SNORM: return VK_FORMAT_R32_UINT;
|
|
case VK_FORMAT_R16G16B16A16_UNORM: case VK_FORMAT_R16G16B16A16_SNORM: return VK_FORMAT_R32G32_UINT;
|
|
case VK_FORMAT_R8G8_SNORM: case VK_FORMAT_R16_SNORM: return VK_FORMAT_R16_UINT;
|
|
case VK_FORMAT_R8_SNORM: return VK_FORMAT_R8_UINT;
|
|
// (compressed: one block a texel, written raw by the game's encoders)
|
|
case VK_FORMAT_BC1_RGB_UNORM_BLOCK: case VK_FORMAT_BC1_RGBA_UNORM_BLOCK: case VK_FORMAT_BC4_UNORM_BLOCK:
|
|
case VK_FORMAT_BC4_SNORM_BLOCK: return VK_FORMAT_R32G32_UINT;
|
|
case VK_FORMAT_BC2_UNORM_BLOCK: case VK_FORMAT_BC3_UNORM_BLOCK: case VK_FORMAT_BC5_UNORM_BLOCK:
|
|
case VK_FORMAT_BC5_SNORM_BLOCK: case VK_FORMAT_BC6H_UFLOAT_BLOCK: case VK_FORMAT_BC6H_SFLOAT_BLOCK:
|
|
case VK_FORMAT_BC7_UNORM_BLOCK: return VK_FORMAT_R32G32B32A32_UINT;
|
|
default: return f;
|
|
}
|
|
}
|
|
/// The class an image format shares images by (see the top of the file)
|
|
uint32_t format_class(VkFormat vf, int nvn_format, int bpb, bool astc_decoded) {
|
|
if (astc_decoded) return 0x30000u | (uint32_t)(nvn_format - 0x79) % 14;
|
|
if (is_depth_format(vf)) return 0x10000u | (uint32_t)vf;
|
|
if (is_bc(vf) || is_astc(vf)) return 0x20000u | (uint32_t)linear_twin(vf);
|
|
int from = 0;
|
|
widened(vf, &from);
|
|
return 0x40000u | (uint32_t)bpb | (from ? 0x100u : 0u);
|
|
}
|
|
|
|
VkComponentSwizzle component(unsigned s) {
|
|
switch (s) {
|
|
case NVN_TEXTURE_SWIZZLE_ZERO: return VK_COMPONENT_SWIZZLE_ZERO;
|
|
case NVN_TEXTURE_SWIZZLE_ONE: return VK_COMPONENT_SWIZZLE_ONE;
|
|
case NVN_TEXTURE_SWIZZLE_R: return VK_COMPONENT_SWIZZLE_R;
|
|
case NVN_TEXTURE_SWIZZLE_G: return VK_COMPONENT_SWIZZLE_G;
|
|
case NVN_TEXTURE_SWIZZLE_B: return VK_COMPONENT_SWIZZLE_B;
|
|
case NVN_TEXTURE_SWIZZLE_A: return VK_COMPONENT_SWIZZLE_A;
|
|
default: return VK_COMPONENT_SWIZZLE_IDENTITY;
|
|
}
|
|
}
|
|
|
|
// ---- the cache ----
|
|
struct Key {
|
|
uint64_t gpu;
|
|
uint32_t target, width, height, depth, levels, samples, layout, cls;
|
|
bool operator==(const Key& o) const {
|
|
return gpu == o.gpu && target == o.target && width == o.width && height == o.height && depth == o.depth &&
|
|
levels == o.levels && samples == o.samples && layout == o.layout && cls == o.cls;
|
|
}
|
|
};
|
|
struct KeyHash {
|
|
size_t operator()(const Key& k) const {
|
|
uint64_t h = mix(k.gpu, k.target);
|
|
h = mix(h, (uint64_t)k.width << 32 | k.height);
|
|
h = mix(h, (uint64_t)k.depth << 32 | k.levels);
|
|
h = mix(h, (uint64_t)k.samples << 40 | (uint64_t)k.layout << 20 | k.cls);
|
|
return (size_t)h;
|
|
}
|
|
};
|
|
std::unordered_map<Key, std::unique_ptr<Texture>, KeyHash> textures;
|
|
// (Triforce, texture detail: the keys made whole whatever the card's detail level, for the session, and the textures asked
|
|
// whole since the last sweep: texture_want_whole)
|
|
std::unordered_set<Key, KeyHash> whole_keys;
|
|
std::vector<Texture*> whole_pending;
|
|
std::unordered_multimap<uint64_t, Texture*> by_gpu;
|
|
uint64_t frame_no = 1;
|
|
uint64_t total_bytes = 0;
|
|
|
|
// (RX_VK_LOG_REMADE=1) Textures evicted and made again within 600 frames, and those of them that were then sampled
|
|
// empty (storage the GPU rendered, not uploaded): what the eviction's minimum age below is measured by
|
|
struct Evicted {
|
|
uint64_t frame;
|
|
bool rendered;
|
|
};
|
|
std::mutex evicted_mu;
|
|
std::unordered_map<uint64_t, Evicted> evicted_at; // (storage, size and format -> when it went)
|
|
std::unordered_map<const Texture*, uint64_t> remade; // (made again: how many frames after it went)
|
|
uint64_t evicted_key(uint64_t gpu, uint32_t w, uint32_t h, uint32_t fmt) {
|
|
return gpu ^ ((uint64_t)w << 48) ^ ((uint64_t)h << 32) ^ ((uint64_t)fmt << 20);
|
|
}
|
|
bool remade_log_on() {
|
|
static const bool on = getenv("RX_VK_LOG_REMADE") && atoi(getenv("RX_VK_LOG_REMADE")) != 0;
|
|
return on;
|
|
}
|
|
// (hookshot) Textures uploaded from the game's memory (never drawn by the GPU) that the budget evicted: storage, size and format.
|
|
// Made again, such a texture is uploaded from that memory whatever ranges render targets once took over it (upload's "storage
|
|
// the GPU rendered" skip below). Skipped, its new image was never filled: whatever the VRAM held before, random-colour blocks on
|
|
// characters, shop shelves, floors, which spread over play time on 4-8 GB cards (the ones that evict) and changed each time a
|
|
// menu was opened and closed (issue 993). RX_VK_REMADE_SKIP=1: as before
|
|
std::mutex from_memory_mu;
|
|
std::unordered_map<uint64_t, uint64_t> evicted_from_memory; // (evicted_key -> guest size)
|
|
std::atomic<uint64_t> st_remade_uploaded{0};
|
|
bool remade_skip_old() {
|
|
static const bool on = getenv("RX_VK_REMADE_SKIP") && atoi(getenv("RX_VK_REMADE_SKIP")) != 0;
|
|
return on;
|
|
}
|
|
|
|
struct Dead {
|
|
uint64_t value;
|
|
VkImage image;
|
|
VmaAllocation allocation;
|
|
std::vector<VkImageView> views;
|
|
std::vector<VkDeviceMemory> memory; // (a sparse image's)
|
|
VkBuffer buffer = VK_NULL_HANDLE; // (the BC3 scratch's output buffer: given back with its image)
|
|
VmaAllocation buffer_alloc = nullptr;
|
|
uint64_t pool_key = 0; // (a render target's image: to the pool below, with its views, not destroyed)
|
|
std::unordered_map<uint64_t, VkImageView> view_map;
|
|
uint64_t bytes = 0;
|
|
};
|
|
std::deque<Dead> dead;
|
|
|
|
// ---- (hookshot) Render target images, reused. OoT makes a frame's transient targets in a scratch heap whose offsets shift with
|
|
// what came before them in the frame (the dead targets, texture_frame): each shifted one was a texture of its own with a new image,
|
|
// and every 60 presents the sweep let go of the dead ones, 300-700 at once, destroyed in the next present: 20-50 ms on the thread
|
|
// replaying, once a second while the camera moves (RTX 5070, Kokiri Forest: 10,500 images made and destroyed in 4.5 minutes,
|
|
// 17 GB). Now a render target's image whose texture is gone waits in a pool by its create info, views and all, and a new texture
|
|
// of the same create info takes it; it is put into GENERAL from UNDEFINED and cleared as a new one is (create), so nothing it held
|
|
// shows. Only images made for a target (texture_get's !for_sampling) are pooled. The pool is bounded (RX_VK_TARGET_POOL_MB, default
|
|
// 256; 0: none, every image destroyed as before; and none while VRAM is near what the driver allows), and an image unused in it for
|
|
// 120 presents is destroyed, at most 16 a present ----
|
|
struct Pooled {
|
|
VkImage image;
|
|
VmaAllocation allocation;
|
|
std::unordered_map<uint64_t, VkImageView> views;
|
|
uint64_t bytes, since;
|
|
};
|
|
std::mutex pool_mu;
|
|
std::unordered_map<uint64_t, std::vector<Pooled>> target_pool; // by pool_key_of
|
|
uint64_t pool_bytes, pool_n, st_pool_reused, st_pool_made, st_pool_destroyed;
|
|
bool pool_pressed; // (VRAM near what the driver allows: texture_frame's budget)
|
|
bool compact_on() { // (hookshot, RX_VK_COMPACT=1: the textures in a pool of their own, compacted: compact_step)
|
|
// (Triforce) on by default on NVIDIA cards (RX_VK_COMPACT=0: off; =1: on, any card). Measured there: an RTX 4070 SUPER
|
|
// played with it since 2026-11-10 (a 584,000-present session, 132 compactions, no fault; 85 -> 80.6 fps after a six-area
|
|
// tour against 81 -> 64 without), and with the allowance held to 7300 MB (RX_VK_TEST_ALLOWED_MB, an 8 GB card's) a tour of
|
|
// Hyrule Field, Kakariko, Lake Hylia and Kokiri peaked at 7165 MB of VRAM against 8565 without (over the allowance in 2 of
|
|
// 12 reports against 11) at 69 fps against ~64. On an 8 GB card what is over the allowance is paged to system memory, and
|
|
// the textures and render targets evicted to get back under it are made and uploaded again, 58 to 190 ms a frame (an
|
|
// RX 5700 XT's and an RTX 3060's logs, 2026-11-11). Not measured on AMD or Intel: as it was there until someone does
|
|
// (Triforce, later that day) and on trial on the other makes' cards: memory_trial_on (vk_memory.cpp), which turns it off by
|
|
// itself on a PC where a session with it loses the GPU or two end in a crash
|
|
static const bool v = [] {
|
|
if (const char* e = getenv("RX_VK_COMPACT")) return atoi(e) != 0;
|
|
const bool on = memory_trial_on();
|
|
if (on) logf("vk: the textures' VRAM is compacted (RX_VK_COMPACT=0: off)");
|
|
return on;
|
|
}();
|
|
return v;
|
|
}
|
|
uint64_t pool_cap() {
|
|
static const uint64_t mb = getenv("RX_VK_TARGET_POOL_MB") ? (uint64_t)std::max(0, atoi(getenv("RX_VK_TARGET_POOL_MB"))) : 256;
|
|
return mb << 20;
|
|
}
|
|
uint64_t pool_key_of(const VkImageCreateInfo& ci) {
|
|
uint64_t k = mix((uint64_t)ci.flags, (uint64_t)ci.imageType << 32 | (uint64_t)ci.format);
|
|
k = mix(k, (uint64_t)ci.extent.width << 32 | ci.extent.height);
|
|
k = mix(k, (uint64_t)ci.extent.depth << 32 | ci.mipLevels);
|
|
k = mix(k, (uint64_t)ci.arrayLayers << 32 | (uint64_t)ci.samples);
|
|
k = mix(k, (uint64_t)ci.usage << 8 | (uint64_t)ci.tiling);
|
|
return k | 1; // (0: not pooled)
|
|
}
|
|
bool pool_take(uint64_t key, VkImage& image, VmaAllocation& allocation, std::unordered_map<uint64_t, VkImageView>& views) {
|
|
std::lock_guard lk(pool_mu);
|
|
auto it = target_pool.find(key);
|
|
if (it == target_pool.end() || it->second.empty()) return false;
|
|
Pooled& p = it->second.back();
|
|
image = p.image, allocation = p.allocation, views = std::move(p.views);
|
|
pool_bytes -= std::min(pool_bytes, p.bytes), pool_n--;
|
|
it->second.pop_back();
|
|
if (it->second.empty()) target_pool.erase(it);
|
|
st_pool_reused++;
|
|
return true;
|
|
}
|
|
void pool_destroy(Pooled& p) {
|
|
for (auto& kv : p.views)
|
|
if (kv.second) vkDestroyImageView(dev.device, kv.second, nullptr);
|
|
destroy_image(p.image, p.allocation);
|
|
st_pool_destroyed++;
|
|
}
|
|
/// (texture_frame) a dead render target's image the GPU is done with, into the pool
|
|
void pool_put(Dead& d) {
|
|
std::lock_guard lk(pool_mu);
|
|
target_pool[d.pool_key].push_back(Pooled{d.image, d.allocation, std::move(d.view_map), d.bytes, frame_no});
|
|
pool_bytes += d.bytes, pool_n++;
|
|
}
|
|
/// (texture_frame, each present) the pool over its bound, or all of it while VRAM is short, and what waits in it unused for 120
|
|
/// presents: destroyed, at most 16 a present (64 while VRAM is short). Returns how many
|
|
size_t pool_trim() {
|
|
std::lock_guard lk(pool_mu);
|
|
// (Triforce, 11-11: with a floor of blocks kept (target_floor_kept) the pool is not emptied while VRAM is short: what it gave
|
|
// back went into blocks the floor keeps, the driver's count did not move, and every target of the next frames was made new)
|
|
const bool give_back = pool_pressed && !target_floor_kept();
|
|
const uint64_t cap = give_back ? 0 : pool_cap();
|
|
const size_t most = give_back ? 64 : 16;
|
|
size_t n = 0;
|
|
for (auto it = target_pool.begin(); it != target_pool.end() && n < most;) {
|
|
auto& v = it->second;
|
|
for (size_t i = 0; i < v.size() && n < most;) { // (the oldest of a shape first: the front)
|
|
if (pool_bytes <= cap && frame_no - v[i].since <= 120) {
|
|
++i;
|
|
continue;
|
|
}
|
|
pool_destroy(v[i]);
|
|
pool_bytes -= std::min(pool_bytes, v[i].bytes), pool_n--, n++;
|
|
v.erase(v.begin() + (ptrdiff_t)i);
|
|
}
|
|
it = v.empty() ? target_pool.erase(it) : std::next(it);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// ---- the recording's upload command buffer ----
|
|
thread_local VkCommandBuffer up_cmd = VK_NULL_HANDLE; // (each recording thread's: a parallel replay's segments
|
|
uint64_t rec_value = 0; // record their own uploads)
|
|
thread_local bool up_used = false;
|
|
thread_local uint64_t tl_seg = 0;
|
|
uint32_t epoch = 0;
|
|
int64_t st_uploads, st_astc, st_bytes;
|
|
|
|
VkCommandBuffer upload_cmd() {
|
|
if (!up_used) {
|
|
up_used = true;
|
|
full_barrier(up_cmd); // (after the work submitted before: it may still read what is replaced)
|
|
}
|
|
return up_cmd;
|
|
}
|
|
|
|
// ---- (parallel replay) a new image's first layout change, in queue order ----
|
|
// The segments' workers record at once, and the one that makes an image first need not be the first in the queue to use
|
|
// it: a later segment's worker made it (its UNDEFINED -> GENERAL change in its own uploads, submitted after the earlier
|
|
// segment), the earlier one used it (a copy into it again, the BC3 encoder's dispatch: its uploads, at the start of its
|
|
// command buffers), and the queue reached that use with the image UNDEFINED (VUID-vkCmdDraw-None-09600, RX_VK_PARALLEL
|
|
// only). The change goes to the init command buffer of the earliest segment still being recorded instead: those before
|
|
// it were done before the image was made, so never used it, and each segment's init is submitted before its uploads.
|
|
struct SegInit {
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
bool used = false, open = false;
|
|
};
|
|
std::mutex seg_init_mu;
|
|
std::vector<SegInit> seg_inits; // (the submission's segments, in queue order; seg_init_mu)
|
|
thread_local int tl_seg_index = -1; // (the segment this thread records; -1: none)
|
|
/// While it lives, upload_cmd() is the earliest open segment's init command buffer (a segment's worker only)
|
|
struct InitRedirect {
|
|
std::unique_lock<std::mutex> lk;
|
|
SegInit* to = nullptr;
|
|
VkCommandBuffer saved_cmd = VK_NULL_HANDLE;
|
|
bool saved_used = false;
|
|
explicit InitRedirect(bool want = true) {
|
|
if (!want || tl_seg_index < 0) return;
|
|
lk = std::unique_lock<std::mutex>(seg_init_mu);
|
|
for (int i = 0; i <= tl_seg_index && i < (int)seg_inits.size(); i++)
|
|
if (seg_inits[i].open) { to = &seg_inits[i]; break; }
|
|
if (!to) { lk.unlock(); return; } // (none open: its own is, while it records)
|
|
saved_cmd = up_cmd, saved_used = up_used;
|
|
up_cmd = to->cmd, up_used = to->used;
|
|
}
|
|
void done() {
|
|
if (to) {
|
|
to->used = up_used;
|
|
up_cmd = saved_cmd, up_used = saved_used;
|
|
to = nullptr;
|
|
}
|
|
if (lk.owns_lock()) lk.unlock();
|
|
}
|
|
~InitRedirect() { done(); }
|
|
InitRedirect(const InitRedirect&) = delete;
|
|
InitRedirect& operator=(const InitRedirect&) = delete;
|
|
};
|
|
|
|
// ---- staging: host-visible arenas, reused once the submission that filled them completed ----
|
|
struct Arena {
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
VmaAllocation allocation = nullptr;
|
|
uint8_t* mapped = nullptr;
|
|
uint64_t size = 0, used = 0, value = 0; // value: the last submission that reads it
|
|
std::chrono::steady_clock::time_point used_at; // (when it last staged something)
|
|
};
|
|
std::vector<Arena> arenas;
|
|
constexpr uint64_t kArena = 64ull << 20;
|
|
|
|
struct Staging {
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
uint64_t offset = 0;
|
|
uint8_t* data = nullptr;
|
|
VmaAllocation allocation = nullptr;
|
|
};
|
|
uint64_t completed_value = 0;
|
|
// (GPU writes into guest memory by buffers: GPU address -> (end, the submission that wrote it last))
|
|
std::map<uint64_t, std::pair<uint64_t, uint64_t>> gpu_wrote;
|
|
std::mutex gpu_wrote_mu;
|
|
} // namespace
|
|
bool gpu_wrote_into(Texture& tex);
|
|
namespace {
|
|
Staging staging(uint64_t n, uint64_t align = 256) {
|
|
// Separate slices may already be read by submitted parallel segments. A flush must not share their cache atom.
|
|
align = std::max<uint64_t>(align, dev.props.limits.nonCoherentAtomSize);
|
|
for (Arena& a : arenas) { // the current one first, then any whose readers completed
|
|
if (a.value == rec_value) {
|
|
const uint64_t at = (a.used + align - 1) & ~(align - 1);
|
|
if (at + n <= a.size) {
|
|
a.used = at + n;
|
|
a.used_at = std::chrono::steady_clock::now();
|
|
return {a.buffer, at, a.mapped + at, a.allocation};
|
|
}
|
|
}
|
|
}
|
|
for (Arena& a : arenas) {
|
|
if (a.value != rec_value && a.value <= completed_value && a.size >= n) {
|
|
a.value = rec_value, a.used = n;
|
|
a.used_at = std::chrono::steady_clock::now();
|
|
return {a.buffer, 0, a.mapped, a.allocation};
|
|
}
|
|
}
|
|
Arena a;
|
|
void* m = nullptr;
|
|
const auto make = [&](uint64_t size) {
|
|
a.size = size, m = nullptr;
|
|
if (!create_buffer(size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, true, a.buffer,
|
|
a.allocation, &m))
|
|
return false;
|
|
if (m) return true;
|
|
destroy_buffer(a.buffer, a.allocation); // (made but not mapped: of no use here)
|
|
a.buffer = VK_NULL_HANDLE, a.allocation = nullptr;
|
|
return false;
|
|
};
|
|
const auto t_make = std::chrono::steady_clock::now();
|
|
// (hookshot) and one of the size asked for when a 64 MB one cannot be made: without resizable BAR an arena is new system
|
|
// memory the driver must pin. On a 16 GB laptop with the game's pools imported and RAM nearly gone, no 64 MB arena could
|
|
// be made even for a 36 KB upload, the idle ones having been given back (texture_frame), and every upload failed until one
|
|
// was free: BC3 textures drawn as noise (RX 5500M, 2.9 GB of RAM available)
|
|
const uint64_t need = (n + 0xFFFFF) & ~0xFFFFFull;
|
|
if (!make(std::max(kArena, need))) {
|
|
const bool small = need < kArena && make(need);
|
|
static std::atomic<int> told{0};
|
|
if (told++ < 8)
|
|
logf("vk: staging: no %llu MB arena could be made (%zu kept); %s", (unsigned long long)(kArena >> 20), arenas.size(),
|
|
small ? "one of the size asked for instead" : "the upload waits");
|
|
if (!small) return {};
|
|
}
|
|
{ // (hookshot: what making a staging arena costs the replay thread: on a card with resizable BAR it is VRAM the driver hands
|
|
// out, and a scene change made several right after the idle ones were freed)
|
|
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t_make).count();
|
|
static int slow_logged;
|
|
if (ms > 5 && slow_logged++ < 24) logf("vk: a staging arena of %llu MB took %.1f ms to make", (unsigned long long)(a.size >> 20), ms);
|
|
}
|
|
a.mapped = (uint8_t*)m;
|
|
a.value = rec_value, a.used = n;
|
|
a.used_at = std::chrono::steady_clock::now();
|
|
arenas.push_back(a);
|
|
static int logged;
|
|
if (logged++ < 4 || arenas.size() % 16 == 0)
|
|
logf("vk: staging arenas: %zu (%llu MB in this one)", arenas.size(), (unsigned long long)(a.size >> 20));
|
|
return {a.buffer, 0, a.mapped, a.allocation};
|
|
}
|
|
|
|
// ---- the GPU ASTC decoder (Eden's) ----
|
|
struct AstcPush {
|
|
uint32_t block_dims[2], layer_stride, block_size, x_shift, block_height, block_height_mask, native_tiling;
|
|
};
|
|
VkDescriptorSetLayout astc_set = VK_NULL_HANDLE;
|
|
VkPipelineLayout astc_layout = VK_NULL_HANDLE;
|
|
VkPipeline astc_pipe = VK_NULL_HANDLE;
|
|
bool astc_ready() {
|
|
static int state = 0; // 0 untried, 1 ready, -1 unavailable
|
|
if (state) return state > 0;
|
|
state = -1;
|
|
if (!dev.push_descriptor || !vkCmdPushDescriptorSetKHR || getenv("RX_VK_ASTC_CPU")) return false;
|
|
const VkDescriptorSetLayoutBinding b[2] = {
|
|
{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
|
|
{1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}};
|
|
VkDescriptorSetLayoutCreateInfo li{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
|
|
li.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
|
|
li.bindingCount = 2;
|
|
li.pBindings = b;
|
|
if (vkCreateDescriptorSetLayout(dev.device, &li, nullptr, &astc_set) != VK_SUCCESS) return false;
|
|
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(AstcPush)};
|
|
VkPipelineLayoutCreateInfo pi{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
|
|
pi.setLayoutCount = 1;
|
|
pi.pSetLayouts = &astc_set;
|
|
pi.pushConstantRangeCount = 1;
|
|
pi.pPushConstantRanges = ≺
|
|
if (vkCreatePipelineLayout(dev.device, &pi, nullptr, &astc_layout) != VK_SUCCESS) return false;
|
|
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
|
mi.codeSize = sizeof ASTC_DECODER_COMP_SPV;
|
|
mi.pCode = ASTC_DECODER_COMP_SPV;
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) != VK_SUCCESS) return false;
|
|
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
|
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
|
|
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
|
ci.stage.module = m;
|
|
ci.stage.pName = "main";
|
|
ci.layout = astc_layout;
|
|
const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &astc_pipe);
|
|
vkDestroyShaderModule(dev.device, m, nullptr);
|
|
if (r != VK_SUCCESS) return false;
|
|
state = 1;
|
|
logf("vk: ASTC is decoded on the GPU (Eden's decoder, Switch 2 tiling)");
|
|
return true;
|
|
}
|
|
|
|
VkImageView make_view(Texture* tex, VkImageViewType type, VkFormat f, VkImageAspectFlags aspect, uint32_t level0,
|
|
uint32_t nlevels, uint32_t layer0, uint32_t nlayers, const VkComponentMapping& cm,
|
|
VkImageUsageFlags usage) {
|
|
if (!usage || (usage & tex->usage) != usage) return VK_NULL_HANDLE;
|
|
// (block-texel views of a compressed image: one level; several layers where the driver takes them)
|
|
const bool block_view = is_bc(tex->format) && !is_bc(f);
|
|
if (block_view && nlevels != 1) return VK_NULL_HANDLE;
|
|
VkImageViewUsageCreateInfo uci{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO};
|
|
uci.usage = usage & tex->usage;
|
|
VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
|
vi.pNext = &uci;
|
|
vi.image = tex->image;
|
|
vi.viewType = type;
|
|
vi.format = f;
|
|
vi.components = cm;
|
|
vi.subresourceRange = {aspect, level0, nlevels, layer0, nlayers};
|
|
VkImageView v = VK_NULL_HANDLE;
|
|
if (vkCreateImageView(dev.device, &vi, nullptr, &v) == VK_SUCCESS) return v;
|
|
if (!block_view || nlayers == 1) return VK_NULL_HANDLE;
|
|
// (Vulkan 1.1 says a block-texel view is of a single layer; some drivers hold to it)
|
|
vi.subresourceRange.layerCount = 1;
|
|
if (vi.viewType == VK_IMAGE_VIEW_TYPE_2D_ARRAY) vi.viewType = VK_IMAGE_VIEW_TYPE_2D;
|
|
if (vkCreateImageView(dev.device, &vi, nullptr, &v) != VK_SUCCESS) return VK_NULL_HANDLE;
|
|
return v;
|
|
}
|
|
|
|
// ---- big ASTC textures kept as BC3 (8 bits a texel, a quarter of RGBA8): decoded into a scratch image a few
|
|
// layers at a time, re-encoded by Eden's encoder (host_shaders/bc_encoder.comp: bounding box along the colours'
|
|
// covariance, inset; GPL, as the decoder) into a buffer, copied into the image. Decoded RGBA8 took 6 GB of VRAM at
|
|
// the title (the terrain's material arrays, 2048 x 2048 x 68, 1.45 GB each): more than the driver allowed, which
|
|
// paged VRAM out and evicted textures that were made and uploaded again. RX_VK_ASTC_RGBA8=1: all as RGBA8.
|
|
struct BcPush {
|
|
uint32_t blocks_x, blocks_y, width, height, bc3, out_offset;
|
|
};
|
|
VkDescriptorSetLayout bc_set = VK_NULL_HANDLE;
|
|
VkPipelineLayout bc_layout = VK_NULL_HANDLE;
|
|
VkPipeline bc_pipe = VK_NULL_HANDLE;
|
|
constexpr uint32_t kBcDim = 2048, kBcLayers = 8; // the scratch: the largest level a BC3 texture may have
|
|
VkImage bc_scratch = VK_NULL_HANDLE;
|
|
VmaAllocation bc_scratch_alloc = nullptr;
|
|
VkImageView bc_scratch_view = VK_NULL_HANDLE;
|
|
VkBuffer bc_out = VK_NULL_HANDLE;
|
|
VmaAllocation bc_out_alloc = nullptr;
|
|
bool bc_scratch_fresh = false; // (its layout still to be set, in the next upload command buffer)
|
|
// (hookshot-amd) a second scratch for textures re-encoded at another size (uneven block levels on AMD): the decoded level
|
|
// is blitted into it, filtered, at the image's size, and encoded from there. Made when an upload of such a texture first
|
|
// needs it, given back with the first scratch.
|
|
VkImage bc_scratch2 = VK_NULL_HANDLE;
|
|
VmaAllocation bc_scratch2_alloc = nullptr;
|
|
VkImageView bc_scratch2_view = VK_NULL_HANDLE;
|
|
bool bc_scratch2_fresh = false;
|
|
// (the scratch and its buffer, 160 MB, are made when an upload first needs them and given back after RX_VK_BC_IDLE_S seconds
|
|
// (20) without one: a title or a load decodes hundreds of textures through them, steady gameplay few (the "given back" line says
|
|
// whether it did).
|
|
// 0: kept for good once made. They have memory objects of their own, so that freeing gives the VRAM back: VMA keeps the empty
|
|
// block of its image pool for the next images. RX_VK_BC_POOL=1: from the pool as before)
|
|
std::mutex bc_mu;
|
|
std::chrono::steady_clock::time_point bc_used_at;
|
|
uint64_t bc_last_value = 0; // (the recording that last used them: they go once it is done)
|
|
std::atomic<bool> bc_have{false};
|
|
uint32_t bc_idle_s() {
|
|
// (hookshot) 0 by default again, kept once made: given back after 20 s, the garbled/rainbow textures of 4-8 GB cards followed
|
|
// (issue 993): every such log has the "given back" line, an RTX 3050 4 GB player's went away with RX_VK_BC_IDLE_S=0 and not with
|
|
// RX_VK_NO_EVICT=1, and they began with v2.0.3, which brought this give-back. Its cost: 160-288 MB of VRAM held after the first
|
|
// decode. RX_VK_BC_IDLE_S=<seconds>: given back as before
|
|
static const uint32_t v = getenv("RX_VK_BC_IDLE_S") ? (uint32_t)atoi(getenv("RX_VK_BC_IDLE_S")) : 0;
|
|
return v;
|
|
}
|
|
bool bc_scratch_make() { // (bc_mu held)
|
|
VkImageCreateInfo ii{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
|
|
ii.imageType = VK_IMAGE_TYPE_2D;
|
|
ii.format = VK_FORMAT_R8G8B8A8_UNORM;
|
|
ii.extent = {kBcDim, kBcDim, 1};
|
|
ii.mipLevels = 1;
|
|
ii.arrayLayers = kBcLayers;
|
|
ii.samples = VK_SAMPLE_COUNT_1_BIT;
|
|
ii.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
|
|
static const bool pool = getenv("RX_VK_BC_POOL") != nullptr;
|
|
if (!create_image(ii, bc_scratch, bc_scratch_alloc, !pool)) return false;
|
|
VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
|
vi.image = bc_scratch;
|
|
vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
|
|
vi.format = VK_FORMAT_R8G8B8A8_UNORM;
|
|
vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers};
|
|
const uint64_t out = (uint64_t)(kBcDim / 4) * (kBcDim / 4) * kBcLayers * 16;
|
|
const VkBufferUsageFlags out_usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
|
|
const auto make_out = [&] {
|
|
if (create_device_buffer(out, out_usage, bc_out, bc_out_alloc, !pool)) return true;
|
|
// (hookshot) VRAM full, the process at its budget (a 4 GB card, the textures evicted after an alt-tab): in system
|
|
// memory, as create_image puts the scratch image there. Only in VRAM, it could not be made, every BC3 texture made
|
|
// again then failed, and they were drawn as coloured noise
|
|
if (!create_buffer(out, out_usage, false, bc_out, bc_out_alloc, nullptr, !pool)) return false;
|
|
static std::atomic<int> told{0};
|
|
if (told++ < 4) logf("vk: VRAM full: the BC3 scratch's output buffer (%llu MB) in system memory", (unsigned long long)(out >> 20));
|
|
return true;
|
|
};
|
|
if (vkCreateImageView(dev.device, &vi, nullptr, &bc_scratch_view) != VK_SUCCESS || !make_out()) {
|
|
if (bc_scratch_view) vkDestroyImageView(dev.device, bc_scratch_view, nullptr);
|
|
destroy_image(bc_scratch, bc_scratch_alloc);
|
|
bc_scratch = VK_NULL_HANDLE, bc_scratch_view = VK_NULL_HANDLE;
|
|
return false;
|
|
}
|
|
bc_scratch_fresh = true;
|
|
bc_have = true;
|
|
return true;
|
|
}
|
|
|
|
bool bc_ready() {
|
|
static int state = 0;
|
|
if (state) return state > 0;
|
|
state = -1;
|
|
if (!astc_ready() || getenv("RX_VK_ASTC_RGBA8") || !dev.features.textureCompressionBC) return false;
|
|
VkFormatProperties fp{};
|
|
vkGetPhysicalDeviceFormatProperties(dev.physical, VK_FORMAT_BC3_UNORM_BLOCK, &fp);
|
|
if (!(fp.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT)) return false;
|
|
const VkDescriptorSetLayoutBinding b[2] = {
|
|
{0, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
|
|
{1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}};
|
|
VkDescriptorSetLayoutCreateInfo li{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
|
|
li.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
|
|
li.bindingCount = 2;
|
|
li.pBindings = b;
|
|
if (vkCreateDescriptorSetLayout(dev.device, &li, nullptr, &bc_set) != VK_SUCCESS) return false;
|
|
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(BcPush)};
|
|
VkPipelineLayoutCreateInfo pi{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
|
|
pi.setLayoutCount = 1;
|
|
pi.pSetLayouts = &bc_set;
|
|
pi.pushConstantRangeCount = 1;
|
|
pi.pPushConstantRanges = ≺
|
|
if (vkCreatePipelineLayout(dev.device, &pi, nullptr, &bc_layout) != VK_SUCCESS) return false;
|
|
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
|
mi.codeSize = sizeof BC_ENCODER_COMP_SPV;
|
|
mi.pCode = BC_ENCODER_COMP_SPV;
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) != VK_SUCCESS) return false;
|
|
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
|
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
|
|
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
|
ci.stage.module = m;
|
|
ci.stage.pName = "main";
|
|
ci.layout = bc_layout;
|
|
const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &bc_pipe);
|
|
vkDestroyShaderModule(dev.device, m, nullptr);
|
|
if (r != VK_SUCCESS) return false;
|
|
state = 1;
|
|
logf("vk: big ASTC textures are kept as BC3 (re-encoded on the GPU; RX_VK_ASTC_RGBA8=1: as RGBA8)");
|
|
return true;
|
|
}
|
|
|
|
/// (hookshot-amd) the second scratch (see bc_scratch2; bc_mu held)
|
|
bool bc_scratch2_make() {
|
|
VkImageCreateInfo ii{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
|
|
ii.imageType = VK_IMAGE_TYPE_2D;
|
|
ii.format = VK_FORMAT_R8G8B8A8_UNORM;
|
|
ii.extent = {kBcDim, kBcDim, 1};
|
|
ii.mipLevels = 1;
|
|
ii.arrayLayers = kBcLayers;
|
|
ii.samples = VK_SAMPLE_COUNT_1_BIT;
|
|
ii.usage = VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
|
|
static const bool pool = getenv("RX_VK_BC_POOL") != nullptr;
|
|
if (!create_image(ii, bc_scratch2, bc_scratch2_alloc, !pool)) return false;
|
|
VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
|
vi.image = bc_scratch2;
|
|
vi.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY;
|
|
vi.format = VK_FORMAT_R8G8B8A8_UNORM;
|
|
vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers};
|
|
if (vkCreateImageView(dev.device, &vi, nullptr, &bc_scratch2_view) != VK_SUCCESS) {
|
|
destroy_image(bc_scratch2, bc_scratch2_alloc);
|
|
bc_scratch2 = VK_NULL_HANDLE, bc_scratch2_view = VK_NULL_HANDLE;
|
|
return false;
|
|
}
|
|
bc_scratch2_fresh = true;
|
|
return true;
|
|
}
|
|
|
|
/// (hookshot-amd) scratch [0, w) x [0, h), layers [0, n), blitted (linear) into scratch2 at ow x oh, then encoded from there
|
|
void bc_encode_resized(VkCommandBuffer cmd, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh, uint32_t n, Texture& dst,
|
|
uint32_t level, uint32_t layer);
|
|
|
|
/// Encodes scratch layers [0, n) (a w x h region at its corner) and copies the blocks into dst's level and layers
|
|
void bc_encode_into(VkCommandBuffer cmd, uint32_t w, uint32_t h, uint32_t n, Texture& dst, uint32_t level,
|
|
uint32_t layer, int dx, int dy, VkImageView from = VK_NULL_HANDLE) {
|
|
full_barrier(cmd);
|
|
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, bc_pipe);
|
|
VkDescriptorImageInfo ii{VK_NULL_HANDLE, from ? from : bc_scratch_view, VK_IMAGE_LAYOUT_GENERAL};
|
|
VkDescriptorBufferInfo bi{bc_out, 0, VK_WHOLE_SIZE};
|
|
VkWriteDescriptorSet wr[2]{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}};
|
|
wr[0].dstBinding = 0, wr[0].descriptorCount = 1, wr[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
wr[0].pImageInfo = ⅈ
|
|
wr[1].dstBinding = 1, wr[1].descriptorCount = 1, wr[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
|
wr[1].pBufferInfo = &bi;
|
|
vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, bc_layout, 0, 2, wr);
|
|
descriptors_disturbed();
|
|
const BcPush pc{(w + 3) / 4, (h + 3) / 4, w, h, 1, 0};
|
|
vkCmdPushConstants(cmd, bc_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
vkCmdDispatch(cmd, (pc.blocks_x + 7) / 8, (pc.blocks_y + 7) / 8, n);
|
|
full_barrier(cmd);
|
|
const VkBufferImageCopy c{0, 0, 0, {VK_IMAGE_ASPECT_COLOR_BIT, level, layer, n}, {dx, dy, 0}, {w, h, 1}};
|
|
vkCmdCopyBufferToImage(cmd, bc_out, dst.image, VK_IMAGE_LAYOUT_GENERAL, 1, &c);
|
|
full_barrier(cmd); // (the scratch and the buffer are the next part's)
|
|
}
|
|
|
|
void bc_encode_resized(VkCommandBuffer cmd, uint32_t w, uint32_t h, uint32_t ow, uint32_t oh, uint32_t n, Texture& dst,
|
|
uint32_t level, uint32_t layer) {
|
|
full_barrier(cmd);
|
|
VkImageBlit bl{};
|
|
bl.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, n};
|
|
bl.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, n};
|
|
bl.srcOffsets[1] = {(int32_t)w, (int32_t)h, 1};
|
|
bl.dstOffsets[1] = {(int32_t)ow, (int32_t)oh, 1};
|
|
vkCmdBlitImage(cmd, bc_scratch, VK_IMAGE_LAYOUT_GENERAL, bc_scratch2, VK_IMAGE_LAYOUT_GENERAL, 1, &bl, VK_FILTER_LINEAR);
|
|
bc_encode_into(cmd, ow, oh, n, dst, level, layer, 0, 0, bc_scratch2_view);
|
|
}
|
|
|
|
/// The scratch for the upload recording now: made if it is not there, and its layout set (false: it could not be made).
|
|
/// resized: the second scratch too (hookshot-amd)
|
|
bool bc_scratch_layout(VkCommandBuffer cmd, bool resized = false) {
|
|
{
|
|
std::lock_guard lk(bc_mu);
|
|
if (!bc_have && !bc_scratch_make()) return false;
|
|
if (resized && !bc_scratch2 && !bc_scratch2_make()) return false;
|
|
bc_used_at = std::chrono::steady_clock::now();
|
|
bc_last_value = rec_value;
|
|
}
|
|
// (a segment's worker: a fresh scratch's layout change in queue order, as a new image's)
|
|
InitRedirect redirect((bc_scratch2_fresh && bc_scratch2) || bc_scratch_fresh);
|
|
if (redirect.to) cmd = upload_cmd();
|
|
if (bc_scratch2_fresh && bc_scratch2) { // (hookshot-amd: the second scratch, once made)
|
|
bc_scratch2_fresh = false;
|
|
VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
b.image = bc_scratch2;
|
|
b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers};
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.imageMemoryBarrierCount = 1;
|
|
di.pImageMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &di);
|
|
}
|
|
if (!bc_scratch_fresh) return true;
|
|
bc_scratch_fresh = false;
|
|
VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
b.image = bc_scratch;
|
|
b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, kBcLayers};
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.imageMemoryBarrierCount = 1;
|
|
di.pImageMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &di);
|
|
return true;
|
|
}
|
|
/// (every present: the scratch unused for RX_VK_BC_IDLE_S seconds goes with the images the GPU is done with)
|
|
void bc_scratch_idle() {
|
|
if (!bc_have.load(std::memory_order_relaxed) || !bc_idle_s()) return;
|
|
std::lock_guard lk(bc_mu);
|
|
if (!bc_have || std::chrono::steady_clock::now() - bc_used_at < std::chrono::seconds(bc_idle_s())) return;
|
|
Dead d{bc_last_value, bc_scratch, bc_scratch_alloc, {bc_scratch_view}, {}};
|
|
d.buffer = bc_out, d.buffer_alloc = bc_out_alloc;
|
|
dead.push_back(std::move(d));
|
|
bc_scratch = VK_NULL_HANDLE, bc_scratch_alloc = nullptr, bc_scratch_view = VK_NULL_HANDLE, bc_out = VK_NULL_HANDLE, bc_out_alloc = nullptr;
|
|
bc_have = false, bc_scratch_fresh = false;
|
|
const bool second = bc_scratch2 != VK_NULL_HANDLE;
|
|
if (second) { // (hookshot-amd: the second scratch, 128 MB, with it)
|
|
dead.push_back(Dead{bc_last_value, bc_scratch2, bc_scratch2_alloc, {bc_scratch2_view}, {}});
|
|
bc_scratch2 = VK_NULL_HANDLE, bc_scratch2_alloc = nullptr, bc_scratch2_view = VK_NULL_HANDLE, bc_scratch2_fresh = false;
|
|
}
|
|
logf("vk: the BC3 scratch (%s) unused for %u s: given back", second ? "288 MB, both" : "160 MB", bc_idle_s());
|
|
}
|
|
|
|
/// The storage view of one level (every layer) the ASTC decoder writes
|
|
VkImageView astc_target_view(Texture* tex, uint32_t level) {
|
|
const uint64_t key = mix(0xA57C, level);
|
|
auto it = tex->views.find(key);
|
|
if (it != tex->views.end()) return it->second;
|
|
const VkImageView v = make_view(tex, VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT,
|
|
level, 1, 0, tex->layers, {}, VK_IMAGE_USAGE_STORAGE_BIT);
|
|
tex->views[key] = v;
|
|
return v;
|
|
}
|
|
|
|
// ---- uploads ----
|
|
struct LevelGeo {
|
|
uint64_t offset; // from the storage's start
|
|
uint32_t w, h, d; // texels
|
|
uint32_t tw, th, td; // blocks
|
|
unsigned tile_h, tile_d; // block height / depth (log2 GOBs)
|
|
uint64_t swizzled; // bytes of one layer of it in the storage
|
|
};
|
|
LevelGeo level_geo(const Texture& tex, uint32_t level) { // (level: the image's; the storage's is level + skip)
|
|
LevelGeo g{};
|
|
const uint32_t gl = level + tex.skip;
|
|
g.offset = host->tex_level_offset(&tex.d, gl);
|
|
const uint32_t w0 = std::max(1u, tex.d.width);
|
|
const uint32_t h0 = tex.type == VK_IMAGE_TYPE_1D ? 1 : std::max(1u, tex.d.height);
|
|
const uint32_t d0 = tex.type == VK_IMAGE_TYPE_3D ? std::max(1u, tex.d.depth) : 1;
|
|
g.w = std::max(1u, w0 >> gl);
|
|
g.h = std::max(1u, h0 >> gl);
|
|
g.d = std::max(1u, d0 >> gl);
|
|
g.tw = (g.w + tex.block_w - 1) / tex.block_w;
|
|
g.th = (g.h + tex.block_h - 1) / tex.block_h;
|
|
g.td = g.d;
|
|
host->tex_level_tiles(&tex.d, gl, &g.tile_h, &g.tile_d);
|
|
const uint64_t gobs_x = ((uint64_t)g.tw * tex.bpb + 63) / 64;
|
|
const uint64_t rows_y = (g.th + (8ull << g.tile_h) - 1) >> (3 + g.tile_h);
|
|
const uint64_t slices = (g.td + (1ull << g.tile_d) - 1) >> g.tile_d;
|
|
g.swizzled = gobs_x * rows_y * slices * (512ull << (g.tile_h + g.tile_d));
|
|
return g;
|
|
}
|
|
|
|
/// The guest bytes [gpu, gpu + n) in host memory: in place when contiguous, else copied into scratch
|
|
const uint8_t* guest_bytes(uint64_t gpu, uint64_t n, std::vector<uint8_t>& scratch) {
|
|
uint64_t avail = 0;
|
|
const uint8_t* p = host->gpu_ptr(gpu, &avail);
|
|
// (below 1 TB the GPU address is the guest's: the storage as it was at the submit being replayed, where the game has
|
|
// rewritten it since (vk_memory.cpp, "texture storage in the capture"))
|
|
if (p && avail >= n) return gpu < (1ull << 40) ? memory_as_submitted(gpu, p, n, scratch) : p;
|
|
if (n >= (32u << 20)) { // (a big sparse texture's pages, gathered on several threads: the course maps' 349 MB)
|
|
scratch.resize(n);
|
|
const uint32_t nt = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 6u);
|
|
const uint64_t per = ((n + nt - 1) / nt + 0xFFFF) & ~0xFFFFull;
|
|
std::vector<std::thread> pool;
|
|
auto part = [&](uint32_t t) {
|
|
const uint64_t a = (uint64_t)t * per, b = std::min(n, a + per);
|
|
for (uint64_t at = a; at < b; at += 0x10000)
|
|
if (!host->gpu_read(gpu + at, scratch.data() + at, std::min<uint64_t>(0x10000, b - at)))
|
|
std::memset(scratch.data() + at, 0, std::min<uint64_t>(0x10000, b - at));
|
|
};
|
|
for (uint32_t t = 1; t < nt; t++) pool.emplace_back(part, t);
|
|
part(0);
|
|
for (std::thread& t : pool) t.join();
|
|
return scratch.data();
|
|
}
|
|
scratch.assign(n, 0);
|
|
if (!host->gpu_read(gpu, scratch.data(), n)) {
|
|
for (uint64_t at = 0; at < n; at += 0x10000) host->gpu_read(gpu + at, scratch.data() + at, std::min<uint64_t>(0x10000, n - at));
|
|
}
|
|
return scratch.data();
|
|
}
|
|
|
|
void widen_rows(uint8_t* dst, const uint8_t* src, size_t texels, int from, int to) {
|
|
const int comp = from / 3;
|
|
for (size_t i = 0; i < texels; i++) {
|
|
std::memcpy(dst + i * to, src + i * from, from);
|
|
uint8_t* a = dst + i * to + from;
|
|
if (comp == 1) a[0] = 0xFF;
|
|
else if (comp == 2) a[0] = 0x00, a[1] = 0x3C; // (half 1.0; UNORM16 would want 0xFFFF: close enough for alpha)
|
|
else { const float one = 1.0f; std::memcpy(a, &one, 4); }
|
|
}
|
|
}
|
|
|
|
/// Byte ranges of a texture's storage, from its start (what the CPU wrote since its last upload)
|
|
using Ranges = std::vector<std::pair<uint64_t, uint64_t>>;
|
|
bool overlaps(const Ranges* dirty, uint64_t a, uint64_t b) {
|
|
if (!dirty) return true;
|
|
for (const auto& r : *dirty)
|
|
if (r.first < b && a < r.second) return true;
|
|
return false;
|
|
}
|
|
|
|
/// The view of one level the ASTC decoder writes: every layer, or one (layer)
|
|
VkImageView astc_level_view(Texture* tex, uint32_t level, uint32_t layer) {
|
|
if (layer == ~0u) return astc_target_view(tex, level);
|
|
const uint64_t key = mix(mix(0xA57D, level), layer);
|
|
auto it = tex->views.find(key);
|
|
if (it != tex->views.end()) return it->second;
|
|
const VkImageView v = make_view(tex, VK_IMAGE_VIEW_TYPE_2D_ARRAY, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT,
|
|
level, 1, layer, 1, {}, VK_IMAGE_USAGE_STORAGE_BIT);
|
|
tex->views[key] = v;
|
|
return v;
|
|
}
|
|
|
|
/// ASTC decoded on the GPU (Eden's decoder): the layers dirty overlaps (all of them without dirty), each layer's
|
|
/// levels from the first one the image keeps (a texture that leaves out its top levels stages only the tail of
|
|
/// every layer's chain: Eden's AssembleInto does the same)
|
|
/// (hookshot) why the last upload_astc_gpu on this thread failed, for the log of a BC3 image left to be tried again
|
|
thread_local const char* astc_fail = nullptr;
|
|
|
|
bool upload_astc_gpu(Texture& tex, uint64_t total, const Ranges* dirty) {
|
|
astc_fail = nullptr;
|
|
if (tex.type != VK_IMAGE_TYPE_2D || tex.samples != VK_SAMPLE_COUNT_1_BIT || !astc_ready()) {
|
|
astc_fail = "not a 2D image the GPU decoder takes";
|
|
return false;
|
|
}
|
|
const uint64_t layer_size = tex.layers > 1 ? tex.L.layer_size : std::max<uint64_t>(tex.L.layer_size, total);
|
|
const uint64_t tail0 = level_geo(tex, 0).offset;
|
|
if (!layer_size || tail0 >= layer_size || (tail0 & 15) || (layer_size & 15)) {
|
|
astc_fail = "its layout";
|
|
return false;
|
|
}
|
|
const uint64_t tail = layer_size - tail0;
|
|
std::vector<uint32_t> layers;
|
|
for (uint32_t layer = 0; layer < tex.layers; layer++)
|
|
if (overlaps(dirty, (uint64_t)layer * layer_size + tail0, (uint64_t)(layer + 1) * layer_size)) layers.push_back(layer);
|
|
if (layers.empty()) return true;
|
|
const bool all = layers.size() == tex.layers;
|
|
const Staging s = staging(tail * layers.size(), std::max<uint64_t>(256, dev.props.limits.minStorageBufferOffsetAlignment));
|
|
if (!s.buffer) {
|
|
astc_fail = "no staging memory for its ASTC blocks";
|
|
return false;
|
|
}
|
|
std::vector<uint8_t> scratch;
|
|
for (size_t k = 0; k < layers.size(); k++) {
|
|
const uint64_t at = (uint64_t)layers[k] * layer_size + tail0;
|
|
if (at >= total) continue;
|
|
const uint64_t n = std::min(tail, total - at);
|
|
std::memcpy(s.data + k * tail, guest_bytes(tex.gpu + at, n, scratch), n);
|
|
}
|
|
const uint64_t staged = tail * layers.size();
|
|
flush_mapped_range(s.allocation, s.offset, staged); // (encoded blocks read by the GPU decoder, not necessarily coherent)
|
|
st_bytes += (int64_t)staged;
|
|
VkCommandBuffer cmd = upload_cmd();
|
|
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, astc_pipe);
|
|
const uint64_t align = std::max<uint64_t>(dev.props.limits.minStorageBufferOffsetAlignment, 4);
|
|
if (tex.bc) { // (decoded into the scratch a run of consecutive layers at a time, re-encoded, copied in)
|
|
if (!bc_scratch_layout(cmd, tex.bc_resized)) {
|
|
astc_fail = "no memory for the BC3 scratch";
|
|
return false;
|
|
}
|
|
for (uint32_t level = 0; level < tex.levels; level++) {
|
|
const LevelGeo g = level_geo(tex, level);
|
|
const uint64_t off = g.offset - tail0;
|
|
if (g.offset < tail0 || off >= tail || (off % align)) continue;
|
|
const uint32_t iw = std::max(1u, tex.width >> level), ih = std::max(1u, tex.height >> level);
|
|
// (hookshot-amd: a resized BC3 image: the level decodes at the game's size, g.w x g.h, and is encoded at iw x ih)
|
|
AstcPush pc{};
|
|
pc.block_dims[0] = (uint32_t)tex.block_w, pc.block_dims[1] = (uint32_t)tex.block_h;
|
|
pc.layer_stride = (uint32_t)tail;
|
|
const uint32_t gobs_x = (g.tw * 16 + 63) >> 6;
|
|
pc.x_shift = 9 + g.tile_h + g.tile_d;
|
|
pc.block_size = gobs_x << pc.x_shift;
|
|
pc.block_height = g.tile_h;
|
|
pc.block_height_mask = (1u << g.tile_h) - 1;
|
|
pc.native_tiling = 1;
|
|
for (size_t k0 = 0; k0 < layers.size();) {
|
|
size_t k1 = k0 + 1;
|
|
// (one layer a run: with runs of up to kBcLayers, an array's layers past the run's first came out wrong in the
|
|
// smaller levels, and Hyrule Field's ground, which samples the terrain's material arrays (1365 x 1365 x 90, ASTC
|
|
// 8x8) at those levels from a distance, had its tiles outlined in dark lines (RX 6600: none with one layer a run,
|
|
// nor with the arrays kept as RGBA8). RX_VK_BC_LAYERS=<n>: up to n a run)
|
|
static const uint32_t run = getenv("RX_VK_BC_LAYERS") ? std::clamp<uint32_t>((uint32_t)atoi(getenv("RX_VK_BC_LAYERS")), 1u, kBcLayers) : 1u;
|
|
while (k1 < layers.size() && k1 - k0 < run && layers[k1] == layers[k1 - 1] + 1) k1++;
|
|
const uint32_t n = (uint32_t)(k1 - k0);
|
|
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, astc_pipe);
|
|
const uint64_t from = k0 * tail + off;
|
|
VkDescriptorBufferInfo bi{s.buffer, s.offset + from, staged - from};
|
|
VkDescriptorImageInfo ii{VK_NULL_HANDLE, bc_scratch_view, VK_IMAGE_LAYOUT_GENERAL};
|
|
VkWriteDescriptorSet w[2]{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}};
|
|
w[0].dstBinding = 0, w[0].descriptorCount = 1, w[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
|
w[0].pBufferInfo = &bi;
|
|
w[1].dstBinding = 1, w[1].descriptorCount = 1, w[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
w[1].pImageInfo = ⅈ
|
|
vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, astc_layout, 0, 2, w);
|
|
vkCmdPushConstants(cmd, astc_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
vkCmdDispatch(cmd, (g.tw + 7) / 8, (g.th + 7) / 8, n);
|
|
if (tex.bc_resized)
|
|
bc_encode_resized(cmd, g.w, g.h, iw, ih, n, tex, level, layers[k0]);
|
|
else
|
|
bc_encode_into(cmd, iw, ih, n, tex, level, layers[k0], 0, 0);
|
|
k0 = k1;
|
|
}
|
|
}
|
|
st_astc++;
|
|
return true;
|
|
}
|
|
for (uint32_t level = 0; level < tex.levels; level++) {
|
|
const LevelGeo g = level_geo(tex, level);
|
|
const uint64_t off = g.offset - tail0;
|
|
if (g.offset < tail0 || off >= tail || (off % align)) continue;
|
|
AstcPush pc{};
|
|
pc.block_dims[0] = (uint32_t)tex.block_w, pc.block_dims[1] = (uint32_t)tex.block_h;
|
|
pc.layer_stride = (uint32_t)tail;
|
|
const uint32_t gobs_x = (g.tw * 16 + 63) >> 6;
|
|
pc.x_shift = 9 + g.tile_h + g.tile_d;
|
|
pc.block_size = gobs_x << pc.x_shift;
|
|
pc.block_height = g.tile_h;
|
|
pc.block_height_mask = (1u << g.tile_h) - 1;
|
|
pc.native_tiling = 1;
|
|
for (size_t k = 0; k < (all ? 1 : layers.size()); k++) {
|
|
const VkImageView view = astc_level_view(&tex, level, all ? ~0u : layers[k]);
|
|
if (!view) {
|
|
astc_fail = "no view of a level";
|
|
return false;
|
|
}
|
|
const uint64_t from = (all ? 0 : k * tail) + off;
|
|
VkDescriptorBufferInfo bi{s.buffer, s.offset + from, staged - from};
|
|
VkDescriptorImageInfo ii{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL};
|
|
VkWriteDescriptorSet w[2]{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}};
|
|
w[0].dstBinding = 0, w[0].descriptorCount = 1, w[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
|
w[0].pBufferInfo = &bi;
|
|
w[1].dstBinding = 1, w[1].descriptorCount = 1, w[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
w[1].pImageInfo = ⅈ
|
|
vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, astc_layout, 0, 2, w);
|
|
vkCmdPushConstants(cmd, astc_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
vkCmdDispatch(cmd, (g.tw + 7) / 8, (g.th + 7) / 8, all ? tex.layers : 1);
|
|
}
|
|
}
|
|
st_astc++;
|
|
return true;
|
|
}
|
|
|
|
// ---- big block-linear textures unswizzled on the GPU (Eden's block_linear_unswizzle_2d.comp, GPL, with the Switch
|
|
// 2 tiling): the shader reads the storage where it is, in its imported pool, and writes the image. The course maps
|
|
// the title loads at each scene change (six of 349 MB) were copied into staging and unswizzled on the CPU: ~200 ms
|
|
// of the replay thread. RX_VK_GPU_UNSWIZZLE=0: on the CPU. ----
|
|
struct UnswizzlePush {
|
|
uint32_t origin[3], pad0;
|
|
int32_t destination[3];
|
|
uint32_t bytes_per_block_log2, layer_stride, block_size, x_shift, block_height, block_height_mask, native_tiling;
|
|
};
|
|
VkDescriptorSetLayout unsw_set = VK_NULL_HANDLE;
|
|
VkPipelineLayout unsw_layout = VK_NULL_HANDLE;
|
|
VkPipeline unsw_pipe = VK_NULL_HANDLE;
|
|
bool unswizzle_ready() {
|
|
static int state = 0;
|
|
if (state) return state > 0;
|
|
state = -1;
|
|
if (!dev.push_descriptor || !vkCmdPushDescriptorSetKHR) return false;
|
|
if (getenv("RX_VK_GPU_UNSWIZZLE") && !atoi(getenv("RX_VK_GPU_UNSWIZZLE"))) return false;
|
|
const VkDescriptorSetLayoutBinding b[2] = {
|
|
{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
|
|
{1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}};
|
|
VkDescriptorSetLayoutCreateInfo li{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
|
|
li.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
|
|
li.bindingCount = 2;
|
|
li.pBindings = b;
|
|
if (vkCreateDescriptorSetLayout(dev.device, &li, nullptr, &unsw_set) != VK_SUCCESS) return false;
|
|
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(UnswizzlePush)};
|
|
VkPipelineLayoutCreateInfo pi{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
|
|
pi.setLayoutCount = 1;
|
|
pi.pSetLayouts = &unsw_set;
|
|
pi.pushConstantRangeCount = 1;
|
|
pi.pPushConstantRanges = ≺
|
|
if (vkCreatePipelineLayout(dev.device, &pi, nullptr, &unsw_layout) != VK_SUCCESS) return false;
|
|
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
|
mi.codeSize = sizeof BLOCK_LINEAR_UNSWIZZLE_2D_COMP_SPV;
|
|
mi.pCode = BLOCK_LINEAR_UNSWIZZLE_2D_COMP_SPV;
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) != VK_SUCCESS) return false;
|
|
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
|
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
|
|
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
|
ci.stage.module = m;
|
|
ci.stage.pName = "main";
|
|
ci.layout = unsw_layout;
|
|
const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &unsw_pipe);
|
|
vkDestroyShaderModule(dev.device, m, nullptr);
|
|
if (r != VK_SUCCESS) return false;
|
|
state = 1;
|
|
logf("vk: big block-linear textures are unswizzled on the GPU from their pools (RX_VK_GPU_UNSWIZZLE=0: the CPU)");
|
|
return true;
|
|
}
|
|
|
|
/// The whole texture (every level and layer dirty overlaps) unswizzled on the GPU; false: not a texture for it
|
|
bool upload_unswizzle_gpu(Texture& tex, uint64_t total, const Ranges* dirty) {
|
|
if (tex.type != VK_IMAGE_TYPE_2D || tex.astc || tex.bc || !(tex.usage & VK_IMAGE_USAGE_STORAGE_BIT) ||
|
|
is_bc(tex.format) || is_astc(tex.format) || is_depth_format(tex.format) || nvn_tex_is_linear(&tex.d))
|
|
return false;
|
|
if (total < (8u << 20) || tex.block_w != 1 || tex.block_h != 1) return false;
|
|
int widen_from = 0;
|
|
widened(tex.d.format ? vk_format(tex.nvn_format).format : tex.format, &widen_from);
|
|
if (widen_from) return false;
|
|
VkFormat vf;
|
|
switch (tex.bpb) {
|
|
case 1: vf = VK_FORMAT_R8_UINT; break;
|
|
case 2: vf = VK_FORMAT_R16_UINT; break;
|
|
case 4: vf = VK_FORMAT_R32_UINT; break;
|
|
case 8: vf = VK_FORMAT_R32G32_UINT; break;
|
|
case 16: vf = VK_FORMAT_R32G32B32A32_UINT; break;
|
|
default: return false;
|
|
}
|
|
if (!unswizzle_ready()) return false;
|
|
const BufferRef src = gpu_buffer(tex.gpu, total);
|
|
const uint64_t align = std::max<uint64_t>(dev.props.limits.minStorageBufferOffsetAlignment, 16);
|
|
if (!src || (src.offset % align)) return false;
|
|
for (uint32_t level = 0; level < tex.levels; level++) { // (every level must be possible before any is done)
|
|
const LevelGeo g = level_geo(tex, level);
|
|
if (g.td != 1 || g.tile_d || (g.offset % align) || g.offset >= total) return false;
|
|
// (the shader reads whole GOB-block rows: a level whose last one runs past the storage, which the CPU path
|
|
// reads from a zero-padded copy, would read past the pool and fault the device)
|
|
if (g.offset + (uint64_t)(tex.layers - 1) * tex.L.layer_size + g.swizzled > total) return false;
|
|
}
|
|
VkCommandBuffer cmd = upload_cmd();
|
|
full_barrier(cmd);
|
|
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, unsw_pipe);
|
|
for (uint32_t level = 0; level < tex.levels; level++) {
|
|
const LevelGeo g = level_geo(tex, level);
|
|
bool any = false;
|
|
for (uint32_t layer = 0; layer < tex.layers && !any; layer++)
|
|
any = overlaps(dirty, g.offset + (uint64_t)layer * tex.L.layer_size,
|
|
g.offset + (uint64_t)layer * tex.L.layer_size + g.swizzled);
|
|
if (!any) continue;
|
|
const uint64_t key = mix(0x5B1Cu, mix(level, (uint64_t)vf));
|
|
VkImageView view;
|
|
if (auto it = tex.views.find(key); it != tex.views.end()) view = it->second;
|
|
else view = tex.views[key] = make_view(&tex, VK_IMAGE_VIEW_TYPE_2D_ARRAY, vf, VK_IMAGE_ASPECT_COLOR_BIT, level, 1, 0,
|
|
tex.layers, {}, VK_IMAGE_USAGE_STORAGE_BIT);
|
|
if (!view) return false;
|
|
UnswizzlePush pc{};
|
|
pc.bytes_per_block_log2 = (uint32_t)std::countr_zero((uint32_t)tex.bpb);
|
|
pc.layer_stride = (uint32_t)tex.L.layer_size;
|
|
const uint32_t gobs_x = (uint32_t)(((uint64_t)g.tw * tex.bpb + 63) >> 6);
|
|
pc.x_shift = 9 + g.tile_h;
|
|
pc.block_size = gobs_x << pc.x_shift;
|
|
pc.block_height = g.tile_h;
|
|
pc.block_height_mask = (1u << g.tile_h) - 1;
|
|
pc.native_tiling = 1;
|
|
VkDescriptorBufferInfo bi{src.buffer, src.offset + g.offset, total - g.offset};
|
|
VkDescriptorImageInfo ii{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL};
|
|
VkWriteDescriptorSet w[2]{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}};
|
|
w[0].dstBinding = 0, w[0].descriptorCount = 1, w[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
|
w[0].pBufferInfo = &bi;
|
|
w[1].dstBinding = 1, w[1].descriptorCount = 1, w[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
|
w[1].pImageInfo = ⅈ
|
|
vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, unsw_layout, 0, 2, w);
|
|
vkCmdPushConstants(cmd, unsw_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, &pc);
|
|
checkpoint_mark(cmd, tex.gpu, 'g');
|
|
vkCmdDispatch(cmd, (g.tw + 31) / 32, (g.th + 31) / 32, tex.layers);
|
|
st_bytes += (int64_t)g.swizzled * tex.layers;
|
|
}
|
|
full_barrier(cmd);
|
|
descriptors_disturbed();
|
|
return true;
|
|
}
|
|
|
|
/// The texture from guest memory into its image: all of it, or (dirty) the layers, levels and block rows the CPU
|
|
/// wrote. Block-linear levels are rows of blocks a GOB-block high, each self-contained: a written page re-uploads
|
|
/// the block rows it falls in.
|
|
uint64_t st_made; // (textures made)
|
|
int64_t st_unswizzle_bytes, st_unswizzle_ns; // (the CPU's unswizzling)
|
|
std::map<uint64_t, std::pair<uint64_t, uint32_t>> gpu_owned; // GPU address -> (end, memory generation then)
|
|
thread_local uint64_t tl_watch_prog;
|
|
/// (RX_VK_VERIFY_DIM=WxH: the guest memory of textures that size, hashed when uploaded and every 300 frames: a change
|
|
/// the write tracking did not report is logged as a missed write)
|
|
std::pair<uint32_t, uint32_t> verify_dim() {
|
|
static const std::pair<uint32_t, uint32_t> d = [] {
|
|
unsigned w = 0, h = 0;
|
|
if (const char* e = getenv("RX_VK_VERIFY_DIM")) sscanf(e, "%ux%u", &w, &h);
|
|
return std::pair<uint32_t, uint32_t>{w, h};
|
|
}();
|
|
return d;
|
|
}
|
|
std::unordered_map<uint32_t, std::vector<uint8_t>> verify_snap; // (each verified texture's memory when uploaded)
|
|
uint64_t guest_hash(const Texture& tex, bool snapshot = false) {
|
|
std::vector<uint8_t> b(tex.guest_size);
|
|
if (!host->gpu_read(tex.gpu, b.data(), b.size())) return 0;
|
|
uint64_t h = 1469598103934665603ull;
|
|
for (uint8_t c : b) h = (h ^ c) * 1099511628211ull;
|
|
if (snapshot) verify_snap[tex.id] = std::move(b);
|
|
return h;
|
|
}
|
|
/// (a missed write: which 4 KB pages changed since the upload, and the CPU's next writes to them watched)
|
|
void verify_report(const Texture& tex) {
|
|
const auto it = verify_snap.find(tex.id);
|
|
if (it == verify_snap.end()) return;
|
|
std::vector<uint8_t> now(tex.guest_size);
|
|
if (!host->gpu_read(tex.gpu, now.data(), now.size())) return;
|
|
std::string runs;
|
|
int nruns = 0, watched = 0;
|
|
uint64_t changed = 0;
|
|
for (uint64_t off = 0; off < tex.guest_size; off += 4096) {
|
|
const uint64_t n = std::min<uint64_t>(4096, tex.guest_size - off);
|
|
if (!std::memcmp(now.data() + off, it->second.data() + off, n)) continue;
|
|
changed++;
|
|
if (nruns < 12) {
|
|
char b[48];
|
|
snprintf(b, sizeof b, " +%#llx", (unsigned long long)off);
|
|
runs += b, nruns++;
|
|
}
|
|
static int watches;
|
|
if (host->watch_guest_writes && watches < 4 && watched < 2) {
|
|
const uint64_t g = host->gpu_guest(tex.gpu + off);
|
|
if (g) host->watch_guest_writes(g, 4096), watches++, watched++;
|
|
}
|
|
}
|
|
logf("vk: verify: texture %#llx: %llu pages changed since the upload:%s%s", (unsigned long long)tex.gpu,
|
|
(unsigned long long)changed, runs.c_str(), watched ? " (the next CPU writes to two of them watched)" : "");
|
|
{ // (the first changed page as the write tracking has it: ever written, written since the upload, and when)
|
|
for (uint64_t off = 0; off < tex.guest_size; off += 4096) {
|
|
const uint64_t n = std::min<uint64_t>(4096, tex.guest_size - off);
|
|
if (!std::memcmp(now.data() + off, it->second.data() + off, n)) continue;
|
|
const uint64_t g = host->gpu_guest(tex.gpu + off);
|
|
std::vector<std::pair<uint64_t, uint64_t>> ever, since;
|
|
memory_written_runs(g, 4096, 0, ever);
|
|
memory_written_runs(g, 4096, tex.upload_gen, since);
|
|
logf("vk: verify: its page +%#llx (guest %#llx): stamped ever %d, since the upload (gen %u) %d; generation now %u",
|
|
(unsigned long long)off, (unsigned long long)g, (int)!ever.empty(), tex.upload_gen, (int)!since.empty(),
|
|
memory_generation());
|
|
break;
|
|
}
|
|
}
|
|
it->second = std::move(now);
|
|
} // (the draw or dispatch the replay is at, for watch_tex)
|
|
/// (RX_VK_WATCH_TEX=addr[,addr...]: what happens to the textures over those GPU addresses, 600 lines)
|
|
void watch_tex(const Texture* tex, const char* what) {
|
|
static std::vector<uint64_t> addrs = [] {
|
|
std::vector<uint64_t> v;
|
|
if (const char* e = getenv("RX_VK_WATCH_TEX"))
|
|
for (const char* p = e; *p;) {
|
|
char* end;
|
|
const uint64_t a = strtoull(p, &end, 16);
|
|
if (end == p) break;
|
|
v.push_back(a);
|
|
p = *end ? end + 1 : end;
|
|
}
|
|
return v;
|
|
}();
|
|
static const std::pair<uint32_t, uint32_t> dim = [] {
|
|
unsigned w = 0, h = 0;
|
|
if (const char* e = getenv("RX_VK_WATCH_DIM")) sscanf(e, "%ux%u", &w, &h);
|
|
return std::pair<uint32_t, uint32_t>{w, h};
|
|
}();
|
|
static const std::pair<uint64_t, uint64_t> range = [] { // (RX_VK_WATCH_RANGE=<lo>:<hi>: textures overlapping it)
|
|
unsigned long long a = 0, b = 0;
|
|
if (const char* e = getenv("RX_VK_WATCH_RANGE")) sscanf(e, "%llx:%llx", &a, &b);
|
|
return std::pair<uint64_t, uint64_t>{a, b};
|
|
}();
|
|
if (tex && range.second && tex->gpu < range.second && tex->gpu + std::max<uint64_t>(tex->guest_size, 1) > range.first) {
|
|
static int rlogged;
|
|
if (rlogged++ < 600)
|
|
logf("vk: watch range frame %llu prog %016llx %s #%u %#llx %ux%ux%u x%u levels %u NVN %#x target %d size %llu%s%s",
|
|
(unsigned long long)frame_no, (unsigned long long)tl_watch_prog, what, tex->id, (unsigned long long)tex->gpu,
|
|
tex->width, tex->height, tex->depth, tex->layers, tex->levels, tex->nvn_format, tex->d.target,
|
|
(unsigned long long)tex->guest_size, tex->gpu_written ? " rendered" : "", tex->uploaded ? " uploaded" : "");
|
|
}
|
|
if ((addrs.empty() && !dim.first) || !tex) return;
|
|
{ // (RX_VK_DUMP_ON_WATCH=<frames>: a watched texture made after boot dumps the frames after it, once)
|
|
static const int on_watch = getenv("RX_VK_DUMP_ON_WATCH") ? atoi(getenv("RX_VK_DUMP_ON_WATCH")) : 0;
|
|
static bool requested = false;
|
|
if (on_watch && !requested && frame_no > 1000 && !strcmp(what, "made")) {
|
|
for (const uint64_t a : addrs)
|
|
if (a == tex->gpu) {
|
|
requested = true;
|
|
dump_request((int64_t)frame_no + 1, on_watch);
|
|
logf("vk: watch: texture %#llx made at frame %llu: the next %d frames dumped", (unsigned long long)a,
|
|
(unsigned long long)frame_no, on_watch);
|
|
}
|
|
}
|
|
}
|
|
static int logged;
|
|
static std::unordered_map<uint32_t, uint64_t> told; // (a texture's events once a frame)
|
|
const bool by_dim = dim.first && tex->width == dim.first && tex->height == dim.second;
|
|
if (by_dim && !strcmp(what, "written")) { // ("written" once a frame; the rest each time)
|
|
uint64_t& f = told[tex->id];
|
|
if (f == frame_no || logged >= 20000) return;
|
|
f = frame_no;
|
|
}
|
|
const uint64_t hi = tex->gpu + std::max<uint64_t>(tex->guest_size, 1);
|
|
for (const uint64_t a : by_dim ? std::vector<uint64_t>{tex->gpu} : addrs) {
|
|
if (a < tex->gpu || a >= hi || logged >= (by_dim ? 20000 : 600)) continue;
|
|
logged++;
|
|
// (Triforce) one about to be uploaded from the game's memory: how much of that memory is not zero. A texture the game
|
|
// expects the GPU's texels in (a tile's normal map made again from its tile cache) showed 0 here before bake_writeback_step
|
|
char guest[64] = "";
|
|
if (by_dim && !strcmp(what, "sampled, uploaded") && tex->guest_size && tex->guest_size <= (4u << 20)) {
|
|
static thread_local std::vector<uint8_t> mem;
|
|
mem.resize((size_t)tex->guest_size);
|
|
uint64_t nz = 0;
|
|
if (host->gpu_read(tex->gpu, mem.data(), mem.size()))
|
|
for (uint8_t c : mem) nz += c != 0;
|
|
snprintf(guest, sizeof guest, " memory: %llu of %llu bytes not zero", (unsigned long long)nz,
|
|
(unsigned long long)tex->guest_size);
|
|
}
|
|
logf("vk: watch tex frame %llu prog %016llx %s #%u %#llx %ux%ux%u x%u levels %u NVN %#x layout %#x class %#x target %d "
|
|
"size %llu%s%s%s", (unsigned long long)frame_no, (unsigned long long)tl_watch_prog, what, tex->id, (unsigned long long)tex->gpu, tex->width,
|
|
tex->height, tex->depth, tex->layers, tex->levels, tex->nvn_format, tex->key_layout, tex->key_cls,
|
|
tex->d.target, (unsigned long long)tex->guest_size, tex->gpu_written ? " rendered" : "",
|
|
tex->uploaded ? " uploaded" : "", guest);
|
|
return;
|
|
}
|
|
}
|
|
/// Whether [gpu, gpu + n) is storage the GPU's images rendered last (covered by ranges render targets took), and the
|
|
/// generation from which the CPU's writes would count (0: not the GPU's)
|
|
/// (hookshot) The ranges can overlap (texture_rendered drops only the ones inside a new range), so the bytes they cover are
|
|
/// counted once each: summed, two overlapping ranges counted their overlap twice and a range with a gap the GPU never drew
|
|
/// read as covered, and a texture over it was never uploaded
|
|
uint32_t gpu_owned_since(uint64_t gpu, uint64_t n) {
|
|
uint64_t covered = 0, reach = gpu; // (reach: the end of what is counted so far; the ranges come in address order)
|
|
uint32_t since = UINT32_MAX;
|
|
auto it = gpu_owned.upper_bound(gpu);
|
|
if (it != gpu_owned.begin()) --it;
|
|
for (; it != gpu_owned.end() && it->first < gpu + n; ++it) {
|
|
const uint64_t a = std::max(it->first, gpu), b = std::min(it->second.first, gpu + n);
|
|
if (b <= a) continue;
|
|
since = std::min(since, it->second.second);
|
|
if (b > std::max(a, reach)) covered += b - std::max(a, reach), reach = b;
|
|
}
|
|
return covered >= n && since != UINT32_MAX ? std::max(since, 1u) : 0;
|
|
}
|
|
|
|
// (hookshot, smoothness) What a frame's uploads cost by the way they went: when a frame's uploads take over 50 ms (the stutters at
|
|
// area loads and in walking, 180-360 ms frames of 250-450 MB: Discord and the frame trace, 2026-11-09), one log line of the routes
|
|
// (ASTC on the GPU, unswizzled on the GPU, on the CPU), their count, MB and ms, and the slowest textures
|
|
enum UpRoute { UP_ASTC, UP_GPU, UP_CPU, UP_COUNT };
|
|
thread_local int up_route = UP_CPU;
|
|
struct UpStat { int64_t n = 0, bytes = 0, ns = 0; };
|
|
UpStat up_frame[UP_COUNT];
|
|
struct UpSlow { int64_t ns; uint32_t w, h, layers, nvn; int route; uint64_t bytes; };
|
|
std::vector<UpSlow> up_slowest;
|
|
std::mutex up_mu; // (uploads may come from a parallel replay's threads)
|
|
std::atomic<int64_t> frame_upload_ns{0}; // (this frame's time in upload(), every thread's)
|
|
std::atomic<int> frame_astc_admitted{0}; // (Triforce: this frame's new ASTC textures asked about so far: upload_defer)
|
|
std::atomic<int64_t> st_deferred{0}, st_deferred_frame{0};
|
|
void upload_frame_report() {
|
|
std::lock_guard lk(up_mu);
|
|
int64_t ns = 0;
|
|
for (const UpStat& u : up_frame) ns += u.ns;
|
|
static int logged;
|
|
if (ns >= 50000000 && logged < 300) {
|
|
logged++;
|
|
static const char* names[UP_COUNT] = {"ASTC on the GPU", "unswizzled on the GPU", "on the CPU"};
|
|
char line[900];
|
|
int k = snprintf(line, sizeof line, "vk: a frame's uploads took %.0f ms:", ns / 1e6);
|
|
for (int r = 0; r < UP_COUNT; r++)
|
|
if (up_frame[r].n)
|
|
k += snprintf(line + k, sizeof line - k, " %s %lld (%.1f MB, %.0f ms);", names[r], (long long)up_frame[r].n,
|
|
up_frame[r].bytes / 1048576.0, up_frame[r].ns / 1e6);
|
|
std::sort(up_slowest.begin(), up_slowest.end(), [](const UpSlow& a, const UpSlow& b) { return a.ns > b.ns; });
|
|
for (size_t i = 0; i < up_slowest.size() && i < 3 && k < (int)sizeof line - 80; i++)
|
|
k += snprintf(line + k, sizeof line - k, " [%ux%u x%u NVN %#x %s %.1f MB %.0f ms]", up_slowest[i].w, up_slowest[i].h,
|
|
up_slowest[i].layers, up_slowest[i].nvn, names[up_slowest[i].route], up_slowest[i].bytes / 1048576.0,
|
|
up_slowest[i].ns / 1e6);
|
|
logf("%s", line);
|
|
}
|
|
for (UpStat& u : up_frame) u = {};
|
|
up_slowest.clear();
|
|
}
|
|
bool upload_inner(Texture& tex, const Ranges* dirty);
|
|
bool upload(Texture& tex, const Ranges* dirty) {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
up_route = UP_CPU;
|
|
const bool ok = upload_inner(tex, dirty);
|
|
const int64_t ns = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
|
|
frame_upload_ns.fetch_add(ns, std::memory_order_relaxed);
|
|
std::lock_guard lk(up_mu);
|
|
UpStat& u = up_frame[up_route];
|
|
u.n++, u.bytes += (int64_t)tex.guest_size, u.ns += ns;
|
|
if (ns > 2000000) {
|
|
up_slowest.push_back({ns, tex.width, tex.height, tex.layers, tex.nvn_format, up_route, tex.guest_size});
|
|
if (up_slowest.size() > 64) {
|
|
std::sort(up_slowest.begin(), up_slowest.end(), [](const UpSlow& a, const UpSlow& b) { return a.ns > b.ns; });
|
|
up_slowest.resize(8);
|
|
}
|
|
}
|
|
return ok;
|
|
}
|
|
/// (hookshot) A big texture whose whole storage is zero: the image cleared to zero on the GPU, not unswizzled on the CPU. All-zero
|
|
/// bits are 0 in every format this path takes (not ASTC or BC, not one widened on the way), so the clear is what the unswizzle
|
|
/// would have made. The game's shadow map (12288 x 12288 D16, 288 MB, its storage fresh and zero) took 73 ms of the replay
|
|
/// thread at a scene change; the check reads the storage once, on several threads, and stops at the first byte that is not 0.
|
|
/// RX_VK_ZERO_UPLOAD=0: off
|
|
bool upload_all_zero(Texture& tex, uint64_t total, const Ranges* dirty, bool widened_format) {
|
|
static const bool off = getenv("RX_VK_ZERO_UPLOAD") && atoi(getenv("RX_VK_ZERO_UPLOAD")) == 0;
|
|
if (off || total < (32ull << 20) || tex.astc || tex.bc || tex.sparse || widened_format || !tex.image ||
|
|
tex.type == VK_IMAGE_TYPE_3D || is_bc(tex.format) || is_astc(tex.format))
|
|
return false;
|
|
if (dirty) { // (only an upload of the whole storage: a part of it says nothing of the rest)
|
|
uint64_t covered = 0;
|
|
for (const auto& r : *dirty) covered += std::min(r.second, total) - std::min(r.first, total);
|
|
if (covered < total) return false;
|
|
}
|
|
// (live memory, as the GPU unswizzle reads it: not a storage the capture kept as it was at its submit, which the CPU path reads)
|
|
if (tex.gpu >= (1ull << 40) || memory_has_shadow(tex.gpu, total)) return false;
|
|
uint64_t avail = 0;
|
|
const uint8_t* p = host->gpu_ptr(tex.gpu, &avail);
|
|
if (!p || avail < total) return false;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::atomic<bool> found{false};
|
|
const uint32_t nthreads = std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u);
|
|
const uint64_t per = ((total / nthreads) + 4095) & ~4095ull;
|
|
auto part = [&](uint32_t t) {
|
|
const uint64_t a = std::min<uint64_t>(total, (uint64_t)t * per), b = std::min<uint64_t>(total, a + per);
|
|
for (uint64_t at = a; at < b && !found.load(std::memory_order_relaxed); at += 65536) {
|
|
const uint64_t end = std::min<uint64_t>(b, at + 65536);
|
|
uint64_t acc = 0;
|
|
uint64_t k = at;
|
|
for (; k + 8 <= end; k += 8) {
|
|
uint64_t w;
|
|
std::memcpy(&w, p + k, 8);
|
|
acc |= w;
|
|
}
|
|
for (; k < end; k++) acc |= p[k];
|
|
if (acc) found.store(true, std::memory_order_relaxed);
|
|
}
|
|
};
|
|
if (nthreads > 1) {
|
|
std::vector<std::thread> th;
|
|
for (uint32_t t = 1; t < nthreads; t++) th.emplace_back(part, t);
|
|
part(0);
|
|
for (auto& x : th) x.join();
|
|
} else {
|
|
part(0);
|
|
}
|
|
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
|
static std::atomic<int> told{0};
|
|
if (found.load()) {
|
|
if (told.fetch_add(1) < 4)
|
|
logf("vk: upload: %ux%u NVN %#x (%llu MB) is not all zero (checked in %.1f ms): unswizzled as before", tex.width,
|
|
tex.height, tex.nvn_format, (unsigned long long)(total >> 20), ms);
|
|
return false;
|
|
}
|
|
texture_resident_flush(&tex);
|
|
checkpoint_mark(upload_cmd(), tex.gpu, 'u');
|
|
const VkImageSubresourceRange range{tex.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
|
|
if (tex.aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
|
|
const VkClearDepthStencilValue z{0.0f, 0};
|
|
vkCmdClearDepthStencilImage(upload_cmd(), tex.image, VK_IMAGE_LAYOUT_GENERAL, &z, 1, &range);
|
|
} else {
|
|
const VkClearColorValue z{};
|
|
vkCmdClearColorImage(upload_cmd(), tex.image, VK_IMAGE_LAYOUT_GENERAL, &z, 1, &range);
|
|
}
|
|
if (told.fetch_add(1) < 4)
|
|
logf("vk: upload: %ux%u NVN %#x (%llu MB) is all zero: cleared on the GPU (checked in %.1f ms)", tex.width, tex.height,
|
|
tex.nvn_format, (unsigned long long)(total >> 20), ms);
|
|
return true;
|
|
}
|
|
bool upload_inner(Texture& tex, const Ranges* dirty) {
|
|
PROF(PROF_UPLOAD);
|
|
if (!tex.gpu || tex.samples != VK_SAMPLE_COUNT_1_BIT) return false;
|
|
const uint64_t total = tex.guest_size;
|
|
if (!total || (total > (1ull << 31) && !tex.sparse)) return false; // (a sparse image: only what holds texels)
|
|
// (depth-stencil: the depth aspect only; packed D24S8 storage is not copied into it as is: such targets are
|
|
// rendered, not uploaded)
|
|
if (tex.aspect == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) return false;
|
|
st_uploads++;
|
|
{ // RX_VK_LOG_UPLOADS=<present>: the uploads from that present on (200 of them; what churns)
|
|
static const int64_t from = getenv("RX_VK_LOG_UPLOADS") ? atoll(getenv("RX_VK_LOG_UPLOADS")) : -1;
|
|
static int logged;
|
|
if (from >= 0 && (int64_t)frame_no >= from && logged < 200) {
|
|
logged++;
|
|
uint64_t part = 0;
|
|
if (dirty)
|
|
for (const auto& r : *dirty) part += r.second - r.first;
|
|
logf("vk: upload #%u %#llx %ux%ux%u x%u levels %u NVN %#x%s: %llu KB of %llu KB (%s), made at present %llu",
|
|
tex.id, (unsigned long long)tex.gpu, tex.width, tex.height, tex.depth, tex.layers, tex.levels,
|
|
tex.nvn_format, tex.astc ? " ASTC" : "", (unsigned long long)((dirty ? part : total) >> 10),
|
|
(unsigned long long)(total >> 10), !tex.uploaded ? "new" : dirty ? "written" : "whole",
|
|
(unsigned long long)tex.first_seen);
|
|
}
|
|
}
|
|
if (tex.astc && (up_route = UP_ASTC, upload_astc_gpu(tex, total, dirty))) return true;
|
|
up_route = UP_CPU;
|
|
if (tex.bc) return false; // (no CPU path into BC3)
|
|
// (the GPU's unswizzle reads guest memory as it is: not where the capture kept the storage as it was at its submit)
|
|
if (!(tex.gpu < (1ull << 40) && memory_has_shadow(tex.gpu, total)) && upload_unswizzle_gpu(tex, total, dirty))
|
|
return up_route = UP_GPU, true;
|
|
const bool linear = nvn_tex_is_linear(&tex.d);
|
|
int widen_from = 0;
|
|
widened(tex.d.format ? vk_format(tex.nvn_format).format : tex.format, &widen_from);
|
|
if (upload_all_zero(tex, total, dirty, widen_from != 0)) return true;
|
|
const int src_bpb = tex.bpb;
|
|
const int out_bpb = tex.astc ? 4 : widen_from ? (widen_from == 3 ? 4 : widen_from == 6 ? 8 : 16) : src_bpb;
|
|
const uint32_t copy_layers = tex.type == VK_IMAGE_TYPE_3D ? 1 : tex.layers;
|
|
const VkImageAspectFlags aspect = tex.aspect;
|
|
std::vector<VkBufferImageCopy> copies;
|
|
std::vector<uint8_t> scratch, lin;
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
for (uint32_t level = 0; level < tex.levels; level++) {
|
|
const LevelGeo g = level_geo(tex, level);
|
|
for (uint32_t layer = 0; layer < copy_layers; layer++) {
|
|
const uint64_t src_off = g.offset + (uint64_t)layer * tex.L.layer_size;
|
|
if (src_off >= total) continue;
|
|
const uint32_t stride = linear ? (tex.d.stride ? tex.d.stride : host->tex_linear_stride(&tex.d)) : 0;
|
|
const uint64_t span = linear ? (uint64_t)stride * g.th : std::max<uint64_t>(g.swizzled, 1);
|
|
if (!overlaps(dirty, src_off, src_off + span)) continue;
|
|
// the block rows to upload: all of them, or those a written page falls in (2D block-linear levels)
|
|
const uint32_t rows_per = 8u << g.tile_h; // (block rows of a GOB block)
|
|
const uint32_t nrows = (g.th + rows_per - 1) / rows_per;
|
|
const uint64_t gobs_x = ((uint64_t)g.tw * src_bpb + 63) / 64;
|
|
const uint64_t row_bytes = gobs_x * (512ull << (g.tile_h + g.tile_d));
|
|
std::vector<std::pair<uint32_t, uint32_t>> runs; // (first row, rows)
|
|
if (!dirty || linear || tex.astc || widen_from || g.td > 1 || g.tile_d) {
|
|
runs.push_back({0, nrows});
|
|
} else {
|
|
std::vector<uint8_t> mark(nrows, 0);
|
|
for (const auto& r : *dirty) {
|
|
const uint64_t lo = std::max(r.first, src_off), hi = std::min(r.second, src_off + span);
|
|
if (lo >= hi) continue;
|
|
const uint64_t r0 = (lo - src_off) / row_bytes, r1 = std::min<uint64_t>((hi - 1 - src_off) / row_bytes, nrows - 1);
|
|
for (uint64_t k = r0; k <= r1; k++) mark[k] = 1;
|
|
}
|
|
for (uint32_t k = 0; k < nrows;) {
|
|
if (!mark[k]) { k++; continue; }
|
|
uint32_t e = k;
|
|
while (e < nrows && mark[e]) e++;
|
|
runs.push_back({k, e - k});
|
|
k = e;
|
|
}
|
|
}
|
|
for (const auto& [row0, nrow] : runs) {
|
|
const uint32_t y0 = row0 * rows_per; // (blocks)
|
|
const uint32_t th = std::min(g.th - y0, nrow * rows_per); // (blocks)
|
|
const uint64_t block_bytes = (uint64_t)g.tw * th * g.td * src_bpb;
|
|
const uint32_t px_h = std::min(g.h - y0 * (uint32_t)tex.block_h, th * (uint32_t)tex.block_h);
|
|
const uint64_t out_bytes = tex.astc ? (uint64_t)g.w * px_h * g.d * 4 : (uint64_t)g.tw * th * g.td * out_bpb;
|
|
const Staging st = staging((out_bytes + 15) & ~15ull, 16);
|
|
if (!st.buffer) return false;
|
|
if (buffer && st.buffer != buffer && !copies.empty()) { // (a new arena: copy what the last one holds)
|
|
texture_resident_flush(&tex);
|
|
checkpoint_mark(upload_cmd(), tex.gpu, 'u');
|
|
vkCmdCopyBufferToImage(upload_cmd(), buffer, tex.image, VK_IMAGE_LAYOUT_GENERAL,
|
|
(uint32_t)copies.size(), copies.data());
|
|
copies.clear();
|
|
}
|
|
buffer = st.buffer;
|
|
uint8_t* out = st.data;
|
|
uint8_t* blocks = (tex.astc || widen_from) ? (lin.resize(block_bytes), lin.data()) : out;
|
|
if (linear) {
|
|
const uint64_t row = (uint64_t)g.tw * src_bpb;
|
|
const uint8_t* src = guest_bytes(tex.gpu + src_off, (uint64_t)stride * (g.th - 1) + row, scratch);
|
|
for (uint32_t y = 0; y < g.th; y++) std::memcpy(blocks + y * row, src + (uint64_t)y * stride, row);
|
|
} else {
|
|
// (suyu's unswizzle reads the whole swizzled span without bounds checks: a span the storage
|
|
// ends in is read from a zero-padded copy)
|
|
const uint64_t from = src_off + (uint64_t)row0 * row_bytes;
|
|
const uint64_t need = runs.size() == 1 && row0 == 0 && nrow == nrows ? span : (uint64_t)nrow * row_bytes;
|
|
// (a big 2D level: its GOB-block rows unswizzled on several threads, each row on its own. The
|
|
// course maps the title loads at each scene change, 6 of 349 MB, took ~100 ms each on one. Their
|
|
// storage is sparse: each thread gathers its own rows, 8 MB at a time, rather than all of it
|
|
// being copied first (a 349 MB scratch beside the 349 MB staged))
|
|
const uint32_t nthreads = block_bytes >= (16u << 20) && g.td == 1 && !g.tile_d
|
|
? std::clamp(std::thread::hardware_concurrency() / 2, 1u, 8u) : 1u;
|
|
uint64_t avail = 0;
|
|
const uint8_t* direct = host->gpu_ptr(tex.gpu + from, &avail);
|
|
const bool gather_parts = nthreads > 1 && !(direct && avail >= need) && from < total;
|
|
const uint8_t* src = nullptr;
|
|
if (gather_parts) {
|
|
} else if (from < total && total - from >= need) {
|
|
src = guest_bytes(tex.gpu + from, need, scratch);
|
|
} else {
|
|
scratch.assign(need, 0);
|
|
if (from < total) host->gpu_read(tex.gpu + from, scratch.data(), total - from);
|
|
src = scratch.data();
|
|
}
|
|
const auto u0 = std::chrono::steady_clock::now();
|
|
if (nthreads > 1) {
|
|
const uint64_t out_row = (uint64_t)g.tw * rows_per * src_bpb; // (bytes per GOB-block row)
|
|
const uint32_t per = (nrow + nthreads - 1) / nthreads;
|
|
const uint32_t chunk = (uint32_t)std::max<uint64_t>(1, (8u << 20) / row_bytes);
|
|
auto part = [&](uint32_t t) {
|
|
const uint32_t a0 = t * per, b0 = std::min(nrow, a0 + per);
|
|
std::vector<uint8_t> mine;
|
|
for (uint32_t a = a0; a < b0; a += chunk) {
|
|
const uint32_t b = std::min(b0, a + chunk);
|
|
const uint32_t h = std::min(th - a * rows_per, (b - a) * rows_per);
|
|
const uint8_t* rows;
|
|
uint64_t rows_n;
|
|
if (gather_parts) { // (its rows, zero past the storage's end)
|
|
rows_n = (uint64_t)(b - a) * row_bytes;
|
|
mine.assign(rows_n, 0);
|
|
const uint64_t at = from + (uint64_t)a * row_bytes;
|
|
for (uint64_t o = 0; o < rows_n && at + o < total; o += 0x10000) {
|
|
const uint64_t k = std::min<uint64_t>({0x10000, rows_n - o, total - at - o});
|
|
host->gpu_read(tex.gpu + at + o, mine.data() + o, k);
|
|
}
|
|
rows = mine.data();
|
|
} else {
|
|
rows = src + (uint64_t)a * row_bytes, rows_n = need - (uint64_t)a * row_bytes;
|
|
}
|
|
Tegra::Texture::UnswizzleTexture(
|
|
std::span<uint8_t>(blocks + a * out_row, (uint64_t)g.tw * h * src_bpb),
|
|
std::span<const uint8_t>(rows, rows_n), (u32)src_bpb, g.tw, h, 1, g.tile_h, 0, 1);
|
|
}
|
|
};
|
|
std::vector<std::thread> pool;
|
|
for (uint32_t t = 1; t < nthreads; t++) pool.emplace_back(part, t);
|
|
part(0);
|
|
for (std::thread& t : pool) t.join();
|
|
static const bool check = getenv("RX_VK_UNSWIZZLE_CHECK") != nullptr; // (against one thread)
|
|
if (check && src) {
|
|
std::vector<uint8_t> ref(block_bytes);
|
|
Tegra::Texture::UnswizzleTexture(std::span<uint8_t>(ref.data(), block_bytes),
|
|
std::span<const uint8_t>(src, need), (u32)src_bpb, g.tw,
|
|
th, g.td, g.tile_h, g.tile_d, 1);
|
|
const bool same = !std::memcmp(ref.data(), blocks, block_bytes);
|
|
logf("vk: unswizzle check %#llx level %u rows %u+%u: %s", (unsigned long long)tex.gpu,
|
|
level, row0, nrow, same ? "same" : "DIFFERENT");
|
|
}
|
|
} else {
|
|
Tegra::Texture::UnswizzleTexture(std::span<uint8_t>(blocks, block_bytes),
|
|
std::span<const uint8_t>(src, need), (u32)src_bpb, g.tw, th,
|
|
g.td, g.tile_h, g.tile_d, 1);
|
|
}
|
|
st_unswizzle_bytes += (int64_t)block_bytes;
|
|
st_unswizzle_ns +=
|
|
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - u0).count();
|
|
}
|
|
if (tex.astc) {
|
|
Tegra::Texture::ASTC::DecompressSerial(std::span<const uint8_t>(blocks, block_bytes), g.w, px_h, g.d,
|
|
(u32)tex.block_w, (u32)tex.block_h,
|
|
std::span<uint8_t>(out, (size_t)g.w * px_h * g.d * 4));
|
|
} else if (widen_from) {
|
|
widen_rows(out, blocks, (size_t)g.tw * th * g.td, src_bpb, out_bpb);
|
|
}
|
|
st_bytes += (int64_t)out_bytes;
|
|
// (a sparse image: only its blocks this holds texels other than zero get memory; the others read
|
|
// zero unbound, and what the copy writes into them is dropped)
|
|
if (tex.sparse) {
|
|
const VkExtent3D gr = tex.sparse_req.formatProperties.imageGranularity;
|
|
const uint32_t bw = (uint32_t)tex.block_w, bh = (uint32_t)tex.block_h;
|
|
const uint64_t row = (uint64_t)g.tw * out_bpb; // (bytes of a row of blocks)
|
|
const uint32_t tbw = std::max(1u, gr.width / bw); // (blocks across a sparse block)
|
|
const uint32_t ntx = (g.tw + tbw - 1) / tbw;
|
|
std::vector<uint8_t> hit; // (sparse block rows x columns this reaches)
|
|
const uint32_t ty0 = y0 * bh / gr.height, ty1 = ((y0 + th) * bh - 1) / gr.height;
|
|
hit.assign((size_t)(ty1 - ty0 + 1) * ntx, 0);
|
|
if (level < tex.sparse_req.imageMipTailFirstLod) {
|
|
for (uint32_t br = 0; br < th; br++) {
|
|
const uint8_t* p = out + br * row;
|
|
const uint32_t ty = (y0 + br) * bh / gr.height - ty0;
|
|
for (uint32_t tx = 0; tx < ntx; tx++) {
|
|
uint8_t& h = hit[(size_t)ty * ntx + tx];
|
|
if (h) continue;
|
|
const uint64_t a = (uint64_t)tx * tbw * out_bpb, e = std::min<uint64_t>(row, a + (uint64_t)tbw * out_bpb);
|
|
uint64_t any = 0;
|
|
for (uint64_t i = a; i + 8 <= e; i += 8) {
|
|
uint64_t v;
|
|
std::memcpy(&v, p + i, 8);
|
|
any |= v;
|
|
}
|
|
h = any != 0;
|
|
}
|
|
}
|
|
} else {
|
|
uint64_t any = 0;
|
|
for (uint64_t i = 0; i + 8 <= out_bytes && !any; i += 8) {
|
|
uint64_t v;
|
|
std::memcpy(&v, out + i, 8);
|
|
any |= v;
|
|
}
|
|
if (any) hit.assign(hit.size(), 1);
|
|
}
|
|
bool some = false;
|
|
for (uint32_t ty = 0; ty < ty1 - ty0 + 1; ty++)
|
|
for (uint32_t tx = 0; tx < ntx; tx++)
|
|
if (hit[(size_t)ty * ntx + tx]) {
|
|
some = true;
|
|
texture_resident_queue(&tex, level, layer, tx * gr.width, (ty0 + ty) * gr.height,
|
|
gr.width, gr.height);
|
|
}
|
|
if (!some) continue;
|
|
}
|
|
flush_mapped_range(st.allocation, st.offset, out_bytes); // (all CPU conversion workers joined before publication)
|
|
VkBufferImageCopy c{};
|
|
c.bufferOffset = st.offset;
|
|
c.bufferRowLength = tex.astc ? g.w : g.tw * tex.block_w;
|
|
c.bufferImageHeight = tex.astc ? px_h : th * tex.block_h;
|
|
c.imageSubresource = {aspect, level, tex.type == VK_IMAGE_TYPE_3D ? 0 : layer, 1};
|
|
c.imageOffset = {0, (int32_t)(y0 * (uint32_t)tex.block_h), 0};
|
|
c.imageExtent = {g.w, px_h, g.d};
|
|
copies.push_back(c);
|
|
}
|
|
}
|
|
}
|
|
if (!copies.empty()) {
|
|
texture_resident_flush(&tex);
|
|
checkpoint_mark(upload_cmd(), tex.gpu, 'u');
|
|
vkCmdCopyBufferToImage(upload_cmd(), buffer, tex.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)copies.size(),
|
|
copies.data());
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// What the CPU wrote of a texture's storage since gen, as byte ranges from its start. A texture in a virtual
|
|
/// (sparse) pool is looked at page by page through its mapping, and is all dirty when its mapping changed.
|
|
/// same_ok (Triforce): a capture's stamp of a page with the bytes of its last copy is no write (vk_memory.cpp page_gen_tex): only
|
|
/// for an uploaded texture's "written since my upload?"
|
|
bool texture_dirty(Texture& tex, uint32_t gen, Ranges& out, bool same_ok = false) {
|
|
out.clear();
|
|
constexpr uint64_t kVirtual = 1ull << 40, kVPage = 1ull << 16;
|
|
if (tex.gpu < kVirtual) {
|
|
memory_written_runs(tex.guest, tex.guest_size, gen, out, same_ok);
|
|
return !out.empty();
|
|
}
|
|
// (the page walk only when the GPU mappings changed: the terrain's sparse arrays span over 1 GB, 64 KB pages,
|
|
// and were walked every recording; between changes, runs of contiguous guest memory are checked)
|
|
const uint32_t maps_gen = host->maps_generation() + host->vmap_generation();
|
|
if (tex.runs_maps_gen != maps_gen) {
|
|
tex.runs_maps_gen = maps_gen;
|
|
tex.runs.clear();
|
|
uint64_t sig = 0x5157;
|
|
// (the pages whose mapping changed: only those are dirty. The course maps' sparse pools are remapped every
|
|
// ~15 s at the title, a few pages at a time, and each remap re-uploaded all of six 349 MB maps)
|
|
const size_t npages = (size_t)((tex.guest_size + kVPage - 1) / kVPage);
|
|
const bool had_map = tex.page_map.size() == npages;
|
|
if (!had_map) tex.page_map.assign(npages, 0);
|
|
Ranges remapped;
|
|
for (uint64_t off = 0; off < tex.guest_size; off += kVPage) {
|
|
const uint64_t n = std::min(kVPage, tex.guest_size - off);
|
|
const uint64_t g = host->gpu_guest(tex.gpu + off);
|
|
sig = mix(sig, g);
|
|
uint64_t& was = tex.page_map[(size_t)(off / kVPage)];
|
|
if (had_map && was != g) {
|
|
if (!remapped.empty() && remapped.back().second == off) remapped.back().second = off + n;
|
|
else remapped.push_back({off, off + n});
|
|
}
|
|
was = g;
|
|
if (!g) continue;
|
|
Texture::Run& last = tex.runs.empty() ? tex.runs.emplace_back(Texture::Run{off, g, 0}) : tex.runs.back();
|
|
if (last.off + last.n == off && last.guest + last.n == g) last.n += n;
|
|
else tex.runs.push_back({off, g, n});
|
|
}
|
|
std::erase_if(tex.runs, [](const Texture::Run& r) { return r.n == 0; });
|
|
{ // (RX_VK_LOG_REMAP=1: a big texture's pages mapped anew: to which guest memory, and what images are there)
|
|
static const bool log_remap = getenv("RX_VK_LOG_REMAP") != nullptr;
|
|
static int logged;
|
|
if (log_remap && had_map && !remapped.empty() && tex.guest_size >= (32ull << 20) && logged < 120) {
|
|
for (size_t i = 0; i < remapped.size() && i < 4 && logged < 120; i++, logged++) {
|
|
const uint64_t g = host->gpu_guest(tex.gpu + remapped[i].first);
|
|
std::string there;
|
|
for (const auto& kv : textures) { // (images over that guest memory, by their storage)
|
|
const Texture* o = kv.second.get();
|
|
if (o == &tex || !g || o->gpu >= (1ull << 40) || g < o->gpu || g >= o->gpu + o->guest_size) continue;
|
|
char b[160];
|
|
snprintf(b, sizeof b, " #%u %#llx %ux%u x%u NVN %#x%s%s;", o->id, (unsigned long long)o->gpu, o->width,
|
|
o->height, o->layers, o->nvn_format, o->gpu_written ? " rendered" : "",
|
|
o->uploaded ? " uploaded" : "");
|
|
if (there.size() < 400) there += b;
|
|
}
|
|
logf("vk: remap of #%u %#llx %ux%u NVN %#x: +%#llx..+%#llx now at guest %#llx%s%s", tex.id,
|
|
(unsigned long long)tex.gpu, tex.width, tex.height, tex.nvn_format,
|
|
(unsigned long long)remapped[i].first, (unsigned long long)remapped[i].second, (unsigned long long)g,
|
|
g ? "" : " (unmapped)", there.empty() ? " (no image there)" : there.c_str());
|
|
}
|
|
}
|
|
}
|
|
if (sig != tex.map_sig) { // (pages mapped or unmapped since; gen 0 asks what was ever written)
|
|
tex.map_sig = sig;
|
|
if (gen && !had_map) {
|
|
out.assign(1, {0, tex.guest_size});
|
|
return true;
|
|
}
|
|
if (gen) out = remapped; // (and below, what the CPU wrote in the pages mapped all along)
|
|
}
|
|
}
|
|
static thread_local Ranges in_run;
|
|
const size_t remaps = out.size();
|
|
for (const Texture::Run& r : tex.runs) {
|
|
in_run.clear();
|
|
memory_written_runs(r.guest, r.n, gen, in_run, same_ok);
|
|
for (const auto& [lo, hi] : in_run) {
|
|
if (out.size() > remaps && out.back().second == r.off + lo) out.back().second = r.off + hi;
|
|
else out.push_back({r.off + lo, r.off + hi});
|
|
}
|
|
}
|
|
if (remaps && out.size() > remaps) { // (the two lists in order, overlaps merged)
|
|
std::sort(out.begin(), out.end());
|
|
Ranges merged;
|
|
for (const auto& r : out) {
|
|
if (!merged.empty() && r.first <= merged.back().second) merged.back().second = std::max(merged.back().second, r.second);
|
|
else merged.push_back(r);
|
|
}
|
|
out.swap(merged);
|
|
}
|
|
return !out.empty();
|
|
}
|
|
|
|
uint64_t destroy_gen = 1; // (bumped when textures go: what was resolved to them goes stale)
|
|
texture_gone::Record gone_record; // (which ones went: texture_destroyed_since, for a pointer kept across draws)
|
|
|
|
void destroy_later(Texture& tex) {
|
|
destroy_gen++;
|
|
gone_record.went(tex.id);
|
|
Dead d{tex.last_value, tex.image, tex.allocation, {}, {}};
|
|
if (tex.pool_key && tex.allocation && tex.sparse_chunks.empty() && pool_cap()) { // (a render target's: to the pool, its views too)
|
|
d.pool_key = tex.pool_key, d.view_map = std::move(tex.views), d.bytes = tex.bytes + tex.pad;
|
|
tex.views.clear();
|
|
} else {
|
|
for (auto& kv : tex.views)
|
|
if (kv.second) d.views.push_back(kv.second);
|
|
}
|
|
d.memory = tex.sparse_chunks;
|
|
dead.push_back(std::move(d));
|
|
if (tex.twin3d) { // (Triforce: its one-slice 3D copy goes with it)
|
|
Dead t{tex.last_value, tex.twin3d, tex.twin3d_alloc, {}, {}};
|
|
for (auto& kv : tex.twin3d_views)
|
|
if (kv.second) t.views.push_back(kv.second);
|
|
dead.push_back(std::move(t));
|
|
tex.twin3d = VK_NULL_HANDLE, tex.twin3d_alloc = nullptr;
|
|
tex.twin3d_views.clear();
|
|
}
|
|
total_bytes -= std::min(total_bytes, tex.bytes + tex.pad);
|
|
}
|
|
|
|
void vw_forget(const Texture& t); // (below: a destroyed texture leaves the list of GPU-written ones)
|
|
void forget_gpu(Texture* tex) {
|
|
vw_forget(*tex);
|
|
auto [a, b] = by_gpu.equal_range(tex->gpu);
|
|
for (auto it = a; it != b; ++it)
|
|
if (it->second == tex) {
|
|
by_gpu.erase(it);
|
|
break;
|
|
}
|
|
}
|
|
|
|
/// (as_target: texture_get's for a render target or a copy's destination, not for sampling: its image goes to the render targets'
|
|
/// blocks, and to the target pool when its texture goes)
|
|
/// (hookshot) a size and sample count Vulkan can make on this GPU: they come from the game's texture (its width a u32 it
|
|
/// reads from a file), and past the device's limits vkCreateImage is undefined, and the uploads' buffers grew with them
|
|
bool texture_size_ok(VkImageType type, uint32_t w, uint32_t h, uint32_t z, uint32_t layers, int samples) {
|
|
const VkPhysicalDeviceLimits& lim = dev.props.limits;
|
|
const uint32_t most = type == VK_IMAGE_TYPE_1D ? lim.maxImageDimension1D
|
|
: type == VK_IMAGE_TYPE_3D ? lim.maxImageDimension3D : lim.maxImageDimension2D;
|
|
return w <= most && h <= most && z <= lim.maxImageDimension3D && layers <= lim.maxImageArrayLayers &&
|
|
(samples <= 1 || samples == 2 || samples == 4 || samples == 8 || samples == 16);
|
|
}
|
|
Texture* create(const Key& key, const nvn_tex* t, VkFormat vf, bool astc, int bw, int bh, int bpb, bool as_target) {
|
|
PROF_EXACT(); // (texture_get's slow path: PROF_TEXGET samples the rest)
|
|
auto tex = std::make_unique<Texture>();
|
|
Texture& x = *tex;
|
|
x.gpu = t->gpu, x.d = t->d, x.L = t->L, x.nvn_format = t->d.format;
|
|
x.guest = host->gpu_guest(t->gpu);
|
|
x.guest_size = t->L.size;
|
|
x.astc = astc, x.block_w = bw, x.block_h = bh, x.bpb = bpb;
|
|
int from = 0;
|
|
vf = widened(vf, &from);
|
|
const bool depth = is_depth_format(vf);
|
|
x.format = depth ? vf : astc ? VK_FORMAT_R8G8B8A8_UNORM : linear_twin(vf);
|
|
x.aspect = !depth ? VK_IMAGE_ASPECT_COLOR_BIT
|
|
: vf == VK_FORMAT_S8_UINT ? VK_IMAGE_ASPECT_STENCIL_BIT
|
|
: has_stencil(vf) ? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT
|
|
: VK_IMAGE_ASPECT_DEPTH_BIT;
|
|
const nvn_texdesc& d = t->d;
|
|
x.width = std::max(1u, d.width);
|
|
x.height = std::max(1u, d.height);
|
|
switch (d.target) {
|
|
case NVN_TEXTURE_TARGET_1D: x.type = VK_IMAGE_TYPE_1D, x.height = 1; break;
|
|
case NVN_TEXTURE_TARGET_1D_ARRAY: x.type = VK_IMAGE_TYPE_1D, x.layers = std::max(1u, d.height), x.height = 1; break;
|
|
case NVN_TEXTURE_TARGET_2D_ARRAY: case NVN_TEXTURE_TARGET_2D_MULTISAMPLE_ARRAY: x.layers = std::max(1u, d.depth); break;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP: x.layers = 6; break;
|
|
case NVN_TEXTURE_TARGET_CUBEMAP_ARRAY: x.layers = std::max(6u, d.depth); break;
|
|
case NVN_TEXTURE_TARGET_3D: x.type = VK_IMAGE_TYPE_3D, x.depth = std::max(1u, d.depth); break;
|
|
default: break;
|
|
}
|
|
const uint32_t max_dim = std::max({x.width, x.height, x.depth});
|
|
const uint32_t max_levels = 1 + (uint32_t)std::floor(std::log2((double)max_dim));
|
|
x.levels = nvn_tex_is_linear(&d) ? 1 : std::clamp<uint32_t>(d.levels, 1, max_levels);
|
|
bool by_detail = false; // (its top levels are left out by the card's texture detail alone: tex_detail.h)
|
|
{ // huge textures leave out their top levels: wider than RX_VK_TEX_MAX_DIM (default 2048) once they would take
|
|
// 64 MB, and down to RX_VK_TEX_MAX_MB (default 512) whatever their size (MKW's terrain material arrays are
|
|
// 8192 x 8192 x 68 ASTC: 24 GB as RGBA8)
|
|
static const uint32_t max_dim = getenv("RX_VK_TEX_MAX_DIM") ? (uint32_t)atoi(getenv("RX_VK_TEX_MAX_DIM")) : 2048;
|
|
static const uint64_t max_bytes = (uint64_t)(getenv("RX_VK_TEX_MAX_MB") ? atoi(getenv("RX_VK_TEX_MAX_MB")) : 1536) << 20;
|
|
const uint32_t texel = astc ? 4 : std::max(1, bpb / std::max(1, bw * bh));
|
|
auto bytes_at = [&](uint32_t skip) {
|
|
uint64_t px = 0;
|
|
for (uint32_t l = skip; l < x.levels; l++)
|
|
px += (uint64_t)std::max(1u, x.width >> l) * std::max(1u, x.height >> l) * std::max(1u, x.depth >> l);
|
|
return px * x.layers * texel * std::max(1, d.samples);
|
|
};
|
|
// (decoded ASTC only by size: uncompressed maps may be read at exact texel coordinates, and the GPU may
|
|
// render into them, where a level left out is wrong)
|
|
// (a 2D array too big to keep whole, in a format with sparse residency: kept whole, as a sparse image)
|
|
if (!astc && !depth && dev.sparse_images && x.type == VK_IMAGE_TYPE_2D && x.layers > 1 && d.samples <= 1 &&
|
|
bytes_at(0) > max_bytes) {
|
|
VkImageUsageFlags u = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
|
|
if (!is_bc(x.format)) u |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT;
|
|
uint32_t np = 0;
|
|
vkGetPhysicalDeviceSparseImageFormatProperties(dev.physical, x.format, VK_IMAGE_TYPE_2D, VK_SAMPLE_COUNT_1_BIT,
|
|
u, VK_IMAGE_TILING_OPTIMAL, &np, nullptr);
|
|
x.sparse = np > 0;
|
|
}
|
|
// (RX_VK_TEX_SKIP_MB: from how many MB as RGBA8 a decoded ASTC texture wider than max_dim loses its top levels;
|
|
// 64 by default. The Steam Deck's 16 GB are shared with the GPU: lower there, with RX_VK_TEX_MAX_DIM=1024)
|
|
static const uint64_t skip_from = (uint64_t)(getenv("RX_VK_TEX_SKIP_MB") ? atoi(getenv("RX_VK_TEX_SKIP_MB")) : 64) << 20;
|
|
while (!x.sparse && astc && x.skip + 1 < x.levels && bytes_at(x.skip) > skip_from &&
|
|
(std::max(x.width, x.height) >> x.skip) > max_dim)
|
|
x.skip++;
|
|
while (!x.sparse && x.skip + 1 < x.levels && bytes_at(x.skip) > max_bytes) x.skip++;
|
|
if (x.skip) {
|
|
static int logged;
|
|
if (logged++ < 40)
|
|
logf("vk: texture %#llx %ux%ux%u x%u (NVN %#x, %llu MB): its %u top level(s) left out (%llu MB)",
|
|
(unsigned long long)t->gpu, x.width, x.height, x.depth, x.layers, d.format,
|
|
(unsigned long long)(bytes_at(0) >> 20), x.skip, (unsigned long long)(bytes_at(x.skip) >> 20));
|
|
}
|
|
// (texture detail, tex_detail.h: on a card the driver allows under 10 GB a material texture is made without its
|
|
// largest level, under 6 GB without two. Only a decoded ASTC texture the game samples: 2D or a 2D array, one
|
|
// sample, mipmapped, 256 texels or more each way, not one first met as a render target or a copy's destination.
|
|
// RX_VK_TEX_DETAIL=full|reduced|low says so on any card; read at the first texture)
|
|
// (nor one a shader reads by texel position or asks the size of: whole_keys, texture_want_whole)
|
|
const auto detail_level = [] { return texture_detail_level(); };
|
|
if (!as_target && astc && !x.sparse && x.type == VK_IMAGE_TYPE_2D && d.samples <= 1 &&
|
|
(d.target == NVN_TEXTURE_TARGET_2D || d.target == NVN_TEXTURE_TARGET_2D_ARRAY) && !whole_keys.count(key)) {
|
|
const uint32_t by = tex_detail::skip(detail_level(), x.width, x.height, x.levels);
|
|
if (by > x.skip) {
|
|
x.detail_skip = true;
|
|
static std::atomic<uint64_t> n{0}, whole{0}, kept{0};
|
|
by_detail = !x.skip;
|
|
const uint64_t c = ++n;
|
|
whole += bytes_at(0) >> 2, kept += bytes_at(by) >> 2; // (texels)
|
|
x.skip = by;
|
|
if (c >= 64 && (c & (c - 1)) == 0)
|
|
logf("vk: texture detail: %llu textures made without their largest level(s) so far, %llu M texels "
|
|
"kept of %llu M", (unsigned long long)c, (unsigned long long)(kept.load() >> 20),
|
|
(unsigned long long)(whole.load() >> 20));
|
|
}
|
|
}
|
|
if (x.skip) {
|
|
x.width = std::max(1u, x.width >> x.skip), x.height = std::max(1u, x.height >> x.skip);
|
|
x.depth = std::max(1u, x.depth >> x.skip);
|
|
x.levels -= x.skip;
|
|
}
|
|
}
|
|
if (!texture_size_ok(x.type, x.width, x.height, x.depth, x.layers, d.samples)) {
|
|
static int logged;
|
|
if (logged++ < 16)
|
|
logf("vk: texture %#llx %ux%ux%u x%u samples %d (NVN %#x): larger than this GPU's images, not made",
|
|
(unsigned long long)t->gpu, x.width, x.height, x.depth, x.layers, d.samples, d.format);
|
|
return nullptr;
|
|
}
|
|
x.samples = d.samples > 1 ? (VkSampleCountFlagBits)d.samples : VK_SAMPLE_COUNT_1_BIT;
|
|
{ // (decoded ASTC as BC3: 2D, at most the scratch's size; from blocks of 6 x 6 texels up (2.25 bits a texel or
|
|
// less: BC3's 8 keep what they have) from 2 MB as RGBA8 (RX_VK_ASTC_BC_MB), the finer ones (UI, characters)
|
|
// only from 64 MB)
|
|
static const uint64_t low = (uint64_t)(getenv("RX_VK_ASTC_BC_MB") ? atoi(getenv("RX_VK_ASTC_BC_MB")) : 2) << 20;
|
|
// (issue 519: on a card the driver gives under 6 GB, the finer ones too. An RTX 3050 Laptop (4 GB, 3.3 allowed)
|
|
// held 1.75 GB of decoded ASTC in Goron City, drawn every frame: 5.6 GB used of 3.3, paged, 4 fps. As BC3, a
|
|
// quarter. Now 8 GB cards too: astc_bc_policy.h. RX_VK_ASTC_BC_ALL=1 or 0 says so on any card)
|
|
static const bool all = [] {
|
|
if (const char* e = getenv("RX_VK_ASTC_BC_ALL")) return e[0] == '1';
|
|
uint64_t used = 0, allowed = 0;
|
|
const bool small = vram_usage(&used, &allowed) && astc_bc_policy::all_by_vram(allowed);
|
|
if (small) logf("vk: %llu MB of VRAM allowed: all large ASTC textures kept as BC3", (unsigned long long)(allowed >> 20));
|
|
return small;
|
|
}();
|
|
const uint64_t min_bytes = (bw * bh >= 36 || all) ? low : 64ull << 20;
|
|
uint64_t px = 0;
|
|
for (uint32_t l = 0; l < x.levels; l++)
|
|
px += (uint64_t)std::max(1u, x.width >> l) * std::max(1u, x.height >> l);
|
|
// (not one with its top levels left out and a single layer: copies into those levels are blitted into its
|
|
// first, which BC3 cannot take; trimmed, it is small)
|
|
// (hookshot-amd) AMD's drivers sample BC levels wrong when a level's block grid is not level 0's halved: 1365 texels
|
|
// are 342 blocks, but level 2 (341 texels) is 86 blocks, not 342 >> 2 = 85. Ocarina's terrain material arrays
|
|
// (1365 x 1365 x 90) then draw a grid of dark lines along the terrain's seams, worse in the distance (Lichii #448 found
|
|
// it on an RX 6600 XT; an RX 6700 XT here). On AMD such a texture is encoded at the nearest size whose blocks halve
|
|
// evenly down to its 16-texel levels (1365 -> 1536, 683 -> 768; 768 and 1536 stay) and keeps only the levels that
|
|
// halve evenly: the game samples with normalized coordinates, so a resized level covers the same area. Where that
|
|
// would grow it by more than a quarter, or past the scratch, it stays RGBA8. RX_VK_UNEVEN_RGBA8=1: RGBA8 instead of resizing.
|
|
// OFF by default since v2.0.6: in v2.0.5 an RX 6700 XT's GPU time went from 10-13 ms to 70-100 ms a frame in play
|
|
// with it (issue 823, logs of both versions), and AMD players saw rainbow ground (Discord). RX_VK_UNEVEN_BC=0: on
|
|
// (every card); unset or 1: BC3 as in v2.0.4.
|
|
using bc_levels::first_uneven;
|
|
using bc_levels::even_size;
|
|
const bool uneven = first_uneven(x.width, x.height, x.levels) < x.levels;
|
|
static const bool amd_fix = [] {
|
|
if (const char* e = getenv("RX_VK_UNEVEN_BC")) return e[0] == '0';
|
|
return false;
|
|
}();
|
|
static const bool rgba8_instead = getenv("RX_VK_UNEVEN_RGBA8") != nullptr;
|
|
// (one whose levels the texture detail left out is BC3 all the same: a copy into a level it lacks is left out)
|
|
const bool bc_candidate = astc && !(x.skip && x.layers == 1 && !by_detail) && x.type == VK_IMAGE_TYPE_2D &&
|
|
x.samples == VK_SAMPLE_COUNT_1_BIT && x.width <= kBcDim && x.height <= kBcDim &&
|
|
px * x.layers * 4 >= min_bytes;
|
|
if (bc_candidate && uneven && amd_fix) {
|
|
const uint32_t ew = even_size(x.width, x.levels), eh = even_size(x.height, x.levels);
|
|
const uint32_t el = first_uneven(ew, eh, x.levels);
|
|
const bool resize = ew != x.width || eh != x.height;
|
|
static int n_resized, n_trimmed, n_rgba8;
|
|
static uint64_t mb_saved;
|
|
if (!rgba8_instead && ew <= kBcDim && eh <= kBcDim && ew * 4 <= x.width * 5 && eh * 4 <= x.height * 5 &&
|
|
el >= 1 && bc_ready()) {
|
|
uint64_t epx = 0;
|
|
for (uint32_t l = 0; l < el; l++) epx += (uint64_t)std::max(1u, ew >> l) * std::max(1u, eh >> l);
|
|
mb_saved += (px * x.layers * 4 - epx * x.layers) >> 20;
|
|
int& n = resize ? n_resized : n_trimmed;
|
|
n++;
|
|
if (n <= 4 || (n & (n - 1)) == 0)
|
|
logf("vk: texture %#llx %ux%u x%u, %u levels: uneven BC block levels on AMD, as BC3 at %ux%u, %u levels "
|
|
"(%d resized, %d with fewer levels, %d as RGBA8; %llu MB less than RGBA8 so far)",
|
|
(unsigned long long)t->gpu, x.width, x.height, x.layers, x.levels, ew, eh, el, n_resized, n_trimmed,
|
|
n_rgba8, (unsigned long long)mb_saved);
|
|
x.width = ew, x.height = eh, x.levels = el;
|
|
x.bc = true, x.bc_resized = resize;
|
|
x.format = VK_FORMAT_BC3_UNORM_BLOCK;
|
|
} else {
|
|
n_rgba8++;
|
|
if (n_rgba8 <= 4 || (n_rgba8 & (n_rgba8 - 1)) == 0)
|
|
logf("vk: texture %#llx %ux%u x%u, %u levels: uneven BC block levels on AMD, kept as RGBA8 (%d so far)",
|
|
(unsigned long long)t->gpu, x.width, x.height, x.layers, x.levels, n_rgba8);
|
|
}
|
|
} else if (bc_candidate && bc_ready()) {
|
|
x.bc = true;
|
|
x.format = VK_FORMAT_BC3_UNORM_BLOCK;
|
|
}
|
|
}
|
|
const VkFormatProperties fp = format_props(x.format);
|
|
x.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
|
|
if (fp.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT) x.usage |= VK_IMAGE_USAGE_SAMPLED_BIT;
|
|
VkImageCreateFlags flags = 0;
|
|
if (depth) {
|
|
x.usage |= VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
|
|
} else if (!is_bc(x.format) && !is_astc(x.format)) {
|
|
// (any format of the class may be a view: attachments and storage where one of them can)
|
|
flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
|
|
x.usage |= VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
|
|
if (x.samples == VK_SAMPLE_COUNT_1_BIT) x.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
|
|
} else {
|
|
// Raw storage views address one compressed block per texel (8 or 16 bytes).
|
|
flags |= VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT | VK_IMAGE_CREATE_BLOCK_TEXEL_VIEW_COMPATIBLE_BIT |
|
|
VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
|
|
const VkFormat raw = bpb == 8 ? VK_FORMAT_R32G32_UINT : VK_FORMAT_R32G32B32A32_UINT;
|
|
VkImageFormatProperties limits{};
|
|
// (asked without STORAGE: AMD's driver checks that against the compressed format and says no, though it makes
|
|
// the image and its raw views (RX 6600 XT, amdvlk); without them the game's BC encoders wrote nowhere and
|
|
// Link's clothes showed stale blocks. If the image is then refused, it is made again without)
|
|
// (RX_VK_BC_STORAGE=0: compressed images without it, as before v2.0.8 on AMD: issue 1278, an RX 6700 XT's GPU time 6-12 ms
|
|
// a frame on v2.0.6 and 60-100 ms on v2.0.11, the game's BC encoders running there since 04675cf being the lead)
|
|
static const bool bc_storage = !getenv("RX_VK_BC_STORAGE") || atoi(getenv("RX_VK_BC_STORAGE")) != 0;
|
|
if (bc_storage && (format_props(raw).optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) &&
|
|
vkGetPhysicalDeviceImageFormatProperties(dev.physical, x.format, x.type, VK_IMAGE_TILING_OPTIMAL,
|
|
x.usage, flags, &limits) == VK_SUCCESS)
|
|
x.usage |= VK_IMAGE_USAGE_STORAGE_BIT;
|
|
// (and colour attachment views, raw too: OoT renders the slices of two 64^3 BC4 noise volumes as RGBA16UI targets, one
|
|
// texel a block (53509044401d78d2, while an area loads). Without the usage those views were not made, the volumes
|
|
// stayed zero, and the foliage's dithered LOD crossfade, which reads them, left trees bare at mid distance, #1151.
|
|
// 3D images take the usage on the driver's word for the 2D one: the query does not cover 2D_ARRAY_COMPATIBLE's
|
|
// per-slice attachment views)
|
|
if (format_props(raw).optimalTilingFeatures & VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT) {
|
|
// (asked with the flags a 3D image is made with below; a refused image is made again without the usage)
|
|
const VkImageCreateFlags asked = flags | (x.type != VK_IMAGE_TYPE_3D ? 0u
|
|
: VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT | (dev.view_2d_of_3d ? VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT : 0u));
|
|
const VkResult r = vkGetPhysicalDeviceImageFormatProperties(dev.physical, x.format, x.type, VK_IMAGE_TILING_OPTIMAL,
|
|
x.usage | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, asked, &limits);
|
|
if (r == VK_SUCCESS || x.type == VK_IMAGE_TYPE_3D) x.usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
|
|
static int told;
|
|
if (r != VK_SUCCESS && told++ < 4)
|
|
logf("vk: BC image %ux%ux%u (vk %d) as a raw colour attachment: the driver says %d%s", x.width, x.height, x.depth,
|
|
(int)x.format, (int)r, x.type == VK_IMAGE_TYPE_3D ? " (3D: taken)" : "");
|
|
}
|
|
}
|
|
if (x.type == VK_IMAGE_TYPE_2D && x.layers >= 6 && x.width == x.height && x.samples == VK_SAMPLE_COUNT_1_BIT)
|
|
flags |= VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT, x.cube = true;
|
|
if (x.type == VK_IMAGE_TYPE_3D) {
|
|
flags |= VK_IMAGE_CREATE_2D_ARRAY_COMPATIBLE_BIT;
|
|
if (dev.view_2d_of_3d) flags |= VK_IMAGE_CREATE_2D_VIEW_COMPATIBLE_BIT_EXT;
|
|
}
|
|
VkImageCreateInfo ci{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
|
|
ci.flags = flags;
|
|
ci.imageType = x.type;
|
|
ci.format = x.format;
|
|
ci.extent = {x.width, x.height, x.depth};
|
|
ci.mipLevels = x.levels;
|
|
ci.arrayLayers = x.layers;
|
|
ci.samples = x.samples;
|
|
ci.tiling = VK_IMAGE_TILING_OPTIMAL;
|
|
ci.usage = x.usage;
|
|
ci.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
// (a render target's image, not compressed: from the pool when one of this create info waits there, above)
|
|
const uint64_t pool_key = as_target && !x.sparse && pool_cap() && !is_bc(x.format) && !is_astc(x.format) &&
|
|
(x.usage & (VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT))
|
|
? pool_key_of(ci)
|
|
: 0;
|
|
x.pool_key = pool_key;
|
|
x.create_flags = ci.flags;
|
|
// (hookshot, RX_VK_COMPACT) an image that is not a render target's: from the textures' own pool, the one compacted
|
|
const bool in_textures = compact_on() && !x.sparse && !(as_target && !is_bc(x.format) && !is_astc(x.format));
|
|
if (x.sparse) {
|
|
ci.flags |= VK_IMAGE_CREATE_SPARSE_BINDING_BIT | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
|
|
bool ok = vkCreateImage(dev.device, &ci, nullptr, &x.image) == VK_SUCCESS;
|
|
if (ok) {
|
|
VkMemoryRequirements mr{};
|
|
vkGetImageMemoryRequirements(dev.device, x.image, &mr);
|
|
uint32_t n = 0;
|
|
vkGetImageSparseMemoryRequirements(dev.device, x.image, &n, nullptr);
|
|
std::vector<VkSparseImageMemoryRequirements> reqs(n);
|
|
vkGetImageSparseMemoryRequirements(dev.device, x.image, &n, reqs.data());
|
|
ok = false;
|
|
for (const auto& r : reqs)
|
|
if (r.formatProperties.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) x.sparse_req = r, ok = true;
|
|
x.sparse_align = mr.alignment;
|
|
x.sparse_type = UINT32_MAX;
|
|
for (uint32_t i = 0; i < dev.memory.memoryTypeCount && x.sparse_type == UINT32_MAX; i++)
|
|
if ((mr.memoryTypeBits >> i & 1) && (dev.memory.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
|
|
x.sparse_type = i;
|
|
ok = ok && x.sparse_type != UINT32_MAX && x.sparse_align;
|
|
if (!ok) vkDestroyImage(dev.device, x.image, nullptr), x.image = VK_NULL_HANDLE;
|
|
}
|
|
if (!ok) {
|
|
logf("vk: sparse image %ux%u x%u levels %u format %d not made", x.width, x.height, x.layers, x.levels,
|
|
(int)x.format);
|
|
return nullptr;
|
|
}
|
|
x.tiles.resize((size_t)x.layers * x.levels);
|
|
x.tail_bound.assign(x.layers, 0);
|
|
logf("vk: texture %#llx %ux%u x%u levels %u (NVN %#x): sparse, layers made resident as they are written "
|
|
"(blocks %ux%u, mip tail from level %u, %llu KB%s)",
|
|
(unsigned long long)t->gpu, x.width, x.height, x.layers, x.levels, d.format,
|
|
x.sparse_req.formatProperties.imageGranularity.width, x.sparse_req.formatProperties.imageGranularity.height,
|
|
x.sparse_req.imageMipTailFirstLod, (unsigned long long)(x.sparse_req.imageMipTailSize >> 10),
|
|
x.sparse_req.formatProperties.flags & VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT ? ", one for all layers" : " a layer");
|
|
} else if (in_textures && create_image(ci, x.image, x.allocation, false, false, true)) {
|
|
// (made in the textures' pool; one refused there is made as any other, below)
|
|
} else if (!(pool_key && pool_take(pool_key, x.image, x.allocation, x.views)) && (st_pool_made += pool_key != 0, true) &&
|
|
!create_image(ci, x.image, x.allocation, false, as_target && !is_bc(x.format) && !is_astc(x.format)) &&
|
|
// (a compressed image refused: made again without its raw attachment usage, then without its raw storage
|
|
// usage, as #1332 does; its raw views are then not made, as before)
|
|
!((ci.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) && is_bc(x.format) &&
|
|
(ci.usage &= ~VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, x.usage = ci.usage, create_image(ci, x.image, x.allocation))) &&
|
|
!((ci.usage & VK_IMAGE_USAGE_STORAGE_BIT) && (is_bc(x.format) || is_astc(x.format)) &&
|
|
(ci.usage &= ~VK_IMAGE_USAGE_STORAGE_BIT, x.usage = ci.usage, create_image(ci, x.image, x.allocation)))) {
|
|
static int logged;
|
|
if (logged++ < 16)
|
|
logf("vk: image %ux%ux%u x%u levels %u format %d (NVN %#x) not made", x.width, x.height, x.depth, x.layers,
|
|
x.levels, (int)x.format, d.format);
|
|
return nullptr;
|
|
}
|
|
// what it costs: the level chain's texels
|
|
uint64_t px = 0;
|
|
for (uint32_t l = 0; l < x.levels; l++)
|
|
px += (uint64_t)std::max(1u, x.width >> l) * std::max(1u, x.height >> l) * std::max(1u, x.depth >> l);
|
|
const uint32_t texel = x.bc || is_bc(x.format) ? 1 : astc ? 4 : std::max(1, from ? (from == 3 ? 4 : from == 6 ? 8 : 16) : bpb);
|
|
x.bytes = x.sparse ? 0 : px * x.layers * texel * x.samples; // (a sparse image's: as its layers are bound)
|
|
// (the budget counts what the allocator gave it: the texels' 8% short of that over a scene's 2000 textures, 3228 MB counted for 3504
|
|
// MB held, and a budget of 4035 MB took 4300. RX_VK_COUNT_BY_ESTIMATE=1: the texels)
|
|
static const bool by_estimate = getenv("RX_VK_COUNT_BY_ESTIMATE") != nullptr;
|
|
if (!x.sparse && !by_estimate) x.pad = std::max(x.bytes, allocation_size(x.allocation)) - x.bytes;
|
|
total_bytes += x.bytes + x.pad;
|
|
InitRedirect redirect; // (a parallel replay's worker: into the earliest open segment's init command buffer)
|
|
// into GENERAL, and cleared (a target the game renders into before writing it all reads zeros, not garbage)
|
|
VkCommandBuffer cmd = upload_cmd();
|
|
VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
b.image = x.image;
|
|
b.subresourceRange = {x.aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.imageMemoryBarrierCount = 1;
|
|
di.pImageMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &di);
|
|
if (depth) {
|
|
const VkClearDepthStencilValue z{0.0f, 0};
|
|
vkCmdClearDepthStencilImage(cmd, x.image, VK_IMAGE_LAYOUT_GENERAL, &z, 1, &b.subresourceRange);
|
|
} else if (!is_bc(x.format) && !is_astc(x.format)) {
|
|
const VkClearColorValue z{};
|
|
vkCmdClearColorImage(cmd, x.image, VK_IMAGE_LAYOUT_GENERAL, &z, 1, &b.subresourceRange);
|
|
}
|
|
if (!is_bc(x.format) && !is_astc(x.format)) {
|
|
// (the clear finished before anything after it in this command buffer writes the image: the upload of its contents
|
|
// comes right after, a copy into the same image. Two transfer writes with no barrier between them may run in either
|
|
// order; NVIDIA's ran in order, AMD's (a fast clear) landed last, so a texture uploaded in the recording that made it
|
|
// read zeros there. The menus' text is that case: the CPU writes each string's letters into a new R8 texture, which is
|
|
// drawn once into the string's label, so every menu and title text was blank on AMD cards)
|
|
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
|
|
b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.oldLayout = b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
vkCmdPipelineBarrier2(cmd, &di);
|
|
}
|
|
Texture* p = tex.get();
|
|
redirect.done();
|
|
static uint32_t ids;
|
|
p->id = ++ids;
|
|
p->first_seen = frame_no;
|
|
st_made++;
|
|
p->key_layout = key.layout, p->key_cls = key.cls;
|
|
{ // (another image at this storage: what tells them apart; content one renders the other does not see)
|
|
static int told;
|
|
auto [a, e] = by_gpu.equal_range(p->gpu);
|
|
for (auto it = a; it != e && told < 80; ++it) {
|
|
const Texture* o = it->second;
|
|
told++;
|
|
logf("vk: texture #%u at %#llx beside #%u: %ux%ux%u x%u levels %u layout %#x class %#x NVN %#x target %d, "
|
|
"the other %ux%ux%u x%u levels %u layout %#x class %#x NVN %#x target %d%s",
|
|
p->id, (unsigned long long)p->gpu, o->id, p->width, p->height, p->depth, p->layers, p->levels,
|
|
key.layout, key.cls, d.format, d.target, o->width, o->height, o->depth, o->layers, o->levels,
|
|
o->key_layout, o->key_cls, o->nvn_format, o->d.target, o->gpu_written ? " (rendered)" : "");
|
|
}
|
|
}
|
|
textures.emplace(key, std::move(tex));
|
|
by_gpu.emplace(p->gpu, p);
|
|
watch_tex(p, "made");
|
|
if (remade_log_on()) { // (one evicted not long ago, made again)
|
|
std::lock_guard lk(evicted_mu);
|
|
auto ev = evicted_at.find(evicted_key(p->gpu, p->width, p->height, p->nvn_format));
|
|
if (ev != evicted_at.end()) {
|
|
const uint64_t ago = frame_no - ev->second.frame;
|
|
if (ago < 600) {
|
|
remade[p] = ago;
|
|
static int told;
|
|
if (told++ < 300)
|
|
logf("vk: texture #%u %#llx %ux%u NVN %#x made again %llu frames after it was evicted (%s then)", p->id,
|
|
(unsigned long long)p->gpu, p->width, p->height, p->nvn_format, (unsigned long long)ago,
|
|
ev->second.rendered ? "rendered" : "not rendered");
|
|
}
|
|
evicted_at.erase(ev);
|
|
}
|
|
}
|
|
static int logged;
|
|
if (logged++ < 40 || textures.size() % 500 == 0 || p->bytes > (64ull << 20))
|
|
logf("vk: texture %#llx %ux%ux%u x%u levels %u NVN %#x -> vk %d%s (%zu textures, %llu MB)",
|
|
(unsigned long long)p->gpu, p->width, p->height, p->depth, p->layers, p->levels, d.format, (int)p->format,
|
|
astc ? " (ASTC decoded)" : "", textures.size(), (unsigned long long)(total_bytes >> 20));
|
|
return p;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
std::atomic<uint64_t> work_epoch{0};
|
|
namespace {
|
|
// (each thread's own: the display thread's present records barriers too)
|
|
thread_local VkCommandBuffer last_barrier_cmd = VK_NULL_HANDLE; // (the command buffer this thread's last full barrier went into, and the work count then)
|
|
thread_local uint64_t last_barrier_epoch = 0;
|
|
std::atomic<int64_t> st_barriers_made{0}, st_barriers_left_out{0};
|
|
} // namespace
|
|
void barrier_stats(int64_t* made, int64_t* left_out) {
|
|
*made = st_barriers_made.load(std::memory_order_relaxed), *left_out = st_barriers_left_out.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
// (hookshot) A full barrier right after another one on the same command buffer, no command that does or orders GPU work in between
|
|
// (vk_work.h counts them), orders nothing the first did not: left out. The game's own barrier (end of a pass), then the next pass's
|
|
// start barrier, then a dispatch's, were two or three in a row. RX_VK_BARRIER_DEDUPE_OFF=1: all recorded, as before.
|
|
void full_barrier(VkCommandBuffer cmd, const char* why) {
|
|
static const bool off = getenv("RX_VK_BARRIER_DEDUPE_OFF") != nullptr;
|
|
const bool dedupe = !off && !par_active.load(std::memory_order_relaxed); // (a parallel replay's threads share command buffers and the count)
|
|
if (dedupe && cmd == last_barrier_cmd && work_epoch.load(std::memory_order_relaxed) == last_barrier_epoch) {
|
|
st_barriers_left_out.fetch_add(1, std::memory_order_relaxed);
|
|
return;
|
|
}
|
|
st_barriers_made.fetch_add(1, std::memory_order_relaxed);
|
|
GprofSpan gp(cmd, why); // (RX_VK_PASS_TIMING: what a barrier costs the GPU that has nothing left to wait for)
|
|
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
d.memoryBarrierCount = 1;
|
|
d.pMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &d);
|
|
last_barrier_cmd = cmd, last_barrier_epoch = work_epoch.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
namespace {
|
|
thread_local VkCommandBuffer last_pass_end_cmd = VK_NULL_HANDLE; // (the command buffer this thread's last draw pass ended on, and the work count then)
|
|
thread_local uint64_t last_pass_end_epoch = 0;
|
|
std::atomic<int64_t> st_pass_relaxed{0};
|
|
} // namespace
|
|
void note_pass_end(VkCommandBuffer cmd) {
|
|
last_pass_end_cmd = cmd, last_pass_end_epoch = work_epoch.load(std::memory_order_relaxed);
|
|
}
|
|
int64_t pass_barriers_relaxed() { return st_pass_relaxed.load(std::memory_order_relaxed); }
|
|
void pass_start_barrier(VkCommandBuffer cmd) {
|
|
static const bool game = [] {
|
|
const char* e = getenv("RX_VK_PASS_BARRIER");
|
|
const bool on = e && !strcmp(e, "game");
|
|
if (on) logf("vk: RX_VK_PASS_BARRIER=game: a draw pass right after another, no barrier of the game's between, waits in its fragment stages only");
|
|
return on;
|
|
}();
|
|
if (!game || par_active.load(std::memory_order_relaxed) || cmd != last_pass_end_cmd ||
|
|
work_epoch.load(std::memory_order_relaxed) != last_pass_end_epoch) {
|
|
full_barrier(cmd, "barrier (pass start)");
|
|
return;
|
|
}
|
|
// (the previous pass's writes, all stages, made visible to this pass's fragment-side stages; its vertex, task and mesh stages run
|
|
// as soon as they can, as on the console's GPU)
|
|
st_pass_relaxed.fetch_add(1, std::memory_order_relaxed);
|
|
GprofSpan gp(cmd, "barrier (pass start, fragments)");
|
|
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT |
|
|
VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
d.memoryBarrierCount = 1;
|
|
d.pMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &d);
|
|
last_barrier_cmd = VK_NULL_HANDLE; // (not a full one: the next full barrier is not left out after it)
|
|
}
|
|
|
|
void texture_begin_recording(VkCommandBuffer upload, uint64_t value) {
|
|
last_barrier_cmd = VK_NULL_HANDLE;
|
|
last_pass_end_cmd = VK_NULL_HANDLE; // (a new recording: its command buffers start afresh, whatever their handles held)
|
|
up_cmd = upload;
|
|
if (value != rec_value) epoch++; // (a submission's parts check a texture's memory once: nothing is collected between)
|
|
rec_value = value;
|
|
up_used = false;
|
|
memory_table_begin();
|
|
memory_rb_recording_begun(); // (which readback pages its uploads may refresh: memory_rb_refresh)
|
|
}
|
|
VkCommandBuffer upload_command_buffer() {
|
|
return upload_cmd();
|
|
}
|
|
bool texture_uploads_recorded() {
|
|
if (memory_rb_refresh(up_cmd)) up_used = true; // (PR 430: readback pages from guest memory, last of the uploads)
|
|
memory_mirror_binds_flush(); // (Triforce: the memory of the mirrors' new pieces, bound before this recording is submitted)
|
|
memory_table_flush(); // (vk_memory.cpp: the page table's changes, after this recording's mirror copies)
|
|
if (up_used) full_barrier(up_cmd); // (before the recording that reads them)
|
|
return up_used;
|
|
}
|
|
|
|
int view_nvn_format(const nvn_tex* t, const nvn_view* v) {
|
|
return v && v->set_format ? v->format : t->d.format;
|
|
}
|
|
unsigned view_level(const nvn_view* v) {
|
|
return v && v->set_levels ? v->base_level : 0;
|
|
}
|
|
unsigned image_level(const Texture* tex, unsigned level) {
|
|
return level > tex->skip ? std::min(level - tex->skip, tex->levels - 1) : 0;
|
|
}
|
|
unsigned view_layer(const nvn_view* v) {
|
|
return v && v->set_layers ? v->min_layer : 0;
|
|
}
|
|
|
|
int texture_detail_level() {
|
|
static const int level = [] {
|
|
const int asked = tex_detail::parse(getenv("RX_VK_TEX_DETAIL"));
|
|
uint64_t used = 0, allowed = 0;
|
|
const bool known = vram_usage(&used, &allowed);
|
|
const bool integrated = dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
|
|
const int l = asked >= 0 ? asked : tex_detail::by_vram(known ? allowed : 0, integrated);
|
|
if (asked >= 0)
|
|
logf("vk: texture detail: %s (RX_VK_TEX_DETAIL)", tex_detail::name(l));
|
|
else if (l)
|
|
logf("vk: texture detail: %s, for the %llu MB of VRAM allowed here%s (full from %llu MB, low under %llu; "
|
|
"RX_VK_TEX_DETAIL=full|reduced|low says otherwise)", tex_detail::name(l),
|
|
(unsigned long long)(allowed >> 20), integrated ? " on an integrated GPU" : "",
|
|
(unsigned long long)tex_detail::kFullMb, (unsigned long long)tex_detail::kLowMb);
|
|
return l;
|
|
}();
|
|
return level;
|
|
}
|
|
void texture_want_whole(Texture* tex) {
|
|
if (!tex || !tex->detail_skip || tex->whole_asked) return;
|
|
ParLock lk;
|
|
if (tex->whole_asked) return;
|
|
tex->whole_asked = true;
|
|
whole_pending.push_back(tex);
|
|
}
|
|
|
|
/// The image the game's texture has: its storage and what makes two textures over one storage different images
|
|
static Key key_of(const nvn_tex* t, const VkFormatInfo& fi, int bpb) {
|
|
Key k{};
|
|
k.gpu = t->gpu;
|
|
k.target = (uint32_t)t->d.target;
|
|
// (a cube map and a 2D array of 6 layers over the same storage are one image; MS targets stay apart)
|
|
if (k.target == NVN_TEXTURE_TARGET_CUBEMAP) k.target = NVN_TEXTURE_TARGET_2D_ARRAY;
|
|
if (k.target == NVN_TEXTURE_TARGET_CUBEMAP_ARRAY) k.target = NVN_TEXTURE_TARGET_2D_ARRAY;
|
|
if (k.target == NVN_TEXTURE_TARGET_RECTANGLE) k.target = NVN_TEXTURE_TARGET_2D;
|
|
k.width = t->d.width, k.height = t->d.height;
|
|
k.depth = t->d.target == NVN_TEXTURE_TARGET_CUBEMAP ? 6 : t->d.depth;
|
|
if (k.target == NVN_TEXTURE_TARGET_2D) k.depth = 1;
|
|
k.levels = std::max(1u, t->d.levels);
|
|
k.samples = std::max(1, t->d.samples);
|
|
k.layout = nvn_tex_is_linear(&t->d) ? 0x80000000u | (t->d.stride ? t->d.stride : host->tex_linear_stride(&t->d))
|
|
: t->L.tileH | t->L.tileD << 4;
|
|
k.cls = format_class(fi.format, t->d.format, bpb, fi.astc);
|
|
return k;
|
|
}
|
|
|
|
Texture* texture_get(const nvn_tex* t, bool for_sampling) {
|
|
PROF_HOT(PROF_TEXGET);
|
|
if (!t || !t->gpu || t->d.target == NVN_TEXTURE_TARGET_BUFFER) return nullptr;
|
|
const VkFormatInfo fi = vk_format(t->d.format);
|
|
if (fi.format == VK_FORMAT_UNDEFINED) {
|
|
static int seen[256];
|
|
if (t->d.format >= 0 && t->d.format < 256 && seen[t->d.format]++ < 2)
|
|
logf("vk: NVN format %#x has no Vulkan format: its textures are left out", t->d.format);
|
|
return nullptr;
|
|
}
|
|
int bw = 1, bh = 1;
|
|
const int bpb = nvn_block_size(t->d.format, &bw, &bh);
|
|
if (bpb <= 0) return nullptr;
|
|
const Key k = key_of(t, fi, bpb);
|
|
Texture* tex;
|
|
auto it = textures.find(k);
|
|
if (it == textures.end()) {
|
|
tex = create(k, t, fi.format, fi.astc, bw, bh, bpb, !for_sampling);
|
|
if (!tex) return nullptr;
|
|
static const int64_t log_from = getenv("RX_VK_LOG_MADE") ? atoll(getenv("RX_VK_LOG_MADE")) : -1;
|
|
static int made_logged;
|
|
if (log_from >= 0 && (int64_t)frame_no >= log_from && made_logged++ < 300) {
|
|
auto [a, e] = by_gpu.equal_range(tex->gpu);
|
|
int others = 0;
|
|
for (auto it = a; it != e; ++it) others += it->second != tex;
|
|
logf("vk: made #%u %#llx %ux%ux%u x%u levels %u NVN %#x target %d tileH %u, first %s; %d other images at it",
|
|
tex->id, (unsigned long long)tex->gpu, t->d.width, t->d.height, t->d.depth, tex->layers, t->d.levels,
|
|
t->d.format, t->d.target, t->L.tileH, for_sampling ? "sampled" : "rendered", others);
|
|
}
|
|
} else {
|
|
tex = it->second.get();
|
|
}
|
|
return texture_touch(tex, for_sampling);
|
|
}
|
|
|
|
uint64_t texture_destroy_generation() {
|
|
return destroy_gen;
|
|
}
|
|
|
|
uint64_t texture_destroyed_count() {
|
|
return gone_record.count();
|
|
}
|
|
|
|
bool texture_destroyed_since(uint32_t id, uint64_t since) {
|
|
return gone_record.gone_since(id, since);
|
|
}
|
|
|
|
uint32_t texture_epoch() {
|
|
return epoch;
|
|
}
|
|
|
|
// ---- (Triforce, diagnostics) RX_VK_AUDIT_GPU_ONLY=1: where the game makes textures out of memory only the GPU drew into ----
|
|
// On the console what the GPU draws into a texture is in that texture's storage, and the game may use those bytes later: copy
|
|
// them, keep them in a cache, make another texture over them (the terrain's normal maps: bake_writeback_step). Here they are
|
|
// only in the image unless something writes them back. With this switch a texture's storage is filled with a mark when the GPU
|
|
// first draws into it (the word "GPUo" and the texture's number, over and over), and every texture uploaded from memory is
|
|
// searched for marks: one that holds them is a texture the game expects the GPU's texels in, wherever the game has moved
|
|
// them since, and is logged with the texture the marks are of. A lab switch: the marks replace what that memory held, so a
|
|
// game that reads it on the CPU gets them too. Not marked: a texture that had the CPU's texels before the GPU's (mixed
|
|
// content), virtual pools, storage over 64 MB.
|
|
void bake_note_written(Texture* t); // (below: what the GPU baked, into the game's memory)
|
|
constexpr uint32_t kAuditMark = 0x6f555047; // ("GPUo")
|
|
bool audit_on() {
|
|
static const bool on = getenv("RX_VK_AUDIT_GPU_ONLY") && atoi(getenv("RX_VK_AUDIT_GPU_ONLY")) != 0;
|
|
return on;
|
|
}
|
|
struct AuditSource { uint32_t w, h, levels, layers; int format; uint64_t gpu, frame; };
|
|
std::unordered_map<uint32_t, AuditSource> audit_sources; // (a marked texture's number -> what it was)
|
|
void audit_mark(const Texture& t) {
|
|
if (!t.guest_size || t.guest_size > (64ull << 20) || t.gpu >= (1ull << 39)) return;
|
|
static thread_local std::vector<uint32_t> words;
|
|
words.resize((size_t)(t.guest_size / 8) * 2);
|
|
for (size_t i = 0; i + 1 < words.size(); i += 2) words[i] = kAuditMark, words[i + 1] = t.id;
|
|
host->gpu_write(t.gpu, words.data(), words.size() * 4);
|
|
audit_sources[t.id] = {t.width, t.height, t.levels, t.layers, t.nvn_format, t.gpu, frame_no};
|
|
}
|
|
void audit_scan(const Texture& t) {
|
|
if (!audit_on() || t.guest_size < 16 || t.guest_size > (64ull << 20) || t.gpu >= (1ull << 39)) return;
|
|
static thread_local std::vector<uint8_t> mem;
|
|
mem.resize((size_t)t.guest_size);
|
|
if (!host->gpu_read(t.gpu, mem.data(), mem.size())) return;
|
|
std::unordered_map<uint32_t, uint64_t> hits; // (the marked texture's number -> bytes of its marks here)
|
|
const size_t n = mem.size() / 4;
|
|
for (size_t i = 0; i + 1 < n; i++) {
|
|
uint32_t w[2];
|
|
memcpy(w, mem.data() + i * 4, 8);
|
|
if (w[0] == kAuditMark) hits[w[1]] += 8, i++;
|
|
}
|
|
static std::unordered_set<uint64_t> told;
|
|
for (const auto& [id, bytes] : hits) {
|
|
if (bytes < 256 || told.size() >= 800 || !told.insert((uint64_t)t.id << 32 | id).second) continue;
|
|
const auto s = audit_sources.find(id);
|
|
char src[160] = "";
|
|
if (s != audit_sources.end())
|
|
snprintf(src, sizeof src, " (%ux%u x%u levels %u NVN %#x at %#llx, first drawn at frame %llu%s)", s->second.w, s->second.h,
|
|
s->second.layers, s->second.levels, s->second.format, (unsigned long long)s->second.gpu,
|
|
(unsigned long long)s->second.frame, s->second.gpu == t.gpu ? "; the same storage" : "; other storage: the bytes were moved");
|
|
logf("vk: audit: texture #%u %#llx %ux%ux%u x%u levels %u NVN %#x, sampled by %016llx at frame %llu, is uploaded from memory "
|
|
"that holds %llu of %llu KB of what only the GPU drew: texture #%u%s", t.id, (unsigned long long)t.gpu, t.width, t.height,
|
|
t.depth, t.layers, t.levels, t.nvn_format, (unsigned long long)tl_watch_prog, (unsigned long long)frame_no,
|
|
(unsigned long long)(bytes >> 10), (unsigned long long)(t.guest_size >> 10), id, src);
|
|
}
|
|
}
|
|
|
|
thread_local bool upload_defer_ok = false;
|
|
namespace {
|
|
/// (hookshot) A new ASTC texture a draw samples, met after this frame's uploads took their budget: not uploaded now, the draw
|
|
/// samples the placeholder, and the next frame's first look at it uploads it. An area's load made 300-800 ASTC uploads in one
|
|
/// frame, 200-600 ms of the replay thread (the "a frame's uploads took" lines); now that is spread over several frames, the
|
|
/// textures coming in a few frames late. ASTC only: those are the materials' colour, normal and mask textures; uncompressed
|
|
/// ones can be data (height maps, look-up tables) that a frame without would get wrong. RX_VK_UPLOAD_BUDGET_MS=<n>: the
|
|
/// budget (40: only an area load's burst; at 8 the ground's streamed textures waited a frame in ordinary play and the ground
|
|
/// flickered, Discord 2026-11-09), 0: no deferral, as before
|
|
bool upload_defer(const Texture& tex) {
|
|
static const int64_t budget_ns = (int64_t)(getenv("RX_VK_UPLOAD_BUDGET_MS") ? atof(getenv("RX_VK_UPLOAD_BUDGET_MS")) : 40.0) * 1000000;
|
|
if (!budget_ns || !upload_defer_ok || tex.uploaded || !tex.astc || tex.gpu_written) return false;
|
|
// (Triforce) and the first RX_VK_UPLOAD_MIN_ASTC of a frame (24) are never left for later. The budget counts every upload of
|
|
// the frame, and a frame whose other uploads took it all left every new ASTC texture on the placeholder: on a card at its
|
|
// VRAM allowance the render targets evicted and made again are uploaded from memory on the CPU, 58 to 190 ms a frame for
|
|
// frames on end (an RX 5700 XT's log, 2026-11-11: 8,586 textures left for later in one session), and the materials stayed
|
|
// flat white or blue for as long as that went on. An ASTC upload is decoded on the GPU: its part of such a frame is small
|
|
static const int min_astc = getenv("RX_VK_UPLOAD_MIN_ASTC") ? std::max(0, atoi(getenv("RX_VK_UPLOAD_MIN_ASTC"))) : 24;
|
|
if (frame_astc_admitted.fetch_add(1, std::memory_order_relaxed) < min_astc) return false;
|
|
if (frame_upload_ns.load(std::memory_order_relaxed) < budget_ns) return false;
|
|
st_deferred.fetch_add(1, std::memory_order_relaxed), st_deferred_frame.fetch_add(1, std::memory_order_relaxed);
|
|
return true;
|
|
}
|
|
} // namespace
|
|
Texture* texture_touch(Texture* tex, bool for_sampling) {
|
|
tex->last_frame = frame_no;
|
|
tex->let_go = 0; // (in use: texture_sweep)
|
|
tex->last_value = rec_value;
|
|
if (!for_sampling) return tex;
|
|
if (upload_defer(*tex)) return nullptr;
|
|
// contents: from guest memory unless the GPU wrote the image (then it is the content, unless the texture was
|
|
// finalized since and the CPU writes its memory again). What the CPU writes into it after the GPU's write is taken
|
|
// all the same, those ranges alone, the rest staying the GPU's: Eden's texture cache takes a CPU write over the
|
|
// GPU's contents, and MKW's font cache is cleared on the GPU with its glyphs written by the CPU as text appears
|
|
// (the menus were missing every letter added later). Only whole pages of its storage: a write to another variable
|
|
// sharing an edge page once re-uploaded garbage over a drawn LUT (the rainbow sky). Opt-in (RX_VK_CPU_OVER_GPU=1): transient render targets share memory the CPU writes for other things, and took that over their texels, and the menus' letters stayed missing with it.
|
|
if (tex->gpu_written && !tex->released) {
|
|
// (hookshot) and a texture the GPU only cleared: the game's font cache, whose letters the CPU writes as text appears. Its menu text
|
|
// went missing at random (Hookshot issues 40 and 44), whenever the clear reached the image first
|
|
static const bool take = getenv("RX_VK_CPU_OVER_GPU") != nullptr;
|
|
if ((!take && !tex->only_cleared) || tex->checked_epoch == epoch) return tex;
|
|
tex->checked_epoch = epoch;
|
|
static thread_local Ranges cpu, inner;
|
|
if (!texture_dirty(*tex, std::max(tex->upload_gen, tex->gpu_gen), cpu)) return tex;
|
|
const uint64_t first = (0x1000 - (tex->gpu & 0xFFF)) & 0xFFF;
|
|
const uint64_t last = ((tex->gpu + tex->guest_size) & ~0xFFFull) - tex->gpu;
|
|
inner.clear();
|
|
uint64_t bytes = 0;
|
|
for (const auto& r : cpu) {
|
|
const uint64_t a = std::max<uint64_t>(r.first, first), b = std::min<uint64_t>(r.second, last);
|
|
if (a < b) inner.push_back({a, b}), bytes += b - a;
|
|
}
|
|
const uint32_t gen = memory_generation();
|
|
if (!inner.empty() && upload(*tex, &inner)) {
|
|
tex->upload_value = rec_value, tex->upload_seg = tl_seg;
|
|
static int logged;
|
|
if (logged++ < 20)
|
|
logf("vk: texture %#llx %ux%u x%u NVN %#x: %llu KB the CPU wrote after the GPU taken", (unsigned long long)tex->gpu,
|
|
tex->width, tex->height, tex->layers, tex->nvn_format, (unsigned long long)(bytes >> 10));
|
|
}
|
|
tex->upload_gen = gen;
|
|
return tex;
|
|
}
|
|
{ // (RX_VK_REUPLOAD_DIM=<w>x<h> (debugging): textures that size uploaded whole again every frame: is what a
|
|
// sampled texture misses in guest memory, unseen by the write tracking, or not there at all?)
|
|
static uint32_t rw = 0, rh = 0;
|
|
static const bool parsed = [] {
|
|
if (const char* e = getenv("RX_VK_REUPLOAD_DIM")) sscanf(e, "%ux%u", &rw, &rh);
|
|
return true;
|
|
}();
|
|
(void)parsed;
|
|
static thread_local uint64_t last;
|
|
if (rw && tex->width == rw && tex->height == rh && tex->uploaded && tex->checked_epoch != epoch &&
|
|
last != frame_no) {
|
|
last = frame_no;
|
|
tex->checked_epoch = epoch;
|
|
upload(*tex, nullptr);
|
|
return tex;
|
|
}
|
|
}
|
|
if (tex->uploaded && tex->upload_value == rec_value && tex->upload_seg > tl_seg) {
|
|
// (a parallel replay: uploaded in this submission by a segment after this one, which reads it first in stream
|
|
// order: uploaded again, the same data, in this segment's uploads)
|
|
upload(*tex, nullptr);
|
|
tex->upload_seg = tl_seg;
|
|
return tex;
|
|
}
|
|
if (tex->uploaded && tex->checked_epoch == epoch) return tex;
|
|
tex->checked_epoch = epoch;
|
|
static thread_local Ranges dirty;
|
|
static const bool no_partial = getenv("RX_VK_NO_PARTIAL") != nullptr; // (A/B: whole re-uploads)
|
|
// (a buffer the GPU wrote over its storage, done since: all of it again; the tiles of MKW's terrain are filled so
|
|
// and stayed the flat placeholder they were made with)
|
|
const bool gpu_dirty = tex->uploaded && gpu_wrote_into(*tex);
|
|
const bool partial = !no_partial && !gpu_dirty && tex->uploaded && !tex->gpu_written && !tex->released;
|
|
// (a texture the GPU drew and the game finalized: only what the CPU wrote since then. Its memory never got the
|
|
// GPU's texels, and a write before (another variable in the same page) re-uploaded garbage over a drawn LUT:
|
|
// the rainbow sky)
|
|
const uint32_t since = tex->gpu_written && tex->released ? std::max(tex->upload_gen, tex->release_gen) : tex->upload_gen;
|
|
if (tex->gpu_written && tex->released && frame_no > tex->last_written + 60) {
|
|
// (hookshot: one the game finalized, sampled again well after it was drawn: what the eviction keeps them for)
|
|
static std::unordered_set<uint32_t> told;
|
|
if (told.size() < 200 && told.insert(tex->id).second)
|
|
logf("vk: texture #%u %#llx %ux%u x%u NVN %#x the GPU drew, finalized by the game, sampled again at frame %llu (drawn at "
|
|
"%llu)", tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, tex->layers, tex->nvn_format,
|
|
(unsigned long long)frame_no, (unsigned long long)tex->last_written);
|
|
}
|
|
if (tex->uploaded && !gpu_dirty && !texture_dirty(*tex, since, dirty, !tex->gpu_written)) return tex;
|
|
const uint32_t gen = memory_generation();
|
|
// (storage render targets took last, which the CPU has not written since: nothing to upload, the memory holds
|
|
// nothing the GPU's images did not replace. The game's transient heaps move with the dynamic resolution, and a
|
|
// render target sampled before it is rendered at its new address (the culling reads last frame's shadow map)
|
|
// was unswizzled from that memory: 55 MB for the 3072 x 3072 x 3 shadow map, frame after frame. Not "storage
|
|
// the write watch never saw written": the game fills memory before it makes a pool of it.)
|
|
static const bool upload_all = getenv("RX_VK_UPLOAD_ALL") != nullptr; // (A/B: such storage uploaded too)
|
|
// (from Shakoquo/Mako ff479e988d) never for a block-compressed texture (ASTC, BC, ETC): the GPU cannot render into one, so
|
|
// its texels come from memory whatever render targets used that storage before. Skipped, a boss or a new enemy met for
|
|
// the first time (its textures loaded into the transient heaps' memory) was drawn with an empty texture, all blue, for
|
|
// the whole cutscene. RX_VK_SKIP_COMPRESSED_OVER_RT=1: as before
|
|
// (Triforce: this tree has no env_on yet; the switch is on when set to anything but 0)
|
|
static const bool skip_compressed = getenv("RX_VK_SKIP_COMPRESSED_OVER_RT") && atoi(getenv("RX_VK_SKIP_COMPRESSED_OVER_RT")) != 0;
|
|
const bool compressed = tex->astc || tex->bc ||
|
|
(tex->format >= VK_FORMAT_BC1_RGB_UNORM_BLOCK && tex->format <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK);
|
|
uint32_t owned = !tex->uploaded && !upload_all && (skip_compressed || !compressed) ? gpu_owned_since(tex->gpu, tex->guest_size) : 0;
|
|
if (!tex->uploaded && !upload_all && !skip_compressed && compressed) {
|
|
static std::atomic<int> told{0};
|
|
if (told.load(std::memory_order_relaxed) < 30 && gpu_owned_since(tex->gpu, tex->guest_size) && told++ < 30)
|
|
logf("vk: texture #%u %#llx %ux%u x%u NVN %#x (compressed) over storage render targets took: uploaded from the game's "
|
|
"memory all the same", tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, tex->layers,
|
|
tex->nvn_format);
|
|
}
|
|
if (owned && !remade_skip_old()) { // (one evicted that came from memory: uploaded from it again. Above)
|
|
std::lock_guard lk(from_memory_mu);
|
|
auto e = evicted_from_memory.find(evicted_key(tex->gpu, tex->width, tex->height, tex->nvn_format));
|
|
if (e != evicted_from_memory.end() && e->second == tex->guest_size) {
|
|
evicted_from_memory.erase(e);
|
|
owned = 0;
|
|
const uint64_t n = ++st_remade_uploaded;
|
|
if (n <= 20 || n % 100 == 0)
|
|
logf("vk: texture #%u %#llx %ux%u NVN %#x evicted, made again over storage render targets took: uploaded from "
|
|
"the game's memory (%llu so far)", tex->id, (unsigned long long)tex->gpu, tex->width, tex->height,
|
|
tex->nvn_format, (unsigned long long)n);
|
|
}
|
|
}
|
|
watch_tex(tex, owned ? "sampled, over rendered storage" : "sampled, uploaded");
|
|
audit_scan(*tex); // (RX_VK_AUDIT_GPU_ONLY)
|
|
if (owned && !texture_dirty(*tex, owned, dirty)) {
|
|
tex->uploaded = true;
|
|
tex->upload_gen = gen;
|
|
watch_tex(tex, "NOT uploaded (over storage the GPU rendered)");
|
|
if (audit_on()) { // (the same-storage case: made over what the GPU drew, sampled with nothing in its image)
|
|
static int told;
|
|
if (told++ < 400)
|
|
logf("vk: audit: texture #%u %#llx %ux%ux%u x%u levels %u NVN %#x, sampled by %016llx at frame %llu, is made over "
|
|
"storage the GPU drew and not uploaded: its image holds nothing", tex->id, (unsigned long long)tex->gpu,
|
|
tex->width, tex->height, tex->depth, tex->layers, tex->levels, tex->nvn_format,
|
|
(unsigned long long)tl_watch_prog, (unsigned long long)frame_no);
|
|
}
|
|
if (remade_log_on()) { // (made again after its eviction, and sampled empty)
|
|
std::lock_guard lk(evicted_mu);
|
|
auto r = remade.find(tex);
|
|
if (r != remade.end()) {
|
|
static int told;
|
|
if (told++ < 300)
|
|
logf("vk: texture #%u %#llx %ux%u NVN %#x, made again %llu frames after its eviction, sampled EMPTY "
|
|
"(storage the GPU rendered, not uploaded) at frame %llu", tex->id, (unsigned long long)tex->gpu,
|
|
tex->width, tex->height, tex->nvn_format, (unsigned long long)r->second,
|
|
(unsigned long long)frame_no);
|
|
remade.erase(r);
|
|
}
|
|
}
|
|
return tex;
|
|
}
|
|
if (partial && getenv("RX_VK_LOG_PARTIAL")) {
|
|
static int logged;
|
|
if (logged++ < 200) {
|
|
char line[512];
|
|
int n = snprintf(line, sizeof line, "vk: partial upload %#llx %ux%ux%u x%u levels %u NVN %#x size %llu flags %#x:",
|
|
(unsigned long long)tex->gpu, tex->width, tex->height, tex->depth, tex->layers, tex->levels,
|
|
tex->nvn_format, (unsigned long long)tex->guest_size, tex->d.flags);
|
|
for (size_t i = 0; i < dirty.size() && i < 8 && n < 480; i++)
|
|
n += snprintf(line + n, sizeof line - n, " [%#llx,%#llx)", (unsigned long long)dirty[i].first,
|
|
(unsigned long long)dirty[i].second);
|
|
logf("%s", line);
|
|
}
|
|
}
|
|
if (!tex->uploaded && tex->gpu >= (1ull << 40) && tex->page_map.empty()) {
|
|
// (a sparse texture's mapping as it is uploaded first: its first check after does not take every page for
|
|
// remapped, which uploaded the course maps twice)
|
|
static thread_local Ranges first;
|
|
texture_dirty(*tex, 0, first);
|
|
}
|
|
// (a sparse image, first or whole: only the storage ever written, not all of its gigabytes)
|
|
const bool sparse_written = tex->sparse && !partial;
|
|
if (sparse_written) texture_dirty(*tex, 0, dirty);
|
|
if (tex->gpu_written) { // (an image the GPU drew, finalized and sampled again: from guest memory, 400 lines)
|
|
static int logged;
|
|
uint64_t kb = 0;
|
|
for (const auto& r : dirty) kb += (r.second - r.first) >> 10;
|
|
if (logged++ < 400)
|
|
logf("vk: upload over a drawn image: #%u %#llx %ux%ux%u x%u NVN %#x at present %llu (texture frame %llu), %s, "
|
|
"%llu KB dirty in %zu ranges%s", tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, tex->depth,
|
|
tex->layers, tex->nvn_format, (unsigned long long)(frame_no - 1), (unsigned long long)frame_no,
|
|
tex->released ? "finalized" : "live", (unsigned long long)kb, dirty.size(), gpu_dirty ? ", a GPU buffer wrote it" : "");
|
|
}
|
|
// (hookshot) a BC3 image whose last upload failed waits before the next try: its upload stages the whole ASTC source (some
|
|
// 200 MB for a terrain array), not to be repeated every submission while the memory it needs is not there
|
|
if (tex->bc && !tex->uploaded && tex->bc_retry_frame > frame_no) return tex;
|
|
// (Triforce, RX_VK_STAMP_SRC=1, a diagnostic: a texture of 1 MB or more uploaded again: who wrote its first dirty page (the
|
|
// write watch 1, a capture 2, else a caller of memory_stamp, placed by memory_stamp's own address), and whether its memory
|
|
// holds other bytes than at its last upload at all. 400 lines)
|
|
static const bool stamp_src = getenv("RX_VK_STAMP_SRC") != nullptr;
|
|
if (stamp_src && tex->uploaded && tex->guest_size >= (1u << 20)) {
|
|
static int told;
|
|
static std::unordered_map<uint32_t, int> times; // (RX_VK_STAMP_SRC=<n>: only a texture's n-th such upload and later)
|
|
static const int from_nth = std::max(1, atoi(getenv("RX_VK_STAMP_SRC")));
|
|
const uint64_t h = guest_hash(*tex);
|
|
if (++times[tex->id] >= from_nth && told++ < 400) {
|
|
uint64_t part = 0;
|
|
for (const auto& r : dirty) part += r.second - r.first;
|
|
const uint64_t off = dirty.empty() ? 0 : dirty[0].first;
|
|
const uint64_t guest = tex->gpu < (1ull << 40) ? tex->guest + off : host->gpu_guest(tex->gpu + off);
|
|
uint32_t pg = 0;
|
|
uintptr_t anchor = 0;
|
|
const uintptr_t who = guest ? memory_stamp_who(guest, &pg, &anchor) : 0;
|
|
logf("vk: stamp src: #%u %#llx %ux%u x%u NVN %#x uploaded again (%d) at frame %llu: %llu of %llu KB dirty in %zu ranges%s; bytes %s; "
|
|
"page %#llx gen %u (uploaded at %u) by %s%+lld", tex->id, (unsigned long long)tex->gpu, tex->width, tex->height,
|
|
tex->layers, tex->nvn_format, times[tex->id], (unsigned long long)frame_no, (unsigned long long)(part >> 10),
|
|
(unsigned long long)(tex->guest_size >> 10), dirty.size(), gpu_dirty ? " (a GPU buffer wrote it)" : "",
|
|
!tex->verify_hash ? "not hashed before" : h == tex->verify_hash ? "THE SAME as at its last upload" : "changed",
|
|
(unsigned long long)guest, pg, since, who == 1 ? "the write watch" : who == 2 ? "a capture" : who ? "memory_stamp" : "unknown",
|
|
who > 2 ? (long long)(who - anchor) : 0ll);
|
|
}
|
|
tex->verify_hash = h;
|
|
} else if (stamp_src && tex->guest_size >= (1u << 20)) {
|
|
tex->verify_hash = guest_hash(*tex);
|
|
}
|
|
const auto up0 = std::chrono::steady_clock::now();
|
|
const bool up_ok = upload(*tex, partial || sparse_written ? &dirty : nullptr);
|
|
{ // (an upload over 15 ms: what it was, 60 of them)
|
|
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - up0).count();
|
|
static int slow_logged;
|
|
if (ms > 15.0 && slow_logged++ < 60) {
|
|
uint64_t part = 0;
|
|
if (partial) for (const auto& r : dirty) part += r.second - r.first;
|
|
logf("vk: slow upload %.1f ms: #%u %#llx %ux%ux%u x%u levels %u NVN %#x%s%s, %llu of %llu KB%s", ms, tex->id,
|
|
(unsigned long long)tex->gpu, tex->width, tex->height, tex->depth, tex->layers, tex->levels,
|
|
tex->nvn_format, tex->astc ? " ASTC" : "", tex->bc ? " (BC3)" : "",
|
|
(unsigned long long)((partial ? part : tex->guest_size) >> 10), (unsigned long long)(tex->guest_size >> 10),
|
|
tex->uploaded ? " again" : "");
|
|
}
|
|
}
|
|
if (up_ok) {
|
|
tex->upload_value = rec_value, tex->upload_seg = tl_seg;
|
|
// (its storage joins the submit-time capture: what the game writes into it after a submit is not taken for it)
|
|
// (any size: the capture copies only the pages the CPU wrote, and the big CPU-streamed textures were read as the
|
|
// game had rewritten them a frame or two later. RX_VK_TEX_CAPTURE_MAX_MB=<n>: only up to that, 8 before)
|
|
static const uint64_t cap_max = getenv("RX_VK_TEX_CAPTURE_MAX_MB") ? (uint64_t)atoi(getenv("RX_VK_TEX_CAPTURE_MAX_MB")) << 20 : (2ull << 30);
|
|
if (tex->gpu < (1ull << 40) && tex->guest_size < cap_max) memory_note_texture_upload(tex->gpu, tex->guest_size);
|
|
if (verify_dim().first && tex->width == verify_dim().first && tex->height == verify_dim().second)
|
|
tex->verify_hash = guest_hash(*tex, true);
|
|
if (tex->uploaded) {
|
|
static int logged;
|
|
static std::unordered_map<uint64_t, int> big;
|
|
if (logged++ < 20 || (tex->guest_size > (4u << 20) && big[tex->gpu]++ < 3)) {
|
|
uint64_t part = 0, nz = 0, readable = 0;
|
|
for (const auto& r : dirty) part += r.second - r.first;
|
|
if (partial) { // (what the CPU wrote: how much of it is not zero, and in which levels)
|
|
std::vector<uint8_t> b;
|
|
char lv[96] = "";
|
|
int lvn = 0;
|
|
for (const auto& r : dirty) {
|
|
b.resize((size_t)(r.second - r.first));
|
|
if (!host->gpu_read(tex->gpu + r.first, b.data(), b.size())) continue;
|
|
readable += b.size();
|
|
for (uint8_t c : b) nz += c != 0;
|
|
for (uint32_t l = 0; l < tex->levels && lvn < 90; l++) {
|
|
const uint64_t lo = host->tex_level_offset(&tex->d, l);
|
|
const uint64_t hi = l + 1 < tex->levels ? host->tex_level_offset(&tex->d, l + 1) : tex->guest_size;
|
|
if (r.first < hi && r.second > lo) lvn += snprintf(lv + lvn, sizeof lv - lvn, " L%u", l);
|
|
}
|
|
}
|
|
logf("vk: (its written part: %llu of %llu KB readable, %llu bytes not zero; levels%s)",
|
|
(unsigned long long)(readable >> 10), (unsigned long long)(part >> 10), (unsigned long long)nz, lv);
|
|
}
|
|
logf("vk: texture %#llx %ux%ux%u x%u levels %u NVN %#x (%llu KB of storage) re-uploaded (its memory was "
|
|
"written; flags %#x): %s, %llu KB of it%s%s", (unsigned long long)tex->gpu, tex->width, tex->height,
|
|
tex->depth, tex->layers, tex->levels, tex->nvn_format, (unsigned long long)(tex->guest_size >> 10),
|
|
tex->d.flags, gpu_dirty ? "a GPU write into its storage" : partial ? "what the CPU wrote" : "whole",
|
|
(unsigned long long)((partial || sparse_written ? part : tex->guest_size) >> 10),
|
|
tex->gpu_written ? ", over what the GPU wrote" : "", tex->released ? " (finalized since)" : "");
|
|
}
|
|
}
|
|
tex->uploaded = true;
|
|
tex->upload_gen = gen;
|
|
tex->gpu_written = false;
|
|
tex->released = false;
|
|
} else if (tex->bc) {
|
|
// (hookshot) a BC3 image holds nothing until its encode has run. Marked uploaded as the others are, its blocks as the
|
|
// memory had them were drawn as coloured noise, and never made again unless the game wrote its memory: the path, the
|
|
// trees and Saria after an alt-tab on a 4 GB RX 5500M, its VRAM full and the textures evicted and made again. Not
|
|
// uploaded, it is tried again 30 frames on, until the staging and the scratch it needs can be had
|
|
tex->bc_retry_frame = frame_no + 30;
|
|
static std::atomic<int> told{0};
|
|
if (told++ < 40)
|
|
logf("vk: BC3 texture #%u %#llx %ux%u x%u: its upload failed (%s): tried again in 30 frames", tex->id,
|
|
(unsigned long long)tex->gpu, tex->width, tex->height, tex->layers, astc_fail ? astc_fail : "unknown");
|
|
} else {
|
|
tex->uploaded = true; // (not retried every draw; a write to its memory tries again)
|
|
tex->upload_gen = gen;
|
|
}
|
|
return tex;
|
|
}
|
|
|
|
/// (Triforce, 2026-11-11) A flat texture sampled by a program whose instruction is a 3D sample: a one-slice 3D image with its
|
|
/// texels, and a 3D view of that. Vulkan has no 3D view of a 2D image, so such a binding had no view and read zeros (a null
|
|
/// descriptor). The console's GPU samples by the texture's own header whatever the instruction says: a flat texture's texel at
|
|
/// (u, v), the third coordinate unused, which a one-slice 3D image gives for any w. OoT's water in Zora's River binds one of
|
|
/// the game's 4 x 4 textures so (slot 0x102, 44 lookups in 10 s; its ripples ran in slow motion). Small plain textures only:
|
|
/// one layer copied, up to 1024 x 1024, not depth, not kept as BC3. Copied again at a lookup after the texture's contents
|
|
/// changed (an upload, or the GPU writing it): the lookups are cached by slot, so one the GPU draws every frame is a frame or
|
|
/// more behind here. RX_VK_NO_TWIN3D=1: no such view, as before
|
|
VkImageView twin3d_view(Texture* tex, VkFormat f, uint32_t level0, uint32_t nlevels, uint32_t layer, const VkComponentMapping& cm) {
|
|
static const bool off = getenv("RX_VK_NO_TWIN3D") != nullptr;
|
|
if (off || tex->type != VK_IMAGE_TYPE_2D || tex->sparse || tex->bc || tex->samples != VK_SAMPLE_COUNT_1_BIT ||
|
|
tex->aspect != VK_IMAGE_ASPECT_COLOR_BIT || is_bc(tex->format) || is_astc(tex->format) ||
|
|
(uint64_t)tex->width * tex->height > 1024ull * 1024 || !tex->image)
|
|
return VK_NULL_HANDLE;
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
if (!tex->twin3d) {
|
|
VkImageCreateInfo ci{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
|
|
ci.imageType = VK_IMAGE_TYPE_3D;
|
|
ci.format = tex->format;
|
|
ci.extent = {tex->width, tex->height, 1};
|
|
ci.mipLevels = tex->levels;
|
|
ci.arrayLayers = 1;
|
|
ci.samples = VK_SAMPLE_COUNT_1_BIT;
|
|
ci.tiling = VK_IMAGE_TILING_OPTIMAL;
|
|
ci.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
|
|
ci.flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; // (the view's format may be another of its class: sRGB or not)
|
|
if (!create_image(ci, tex->twin3d, tex->twin3d_alloc, false, false, false)) {
|
|
tex->twin3d = VK_NULL_HANDLE, tex->twin3d_alloc = nullptr;
|
|
return VK_NULL_HANDLE;
|
|
}
|
|
cmd = upload_cmd();
|
|
VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
b.image = tex->twin3d;
|
|
b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, 1};
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.imageMemoryBarrierCount = 1, di.pImageMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &di);
|
|
tex->twin3d_value = ~0ull, tex->twin3d_written = ~0ull;
|
|
static int told;
|
|
if (told++ < 12)
|
|
logf("vk: texture #%u %#llx %ux%u levels %u NVN %#x is sampled as a 3D texture: a one-slice 3D copy of it made", tex->id,
|
|
(unsigned long long)tex->gpu, tex->width, tex->height, tex->levels, tex->nvn_format);
|
|
}
|
|
if (tex->twin3d_value != tex->upload_value || tex->twin3d_written != tex->last_written || tex->twin3d_layer != layer) {
|
|
if (!cmd) cmd = upload_cmd();
|
|
full_barrier(cmd, "barrier (a flat texture's 3D copy: before)");
|
|
std::vector<VkImageCopy> rs;
|
|
for (uint32_t l = 0; l < tex->levels; l++) {
|
|
VkImageCopy r{};
|
|
r.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, l, layer, 1};
|
|
r.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, l, 0, 1};
|
|
r.extent = {std::max(1u, tex->width >> l), std::max(1u, tex->height >> l), 1};
|
|
rs.push_back(r);
|
|
}
|
|
vkCmdCopyImage(cmd, tex->image, VK_IMAGE_LAYOUT_GENERAL, tex->twin3d, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)rs.size(), rs.data());
|
|
full_barrier(cmd, "barrier (a flat texture's 3D copy: after)");
|
|
tex->twin3d_value = tex->upload_value, tex->twin3d_written = tex->last_written, tex->twin3d_layer = layer;
|
|
}
|
|
uint64_t key = mix((uint64_t)f, (uint64_t)level0 << 32 | nlevels);
|
|
key = mix(key, (uint64_t)cm.r << 24 | (uint64_t)cm.g << 16 | (uint64_t)cm.b << 8 | cm.a);
|
|
const auto it = tex->twin3d_views.find(key);
|
|
if (it != tex->twin3d_views.end()) return it->second;
|
|
VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
|
|
vi.image = tex->twin3d;
|
|
vi.viewType = VK_IMAGE_VIEW_TYPE_3D;
|
|
vi.format = f;
|
|
vi.components = cm;
|
|
vi.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, level0, nlevels, 0, 1};
|
|
VkImageView view = VK_NULL_HANDLE;
|
|
if (vkCreateImageView(dev.device, &vi, nullptr, &view) != VK_SUCCESS) view = VK_NULL_HANDLE;
|
|
tex->twin3d_views[key] = view;
|
|
return view;
|
|
}
|
|
|
|
VkImageView texture_view(Texture* tex, const nvn_tex* t, const nvn_view* v, VkImageViewType type, ViewUse use,
|
|
VkFormat* format_out, VkFormat force_format) {
|
|
if (!tex) return VK_NULL_HANDLE;
|
|
const int nf = view_nvn_format(t, v);
|
|
VkFormat f = vk_format(nf).format;
|
|
int from = 0;
|
|
f = widened(f, &from);
|
|
const bool depth = is_depth_format(tex->format);
|
|
if (depth || f == VK_FORMAT_UNDEFINED) f = tex->format;
|
|
if (tex->astc) f = (nf >= 0x87 && nf <= 0x94) ? VK_FORMAT_R8G8B8A8_SRGB : VK_FORMAT_R8G8B8A8_UNORM;
|
|
if (tex->bc) {
|
|
if (use != ViewUse::Sample) return VK_NULL_HANDLE;
|
|
f = f == VK_FORMAT_R8G8B8A8_SRGB ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK;
|
|
}
|
|
if (use == ViewUse::Storage) f = force_format != VK_FORMAT_UNDEFINED ? force_format : storage_view_format(f);
|
|
if (use == ViewUse::Attachment && (is_astc(f) || is_bc(f))) return VK_NULL_HANDLE;
|
|
const uint32_t vl = view_level(v);
|
|
uint32_t level0 = image_level(tex, vl);
|
|
uint32_t nlevels = v && v->set_levels && v->nlevels ? (vl + v->nlevels > tex->skip ? vl + v->nlevels - tex->skip - level0 : 1)
|
|
: tex->levels - level0;
|
|
nlevels = std::clamp<uint32_t>(nlevels, 1, tex->levels - level0);
|
|
const uint32_t total_layers = tex->type == VK_IMAGE_TYPE_3D ? 1 : tex->layers;
|
|
uint32_t layer0 = std::min(view_layer(v), total_layers - 1);
|
|
uint32_t nlayers = v && v->set_layers && v->nlayers ? v->nlayers : total_layers - layer0;
|
|
nlayers = std::clamp<uint32_t>(nlayers, 1, total_layers - layer0);
|
|
{ // (Evil Hookshot, diagnosis) RX_VK_LOG_VIEW_LAYERS=<n>: the views of textures with at least n layers, 40 lines: the
|
|
// texture's target, the game's view (target, layers), the shader's type (11-08: OoT's 1365 x 1365 ground material arrays
|
|
// are 2D arrays of 90 in its own views, as Hookshot makes them)
|
|
static const uint32_t log_n = getenv("RX_VK_LOG_VIEW_LAYERS") ? (uint32_t)atoi(getenv("RX_VK_LOG_VIEW_LAYERS")) : 0;
|
|
static std::atomic<int> logged{0};
|
|
if (log_n && total_layers >= log_n && logged.fetch_add(1) < 40)
|
|
logf("vk: view of #%u %ux%u x%u layers: texture target %d, view %s target %d layers %u+%u, shader type %d, use %d -> "
|
|
"layers %u+%u", tex->id, tex->width, tex->height, total_layers, (int)t->d.target, v ? "yes" : "no",
|
|
v && v->set_target ? (int)v->target : -1, v && v->set_layers ? v->min_layer : 0u,
|
|
v && v->set_layers ? v->nlayers : 0u, (int)type, (int)use, layer0, nlayers);
|
|
}
|
|
VkImageAspectFlags aspect = tex->aspect;
|
|
if (use != ViewUse::Attachment && aspect == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT))
|
|
aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
|
|
if (use == ViewUse::Attachment) {
|
|
nlevels = 1, nlayers = 1;
|
|
type = tex->type == VK_IMAGE_TYPE_1D ? VK_IMAGE_VIEW_TYPE_1D : VK_IMAGE_VIEW_TYPE_2D;
|
|
// (a slice of a 3D target: a 2D view of it, 2D_ARRAY_COMPATIBLE, the slice as its layer)
|
|
if (tex->type == VK_IMAGE_TYPE_3D) layer0 = std::min(view_layer(v), std::max(1u, tex->depth >> level0) - 1);
|
|
} else if (use == ViewUse::Storage) {
|
|
nlevels = 1;
|
|
if (tex->type == VK_IMAGE_TYPE_3D) type = VK_IMAGE_VIEW_TYPE_3D, layer0 = 0, nlayers = 1;
|
|
else if (type == VK_IMAGE_VIEW_TYPE_2D && nlayers > 1) nlayers = 1;
|
|
else if (type == VK_IMAGE_VIEW_TYPE_1D && nlayers > 1) nlayers = 1;
|
|
}
|
|
int twin_layer = -1; // (Triforce: a flat texture wanted as 3D: the layer its 3D copy is of)
|
|
// the shader's type over the image's: what Vulkan allows of them
|
|
switch (type) {
|
|
case VK_IMAGE_VIEW_TYPE_1D: case VK_IMAGE_VIEW_TYPE_1D_ARRAY:
|
|
if (tex->type != VK_IMAGE_TYPE_1D) return VK_NULL_HANDLE;
|
|
if (type == VK_IMAGE_VIEW_TYPE_1D) nlayers = 1;
|
|
break;
|
|
case VK_IMAGE_VIEW_TYPE_2D: case VK_IMAGE_VIEW_TYPE_2D_ARRAY:
|
|
if (tex->type == VK_IMAGE_TYPE_1D) return VK_NULL_HANDLE;
|
|
if (tex->type == VK_IMAGE_TYPE_3D) {
|
|
// (a 2D sample instruction over a 3D texture, as Eden's recompiler keeps the instruction's type: its first
|
|
// slice, one level (VK_EXT_image_2d_view_of_3d). MKW reads a 64 x 64 x 64 texture so, which was sampled
|
|
// as the dummy)
|
|
if (use == ViewUse::Sample && dev.view_2d_of_3d) {
|
|
type = VK_IMAGE_VIEW_TYPE_2D, nlevels = 1, layer0 = 0, nlayers = 1;
|
|
break;
|
|
}
|
|
if (use != ViewUse::Attachment) return VK_NULL_HANDLE;
|
|
}
|
|
if (type == VK_IMAGE_VIEW_TYPE_2D) nlayers = 1;
|
|
break;
|
|
case VK_IMAGE_VIEW_TYPE_3D:
|
|
if (tex->type != VK_IMAGE_TYPE_3D) {
|
|
// (Triforce: a flat texture sampled as a 3D one: its one-slice 3D copy, twin3d_view above)
|
|
if (use != ViewUse::Sample) return VK_NULL_HANDLE;
|
|
twin_layer = (int)layer0;
|
|
}
|
|
layer0 = 0, nlayers = 1;
|
|
break;
|
|
case VK_IMAGE_VIEW_TYPE_CUBE: case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY:
|
|
if (!tex->cube) return VK_NULL_HANDLE;
|
|
if (type == VK_IMAGE_VIEW_TYPE_CUBE) nlayers = 6;
|
|
else nlayers = std::max(6u, nlayers / 6 * 6);
|
|
if (layer0 + nlayers > tex->layers) return VK_NULL_HANDLE;
|
|
break;
|
|
default: break;
|
|
}
|
|
if (tex->type == VK_IMAGE_TYPE_3D && use == ViewUse::Attachment) type = VK_IMAGE_VIEW_TYPE_2D;
|
|
VkComponentMapping cm{};
|
|
if (use == ViewUse::Sample) {
|
|
const nvn_texdesc& d = t->d;
|
|
unsigned sw[4] = {d.swizzle_r, d.swizzle_g, d.swizzle_b, d.swizzle_a};
|
|
if (v && v->set_swizzle)
|
|
for (int i = 0; i < 4; i++) sw[i] = v->swz[i];
|
|
// (X formats read alpha as one)
|
|
switch (nf) {
|
|
case NVN_FORMAT_RGBX8: case NVN_FORMAT_RGBX8SN: case NVN_FORMAT_RGBX8UI: case NVN_FORMAT_RGBX8I:
|
|
case NVN_FORMAT_RGBX16F: case NVN_FORMAT_RGBX16: case NVN_FORMAT_RGBX16SN: case NVN_FORMAT_RGBX16UI:
|
|
case NVN_FORMAT_RGBX16I: case NVN_FORMAT_RGBX32F: case NVN_FORMAT_RGBX32UI: case NVN_FORMAT_RGBX32I:
|
|
case NVN_FORMAT_RGBX8_SRGB: case NVN_FORMAT_BGRX8: case NVN_FORMAT_BGRX8_SRGB:
|
|
for (unsigned& s : sw)
|
|
if (s == NVN_TEXTURE_SWIZZLE_A) s = NVN_TEXTURE_SWIZZLE_ONE;
|
|
break;
|
|
default: break;
|
|
}
|
|
static const bool swap_9e5 = getenv("RX_VK_9E5_SWAP") != nullptr; // (hookshot, diagnosis: R and B swapped)
|
|
if (swap_9e5 && nf == NVN_FORMAT_RGB9E5F) std::swap(sw[0], sw[2]);
|
|
cm = {component(sw[0]), component(sw[1]), component(sw[2]), component(sw[3])};
|
|
static const bool log_swizzle = getenv("RX_VK_LOG_SWIZZLE") != nullptr; // (hookshot: RGB9E5 and R11G11B10 views)
|
|
static int swizzle_logged;
|
|
if (log_swizzle && nf == NVN_FORMAT_RGB9E5F && swizzle_logged++ < 12)
|
|
logf("vk: sampled view of #%u NVN %#x as vk %d: swizzle %u %u %u %u (descriptor %u %u %u %u, view %d)", tex->id,
|
|
nf, (int)f, sw[0], sw[1], sw[2], sw[3], d.swizzle_r, d.swizzle_g, d.swizzle_b, d.swizzle_a,
|
|
v && v->set_swizzle ? 1 : 0);
|
|
}
|
|
if (twin_layer >= 0) {
|
|
if (format_out) *format_out = f;
|
|
return twin3d_view(tex, f, level0, nlevels, (uint32_t)twin_layer, cm);
|
|
}
|
|
uint64_t key = mix((uint64_t)use, (uint64_t)type);
|
|
key = mix(key, (uint64_t)f << 32 | aspect);
|
|
key = mix(key, (uint64_t)level0 << 48 | (uint64_t)nlevels << 32 | (uint64_t)layer0 << 16 | nlayers);
|
|
key = mix(key, (uint64_t)cm.r << 24 | (uint64_t)cm.g << 16 | (uint64_t)cm.b << 8 | cm.a);
|
|
if (format_out) *format_out = f;
|
|
auto it = tex->views.find(key);
|
|
if (it != tex->views.end()) return it->second;
|
|
const VkImageUsageFlags usage = use == ViewUse::Sample ? VK_IMAGE_USAGE_SAMPLED_BIT
|
|
: use == ViewUse::Storage ? VK_IMAGE_USAGE_STORAGE_BIT
|
|
: depth ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT
|
|
: VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
|
|
const VkImageView view = make_view(tex, type, f, aspect, level0, nlevels, layer0, nlayers, cm, usage);
|
|
if (!view) {
|
|
static int logged;
|
|
if (logged++ < 16)
|
|
logf("vk: view of texture %#llx (%ux%u, vk %d, usage %#x) as vk %d type %d usage %#x "
|
|
"levels %u+%u layers %u+%u not made",
|
|
(unsigned long long)tex->gpu, tex->width, tex->height, (int)tex->format, tex->usage,
|
|
(int)f, (int)type, usage, level0, nlevels, layer0, nlayers);
|
|
}
|
|
tex->views[key] = view;
|
|
return view;
|
|
}
|
|
|
|
bool texture_copy_into_bc(VkCommandBuffer cmd, Texture* src, uint32_t src_level, uint32_t src_layer, int sx, int sy,
|
|
Texture* dst, uint32_t dst_level, uint32_t dst_layer, int dx, int dy, uint32_t w, uint32_t h,
|
|
uint32_t layers) {
|
|
if (!src || !dst || !dst->bc || !src->astc || src->bc || !bc_ready() || w > kBcDim || h > kBcDim || (dx & 3) || (dy & 3))
|
|
return false;
|
|
// (hookshot-amd) into a resized BC3 image only whole levels, resized as their upload is
|
|
const uint32_t ow = std::max(1u, dst->width >> dst_level), oh = std::max(1u, dst->height >> dst_level);
|
|
if (dst->bc_resized) {
|
|
const uint32_t gl = dst_level + dst->skip;
|
|
const uint32_t gw = std::max(1u, dst->d.width >> gl), gh = std::max(1u, dst->d.height >> gl);
|
|
if (dx || dy || w != gw || h != gh) {
|
|
static int logged;
|
|
if (logged++ < 8)
|
|
logf("vk: a copy into part of resized BC3 texture %#llx (level %u, %d,%d %ux%u of %ux%u) left out",
|
|
(unsigned long long)dst->gpu, dst_level, dx, dy, w, h, gw, gh);
|
|
return false;
|
|
}
|
|
}
|
|
if (!bc_scratch_layout(cmd, dst->bc_resized)) return false;
|
|
for (uint32_t l0 = 0; l0 < layers; l0 += kBcLayers) {
|
|
const uint32_t n = std::min(kBcLayers, layers - l0);
|
|
full_barrier(cmd);
|
|
const VkImageCopy cp{{VK_IMAGE_ASPECT_COLOR_BIT, src_level, src_layer + l0, n}, {sx, sy, 0},
|
|
{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, n}, {0, 0, 0}, {w, h, 1}};
|
|
vkCmdCopyImage(cmd, src->image, VK_IMAGE_LAYOUT_GENERAL, bc_scratch, VK_IMAGE_LAYOUT_GENERAL, 1, &cp);
|
|
if (dst->bc_resized)
|
|
bc_encode_resized(cmd, w, h, ow, oh, n, *dst, dst_level, dst_layer + l0);
|
|
else
|
|
bc_encode_into(cmd, w, h, n, *dst, dst_level, dst_layer + l0, dx, dy);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ---- what the GPU wrote into textures of virtual pools, back into their memory ----
|
|
// MK World draws its course maps' tiles (the terrain's height and material maps, 16384 x 16384 R8, 8192 x 8192 R16
|
|
// and RG16, sparse in virtual pools) into small 512 x 512 render targets in virtual pools of their own, finalizes
|
|
// those, unmaps their pools and maps the physical pages into the course map: on the console the texels are in that
|
|
// memory. Here they were only in the small targets' images, the course maps read zeros, and the terrain they shape
|
|
// was missing in places (ground drawn nowhere, the sky through it, snow left floating). Before virtual pages are
|
|
// unmapped or remapped (nvnMemoryPoolMapVirtual, nvnMemoryPoolFinalize: host/nvn_native.c), the textures the GPU
|
|
// wrote there are read back and written into their memory, block linear: the course maps then upload the tiles as
|
|
// they are mapped. The same when the CPU is about to read memory the GPU may have written (nvnMemoryPoolInvalidate-
|
|
// MappedRange, nvnBufferInvalidateMappedRange): MK World reads back a 1 x 1 RGBA32F texture its GPU computes (the
|
|
// scene's luminance), which was never in memory. RX_VK_NO_VMAP_FLUSH=1: off.
|
|
namespace {
|
|
constexpr uint64_t kFlushMax = 16ull << 20; // (the small ones: render targets that size and more are not)
|
|
std::mutex vw_mu;
|
|
std::map<uint64_t, uint64_t> vw; // (storage start -> end of the GPU-written textures of virtual pools that may be flushed)
|
|
bool flushable(const Texture& t) {
|
|
if (!t.guest_size || t.guest_size > kFlushMax || t.astc || t.bc || t.sparse || t.skip ||
|
|
t.samples != VK_SAMPLE_COUNT_1_BIT || t.aspect != VK_IMAGE_ASPECT_COLOR_BIT || t.bpb <= 0 || !t.image ||
|
|
(nvn_tex_is_linear(&t.d) && (t.levels != 1 || t.type == VK_IMAGE_TYPE_3D))) // (linear: one level, rows)
|
|
return false;
|
|
int from = 0;
|
|
widened(t.d.format ? vk_format(t.nvn_format).format : t.format, &from);
|
|
return !from; // (the image's texels as the storage's: no 3-byte formats kept as four)
|
|
}
|
|
void vw_note(Texture& t) {
|
|
static const bool off = getenv("RX_VK_NO_VMAP_FLUSH") != nullptr;
|
|
if (off || t.vw_listed || !flushable(t)) return;
|
|
std::lock_guard lk(vw_mu);
|
|
uint64_t& end = vw[t.gpu];
|
|
end = std::max(end, t.gpu + t.guest_size);
|
|
t.vw_listed = true;
|
|
}
|
|
void vw_forget(const Texture& t) { // (the entry goes; other listed textures at that start stay listed)
|
|
if (!t.vw_listed) return;
|
|
std::lock_guard lk(vw_mu);
|
|
vw.erase(t.gpu);
|
|
auto [a, b] = by_gpu.equal_range(t.gpu);
|
|
for (auto it = a; it != b; ++it)
|
|
if (it->second != &t && it->second->vw_listed) {
|
|
uint64_t& end = vw[t.gpu];
|
|
end = std::max(end, t.gpu + it->second->guest_size);
|
|
}
|
|
}
|
|
// (a command buffer of its own for the read-backs, submitted and waited for at once)
|
|
VkCommandPool rb_pool = VK_NULL_HANDLE;
|
|
VkCommandBuffer rb_cmd = VK_NULL_HANDLE;
|
|
VkFence rb_fence = VK_NULL_HANDLE;
|
|
bool readback_begin() {
|
|
if (!rb_pool) {
|
|
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
|
|
pi.queueFamilyIndex = dev.queue_family;
|
|
pi.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
|
|
if (vkCreateCommandPool(dev.device, &pi, nullptr, &rb_pool) != VK_SUCCESS) return rb_pool = VK_NULL_HANDLE, false;
|
|
VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
|
|
ai.commandPool = rb_pool;
|
|
ai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
|
|
ai.commandBufferCount = 1;
|
|
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
if (vkAllocateCommandBuffers(dev.device, &ai, &rb_cmd) != VK_SUCCESS ||
|
|
vkCreateFence(dev.device, &fi, nullptr, &rb_fence) != VK_SUCCESS)
|
|
return false;
|
|
}
|
|
vkResetCommandPool(dev.device, rb_pool, 0);
|
|
VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
|
|
bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
|
|
return vkBeginCommandBuffer(rb_cmd, &bi) == VK_SUCCESS;
|
|
}
|
|
bool readback_submit_wait() {
|
|
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the copies' writes, for the host)
|
|
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT;
|
|
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
d.memoryBarrierCount = 1;
|
|
d.pMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(rb_cmd, &d);
|
|
if (vkEndCommandBuffer(rb_cmd) != VK_SUCCESS) return false;
|
|
VkCommandBufferSubmitInfo cb{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO};
|
|
cb.commandBuffer = rb_cmd;
|
|
VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
|
|
si.commandBufferInfoCount = 1;
|
|
si.pCommandBufferInfos = &cb;
|
|
vkResetFences(dev.device, 1, &rb_fence);
|
|
{
|
|
std::lock_guard qk(dev.present_mu); // (one queue: submits and presents take turns)
|
|
queue_flush_locked();
|
|
if (vkQueueSubmit2(dev.queue, 1, &si, rb_fence) != VK_SUCCESS) return false;
|
|
}
|
|
return vkWaitForFences(dev.device, 1, &rb_fence, VK_TRUE, 5000000000ull) == VK_SUCCESS;
|
|
}
|
|
/// A texture's write-back: its image's levels and layers read back, swizzled as the storage lays them out, and written into
|
|
/// its memory within [clip_lo, clip_hi). The GPU half (flush_record) goes into rb_cmd, the CPU half (flush_finish) runs once
|
|
/// the read-back completed: flush_textures does a batch of them with one submission and one wait. Each wait drains the
|
|
/// queue, and the retirement sweep wrote back hundreds one by one at a scene load (seconds of stall)
|
|
struct Flush {
|
|
Texture* t;
|
|
uint64_t clip_lo, clip_hi;
|
|
struct Part { uint32_t level, layer; uint64_t at, n; LevelGeo g; };
|
|
std::vector<Part> parts;
|
|
uint64_t total = 0;
|
|
VkBuffer buf = VK_NULL_HANDLE;
|
|
VmaAllocation alloc = nullptr;
|
|
void* map = nullptr;
|
|
bool ok = false;
|
|
bool ignore_cpu = false; // (RX_VK_AUDIT_GPU_ONLY's marks are writes of this process: they do not keep the texels out)
|
|
};
|
|
// (cached host memory, which the CPU reads: the sequential-write kind is write-combined, or VRAM behind the BAR, and every read
|
|
// of it by the unswizzle goes over PCIe uncached. RX_VK_FLUSH_UNCACHED=1: as before)
|
|
bool flush_uncached() {
|
|
static const bool uncached = getenv("RX_VK_FLUSH_UNCACHED") != nullptr;
|
|
return uncached;
|
|
}
|
|
/// The copies of f's image into a staging buffer of its own, recorded into rb_cmd (begun by the caller); false: nothing recorded
|
|
bool flush_record(Flush& f, VkCommandBuffer cmd = VK_NULL_HANDLE) { // (cmd: where the copies go; none: rb_cmd)
|
|
Texture& t = *f.t;
|
|
const uint32_t layers = t.type == VK_IMAGE_TYPE_3D ? 1 : t.layers;
|
|
for (uint32_t level = 0; level < t.levels; level++) {
|
|
const LevelGeo g = level_geo(t, level);
|
|
for (uint32_t layer = 0; layer < layers; layer++) {
|
|
const uint64_t n = (uint64_t)g.tw * g.th * g.td * (uint64_t)t.bpb;
|
|
f.parts.push_back({level, layer, f.total, n, g});
|
|
f.total += (n + 15) & ~15ull;
|
|
}
|
|
}
|
|
if (!f.total) return f.ok = false;
|
|
if (!(flush_uncached() ? create_buffer(f.total, VK_BUFFER_USAGE_TRANSFER_DST_BIT, true, f.buf, f.alloc, &f.map)
|
|
: create_buffer_cached(f.total, VK_BUFFER_USAGE_TRANSFER_DST_BIT, f.buf, f.alloc, &f.map)) || !f.map) {
|
|
if (f.buf) destroy_buffer(f.buf, f.alloc);
|
|
f.buf = VK_NULL_HANDLE, f.alloc = nullptr, f.map = nullptr;
|
|
return f.ok = false;
|
|
}
|
|
std::vector<VkBufferImageCopy> c;
|
|
for (const Flush::Part& p : f.parts) {
|
|
VkBufferImageCopy r{};
|
|
r.bufferOffset = p.at;
|
|
r.bufferRowLength = p.g.tw * (uint32_t)t.block_w;
|
|
r.bufferImageHeight = p.g.th * (uint32_t)t.block_h;
|
|
r.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, p.level, t.type == VK_IMAGE_TYPE_3D ? 0 : p.layer, 1};
|
|
r.imageExtent = {p.g.w, p.g.h, p.g.d};
|
|
c.push_back(r);
|
|
}
|
|
vkCmdCopyImageToBuffer(cmd ? cmd : rb_cmd, t.image, VK_IMAGE_LAYOUT_GENERAL, f.buf, (uint32_t)c.size(), c.data());
|
|
return f.ok = true;
|
|
}
|
|
/// The CPU half, once the batch's read-back completed (gpu_ok) or failed: the texels into the texture's memory, the buffer freed
|
|
bool flush_finish(Flush& f, bool gpu_ok) {
|
|
using Part = Flush::Part;
|
|
Texture& t = *f.t;
|
|
const uint64_t clip_lo = f.clip_lo, clip_hi = f.clip_hi, total = f.total;
|
|
const std::vector<Part>& parts = f.parts;
|
|
void* map = f.map;
|
|
bool ok = f.ok && gpu_ok;
|
|
if (ok && !flush_uncached()) invalidate_mapped(f.alloc, total);
|
|
if (ok && total <= 64) { // (RX_VK_LOG_TINY_FLUSH=1: the values of tiny ones, 400 lines: a 1 x 1 luminance)
|
|
static const bool log_tiny = getenv("RX_VK_LOG_TINY_FLUSH") != nullptr;
|
|
static int logged;
|
|
if (log_tiny && logged++ < 400) {
|
|
const float* f = (const float*)map;
|
|
const uint32_t* u = (const uint32_t*)map;
|
|
logf("vk: tiny write-back #%u %#llx NVN %#x: %g %g %g %g (%08x %08x %08x %08x)", t.id, (unsigned long long)t.gpu,
|
|
t.nvn_format, f[0], f[1], f[2], f[3], u[0], u[1], u[2], u[3]);
|
|
}
|
|
}
|
|
// (hookshot) pages the CPU wrote after the GPU last wrote the texture keep the CPU's bytes: on the console the GPU's
|
|
// writes are in memory as it renders, and a later CPU write is what is there. OoT reuses a render target's memory
|
|
// for scene data and then invalidates a range of it: this wrote the whole texture back over that data, and its
|
|
// SceneLowPriority thread read texels as pointers (an access violation at 0xffffffffffffffff)
|
|
std::vector<std::pair<uint64_t, uint64_t>> cpu_runs; // (offsets from t.gpu)
|
|
if (ok && !f.ignore_cpu && t.gpu < (1ull << 40)) memory_written_runs(t.gpu, t.guest_size, t.gpu_gen, cpu_runs);
|
|
uint64_t kept_cpu = 0;
|
|
// (and only into the pages being remapped: those are what the texture's content would be lost from. The rest of it stays where
|
|
// the game may have put other things since. The watch above does not catch all of those: OoT's SceneLowPriority crash, walking
|
|
// in Kokiri Forest, was texels landing right beside the remapped pages, on scene data)
|
|
const auto write_back = [&](uint64_t whole_gpu, const uint8_t* whole_src, uint64_t whole_n) {
|
|
const uint64_t from = std::max(whole_gpu, clip_lo), to = std::min(whole_gpu + whole_n, clip_hi);
|
|
if (from >= to) return true;
|
|
const uint64_t gpu = from, n = to - from;
|
|
const uint8_t* src = whole_src + (from - whole_gpu);
|
|
const uint64_t lo = gpu - t.gpu, hi = lo + n;
|
|
bool w = true;
|
|
uint64_t at = lo;
|
|
for (const auto& [a, b] : cpu_runs) {
|
|
if (b <= at) continue;
|
|
if (a >= hi) break;
|
|
if (a > at) w &= guest_cpu_write(t.gpu + at, src + (at - lo), a - at) != 0;
|
|
kept_cpu += std::min(b, hi) - std::max(a, at);
|
|
at = std::max(at, b);
|
|
if (at >= hi) break;
|
|
}
|
|
if (at < hi) w &= guest_cpu_write(t.gpu + at, src + (at - lo), hi - at) != 0;
|
|
return w;
|
|
};
|
|
const auto log_kept = [&] {
|
|
static int logged;
|
|
if (kept_cpu && logged++ < 20)
|
|
logf("vk: write-back of #%u %#llx: %llu KB the CPU wrote since the GPU kept", t.id, (unsigned long long)t.gpu,
|
|
(unsigned long long)(kept_cpu >> 10));
|
|
};
|
|
if (ok && nvn_tex_is_linear(&t.d)) { // (a linear texture: its rows at the stride, one level)
|
|
const uint64_t stride = t.d.stride ? t.d.stride : host->tex_linear_stride(&t.d);
|
|
for (const Part& p : parts) {
|
|
const uint64_t row = (uint64_t)p.g.tw * (uint64_t)t.bpb;
|
|
const uint64_t base = (uint64_t)p.layer * t.L.layer_size;
|
|
for (uint32_t y = 0; y < p.g.th; y++) {
|
|
const uint64_t off = base + (uint64_t)y * stride;
|
|
if (off + row > t.guest_size) break;
|
|
if (!write_back(t.gpu + off, (const uint8_t*)map + p.at + (uint64_t)y * row, row)) ok = false;
|
|
}
|
|
}
|
|
log_kept();
|
|
if (f.buf) destroy_buffer(f.buf, f.alloc);
|
|
return ok;
|
|
}
|
|
if (ok) {
|
|
std::vector<uint8_t> sw;
|
|
for (const Part& p : parts) {
|
|
const uint64_t at = t.gpu + p.g.offset + (uint64_t)p.layer * t.L.layer_size;
|
|
if (p.g.offset + (uint64_t)p.layer * t.L.layer_size >= t.guest_size) continue;
|
|
const uint64_t span = std::min<uint64_t>(p.g.swizzled, t.guest_size - p.g.offset - (uint64_t)p.layer * t.L.layer_size);
|
|
sw.assign(std::max<uint64_t>(p.g.swizzled, span), 0);
|
|
host->gpu_read(at, sw.data(), span); // (what is there: the padding between texels kept)
|
|
Tegra::Texture::SwizzleTexture(std::span<uint8_t>(sw.data(), sw.size()),
|
|
std::span<const uint8_t>((const uint8_t*)map + p.at, p.n), (u32)t.bpb, p.g.tw,
|
|
p.g.th, p.g.td, p.g.tile_h, p.g.tile_d, 1);
|
|
if (!write_back(at, sw.data(), span)) ok = false; // (marks the pages written: textures over them upload)
|
|
}
|
|
log_kept();
|
|
}
|
|
if (f.buf) destroy_buffer(f.buf, f.alloc);
|
|
return ok;
|
|
}
|
|
/// A batch of write-backs: one submission and one wait for all of them
|
|
void flush_textures(std::vector<Flush>& fs) {
|
|
bool any = false;
|
|
if (readback_begin()) {
|
|
full_barrier(rb_cmd); // (after what the submissions before wrote)
|
|
for (Flush& f : fs) any |= flush_record(f);
|
|
} else {
|
|
for (Flush& f : fs) f.ok = false;
|
|
}
|
|
const bool gpu_ok = any && readback_submit_wait();
|
|
if (any && !gpu_ok) {
|
|
// (the read-back did not complete: its 5 s wait, which also covers what queue_flush_locked just submitted, or a submit
|
|
// that failed. The copies may still be writing into the staging buffers, so they are not freed here: they go the way a
|
|
// destroyed image does, once a submission made after this one is done. The batch's write-backs are lost, as a single
|
|
// one's was before)
|
|
static int logged;
|
|
if (logged++ < 8)
|
|
logf("vk: a batch of %zu texture write-backs did not complete in time: not written back, its staging buffers kept "
|
|
"until the GPU is past them", fs.size());
|
|
for (Flush& f : fs)
|
|
if (f.buf) {
|
|
Dead d{rec_value + 1, VK_NULL_HANDLE, nullptr, {}, {}};
|
|
d.buffer = f.buf, d.buffer_alloc = f.alloc;
|
|
dead.push_back(std::move(d));
|
|
f.buf = VK_NULL_HANDLE, f.alloc = nullptr, f.map = nullptr;
|
|
}
|
|
}
|
|
for (Flush& f : fs) f.ok = flush_finish(f, gpu_ok);
|
|
}
|
|
bool flush_texture(Texture& t, uint64_t clip_lo, uint64_t clip_hi) {
|
|
std::vector<Flush> one;
|
|
one.push_back(Flush{&t, clip_lo, clip_hi});
|
|
flush_textures(one);
|
|
return one[0].ok;
|
|
}
|
|
} // namespace
|
|
|
|
void texture_describe_written(uint64_t lo, uint64_t size, std::string& out) {
|
|
const uint64_t hi = lo + size;
|
|
int n = 0;
|
|
for (auto& kv : textures) {
|
|
const Texture* t = kv.second.get();
|
|
if (t->gpu >= hi || t->gpu + t->guest_size <= lo || !t->gpu_written) continue;
|
|
if (n++ < 6) {
|
|
char b[160];
|
|
snprintf(b, sizeof b, " #%u %#llx %ux%ux%u x%u NVN %#x %llu KB%s%s;", t->id, (unsigned long long)t->gpu, t->width,
|
|
t->height, t->depth, t->layers, t->nvn_format, (unsigned long long)(t->guest_size >> 10),
|
|
t->released ? " finalized" : "", flushable(*t) ? "" : " (not flushable)");
|
|
out += b;
|
|
}
|
|
}
|
|
if (n > 6) out += " (" + std::to_string(n) + " in all)";
|
|
}
|
|
|
|
bool texture_gpu_written_in(uint64_t lo, uint64_t size) {
|
|
const uint64_t hi = lo + size;
|
|
std::lock_guard lk(vw_mu);
|
|
for (auto it = vw.lower_bound(lo > kFlushMax ? lo - kFlushMax : 0); it != vw.end() && it->first < hi; ++it)
|
|
if (it->second > lo) return true;
|
|
return false;
|
|
}
|
|
|
|
int texture_flush_to_guest(uint64_t lo, uint64_t size) {
|
|
const uint64_t hi = lo + size;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::vector<Texture*> todo;
|
|
for (auto& kv : textures) {
|
|
Texture* t = kv.second.get();
|
|
if (t->gpu >= hi || t->gpu + t->guest_size <= lo) continue;
|
|
t->vw_listed = false;
|
|
if (t->gpu_written && flushable(*t)) todo.push_back(t);
|
|
}
|
|
{
|
|
std::lock_guard lk(vw_mu);
|
|
for (auto it = vw.lower_bound(lo > kFlushMax ? lo - kFlushMax : 0); it != vw.end() && it->first < hi;)
|
|
it = it->second > lo ? vw.erase(it) : std::next(it);
|
|
}
|
|
int done = 0;
|
|
uint64_t bytes = 0;
|
|
for (Texture* t : todo) {
|
|
if (!flush_texture(*t, lo, hi)) continue;
|
|
done++, bytes += t->guest_size;
|
|
if (t->gpu < lo || t->gpu + t->guest_size > hi) continue; // (a part of it: the rest is still only in the image)
|
|
// (its memory is its content now: CPU writes to it count again, and an image made over it uploads it)
|
|
t->gpu_written = false;
|
|
t->uploaded = true;
|
|
t->upload_gen = memory_generation();
|
|
watch_tex(t, "written back into its memory");
|
|
}
|
|
static int64_t flushes, all_bytes;
|
|
static double all_ms;
|
|
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
|
|
flushes += done, all_bytes += (int64_t)bytes, all_ms += ms;
|
|
static int logged;
|
|
if (done && (logged++ < 12 || logged % 200 == 0))
|
|
logf("vk: virtual pages %#llx..%#llx remapped: %d texture(s) the GPU wrote there written back into their memory "
|
|
"(%llu KB, %.1f ms; %lld in all, %lld MB, %.0f ms)", (unsigned long long)lo, (unsigned long long)hi, done,
|
|
(unsigned long long)(bytes >> 10), ms, (long long)flushes, (long long)(all_bytes >> 20), all_ms);
|
|
return done;
|
|
}
|
|
|
|
void texture_written(Texture* tex, const char* why) {
|
|
if (!tex) return;
|
|
if (tex->upload_value == rec_value && tex->upload_seg > tl_seg) { // (a parallel replay's order broken: see dry_mark_written)
|
|
static int64_t n;
|
|
if (n++ < 40 || n % 500 == 0)
|
|
logf("vk: parallel: #%u %#llx %ux%u x%u NVN %#x written in segment %llu, uploaded by the later segment %llu (%lld so); "
|
|
"written by: %s; before this the GPU had %s it%s%s, first seen at frame %llu, now %llu",
|
|
tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, tex->layers, tex->nvn_format,
|
|
(unsigned long long)tl_seg, (unsigned long long)tex->upload_seg, (long long)n, why,
|
|
tex->gpu_written ? "written" : "not written", tex->only_cleared ? " (only cleared)" : "",
|
|
tex->released ? " (released)" : "", (unsigned long long)tex->first_seen, (unsigned long long)frame_no);
|
|
}
|
|
vw_note(*tex);
|
|
tex->only_cleared = false;
|
|
if (tex->sparse && !tex->gpu_written) {
|
|
static int logged;
|
|
if (logged++ < 20) logf("vk: sparse texture %#llx written by the GPU (#%u)", (unsigned long long)tex->gpu, tex->id);
|
|
}
|
|
watch_tex(tex, "written");
|
|
// (RX_VK_AUDIT_GPU_ONLY: its storage marked as the GPU's when the GPU's contents begin; not one the CPU's texels were in first)
|
|
if (audit_on() && (tex->gpu_written ? tex->released : !tex->uploaded)) audit_mark(*tex);
|
|
// (Triforce: a run of frames the GPU draws into it begins when it was not drawn into in the frame before: its first frame, a
|
|
// texture the game finalized and made again, or one the game draws into again later (a terrain tile's normal map baked a
|
|
// second time into the same texture, 363 presents after the first))
|
|
// (and how many such runs began within 30 presents of the one before: a target the game draws into every other frame, or
|
|
// in turns with others, begins a run every few presents for as long as it lives. Written back at each, those were 20,537
|
|
// write-backs and 19.7 GB in a seven-minute session, 0.7 ms of the GPU's frame and 1.1 ms of the replay's (2026-11-10);
|
|
// texture_end_recording stops at the third in a row. Contents that begin anew, or a rest of more than 30 presents, count
|
|
// from none again: a tile baked a second time is taken whenever it comes)
|
|
if (!tex->gpu_written || tex->released) {
|
|
tex->wb_burst = frame_no, tex->wb_quick = 0;
|
|
} else if (tex->last_written + 1 < frame_no) {
|
|
tex->wb_quick = frame_no - tex->wb_burst <= 30 ? tex->wb_quick + 1 : 0;
|
|
tex->wb_burst = frame_no;
|
|
}
|
|
if (!tex->gpu_written || tex->released) tex->first_written = frame_no;
|
|
tex->last_written = frame_no;
|
|
tex->gpu_written = true;
|
|
tex->gpu_gen = memory_generation();
|
|
tex->released = false;
|
|
tex->uploaded = true;
|
|
bake_note_written(tex); // (Triforce: early in such a run, its texels go into its memory at this recording's end)
|
|
}
|
|
|
|
|
|
bool texture_touch_quick(Texture* tex, bool for_sampling) {
|
|
tex->last_frame = frame_no; // (the same values whichever thread stores them)
|
|
tex->let_go = 0;
|
|
tex->last_value = rec_value;
|
|
if (!for_sampling) return true;
|
|
if (tex->gpu_written && !tex->released && !tex->only_cleared) return true;
|
|
if (tex->upload_value == rec_value && tex->upload_seg > tl_seg) return false;
|
|
return tex->uploaded && tex->checked_epoch == epoch;
|
|
}
|
|
void texture_gpu_wrote(uint64_t gpu, uint64_t size) {
|
|
if (!gpu || !size) return;
|
|
const uint64_t lo = gpu, hi = gpu + size;
|
|
{ // (RX_VK_WATCH_GPU_WRITE=<lo hex>[:<hi hex>]: the GPU buffer writes reaching that range, by program, 60 lines)
|
|
static uint64_t wlo = 0, whi = 0;
|
|
static const bool parsed = [] {
|
|
if (const char* e = getenv("RX_VK_WATCH_GPU_WRITE")) {
|
|
unsigned long long a = 0, b = 0;
|
|
if (sscanf(e, "%llx:%llx", &a, &b) < 2) b = a + 1;
|
|
wlo = a, whi = b;
|
|
}
|
|
return true;
|
|
}();
|
|
(void)parsed;
|
|
static int logged;
|
|
if (wlo < hi && lo < whi && logged++ < 60)
|
|
logf("vk: watch: GPU buffer write [%#llx, +%#llx) into [%#llx, %#llx), frame %llu, program %016llx",
|
|
(unsigned long long)lo, (unsigned long long)size, (unsigned long long)wlo, (unsigned long long)whi,
|
|
(unsigned long long)frame_no, (unsigned long long)tl_watch_prog);
|
|
}
|
|
std::lock_guard lk(gpu_wrote_mu);
|
|
auto it = gpu_wrote.find(lo);
|
|
if (it != gpu_wrote.end() && it->second.first >= hi) {
|
|
it->second.second = rec_value;
|
|
return;
|
|
}
|
|
for (auto e = gpu_wrote.lower_bound(lo); e != gpu_wrote.end() && e->first < hi;) // (ranges it covers)
|
|
e = e->second.first <= hi ? gpu_wrote.erase(e) : std::next(e);
|
|
gpu_wrote[lo] = {hi, rec_value};
|
|
if (gpu_wrote.size() > 65536) gpu_wrote.erase(gpu_wrote.begin());
|
|
}
|
|
/// A finished GPU write into the texture's storage it was not uploaded after (and notes it as seen)
|
|
bool gpu_wrote_into(Texture& tex) {
|
|
std::lock_guard lk(gpu_wrote_mu);
|
|
if (gpu_wrote.empty()) return false;
|
|
const uint64_t lo = tex.gpu, hi = tex.gpu + std::max<uint64_t>(tex.guest_size, 1);
|
|
uint64_t newest = 0;
|
|
auto it = gpu_wrote.upper_bound(lo);
|
|
if (it != gpu_wrote.begin()) --it;
|
|
for (; it != gpu_wrote.end() && it->first < hi; ++it)
|
|
if (it->second.first > lo && it->second.second <= completed_value && it->second.second > tex.gpu_wrote_seen)
|
|
newest = std::max(newest, it->second.second);
|
|
if (!newest) return false;
|
|
tex.gpu_wrote_seen = newest;
|
|
static int logged;
|
|
if (logged++ < 40)
|
|
logf("vk: texture #%u %#llx %ux%u NVN %#x: the GPU wrote its storage (submission %llu): uploaded again", tex.id,
|
|
(unsigned long long)tex.gpu, tex.width, tex.height, tex.nvn_format, (unsigned long long)newest);
|
|
return true;
|
|
}
|
|
void texture_watch_prog(uint64_t prog) {
|
|
tl_watch_prog = prog;
|
|
}
|
|
void texture_set_segment(uint64_t seg) {
|
|
tl_seg = seg;
|
|
}
|
|
uint64_t texture_segment() {
|
|
return tl_seg;
|
|
}
|
|
void texture_segments_begin(const VkCommandBuffer* init_cmds, size_t n) {
|
|
std::lock_guard lk(seg_init_mu);
|
|
seg_inits.assign(n, SegInit{});
|
|
for (size_t i = 0; i < n; i++) seg_inits[i].cmd = init_cmds[i], seg_inits[i].open = true;
|
|
}
|
|
void texture_segment_worker(int index) {
|
|
tl_seg_index = index;
|
|
}
|
|
int texture_segment_end(int index) {
|
|
std::lock_guard lk(seg_init_mu);
|
|
if (index < 0 || index >= (int)seg_inits.size()) return 0;
|
|
SegInit& s = seg_inits[index];
|
|
s.open = false; // (no new image's change goes to it from here: it is ended)
|
|
if (s.used) full_barrier(s.cmd); // (before its uploads and its commands)
|
|
if (vkEndCommandBuffer(s.cmd) != VK_SUCCESS) return s.used ? -1 : 0;
|
|
return s.used ? 1 : 0;
|
|
}
|
|
bool texture_marked(const Texture* tex) {
|
|
return !tex || (tex->gpu_written && !tex->released && tex->uploaded && tex->owned_frame == frame_no &&
|
|
tex->first_render_frame);
|
|
}
|
|
|
|
namespace {
|
|
std::mutex sparse_mu;
|
|
std::vector<VkSparseImageMemoryBind> sparse_binds; // (what the next flush binds, under sparse_mu)
|
|
std::vector<VkSparseMemoryBind> sparse_opaque;
|
|
VkDeviceSize sparse_added = 0;
|
|
|
|
/// One sparse block's memory (or n bytes) from the texture's chunks; false when none can be had
|
|
bool sparse_take(Texture* tex, VkDeviceSize n, VkDeviceMemory& mem, VkDeviceSize& off) {
|
|
constexpr VkDeviceSize kChunk = 64ull << 20;
|
|
n = (n + tex->sparse_align - 1) / tex->sparse_align * tex->sparse_align;
|
|
if (tex->sparse_chunks.empty() || tex->chunk_used + n > kChunk) {
|
|
VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
|
ai.allocationSize = std::max(kChunk, n);
|
|
ai.memoryTypeIndex = tex->sparse_type;
|
|
VkDeviceMemory m = VK_NULL_HANDLE;
|
|
if (vkAllocateMemory(dev.device, &ai, nullptr, &m) != VK_SUCCESS) {
|
|
static int logged;
|
|
if (logged++ < 8) logf("vk: sparse texture %#llx: no device memory for its blocks", (unsigned long long)tex->gpu);
|
|
return false;
|
|
}
|
|
tex->sparse_chunks.push_back(m);
|
|
tex->chunk_used = 0;
|
|
tex->bytes += ai.allocationSize;
|
|
sparse_added += ai.allocationSize;
|
|
}
|
|
mem = tex->sparse_chunks.back();
|
|
off = tex->chunk_used;
|
|
tex->chunk_used += n;
|
|
return true;
|
|
}
|
|
|
|
bool sparse_flush(Texture* tex) {
|
|
if (sparse_binds.empty() && sparse_opaque.empty()) return true;
|
|
VkSparseImageMemoryBindInfo ib{tex->image, (uint32_t)sparse_binds.size(), sparse_binds.data()};
|
|
VkSparseImageOpaqueMemoryBindInfo ob{tex->image, (uint32_t)sparse_opaque.size(), sparse_opaque.data()};
|
|
VkBindSparseInfo bi{VK_STRUCTURE_TYPE_BIND_SPARSE_INFO};
|
|
bi.imageBindCount = sparse_binds.empty() ? 0 : 1;
|
|
bi.pImageBinds = &ib;
|
|
bi.imageOpaqueBindCount = sparse_opaque.empty() ? 0 : 1;
|
|
bi.pImageOpaqueBinds = &ob;
|
|
static VkFence fence = VK_NULL_HANDLE;
|
|
if (!fence) {
|
|
VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
|
vkCreateFence(dev.device, &fi, nullptr, &fence);
|
|
}
|
|
VkResult res;
|
|
{ // (bound before any work that writes the blocks is submitted: waited for here, on the recording thread. And
|
|
// after the work submitted before is done: a sparse bind is not ordered with command buffers by their
|
|
// barriers, and one that landed while a pass sampled the image changed what the pass read part-way through
|
|
// its tiles. Opt-in, RX_VK_SPARSE_IDLE=1: MKW binds hundreds of blocks a race, and the replay's stalls at
|
|
// each made its other races worse)
|
|
static const bool no_idle = getenv("RX_VK_SPARSE_IDLE") == nullptr;
|
|
static int64_t binds;
|
|
std::lock_guard qk(dev.present_mu);
|
|
queue_flush_locked();
|
|
if (!no_idle) vkQueueWaitIdle(dev.queue);
|
|
res = vkQueueBindSparse(dev.queue, 1, &bi, fence);
|
|
if (++binds % 200 == 0) logf("vk: %lld sparse binds", (long long)binds);
|
|
}
|
|
if (res == VK_SUCCESS) {
|
|
vkWaitForFences(dev.device, 1, &fence, VK_TRUE, UINT64_MAX);
|
|
vkResetFences(dev.device, 1, &fence);
|
|
} else {
|
|
logf("vk: sparse texture %#llx: binding failed (%d)", (unsigned long long)tex->gpu, (int)res);
|
|
}
|
|
static int logged;
|
|
static uint64_t next_mb = 0;
|
|
if (logged < 20 || (tex->bytes >> 20) >= next_mb) {
|
|
logged++;
|
|
next_mb = (tex->bytes >> 20) + 256;
|
|
logf("vk: sparse texture %#llx: %zu blocks bound (%llu MB of device memory)", (unsigned long long)tex->gpu,
|
|
sparse_binds.size() + sparse_opaque.size(), (unsigned long long)(tex->bytes >> 20));
|
|
}
|
|
total_bytes += sparse_added;
|
|
sparse_added = 0;
|
|
sparse_binds.clear();
|
|
sparse_opaque.clear();
|
|
return res == VK_SUCCESS;
|
|
}
|
|
|
|
/// Queues the binds of what a rectangle reaches that is not bound yet (sparse_mu held)
|
|
bool sparse_queue(Texture* tex, uint32_t level, uint32_t layer, uint32_t x, uint32_t y, uint32_t w, uint32_t h) {
|
|
const VkSparseImageMemoryRequirements& r = tex->sparse_req;
|
|
if (level >= tex->levels || layer >= tex->layers) return true;
|
|
if (level >= r.imageMipTailFirstLod) { // (the mip tail: bound whole)
|
|
const bool single = r.formatProperties.flags & VK_SPARSE_IMAGE_FORMAT_SINGLE_MIPTAIL_BIT;
|
|
uint8_t& bound = tex->tail_bound[single ? 0 : layer];
|
|
if (bound || !r.imageMipTailSize) return true;
|
|
VkDeviceMemory m;
|
|
VkDeviceSize off;
|
|
if (!sparse_take(tex, r.imageMipTailSize, m, off)) return false;
|
|
sparse_opaque.push_back({r.imageMipTailOffset + (single ? 0 : layer * r.imageMipTailStride), r.imageMipTailSize,
|
|
m, off, 0});
|
|
bound = 1;
|
|
return true;
|
|
}
|
|
const VkExtent3D g = r.formatProperties.imageGranularity;
|
|
const uint32_t lw = std::max(1u, tex->width >> level), lh = std::max(1u, tex->height >> level);
|
|
if (x >= lw || y >= lh || !w || !h) return true;
|
|
w = std::min(w, lw - x), h = std::min(h, lh - y);
|
|
const uint32_t ntx = (lw + g.width - 1) / g.width, nty = (lh + g.height - 1) / g.height;
|
|
std::vector<uint8_t>& map = tex->tiles[(size_t)layer * tex->levels + level];
|
|
if (map.empty()) map.assign((size_t)ntx * nty, 0);
|
|
bool ok = true;
|
|
for (uint32_t ty = y / g.height; ty <= (y + h - 1) / g.height; ty++)
|
|
for (uint32_t tx = x / g.width; tx <= (x + w - 1) / g.width; tx++) {
|
|
uint8_t& bound = map[(size_t)ty * ntx + tx];
|
|
if (bound) continue;
|
|
VkDeviceMemory m;
|
|
VkDeviceSize off;
|
|
if (!sparse_take(tex, tex->sparse_align, m, off)) {
|
|
ok = false;
|
|
continue;
|
|
}
|
|
VkSparseImageMemoryBind b{};
|
|
b.subresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, layer};
|
|
b.offset = {(int32_t)(tx * g.width), (int32_t)(ty * g.height), 0};
|
|
b.extent = {std::min(g.width, lw - tx * g.width), std::min(g.height, lh - ty * g.height), 1};
|
|
b.memory = m;
|
|
b.memoryOffset = off;
|
|
sparse_binds.push_back(b);
|
|
bound = 1;
|
|
}
|
|
return ok;
|
|
}
|
|
} // namespace
|
|
|
|
/// (the upload's: queued, and bound together before its copies are recorded)
|
|
void texture_resident_queue(Texture* tex, uint32_t level, uint32_t layer, uint32_t x, uint32_t y, uint32_t w, uint32_t h) {
|
|
std::lock_guard lk(sparse_mu);
|
|
sparse_queue(tex, level, layer, x, y, w, h);
|
|
}
|
|
void texture_resident_flush(Texture* tex) {
|
|
if (!tex || !tex->sparse) return;
|
|
std::lock_guard lk(sparse_mu);
|
|
sparse_flush(tex);
|
|
}
|
|
|
|
bool texture_resident(Texture* tex, uint32_t level, uint32_t first, uint32_t count, uint32_t x, uint32_t y, uint32_t w,
|
|
uint32_t h) {
|
|
if (!tex || !tex->sparse) return true;
|
|
std::lock_guard lk(sparse_mu);
|
|
bool ok = true;
|
|
for (uint32_t layer = first; layer < std::min(tex->layers, first + std::max(1u, count)); layer++)
|
|
ok &= sparse_queue(tex, level, layer, x, y, w, h);
|
|
return sparse_flush(tex) && ok;
|
|
}
|
|
|
|
bool texture_resident(Texture* tex, uint32_t first, uint32_t count) {
|
|
if (!tex || !tex->sparse) return true;
|
|
std::lock_guard lk(sparse_mu);
|
|
bool ok = true;
|
|
for (uint32_t layer = first; layer < std::min(tex->layers, first + std::max(1u, count)); layer++)
|
|
for (uint32_t level = 0; level < tex->levels; level++)
|
|
ok &= sparse_queue(tex, level, layer, 0, 0, ~0u, ~0u);
|
|
return sparse_flush(tex) && ok;
|
|
}
|
|
|
|
void texture_rendered(Texture* tex) {
|
|
if (!tex) return;
|
|
if (tex->sparse && !tex->first_render_frame) texture_resident(tex, 0, tex->layers); // (rendered into: all of it)
|
|
if (!tex->first_render_frame) tex->first_render_frame = frame_no;
|
|
// (its storage's range as the GPU's, once a frame: render targets only. Copies and storage writes registered it
|
|
// too, and a texture made later over such storage (MKW's terrain tiles reuse the slots of a pool) was not uploaded
|
|
// and kept the placeholder it was made with)
|
|
if (tex->owned_frame != frame_no) {
|
|
tex->owned_frame = frame_no;
|
|
const uint64_t lo = tex->gpu, hi = tex->gpu + std::max<uint64_t>(tex->guest_size, 1);
|
|
for (auto it = gpu_owned.lower_bound(lo); it != gpu_owned.end() && it->first < hi;)
|
|
it = it->second.first <= hi ? gpu_owned.erase(it) : std::next(it);
|
|
gpu_owned[lo] = {hi, memory_generation()};
|
|
if (gpu_owned.size() > 65536) gpu_owned.erase(gpu_owned.begin()); // (bounded; the oldest addresses go)
|
|
}
|
|
}
|
|
uint64_t texture_frame_no() {
|
|
return frame_no;
|
|
}
|
|
|
|
VkBufferView texel_buffer_view(const nvn_tex* t, const nvn_view* v, bool storage) {
|
|
if (!t || !t->gpu) return VK_NULL_HANDLE;
|
|
const int nf = view_nvn_format(t, v);
|
|
VkFormat f = vk_format(nf).format;
|
|
if (f == VK_FORMAT_UNDEFINED) return VK_NULL_HANDLE;
|
|
int bw = 1, bh = 1;
|
|
const int bpb = nvn_block_size(nf, &bw, &bh);
|
|
if (bpb <= 0) return VK_NULL_HANDLE;
|
|
const uint64_t size = (uint64_t)std::max(1u, t->d.width) * bpb;
|
|
// (a formatted store into it: textures over that memory are uploaded again once the GPU is done)
|
|
if (storage) texture_gpu_wrote(t->gpu, size);
|
|
static std::unordered_map<uint64_t, VkBufferView> views;
|
|
const uint64_t key = mix(mix(t->gpu, (uint64_t)f << 1 | storage), size);
|
|
auto it = views.find(key);
|
|
if (it != views.end()) return it->second;
|
|
BufferRef b = gpu_buffer(t->gpu, size);
|
|
VkBufferView bv = VK_NULL_HANDLE;
|
|
if (b && b.offset % std::max<uint64_t>(dev.props.limits.minTexelBufferOffsetAlignment, 1) == 0) {
|
|
VkBufferViewCreateInfo ci{VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO};
|
|
ci.buffer = b.buffer;
|
|
ci.format = f;
|
|
ci.offset = b.offset;
|
|
ci.range = std::min<uint64_t>(size, (uint64_t)dev.props.limits.maxTexelBufferElements * bpb);
|
|
vkCreateBufferView(dev.device, &ci, nullptr, &bv);
|
|
}
|
|
views[key] = bv;
|
|
return bv;
|
|
}
|
|
|
|
// ---- (hookshot) VRAM compacted (on by default since the hold lets go of the dead render targets first and of long-unused
|
|
// textures only; RX_VK_COMPACT=0: off, as before) ----
|
|
// An evicted texture's memory stays inside its 128 MB block, and the driver counts the block whole: after some play 0.6 to
|
|
// 1.6 GB of what the driver counts is such free space (3 to 4 GB after the cache was emptied), and evicting more gives
|
|
// none of it back. Compacted, the textures that are not render targets have a pool of their own (vk_vma.cpp; the targets have theirs) and, when RX_VK_COMPACT_MB (512)
|
|
// or more of it is free space, VMA's defragmentation moves textures out of the sparsely used blocks, a few a pass: a new
|
|
// image at the new place, the texels copied on the GPU in the present's upload commands (every draw of the frame is in an
|
|
// earlier recording), the texture's views made again when next asked for, and the old image destroyed once the GPU is past
|
|
// the copy; then VMA frees the blocks that emptied. While a pass is open nothing of the pool is freed (the images the GPU
|
|
// is done with wait a frame or two): VMA holds the allocations it listed.
|
|
struct Compact {
|
|
bool in_pass = false;
|
|
uint64_t value = 0; // (the recording the open pass's copies are in)
|
|
std::vector<VkImage> old_images; // (of the open pass: destroyed when the GPU is past them)
|
|
std::vector<VkImageView> old_views;
|
|
uint64_t next_check = 0; // (the present from which another compaction may start)
|
|
uint64_t moved = 0, moved_bytes = 0, ignored = 0, passes = 0; // (of the compaction under way)
|
|
uint64_t runs = 0, all_moved = 0, all_freed = 0;
|
|
// (the driver's count over 90% of what it allows, texture_frame's look every 10 presents: vram_compact.h)
|
|
bool pressed = false;
|
|
} compact;
|
|
const Texture* present_source = nullptr; // (what the present shows this frame: do_present holds its image and view)
|
|
void compact_step(uint64_t completed) {
|
|
if (!compact_on()) return;
|
|
if (compact.in_pass) {
|
|
if (compact.value > completed) return; // (the GPU has not run the copies yet)
|
|
for (VkImageView v : compact.old_views) vkDestroyImageView(dev.device, v, nullptr);
|
|
for (VkImage i : compact.old_images) vkDestroyImage(dev.device, i, nullptr); // (their memory is VMA's: the pass's end frees it)
|
|
compact.old_views.clear(), compact.old_images.clear();
|
|
compact.in_pass = false;
|
|
uint64_t freed = 0;
|
|
uint32_t blocks = 0;
|
|
if (!image_compact_pass_end(&freed, &blocks)) {
|
|
compact.all_moved += compact.moved, compact.all_freed += freed;
|
|
logf("vk: VRAM compacted: %llu textures moved (%llu MB) in %llu passes, %llu left where they were; %u blocks (%llu MB) "
|
|
"given back to the driver; %llu MB in all so far", (unsigned long long)compact.moved,
|
|
(unsigned long long)(compact.moved_bytes >> 20), (unsigned long long)compact.passes,
|
|
(unsigned long long)compact.ignored, blocks, (unsigned long long)(freed >> 20),
|
|
(unsigned long long)(compact.all_freed >> 20));
|
|
compact.next_check = frame_no + vram_compact::rest(compact.pressed);
|
|
}
|
|
return; // (the images the GPU is done with are freed now, before the next pass opens)
|
|
}
|
|
if (!image_compact_active()) {
|
|
if (frame_no < compact.next_check) return;
|
|
compact.next_check = frame_no + 120;
|
|
static const uint64_t min_free = (uint64_t)(getenv("RX_VK_COMPACT_MB") ? std::max(1, atoi(getenv("RX_VK_COMPACT_MB"))) : 512) << 20;
|
|
uint64_t block_bytes = 0, free_bytes = 0;
|
|
if (!textures_pool_usage(&block_bytes, &free_bytes) ||
|
|
free_bytes < vram_compact::run_from(min_free, compact.pressed))
|
|
return;
|
|
if (!image_compact_begin(64ull << 20, 12)) return;
|
|
compact.runs++, compact.moved = compact.moved_bytes = compact.ignored = compact.passes = 0;
|
|
logf("vk: VRAM compaction %llu begins: the textures' blocks hold %llu MB, %llu MB of it free space",
|
|
(unsigned long long)compact.runs, (unsigned long long)(block_bytes >> 20), (unsigned long long)(free_bytes >> 20));
|
|
}
|
|
static thread_local std::vector<ImageMove> moves;
|
|
uint64_t freed_now = 0;
|
|
uint32_t blocks_now = 0;
|
|
if (!image_compact_pass(moves, &freed_now, &blocks_now)) { // (nothing left to move)
|
|
compact.all_moved += compact.moved, compact.all_freed += freed_now;
|
|
logf("vk: VRAM compaction %llu over: %llu textures moved (%llu MB) in %llu passes, %llu left where they were; %u blocks "
|
|
"(%llu MB) given back to the driver, %llu MB in all so far", (unsigned long long)compact.runs,
|
|
(unsigned long long)compact.moved, (unsigned long long)(compact.moved_bytes >> 20), (unsigned long long)compact.passes,
|
|
(unsigned long long)compact.ignored, blocks_now, (unsigned long long)(freed_now >> 20),
|
|
(unsigned long long)(compact.all_freed >> 20));
|
|
compact.next_check = frame_no + vram_compact::rest(compact.pressed);
|
|
return;
|
|
}
|
|
std::unordered_map<VmaAllocation, Texture*> by_allocation;
|
|
by_allocation.reserve(textures.size());
|
|
for (auto& kv : textures)
|
|
if (kv.second->allocation) by_allocation[kv.second->allocation] = kv.second.get();
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
for (size_t i = 0; i < moves.size(); i++) {
|
|
const auto f = by_allocation.find(moves[i].src);
|
|
Texture* t = f == by_allocation.end() ? nullptr : f->second; // (none: an image the GPU is not done with)
|
|
const VkImageUsageFlags both = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
|
|
bool ok = t && t->image && !t->sparse && t != present_source && t->samples == VK_SAMPLE_COUNT_1_BIT &&
|
|
(t->usage & both) == both;
|
|
VkImage image = VK_NULL_HANDLE;
|
|
if (ok) {
|
|
VkImageCreateInfo ci{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; // (as create() made it)
|
|
ci.flags = t->create_flags;
|
|
ci.imageType = t->type;
|
|
ci.format = t->format;
|
|
ci.extent = {t->width, t->height, t->depth};
|
|
ci.mipLevels = t->levels;
|
|
ci.arrayLayers = t->layers;
|
|
ci.samples = t->samples;
|
|
ci.tiling = VK_IMAGE_TILING_OPTIMAL;
|
|
ci.usage = t->usage;
|
|
ci.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
ok = vkCreateImage(dev.device, &ci, nullptr, &image) == VK_SUCCESS;
|
|
if (ok && !image_compact_bind(i, image)) {
|
|
vkDestroyImage(dev.device, image, nullptr);
|
|
ok = false;
|
|
}
|
|
}
|
|
if (!ok) {
|
|
image_compact_ignore(i);
|
|
compact.ignored++;
|
|
continue;
|
|
}
|
|
if (!cmd) {
|
|
cmd = upload_cmd();
|
|
full_barrier(cmd, "compaction: before the copies");
|
|
}
|
|
VkImageMemoryBarrier2 b{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};
|
|
b.srcStageMask = b.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
b.newLayout = VK_IMAGE_LAYOUT_GENERAL;
|
|
b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
b.image = image;
|
|
b.subresourceRange = {t->aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS};
|
|
VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
di.imageMemoryBarrierCount = 1;
|
|
di.pImageMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &di);
|
|
static thread_local std::vector<VkImageCopy> regions;
|
|
regions.clear();
|
|
for (uint32_t l = 0; l < t->levels; l++) {
|
|
VkImageCopy r{};
|
|
r.srcSubresource = r.dstSubresource = {t->aspect, l, 0, t->layers};
|
|
r.extent = {std::max(1u, t->width >> l), std::max(1u, t->height >> l), std::max(1u, t->depth >> l)};
|
|
regions.push_back(r);
|
|
}
|
|
vkCmdCopyImage(cmd, t->image, VK_IMAGE_LAYOUT_GENERAL, image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)regions.size(), regions.data());
|
|
compact.old_images.push_back(t->image);
|
|
for (auto& kv : t->views)
|
|
if (kv.second) compact.old_views.push_back(kv.second);
|
|
t->views.clear(); // (made again when next asked for)
|
|
t->image = image;
|
|
t->last_value = std::max(t->last_value, rec_value); // (its end, if it goes now, waits for the copy too)
|
|
compact.moved++, compact.moved_bytes += t->bytes + t->pad;
|
|
}
|
|
compact.passes++;
|
|
if (!cmd) { // (nothing of this pass could be moved: ended at once)
|
|
uint64_t freed = 0;
|
|
uint32_t blocks = 0;
|
|
if (!image_compact_pass_end(&freed, &blocks)) {
|
|
compact.all_moved += compact.moved, compact.all_freed += freed;
|
|
logf("vk: VRAM compacted: %llu textures moved (%llu MB) in %llu passes, %llu left where they were; %u blocks (%llu MB) "
|
|
"given back to the driver; %llu MB in all so far", (unsigned long long)compact.moved,
|
|
(unsigned long long)(compact.moved_bytes >> 20), (unsigned long long)compact.passes,
|
|
(unsigned long long)compact.ignored, blocks, (unsigned long long)(freed >> 20),
|
|
(unsigned long long)(compact.all_freed >> 20));
|
|
compact.next_check = frame_no + vram_compact::rest(compact.pressed);
|
|
}
|
|
return;
|
|
}
|
|
full_barrier(cmd, "compaction: after the copies");
|
|
destroy_gen++; // (what was resolved to the moved textures' views is stale)
|
|
compact.in_pass = true, compact.value = rec_value;
|
|
}
|
|
|
|
|
|
// ---- (Triforce) what the GPU baked, in the game's memory too ----
|
|
// On the console a render target's texels are in its storage, and OoT's terrain relies on that. Each tile's 260 x 260 RG8
|
|
// normal map (NVN 0xd, 9 levels) is drawn once from the tile's height map (program aa9a8316); the game's files hold no
|
|
// normal maps (TerrainArc: .hght, .mate, .bake_ao, .bake_gi). When a tile is wanted again the game does not draw it: it
|
|
// makes a texture over the bytes it kept, at the same storage or wherever its tile cache has moved them since, and samples
|
|
// that. Here the texels were only in the image, so that texture was uploaded from memory they never reached: the ground's
|
|
// normals empty, the dirt stretched into streaks over all of Hyrule Field once tiles came back from the cache (2026-11-10,
|
|
// a five-minute run with RX_VK_WATCH_DIM=260x260: 61 of 146 normal maps were sampled without ever being drawn, all at
|
|
// storage no texture had been drawn at; reloading the save, which bakes them again, cleared it). Upstream's "finalized
|
|
// textures kept" covers only a texture made again over the same storage while its image is still there.
|
|
// So a texture the GPU drew in one frame, that the game still has and still samples, gets its texels written into its
|
|
// memory a few presents later: the copy to a staging buffer goes into the present's upload commands (no wait), and the
|
|
// texels are swizzled into the memory once the GPU is past it. Pages the CPU wrote since the draw keep the CPU's bytes
|
|
// (flush_finish), and one finalized or drawn again in between is left alone. RX_VK_BAKE_WRITEBACK=0: off, as before.
|
|
struct BakeItem {
|
|
Flush f;
|
|
uint32_t id;
|
|
uint64_t gpu, written; // (its storage, and the frame it was drawn in, as recorded)
|
|
};
|
|
struct BakeBatch {
|
|
uint64_t value = 0; // (the recording the copies are in)
|
|
std::vector<BakeItem> items;
|
|
};
|
|
std::deque<BakeBatch> bake_batches;
|
|
uint64_t st_bake_wb, st_bake_wb_bytes, st_bake_wb_lost, st_bake_wb_logged;
|
|
uint64_t st_bake_gpu, st_bake_gpu_bytes, swizzle_map_bytes, swizzle_map_n; // (texture_end_recording's, below)
|
|
struct BakeKind { uint32_t w, h, levels; int format; uint64_t n, bytes; };
|
|
std::unordered_map<uint64_t, BakeKind> bake_kinds; // (what it wrote back, by size and format: the log's largest)
|
|
bool bake_writeback_on() {
|
|
static const bool on = !getenv("RX_VK_BAKE_WRITEBACK") || atoi(getenv("RX_VK_BAKE_WRITEBACK")) != 0;
|
|
return on;
|
|
}
|
|
void bake_writeback_step(uint64_t completed) {
|
|
if (!bake_writeback_on()) return;
|
|
while (!bake_batches.empty() && bake_batches.front().value <= completed) { // (the CPU half of what the GPU is past)
|
|
for (BakeItem& it : bake_batches.front().items) {
|
|
Texture* t = nullptr; // (still there: the pointer may be another texture's by now, the id tells)
|
|
auto [a, e] = by_gpu.equal_range(it.gpu);
|
|
for (auto k = a; k != e; ++k)
|
|
if (k->second == it.f.t && k->second->id == it.id) t = k->second;
|
|
if (t && t->gpu_written && !t->released && t->last_written == it.written) {
|
|
if (flush_finish(it.f, true)) {
|
|
t->wb_done_frame = frame_no;
|
|
st_bake_wb++, st_bake_wb_bytes += t->guest_size;
|
|
watch_tex(t, "baked: its texels written into its memory");
|
|
} else {
|
|
t->wb_written = 0, st_bake_wb_lost++;
|
|
}
|
|
} else { // (gone, finalized or drawn again since: its memory is left as it is; drawn again, it is listed again)
|
|
if (it.f.buf) destroy_buffer(it.f.buf, it.f.alloc);
|
|
if (t && t->wb_written == it.written) t->wb_written = 0;
|
|
st_bake_wb_lost++;
|
|
}
|
|
}
|
|
bake_batches.pop_front();
|
|
}
|
|
if (frame_no % 600 == 0 && st_bake_wb + st_bake_wb_lost + st_bake_gpu != st_bake_wb_logged) {
|
|
st_bake_wb_logged = st_bake_wb + st_bake_wb_lost + st_bake_gpu;
|
|
logf("vk: what the GPU drew into small textures, in their memory too: %llu by the recordings that drew them (%llu KB, "
|
|
"%zu layouts' maps of %llu KB); some presents later, by the CPU: %llu (%llu KB), %llu left out (finalized, gone or "
|
|
"drawn again first)", (unsigned long long)st_bake_gpu, (unsigned long long)(st_bake_gpu_bytes >> 10), (size_t)swizzle_map_n,
|
|
(unsigned long long)(swizzle_map_bytes >> 10), (unsigned long long)st_bake_wb, (unsigned long long)(st_bake_wb_bytes >> 10),
|
|
(unsigned long long)st_bake_wb_lost);
|
|
std::vector<const BakeKind*> top; // (the kinds that take the most of it)
|
|
for (const auto& kv : bake_kinds) top.push_back(&kv.second);
|
|
std::sort(top.begin(), top.end(), [](const BakeKind* a, const BakeKind* b) { return a->bytes > b->bytes; });
|
|
char line[400];
|
|
int n = 0;
|
|
for (size_t i = 0; i < top.size() && i < 5 && n < 330; i++)
|
|
n += snprintf(line + n, sizeof line - n, " %ux%u levels %u NVN %#x: %llu (%llu KB);", top[i]->w, top[i]->h, top[i]->levels,
|
|
top[i]->format, (unsigned long long)top[i]->n, (unsigned long long)(top[i]->bytes >> 10));
|
|
if (n) logf("vk: by the recordings, the largest of %zu kinds:%s", bake_kinds.size(), line);
|
|
}
|
|
if (frame_no % 4) return;
|
|
// (drawn in one frame, four presents or more ago; still the game's, and sampled in the last four presents: the game's own
|
|
// draws say its storage is still this texture's. The small ones the eviction calls baked; the guest window's, which the
|
|
// game's code can read)
|
|
BakeBatch nb;
|
|
uint64_t bytes = 0;
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
for (auto& kv : textures) {
|
|
Texture* t = kv.second.get();
|
|
if (!t->gpu_written || t->released || t->only_cleared || !t->first_written || t->first_written != t->last_written ||
|
|
t->wb_written == t->last_written || frame_no < t->last_written + 4 || frame_no - t->last_frame > 4 ||
|
|
t->bytes > (4ull << 20) || t->gpu >= (1ull << 39) || !t->image || !flushable(*t))
|
|
continue;
|
|
if (nb.items.size() >= 24 || bytes + t->guest_size > (8ull << 20)) break; // (the rest four presents on)
|
|
if (!cmd) {
|
|
cmd = upload_cmd();
|
|
full_barrier(cmd, "bake write-back: before the copies");
|
|
}
|
|
BakeItem it{Flush{t, t->gpu, t->gpu + t->guest_size}, t->id, t->gpu, t->last_written};
|
|
it.f.ignore_cpu = audit_on();
|
|
if (!flush_record(it.f, cmd)) continue; // (no staging buffer: tried again four presents on)
|
|
t->wb_written = t->last_written, t->wb_done_frame = 0;
|
|
t->last_value = std::max(t->last_value, rec_value); // (its end, if it goes now, waits for the copy)
|
|
bytes += t->guest_size;
|
|
nb.items.push_back(std::move(it));
|
|
}
|
|
if (nb.items.empty()) return;
|
|
VkMemoryBarrier2 b{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; // (the copies' writes, for the host)
|
|
b.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
|
|
b.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
|
|
b.dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT;
|
|
b.dstAccessMask = VK_ACCESS_2_HOST_READ_BIT;
|
|
VkDependencyInfo d{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
|
|
d.memoryBarrierCount = 1;
|
|
d.pMemoryBarriers = &b;
|
|
vkCmdPipelineBarrier2(cmd, &d);
|
|
nb.value = rec_value;
|
|
bake_batches.push_back(std::move(nb));
|
|
}
|
|
// ---- (Triforce) and at once: on the GPU, by the recording that drew it ----
|
|
// The write-back above reaches the memory some presents after the draw, and the game can be there first: it waits for the
|
|
// GPU to finish the frame and then moves a freshly baked tile into its cache (2026-11-10, with the write-back above alone:
|
|
// 268 of 275 tiles made again from memory had their texels, 7 were empty, each one baked 9 presents or fewer before). On
|
|
// the console the texels are in the storage when the GPU has run the draw. So a recording that draws into a small texture
|
|
// in the texture's first frame ends with commands of its own that put the image's texels into the storage: the levels
|
|
// copied to a buffer row by row, a compute pass that lays them out as the storage does (shaders/swizzle_back.comp, from a
|
|
// map made once per geometry with the CPU's swizzle), and a copy into the imported pool. When the submission is done, so
|
|
// is the memory. RX_VK_BAKE_GPU=0: only the write-back above.
|
|
struct SwizzleMap {
|
|
VkBuffer buf = VK_NULL_HANDLE; // (none: this geometry is not written back on the GPU)
|
|
VmaAllocation alloc = nullptr;
|
|
uint64_t lin_bytes = 0; // (the levels and layers, row by row: the linear copy's size)
|
|
std::vector<VkBufferImageCopy> regions; // (the image into it)
|
|
uint32_t unit_shift = 0; // (log2 of a block's bytes)
|
|
};
|
|
std::unordered_map<uint64_t, SwizzleMap> swizzle_maps; // (by the storage's geometry)
|
|
VkDescriptorSetLayout swz_set = VK_NULL_HANDLE;
|
|
VkPipelineLayout swz_layout = VK_NULL_HANDLE;
|
|
VkPipeline swz_pipe = VK_NULL_HANDLE;
|
|
bool swizzle_back_ready() {
|
|
static int state = 0;
|
|
if (state) return state > 0;
|
|
state = -1;
|
|
if (!dev.push_descriptor || !vkCmdPushDescriptorSetKHR) return false;
|
|
if (getenv("RX_VK_BAKE_GPU") && !atoi(getenv("RX_VK_BAKE_GPU"))) return false;
|
|
const VkDescriptorSetLayoutBinding b[3] = {
|
|
{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
|
|
{1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr},
|
|
{2, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}};
|
|
VkDescriptorSetLayoutCreateInfo li{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
|
|
li.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR;
|
|
li.bindingCount = 3;
|
|
li.pBindings = b;
|
|
if (vkCreateDescriptorSetLayout(dev.device, &li, nullptr, &swz_set) != VK_SUCCESS) return false;
|
|
const VkPushConstantRange pr{VK_SHADER_STAGE_COMPUTE_BIT, 0, 8};
|
|
VkPipelineLayoutCreateInfo pi{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
|
|
pi.setLayoutCount = 1;
|
|
pi.pSetLayouts = &swz_set;
|
|
pi.pushConstantRangeCount = 1;
|
|
pi.pPushConstantRanges = ≺
|
|
if (vkCreatePipelineLayout(dev.device, &pi, nullptr, &swz_layout) != VK_SUCCESS) return false;
|
|
VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
|
mi.codeSize = sizeof SWIZZLE_BACK_COMP_SPV;
|
|
mi.pCode = SWIZZLE_BACK_COMP_SPV;
|
|
VkShaderModule m = VK_NULL_HANDLE;
|
|
if (vkCreateShaderModule(dev.device, &mi, nullptr, &m) != VK_SUCCESS) return false;
|
|
VkComputePipelineCreateInfo ci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
|
ci.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO;
|
|
ci.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
|
ci.stage.module = m;
|
|
ci.stage.pName = "main";
|
|
ci.layout = swz_layout;
|
|
const VkResult r = vkCreateComputePipelines(dev.device, VK_NULL_HANDLE, 1, &ci, nullptr, &swz_pipe);
|
|
vkDestroyShaderModule(dev.device, m, nullptr);
|
|
if (r != VK_SUCCESS) return false;
|
|
state = 1;
|
|
logf("vk: what the GPU draws into a small texture in its first frame goes into the texture's memory with the recording "
|
|
"that drew it (RX_VK_BAKE_GPU=0: some presents later, by the CPU)");
|
|
return true;
|
|
}
|
|
/// The map of t's geometry (made when first asked for), or null when it has none: where in the linear copy each block of the
|
|
/// storage comes from. Found by swizzling, with the swizzle the write-backs use, a level whose blocks hold their own numbers
|
|
const SwizzleMap* swizzle_map_of(const Texture& t) {
|
|
const uint32_t layers = t.type == VK_IMAGE_TYPE_3D ? 1 : t.layers;
|
|
uint64_t key = mix(mix(mix(0x53575Aull, (uint64_t)t.bpb), t.guest_size), mix(t.levels, mix(layers, t.L.layer_size)));
|
|
for (uint32_t level = 0; level < t.levels; level++) {
|
|
const LevelGeo g = level_geo(t, level);
|
|
key = mix(mix(mix(key, g.offset), mix(g.tw, g.th)), mix(mix(g.td, g.tile_h), mix(g.tile_d, mix(g.w, mix(g.h, g.d)))));
|
|
}
|
|
if (const auto it = swizzle_maps.find(key); it != swizzle_maps.end()) return it->second.buf ? &it->second : nullptr;
|
|
SwizzleMap& m = swizzle_maps[key]; // (kept without a buffer when it cannot be made: not tried again)
|
|
const uint64_t bpb = (uint64_t)t.bpb, units = (t.guest_size + bpb - 1) / bpb;
|
|
if (swizzle_maps.size() > 256 || swizzle_map_bytes + units * 4 > (96ull << 20)) return nullptr;
|
|
m.unit_shift = (uint32_t)std::countr_zero((uint32_t)t.bpb);
|
|
std::vector<uint32_t> map(units, 0xffffffffu);
|
|
std::vector<uint8_t> src, dst;
|
|
std::vector<uint32_t> number;
|
|
for (uint32_t level = 0; level < t.levels; level++) {
|
|
const LevelGeo g = level_geo(t, level);
|
|
const uint64_t n = (uint64_t)g.tw * g.th * g.td * bpb, blocks = n / bpb;
|
|
for (uint32_t layer = 0; layer < layers; layer++) {
|
|
const uint64_t at = m.lin_bytes;
|
|
m.lin_bytes += (n + 15) & ~15ull;
|
|
VkBufferImageCopy r{};
|
|
r.bufferOffset = at;
|
|
r.bufferRowLength = g.tw * (uint32_t)t.block_w;
|
|
r.bufferImageHeight = g.th * (uint32_t)t.block_h;
|
|
r.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, t.type == VK_IMAGE_TYPE_3D ? 0 : layer, 1};
|
|
r.imageExtent = {g.w, g.h, g.d};
|
|
m.regions.push_back(r);
|
|
const uint64_t so = g.offset + (uint64_t)layer * t.L.layer_size;
|
|
if (so >= t.guest_size || !blocks) continue;
|
|
if (so % bpb || m.lin_bytes > 0xfffffff0ull) return m.regions.clear(), nullptr;
|
|
const uint64_t span = std::min<uint64_t>(g.swizzled, t.guest_size - so);
|
|
number.assign((size_t)(span / bpb), 0);
|
|
for (uint32_t k = 0; k < 4 && ((blocks + 1) >> (8 * k)) != 0; k++) { // (a block's number + 1, a byte of it a pass)
|
|
src.assign((size_t)n, 0);
|
|
for (uint64_t i = 0; i < blocks; i++) src[(size_t)(i * bpb)] = (uint8_t)((i + 1) >> (8 * k));
|
|
dst.assign((size_t)std::max<uint64_t>(g.swizzled, span), 0);
|
|
Tegra::Texture::SwizzleTexture(std::span<uint8_t>(dst.data(), dst.size()), std::span<const uint8_t>(src.data(), src.size()),
|
|
(u32)t.bpb, g.tw, g.th, g.td, g.tile_h, g.tile_d, 1);
|
|
for (size_t j = 0; j < number.size(); j++) number[j] |= (uint32_t)dst[j * bpb] << (8 * k);
|
|
}
|
|
for (size_t j = 0; j < number.size(); j++)
|
|
if (number[j] && number[j] <= blocks) map[(size_t)(so / bpb) + j] = (uint32_t)(at + (uint64_t)(number[j] - 1) * bpb);
|
|
}
|
|
}
|
|
void* mapped = nullptr;
|
|
if (!m.lin_bytes || !create_buffer(units * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, true, m.buf, m.alloc, &mapped) || !mapped) {
|
|
if (m.buf) destroy_buffer(m.buf, m.alloc);
|
|
m.buf = VK_NULL_HANDLE, m.alloc = nullptr;
|
|
return m.regions.clear(), nullptr;
|
|
}
|
|
std::memcpy(mapped, map.data(), (size_t)units * 4);
|
|
flush_mapped(m.alloc, units * 4);
|
|
swizzle_map_bytes += units * 4, swizzle_map_n++;
|
|
return &m;
|
|
}
|
|
// (The textures drawn into early in a run of frames (the run's first frame and the next: a bake finished a frame later is
|
|
// taken too; a target drawn every frame is written back twice when it is made and no more), by the submission whose recording
|
|
// drew them: recordings of several submissions can be open at once (a queue stopped at an event wait while another's goes
|
|
// on), and the texels must be copied by the recording that holds the draws, after them)
|
|
struct BakeFresh { Texture* t; uint32_t id; uint64_t gpu; };
|
|
std::mutex bake_fresh_mu;
|
|
std::unordered_map<uint64_t, std::vector<BakeFresh>> bake_fresh; // (by rec_value)
|
|
uint64_t bake_rec_no = 1; // (counts the recordings ended: a texture is listed once in each)
|
|
void bake_note_written(Texture* t) { // (texture_written)
|
|
if (frame_no - t->wb_burst > 1 || t->wb_quick > 2 || t->wb_rec == bake_rec_no || !bake_writeback_on()) return;
|
|
t->wb_rec = bake_rec_no;
|
|
std::lock_guard lk(bake_fresh_mu);
|
|
bake_fresh[rec_value].push_back({t, t->id, t->gpu});
|
|
}
|
|
void texture_end_recording(VkCommandBuffer cmd) {
|
|
std::vector<BakeFresh> list;
|
|
{
|
|
std::lock_guard lk(bake_fresh_mu);
|
|
bake_rec_no++;
|
|
if (const auto it = bake_fresh.find(rec_value); it != bake_fresh.end()) list.swap(it->second), bake_fresh.erase(it);
|
|
if (bake_fresh.size() > 64) bake_fresh.clear(); // (recordings that never ended here: a bound)
|
|
}
|
|
if (list.empty() || par_active.load(std::memory_order_relaxed) || !swizzle_back_ready()) return;
|
|
bool any = false;
|
|
std::unordered_set<uint32_t> seen;
|
|
for (const BakeFresh& f : list) {
|
|
Texture* t = nullptr; // (still there: a present may have come between the draw and this end)
|
|
auto [a, e] = by_gpu.equal_range(f.gpu);
|
|
for (auto k = a; k != e; ++k)
|
|
if (k->second == f.t && k->second->id == f.id) t = k->second;
|
|
if (!t || !seen.insert(f.id).second) continue;
|
|
const VkImageUsageFlags need = VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
|
|
if (!t->image || t->released || t->only_cleared || t->bytes > (4ull << 20) ||
|
|
t->gpu >= (1ull << 39) || (t->usage & need) != need || !flushable(*t) || nvn_tex_is_linear(&t->d) || t->bpb > 16 ||
|
|
(t->bpb & (t->bpb - 1)))
|
|
continue;
|
|
const SwizzleMap* m = swizzle_map_of(*t);
|
|
if (!m) continue;
|
|
const BufferRef pool = gpu_buffer(t->gpu, t->guest_size);
|
|
if (!pool) continue;
|
|
const uint64_t words = (t->guest_size + 3) / 4;
|
|
VkBuffer lin = VK_NULL_HANDLE, sto = VK_NULL_HANDLE;
|
|
VmaAllocation lin_a = nullptr, sto_a = nullptr;
|
|
if (!create_device_buffer(m->lin_bytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, lin, lin_a))
|
|
continue;
|
|
if (!create_device_buffer(words * 4, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
|
|
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, sto, sto_a)) {
|
|
destroy_buffer(lin, lin_a);
|
|
continue;
|
|
}
|
|
full_barrier(cmd, "bake write-back: before the copies");
|
|
vkCmdCopyImageToBuffer(cmd, t->image, VK_IMAGE_LAYOUT_GENERAL, lin, (uint32_t)m->regions.size(), m->regions.data());
|
|
if (audit_on()) { // (RX_VK_AUDIT_GPU_ONLY: no mark left between the texels)
|
|
vkCmdFillBuffer(cmd, sto, 0, VK_WHOLE_SIZE, 0);
|
|
} else {
|
|
const VkBufferCopy in{pool.offset, 0, t->guest_size}; // (the storage as it is: what is no texel stays)
|
|
vkCmdCopyBuffer(cmd, pool.buffer, sto, 1, &in);
|
|
}
|
|
full_barrier(cmd, "bake write-back: before the swizzle");
|
|
vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, swz_pipe);
|
|
const VkDescriptorBufferInfo bi[3] = {{m->buf, 0, VK_WHOLE_SIZE}, {lin, 0, VK_WHOLE_SIZE}, {sto, 0, VK_WHOLE_SIZE}};
|
|
VkWriteDescriptorSet w[3]{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET},
|
|
{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}};
|
|
for (uint32_t i = 0; i < 3; i++)
|
|
w[i].dstBinding = i, w[i].descriptorCount = 1, w[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, w[i].pBufferInfo = &bi[i];
|
|
vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, swz_layout, 0, 3, w);
|
|
const uint32_t pc[2] = {(uint32_t)words, m->unit_shift};
|
|
vkCmdPushConstants(cmd, swz_layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc);
|
|
vkCmdDispatch(cmd, (uint32_t)((words + 63) / 64), 1, 1);
|
|
full_barrier(cmd, "bake write-back: before the copy out");
|
|
const VkBufferCopy out{0, pool.offset, t->guest_size};
|
|
vkCmdCopyBuffer(cmd, sto, pool.buffer, 1, &out);
|
|
for (const auto& [buf, alloc] : {std::pair{lin, lin_a}, std::pair{sto, sto_a}}) { // (given back once the GPU is past them)
|
|
Dead d{rec_value, VK_NULL_HANDLE, nullptr, {}, {}};
|
|
d.buffer = buf, d.buffer_alloc = alloc;
|
|
dead.push_back(std::move(d));
|
|
}
|
|
// (the pages count as written from here on: a texture made over them is uploaded from them, a mirror takes them again)
|
|
memory_stamp_gpu(host->gpu_guest(t->gpu), t->guest_size);
|
|
t->wb_written = t->last_written, t->wb_value = rec_value, t->wb_done_frame = frame_no;
|
|
t->last_value = std::max(t->last_value, rec_value);
|
|
st_bake_gpu++, st_bake_gpu_bytes += t->guest_size;
|
|
{
|
|
BakeKind& k = bake_kinds[mix(mix(t->width, t->height), mix(t->levels, (uint64_t)t->nvn_format))];
|
|
k.w = t->width, k.h = t->height, k.levels = t->levels, k.format = t->nvn_format, k.n++, k.bytes += t->guest_size;
|
|
}
|
|
watch_tex(t, "baked: its texels into its memory by this recording");
|
|
any = true;
|
|
}
|
|
if (!any) return;
|
|
full_barrier(cmd, "bake write-back: after");
|
|
descriptors_disturbed();
|
|
}
|
|
// (lab: RX_VK_BAKE_REUPLOAD=1) What the game does to a tile's normal map when the tile comes back from its cache, done here to
|
|
// every bake so that it can be checked at any save, standing still: 24 presents after it was drawn, its image is filled from
|
|
// its memory. With the write-back above the picture stays as it was (the texels made the round trip through the memory, in
|
|
// the storage's layout); with RX_VK_BAKE_WRITEBACK=0 the image gets what the memory held without it: the streaks.
|
|
void bake_reupload_step(uint64_t completed) {
|
|
static unsigned only_w = 0, only_h = 0; // (RX_VK_BAKE_REUPLOAD=<w>x<h>: the bakes of that size alone; 1: all of them)
|
|
static const bool on = [] {
|
|
const char* e = getenv("RX_VK_BAKE_REUPLOAD");
|
|
if (!e) return false;
|
|
if (sscanf(e, "%ux%u", &only_w, &only_h) != 2) only_w = only_h = 0;
|
|
return only_w || atoi(e) != 0;
|
|
}();
|
|
if (!on || frame_no % 4) return;
|
|
static std::unordered_set<uint32_t> done;
|
|
for (auto& kv : textures) {
|
|
Texture* t = kv.second.get();
|
|
if (only_w && (t->width != only_w || t->height != only_h)) continue;
|
|
if (!t->gpu_written || t->released || t->only_cleared || !t->first_written || t->first_written != t->last_written ||
|
|
frame_no < t->last_written + 24 || frame_no - t->last_frame > 4 || t->bytes > (4ull << 20) ||
|
|
t->gpu >= (1ull << 39) || !t->image || !flushable(*t) || done.count(t->id))
|
|
continue;
|
|
// (with the write-back: only once its texels are in the memory, two presents after they got there)
|
|
if (bake_writeback_on() && (t->wb_written != t->last_written || !t->wb_done_frame || frame_no < t->wb_done_frame + 2 ||
|
|
t->wb_value > completed))
|
|
continue;
|
|
done.insert(t->id);
|
|
const bool ok = upload(*t, nullptr);
|
|
if (done.size() <= 60)
|
|
logf("vk: lab: #%u %#llx %ux%u x%u levels %u NVN %#x, drawn at frame %llu: its image filled from its memory again%s",
|
|
t->id, (unsigned long long)t->gpu, t->width, t->height, t->layers, t->levels, t->nvn_format,
|
|
(unsigned long long)t->last_written, ok ? "" : " (the upload failed)");
|
|
}
|
|
}
|
|
|
|
Texture* texture_for_present(const nvn_tex* t) {
|
|
Texture* tex = texture_get(t, false);
|
|
if (tex && tex->gpu_written) return present_source = tex, tex;
|
|
Texture* best = nullptr; // (one rendered at its storage in another description)
|
|
auto [a, b] = by_gpu.equal_range(t ? t->gpu : 0);
|
|
for (auto it = a; it != b; ++it)
|
|
if (it->second->gpu_written && (it->second->aspect & VK_IMAGE_ASPECT_COLOR_BIT) &&
|
|
(!best || it->second->last_frame > best->last_frame))
|
|
best = it->second;
|
|
present_source = best ? best : tex;
|
|
return best ? best : tex;
|
|
}
|
|
|
|
/// (hookshot) The texture budget a quality preset sets, or the card's own (tests/test_preset_texture_budget.py). The preset's
|
|
/// replaced `own` outright, and `own` is where RX_VK_VRAM_MB and an integrated GPU's cap are applied: with a preset on, the
|
|
/// documented RX_VK_VRAM_MB did nothing (the mirrors' RX_VK_MIRROR_MB wins over the preset), and an integrated GPU on Low or
|
|
/// Balanced got 3.6 or 4.8 GB of textures past the 2.5 GB that keeps a 16 GB laptop from running out of RAM. Now
|
|
/// RX_VK_VRAM_MB wins over a preset, and on an integrated GPU the preset's budget is at most `own`
|
|
uint64_t preset_texture_budget(uint64_t preset_mb, bool preset_applies, bool vram_env, bool integrated, uint64_t own) {
|
|
if (!preset_mb || !preset_applies || vram_env) return own;
|
|
const uint64_t b = preset_mb << 20;
|
|
return integrated ? std::min(b, own) : b;
|
|
}
|
|
|
|
uint64_t dead_sweep_next = 0; // (the present of the next pass for the dead targets between texture_frame's looks; 0: none)
|
|
void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evict_to, bool known, uint64_t used, uint64_t allowed,
|
|
bool dead_only, uint64_t hold_to, uint64_t rule_to);
|
|
void texture_frame(uint64_t completed) {
|
|
upload_frame_report();
|
|
frame_upload_ns.store(0, std::memory_order_relaxed);
|
|
frame_astc_admitted.store(0, std::memory_order_relaxed);
|
|
if (const int64_t d = st_deferred_frame.exchange(0, std::memory_order_relaxed)) {
|
|
// (the views and descriptor sets resolved to the placeholder this frame go stale, as when textures go: the next frame
|
|
// looks those textures up again, and uploads them)
|
|
destroy_gen++;
|
|
static int told;
|
|
if (told++ < 40 || st_deferred.load(std::memory_order_relaxed) % 2000 < d)
|
|
logf("vk: %lld new ASTC textures left for a later frame (this frame's uploads took their budget; %lld so far)", (long long)d,
|
|
(long long)st_deferred.load(std::memory_order_relaxed));
|
|
}
|
|
completed_value = completed;
|
|
frame_no++;
|
|
{ // (staging arenas the GPU is done with, beyond 256 MB of them, and any over 64 MB: freed. A scene change stages
|
|
// hundreds of MB to over a GB (the course maps, the terrain's ASTC arrays), and that host memory was kept)
|
|
// (hookshot: 64 on a card of 6 GB or less. On one with resizable BAR the arenas are VRAM: VMA puts a host-visible buffer the GPU
|
|
// reads (STORAGE: the ASTC decoder) in DEVICE_LOCAL | HOST_VISIBLE. The RTX 4050 Laptop held 4 idle ones, 256 MB of 5.7 GB, through
|
|
// all of Kokiri gameplay, which stages ~nothing. A scene change makes the ones it needs again. RX_VK_STAGING_KEEP_MB=256: as before)
|
|
static const uint64_t keep = []() -> uint64_t {
|
|
if (const char* e = getenv("RX_VK_STAGING_KEEP_MB")) return (uint64_t)atoi(e) << 20;
|
|
uint64_t used = 0, allowed = 0;
|
|
return (vram_usage(&used, &allowed) && allowed <= (6ull << 30) + (512ull << 20) ? 64ull : 256ull) << 20;
|
|
}();
|
|
uint64_t kept = 0;
|
|
for (auto it = arenas.begin(); it != arenas.end();) {
|
|
const bool idle = it->value <= completed && it->value != rec_value;
|
|
// (and one nothing has staged into for 30 s: Kokiri Forest's gameplay stages next to nothing, and the 64 MB it kept are VRAM
|
|
// behind the BAR. A load makes one again. RX_VK_STAGING_IDLE_S=<n>: that long, 0: kept)
|
|
static const int64_t idle_s = getenv("RX_VK_STAGING_IDLE_S") ? atoll(getenv("RX_VK_STAGING_IDLE_S")) : 30;
|
|
const bool stale = idle_s && std::chrono::steady_clock::now() - it->used_at > std::chrono::seconds(idle_s);
|
|
if (idle && (it->size > kArena || kept + it->size > keep || stale)) {
|
|
destroy_buffer(it->buffer, it->allocation);
|
|
it = arenas.erase(it);
|
|
continue;
|
|
}
|
|
kept += it->size;
|
|
++it;
|
|
}
|
|
if (frame_no % 600 == 0) { // (the process's memory now and then: what the host side holds)
|
|
PROCESS_MEMORY_COUNTERS_EX pm{};
|
|
pm.cb = sizeof pm;
|
|
if (K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm, sizeof pm))
|
|
logf("vk: memory: working set %llu MB (peak %llu), private %llu MB, staging %llu MB in %zu arenas",
|
|
(unsigned long long)(pm.WorkingSetSize >> 20), (unsigned long long)(pm.PeakWorkingSetSize >> 20),
|
|
(unsigned long long)(pm.PrivateUsage >> 20), (unsigned long long)(kept >> 20), arenas.size());
|
|
#if defined(__linux__) && defined(__GLIBC__)
|
|
// (glibc's heap: what its arenas hold free. The shader translation and the loads' decoding leave their garbage spread through
|
|
// the threads' arenas (26 of them, 1.2 GB resident in Kokiri Forest, the heap's growth in a scene change never going back),
|
|
// and a free chunk's pages are the process's till malloc_trim gives them to the system. Asked on a worker, when 192 MB or
|
|
// more are free: RX_VK_MALLOC_TRIM_MB=<n>: that many, 0: never; the "vk: malloc:" line says what the arenas hold)
|
|
// (every 1800 presents: mallinfo2 walks every free chunk of every arena with its lock held)
|
|
static std::atomic<bool> busy{false};
|
|
if (frame_no % 1800 == 0 && !busy.exchange(true))
|
|
jobs_push([] {
|
|
static const uint64_t trim_mb = getenv("RX_VK_MALLOC_TRIM_MB") ? (uint64_t)atoll(getenv("RX_VK_MALLOC_TRIM_MB")) : 192;
|
|
const struct mallinfo2 a = mallinfo2();
|
|
PROCESS_MEMORY_COUNTERS_EX pm0{};
|
|
pm0.cb = sizeof pm0;
|
|
K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm0, sizeof pm0);
|
|
logf("vk: malloc: %llu MB in arenas (%llu MB in use, %llu MB free, %llu MB of it at the tops), %llu MB mapped",
|
|
(unsigned long long)(a.arena >> 20), (unsigned long long)(a.uordblks >> 20), (unsigned long long)(a.fordblks >> 20),
|
|
(unsigned long long)(a.keepcost >> 20), (unsigned long long)(a.hblkhd >> 20));
|
|
if (trim_mb && (a.fordblks >> 20) >= trim_mb) {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
malloc_trim(0);
|
|
PROCESS_MEMORY_COUNTERS_EX pm1{};
|
|
pm1.cb = sizeof pm1;
|
|
K32GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pm1, sizeof pm1);
|
|
logf("vk: malloc_trim: %llu MB free in the arenas now, the process's private memory %llu -> %llu MB (%.0f ms)",
|
|
(unsigned long long)(mallinfo2().fordblks >> 20), (unsigned long long)(pm0.PrivateUsage >> 20),
|
|
(unsigned long long)(pm1.PrivateUsage >> 20),
|
|
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count());
|
|
}
|
|
busy = false;
|
|
});
|
|
#endif
|
|
}
|
|
}
|
|
bc_scratch_idle();
|
|
bake_writeback_step(completed); // (Triforce: what the GPU baked, into the game's memory too)
|
|
bake_reupload_step(completed); // (lab: RX_VK_BAKE_REUPLOAD)
|
|
compact_step(completed); // (hookshot, RX_VK_COMPACT: while its pass is open nothing of the textures' pool is freed)
|
|
for (auto it = dead.begin(); it != dead.end();) {
|
|
if (compact.in_pass) break; // (hookshot, RX_VK_COMPACT: nothing is freed while a pass is open; a frame or two)
|
|
if (it->value > completed) {
|
|
++it;
|
|
continue;
|
|
}
|
|
if (it->pool_key) { // (a render target's image: kept for the next of its shape, above)
|
|
pool_put(*it);
|
|
it = dead.erase(it);
|
|
continue;
|
|
}
|
|
for (VkImageView v : it->views) vkDestroyImageView(dev.device, v, nullptr);
|
|
destroy_image(it->image, it->allocation);
|
|
if (it->buffer) destroy_buffer(it->buffer, it->buffer_alloc);
|
|
for (VkDeviceMemory m : it->memory) vkFreeMemory(dev.device, m, nullptr);
|
|
it = dead.erase(it);
|
|
}
|
|
pool_trim();
|
|
// A budget: RX_VK_VRAM_MB of textures (default 4096), and less when the driver says the process is near what it
|
|
// may have (other programs use the GPU too). The least recently used go first, down to 7/8 of it; nothing the
|
|
// last 8 presents used, nothing recorded work may still read.
|
|
static uint64_t own = 0;
|
|
static bool last_known; // (the driver's numbers of the last full look, for the passes between)
|
|
static uint64_t last_used, last_allowed;
|
|
{ // (and a present in which a target's image was made new, not taken from the pool, asks for a pass three presents on,
|
|
// when what it replaced can first be found dead (RX_VK_DEAD_MIN_UNUSED): by the passes' own findings alone the first
|
|
// one of a run of new targets came at the next look, and in those ten presents the targets' blocks went 11 -> 22)
|
|
static uint64_t made_seen;
|
|
static const bool dead_quick = !(getenv("RX_VK_DEAD_QUICK") && !atoi(getenv("RX_VK_DEAD_QUICK")));
|
|
if (st_pool_made != made_seen) {
|
|
made_seen = st_pool_made;
|
|
if (dead_quick && (!dead_sweep_next || dead_sweep_next > frame_no + 3)) dead_sweep_next = frame_no + 3;
|
|
}
|
|
}
|
|
if (frame_no % 10) {
|
|
// (Triforce, 11-11) between the looks, a pass for the dead transient targets alone while the game makes them fast
|
|
// (dead_sweep_next, set by texture_sweep): see there
|
|
if (dead_sweep_next && frame_no >= dead_sweep_next)
|
|
texture_sweep(completed, true, ~0ull, 0, last_known, last_used, last_allowed, true, 0, UINT64_MAX); // (no hold in this pass)
|
|
return;
|
|
}
|
|
vma_blocks_ahead_check(); // (Triforce: the blocks made ahead go back while VRAM is short, vk_vma.cpp)
|
|
uint64_t used = 0, allowed = 0, block_free = 0;
|
|
const bool known = vram_usage(&used, &allowed, &block_free);
|
|
last_known = known, last_used = used, last_allowed = allowed;
|
|
if (!own) // (RX_VK_VRAM_MB, else 70% of what the driver allows the process at the first look)
|
|
own = getenv("RX_VK_VRAM_MB") ? (uint64_t)atoi(getenv("RX_VK_VRAM_MB")) << 20
|
|
: known ? std::min<uint64_t>(allowed / 100 * (allowed <= (8704ull << 20) ? 45 : 70), 6144ull << 20)
|
|
: 4096ull << 20;
|
|
// (hookshot: at most 6 GB: a 16 GB card's 10.8 GB filled up, read as a leak: 105, 106. On a card of 8 GB or less, 45% of what the
|
|
// driver allows, not 70%: the game's other VRAM (render targets, the pools' mirrors and imports, buffers, pipelines) is ~3.5-3.8 GB
|
|
// in Hyrule Field, so 70% (5.1 GB of an 8 GB card's 7.3) put the whole past what the card holds. The driver paged VRAM to system
|
|
// memory and the GPU time went from ~13 to 55 ms a frame for good: an RTX 3060 Ti at 14 fps ten seconds into play, #1400's
|
|
// RTX 4050 at 5 fps. RX_VK_VRAM_MB=<MB>: the old size, or any)
|
|
{ // (hookshot) an integrated GPU's "VRAM" is the system's RAM, which the game's own ~10 GB also needs: 70% of what the driver allows
|
|
// was 5.3 GB of textures on a 16 GB laptop with a Radeon 780M, and the PC ran out of memory in Hyrule Field within minutes
|
|
// (Discord, 2026-11-09). At most 2.5 GB there unless RX_VK_VRAM_MB says otherwise
|
|
static bool told;
|
|
if (!getenv("RX_VK_VRAM_MB") && dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU && own > (2560ull << 20)) {
|
|
own = 2560ull << 20;
|
|
if (!told) told = true, logf("vk: an integrated GPU shares the system's RAM: textures at most 2560 MB (RX_VK_VRAM_MB=<MB>: other)");
|
|
}
|
|
}
|
|
// (a smaller preset on a bigger card: the card's own; RX_VK_VRAM_MB and an integrated GPU's cap: preset_texture_budget)
|
|
const bool vram_env = getenv("RX_VK_VRAM_MB") != nullptr;
|
|
const bool preset_budget = opts.vram_mb > 0 && !vram_env && preset_caps_memory();
|
|
uint64_t budget = preset_texture_budget((uint64_t)std::max(opts.vram_mb, 0), preset_budget, vram_env,
|
|
dev.props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, own);
|
|
if (preset_budget && known) {
|
|
// Safety clamp: do not exceed 75% of driver-allowed memory on constrained VRAM (prevents VK_ERROR_DEVICE_LOST)
|
|
budget = std::min(budget, (allowed / 4) * 3);
|
|
}
|
|
// (near what the driver allows, the budget shrinks by the excess. The excess of what the process holds less the
|
|
// free space in VMA's blocks, which new images take before VMA makes a block: measured by the blocks alone, the
|
|
// textures evicted freed space inside them that the driver still counted, so 10 frames later the budget shrank
|
|
// again. A race kept ~3 GB free in the blocks and evicted and made again ~100 textures every 10 frames (1300 at
|
|
// once), mostly the same ones: their uploads and decodes, and images made empty in the frame that drew them.
|
|
// The driver's own count near its limit (97%) still squeezes, whatever the blocks hold free: then it pages VRAM
|
|
// out. RX_VK_BUDGET_BY_BLOCKS=1: as before)
|
|
static const bool by_blocks = getenv("RX_VK_BUDGET_BY_BLOCKS") != nullptr;
|
|
uint64_t evict_to = 0; // (hookshot, RX_VK_COMPACT: what an eviction goes down to when the compaction's hold set the budget)
|
|
uint64_t hold_to = 0; // (and what the hold's own line asks for, when the reserve's lower line set evict_to)
|
|
// (hookshot, RX_VK_COMPACT) compacted, only the textures' pool's free space comes off the count: vram_compact.h
|
|
uint64_t reclaim = block_free;
|
|
if (compact_on()) {
|
|
uint64_t pool_blocks = 0, pool_free = 0;
|
|
if (textures_pool_usage(&pool_blocks, &pool_free)) reclaim = vram_compact::reclaimable(block_free, pool_free);
|
|
}
|
|
const uint64_t pressure = by_blocks ? used : used - std::min(used, reclaim);
|
|
pool_pressed = known && pressure > allowed / 10 * 9; // (the target pool given back meanwhile)
|
|
compact.pressed = known && vram_compact::pressed(used, allowed); // (hookshot, RX_VK_COMPACT: by the driver's own count)
|
|
if (known && pressure > allowed / 10 * 9) {
|
|
const uint64_t over = pressure - allowed / 10 * 9;
|
|
budget = std::min(budget, total_bytes > over ? total_bytes - over : 0);
|
|
}
|
|
// (hookshot, RX_VK_COMPACT: what the budget's size and the 90% rule ask on their own, before the hold's lines; the 97%
|
|
// rule's joins it below. The hold's pass lets a texture unused for the eviction's own time go for that part, as a pass
|
|
// does with the compaction off: vram_compact::hold_target)
|
|
uint64_t rule_budget = budget;
|
|
// (hookshot, RX_VK_COMPACT) and the count is held at RX_VK_COMPACT_PCT% of the allowance (85; 50 to 90), as it will be once a
|
|
// run has given the textures' free space back: within ~0.8 GB of what the driver allows the GPU's frame went from ~10 ms
|
|
// to 13-20 ms with the shadows in place, the driver itself short. At 85 a five-minute walk across Hyrule Field at DLAA
|
|
// 2560 x 1440 had 11-12 frames over 33 ms against 34-49 uncompacted, and a 1% low of 37 fps against 26-29
|
|
if (compact_on() && known && !by_blocks) {
|
|
static const uint64_t pct = getenv("RX_VK_COMPACT_PCT") ? (uint64_t)std::clamp(atoi(getenv("RX_VK_COMPACT_PCT")), 50, 90) : 85;
|
|
const uint64_t held = vram_compact::held_budget(budget, total_bytes, used, reclaim, allowed, pct);
|
|
// (and what goes is the excess alone. Over its budget the cache is cut to 7/8 of it, below: right for a budget that
|
|
// is a size, but this one is "what the textures hold less the excess", crossed again whenever anything else grows.
|
|
// A recorder started beside the game took 0.8 GB of the allowance as play came back from 14 s in the menu: the line
|
|
// was crossed on five passes in a row and the cache went from 4,589 to 2,312 MB (0.875^5) in two seconds, the
|
|
// textures made again at 65-90 ms a frame)
|
|
if (held < budget) budget = evict_to = held;
|
|
hold_to = evict_to;
|
|
// (and a reserve under that line, for the textures long unused: vram_compact.h. The pass below takes the others only
|
|
// for the hold's own line)
|
|
const uint64_t reserve = vram_compact::held_budget(budget, total_bytes, used, reclaim, allowed, vram_compact::reserve_pct(pct));
|
|
if (reserve < budget) budget = evict_to = reserve;
|
|
}
|
|
// (hookshot) not while VMA's blocks hold an eighth of the allowance free: new images take that space, the driver's count does
|
|
// not grow, and what this evicts frees space inside the blocks that the count keeps. Ten frames later it squeezed again: a
|
|
// ratchet. On a 4 GB RX 5500M (3337 MB used of 3337, 1224 MB free in the blocks) a paused game, whose scene draws nothing,
|
|
// lost 1700 MB of textures in seconds, made again on unpausing (the long wait), and the HUD's drawn textures went with them.
|
|
// RX_VK_SQUEEZE_ALWAYS=1: as before
|
|
static const bool squeeze_always = getenv("RX_VK_SQUEEZE_ALWAYS") != nullptr;
|
|
const bool room_in_blocks = !squeeze_always && block_free >= allowed / 8;
|
|
if (known && !by_blocks && used > allowed / 100 * 97 && !room_in_blocks) {
|
|
const uint64_t over = used - allowed / 100 * 97;
|
|
budget = std::min(budget, total_bytes > over ? total_bytes - over : 0);
|
|
rule_budget = std::min(rule_budget, total_bytes > over ? total_bytes - over : 0);
|
|
}
|
|
const uint64_t rule_to = vram_compact::rule_target(total_bytes, rule_budget);
|
|
// (hookshot, RX_VK_COMPACT: the driver's limit asks for more than the reserve does: a pass over the budget, to 7/8 of it, as
|
|
// before. Rare once the reserve is set: its line is 17 points under the limit's and counts the textures' free space out, so
|
|
// this takes more than that share of the allowance free in their pool. Otherwise the pass is the hold's, and what the
|
|
// limit and the 90% rule ask for goes by rule_to)
|
|
if (budget < evict_to) evict_to = hold_to = 0;
|
|
if (frame_no % 600 == 0) {
|
|
memory_trial_tick(); // (Triforce: the memory handling on trial on this card, vk_memory.cpp)
|
|
vma_log_heaps();
|
|
vma_log_image_blocks();
|
|
{
|
|
std::lock_guard lk(pool_mu);
|
|
logf("vk: render target images: %llu reused from the pool, %llu made new, %llu destroyed from it; it holds %llu "
|
|
"(%llu MB)%s", (unsigned long long)st_pool_reused, (unsigned long long)st_pool_made,
|
|
(unsigned long long)st_pool_destroyed, (unsigned long long)pool_n, (unsigned long long)(pool_bytes >> 20),
|
|
pool_cap() ? (pool_pressed ? ", VRAM short: given back" : "") : " (RX_VK_TARGET_POOL_MB=0)");
|
|
}
|
|
logf("vk: VRAM: %llu MB used of %llu MB allowed (%llu MB of it free in the blocks); textures %llu MB (budget %llu "
|
|
"MB), %zu textures%s", (unsigned long long)(used >> 20), (unsigned long long)(allowed >> 20),
|
|
(unsigned long long)(block_free >> 20), (unsigned long long)(total_bytes >> 20),
|
|
(unsigned long long)(budget >> 20), textures.size(),
|
|
known && used > allowed / 100 * 97 && room_in_blocks ? " (at the driver's limit, room in the blocks: not squeezed)" : "");
|
|
// (what takes it: decoded ASTC, the GPU's images, the rest; the largest)
|
|
uint64_t astc_b = 0, rt_b = 0, other_b = 0, recent_b = 0, baked_b = 0;
|
|
size_t baked_n = 0;
|
|
std::vector<const Texture*> big, baked;
|
|
size_t bucket_n[4]{}, bucket_rel[4]{};
|
|
uint64_t bucket_b[4]{};
|
|
for (auto& kv : textures) {
|
|
const Texture* t = kv.second.get();
|
|
(t->astc ? astc_b : t->gpu_written ? rt_b : other_b) += t->bytes;
|
|
if (frame_no - t->last_frame < 8) recent_b += t->bytes;
|
|
if (t->gpu_written && t->first_written && t->first_written == t->last_written && t->bytes <= (4ull << 20)) {
|
|
if (!t->released) baked_n++, baked_b += t->bytes;
|
|
const int k = t->bytes <= (64u << 10) ? 0 : t->bytes <= (256u << 10) ? 1 : t->bytes <= (1u << 20) ? 2 : 3;
|
|
bucket_n[k]++, bucket_b[k] += t->bytes, bucket_rel[k] += t->released;
|
|
baked.push_back(t);
|
|
}
|
|
big.push_back(t);
|
|
}
|
|
if (getenv("RX_VK_LOG_BAKED")) { // (what the textures drawn in one frame are: by size, finalized or not)
|
|
logf("vk: drawn in one frame: <=64 KB %zu (%llu KB, %zu finalized), <=256 KB %zu (%llu KB, %zu), <=1 MB %zu "
|
|
"(%llu KB, %zu), <=4 MB %zu (%llu KB, %zu)", bucket_n[0], (unsigned long long)(bucket_b[0] >> 10),
|
|
bucket_rel[0], bucket_n[1], (unsigned long long)(bucket_b[1] >> 10), bucket_rel[1], bucket_n[2],
|
|
(unsigned long long)(bucket_b[2] >> 10), bucket_rel[2], bucket_n[3], (unsigned long long)(bucket_b[3] >> 10),
|
|
bucket_rel[3]);
|
|
std::sort(baked.begin(), baked.end(), [](const Texture* a, const Texture* b) { return a->bytes > b->bytes; });
|
|
for (size_t i = 0; i < baked.size() && i < 10; i++) {
|
|
const Texture* t = baked[i];
|
|
logf("vk: one frame: #%u %#llx %ux%ux%u x%u levels %u NVN %#x %llu KB, written at %llu, made at %llu, last "
|
|
"used %llu%s%s", t->id, (unsigned long long)t->gpu, t->width, t->height, t->depth, t->layers, t->levels,
|
|
t->nvn_format, (unsigned long long)(t->bytes >> 10), (unsigned long long)t->first_written,
|
|
(unsigned long long)t->first_seen, (unsigned long long)t->last_frame, t->released ? " finalized" : "",
|
|
t->first_render_frame ? " rendered" : " copied or stored");
|
|
}
|
|
}
|
|
std::sort(big.begin(), big.end(), [](const Texture* a, const Texture* b) { return a->bytes > b->bytes; });
|
|
logf("vk: textures: decoded ASTC %llu MB, rendered %llu MB (%zu drawn in one frame, kept: %llu KB), other %llu MB; "
|
|
"used in the last 8 presents %llu MB", (unsigned long long)(astc_b >> 20), (unsigned long long)(rt_b >> 20),
|
|
baked_n, (unsigned long long)(baked_b >> 10), (unsigned long long)(other_b >> 20),
|
|
(unsigned long long)(recent_b >> 20));
|
|
for (size_t i = 0; i < big.size() && i < 8; i++)
|
|
logf("vk: %llu MB: %#llx %ux%ux%u x%u levels %u NVN %#x%s%s, last used %llu presents ago",
|
|
(unsigned long long)(big[i]->bytes >> 20), (unsigned long long)big[i]->gpu, big[i]->width, big[i]->height,
|
|
big[i]->depth, big[i]->layers, big[i]->levels, big[i]->nvn_format, big[i]->astc ? " ASTC" : "",
|
|
big[i]->gpu_written ? " rendered" : "", (unsigned long long)(frame_no - big[i]->last_frame));
|
|
}
|
|
if (verify_dim().first && frame_no % 300 == 0) {
|
|
int checked = 0, changed = 0, missed = 0;
|
|
for (auto& kv : textures) {
|
|
Texture* t = kv.second.get();
|
|
if (t->width != verify_dim().first || t->height != verify_dim().second || !t->verify_hash || t->gpu_written) continue;
|
|
checked++;
|
|
const uint64_t h = guest_hash(*t);
|
|
if (h == t->verify_hash) continue;
|
|
changed++;
|
|
static thread_local Ranges d;
|
|
if (!texture_dirty(*t, t->upload_gen, d)) {
|
|
missed++;
|
|
static int logged;
|
|
if (logged++ < 40) {
|
|
logf("vk: MISSED WRITE: texture #%u %#llx %ux%u NVN %#x: its memory changed since its upload, the write "
|
|
"tracking saw nothing", t->id, (unsigned long long)t->gpu, t->width, t->height, t->nvn_format);
|
|
verify_report(*t);
|
|
}
|
|
}
|
|
}
|
|
logf("vk: verify %ux%u: %d textures, %d changed since uploaded, %d of them unnoticed", verify_dim().first,
|
|
verify_dim().second, checked, changed, missed);
|
|
}
|
|
texture_sweep(completed, total_bytes <= budget, budget, evict_to, known, used, allowed, false, hold_to, rule_to);
|
|
}
|
|
|
|
/// (texture_frame) What goes: over the budget the least recently used, and the render targets drawn over since.
|
|
/// (Triforce, 11-11) dead_only: a pass between texture_frame's looks (every 10 presents) for the dead transient targets alone.
|
|
/// While the camera moves the game makes them at new sizes all the time: 1.2-3 new images a present running across Hyrule
|
|
/// Field in the lab, up to 11 a present between areas in a player's session (6733 in 600 presents). A dead one waited for the
|
|
/// next look, 10 presents plus the 3 it must go unused: at ~20 MB each that is ~650 MB of dead images in the lab's run and
|
|
/// 2.6 GB in that session, on top of what is drawn, their pool's blocks 18 -> 31, and at the driver's limit each new block
|
|
/// took 110-170 ms (ten frames of 115-180 ms in a row). So when a pass finds them coming at one a present and more, the next
|
|
/// pass is the next present, at one in three presents three presents on, else the next look as before: the dead wait 4-6
|
|
/// presents, not 13. RX_VK_DEAD_QUICK=0: only at the looks, as before
|
|
void texture_sweep(uint64_t completed, bool sweep, uint64_t budget, uint64_t evict_to, bool known, uint64_t used, uint64_t allowed,
|
|
bool dead_only, uint64_t hold_to, uint64_t rule_to) {
|
|
static const bool no_evict = getenv("RX_VK_NO_EVICT") != nullptr; // (an experiment: nothing evicted)
|
|
// (hookshot) under the budget, every 600 presents: the render targets drawn over since (below), unused for 120
|
|
// (hookshot-amd) and every 60 presents for the render targets drawn over by one at another start (below): OoT makes a frame's
|
|
// transient targets (768 x 432 RGBA16F, 1536 x 864) in a scratch heap whose offsets shift with what came before them in the
|
|
// frame, so turning the camera made new ones each frame ~1.2 MB from the last, each a new image. Left for the 600-present
|
|
// sweep or the budget, ~200 dead ones (1.2 GB) filled an 8 GB card's VRAM in Hyrule Field, the driver paged it out and the
|
|
// GPU's frame went from 15 to 35 ms. RX_VK_DEAD_TARGETS=0: as before
|
|
static const bool dead_targets = !getenv("RX_VK_DEAD_TARGETS") || atoi(getenv("RX_VK_DEAD_TARGETS")) != 0;
|
|
// (hookshot) every 10 presents (RX_VK_DEAD_EVERY; 60 before): the target pool gets the dead ones as the game makes new ones,
|
|
// not 300-700 at once every second, of which the new ones of the next second missed most. The choice is cheap: 1930 textures
|
|
// ordered in 0.3 ms, 251 chosen in 0.2 (RTX 5070, Ryzen 7 9800X3D)
|
|
static const uint64_t dead_every =
|
|
getenv("RX_VK_DEAD_EVERY") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_DEAD_EVERY"))) : 10;
|
|
if (no_evict || (sweep && !dead_only && frame_no % (dead_targets ? dead_every : 600))) return;
|
|
static const bool dead_quick = !(getenv("RX_VK_DEAD_QUICK") && !atoi(getenv("RX_VK_DEAD_QUICK")));
|
|
static uint64_t dead_pass_last, quick_passes, quick_ns, quick_logged_at;
|
|
const auto pass_t0 = std::chrono::steady_clock::now();
|
|
const uint64_t pass_presents = std::max<uint64_t>(1, frame_no - dead_pass_last);
|
|
dead_pass_last = frame_no;
|
|
dead_sweep_next = 0; // (this pass is the one asked for; the next: below, or a new image's, texture_frame)
|
|
size_t shifted_now = 0;
|
|
std::vector<std::pair<uint64_t, Texture*>> order;
|
|
order.reserve(textures.size());
|
|
for (auto& kv : textures) order.push_back({kv.second->last_frame, kv.second.get()});
|
|
std::sort(order.begin(), order.end(), [](const auto& a, const auto& b) { return a.first < b.first; });
|
|
// (the rendered textures by where their storage starts, for drawn_over_shifted)
|
|
std::vector<const Texture*> by_start;
|
|
if (dead_targets) {
|
|
for (auto& [last, tex] : order)
|
|
if (tex->gpu_written && tex->first_render_frame && tex->guest_size) by_start.push_back(tex);
|
|
std::sort(by_start.begin(), by_start.end(), [](const Texture* a, const Texture* b) { return a->gpu < b->gpu; });
|
|
}
|
|
uint64_t longest = 0;
|
|
for (const Texture* t : by_start) longest = std::max(longest, t->guest_size);
|
|
std::vector<Texture*> gone;
|
|
// (over the budget, what goes has gone unused for RX_VK_EVICT_MIN_UNUSED frames, 60 by default; it was 8. On an
|
|
// 8 GB laptop card the VRAM sits at the driver's limit, the budget shrinks and this runs every 10 frames: with 8,
|
|
// ~300 textures in a few minutes of Kokiri Forest were evicted and made again, all of them within 60 frames (200
|
|
// within 10), uploaded and decoded again each time; and a 260 x 260 one the GPU renders and the game samples every
|
|
// 40 frames came back empty each time (22 times; the lit ground black for a frame, a tiled pattern over the ground).
|
|
// With 60: 2. The textures that are really done with still go, 52 frames later)
|
|
static const uint64_t min_unused =
|
|
getenv("RX_VK_EVICT_MIN_UNUSED") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_EVICT_MIN_UNUSED"))) : 60;
|
|
// (hookshot) a render target another image has since been drawn over (same storage, another size or format:
|
|
// OoT's dynamic resolution makes a new set at each size in the same pool memory) holds nothing any more: it
|
|
// goes like any other. Kept, they were 3.6 GB of 4.5 GB rendered, VRAM near the driver's limit, and the
|
|
// driver's paging made submissions of seconds (5-9 fps for a minute)
|
|
const auto drawn_over = [&](const Texture* tex) {
|
|
if (!tex->gpu_written) return false;
|
|
auto [a, e] = by_gpu.equal_range(tex->gpu);
|
|
for (auto o = a; o != e; ++o)
|
|
if (o->second != tex && o->second->last_written > tex->last_frame) return true;
|
|
return false;
|
|
};
|
|
// (hookshot-amd) a render target of one level and layer whose storage another render target drew over from another start
|
|
// after its last use (target_overlap.h): the transient targets above. Its texels are gone from that memory on the console too
|
|
const auto drawn_over_shifted = [&](const Texture* tex) {
|
|
if (!dead_targets || !tex->gpu_written || !tex->first_render_frame || !tex->guest_size || tex->levels != 1 ||
|
|
tex->layers != 1 || tex->depth > 1)
|
|
return false;
|
|
const uint64_t lo = tex->gpu, hi = tex->gpu + tex->guest_size;
|
|
auto it = std::lower_bound(by_start.begin(), by_start.end(), lo > longest ? lo - longest : 0,
|
|
[](const Texture* t, uint64_t g) { return t->gpu < g; });
|
|
for (; it != by_start.end() && (*it)->gpu < hi; ++it)
|
|
if (*it != tex && target_overlap::shifted_over(lo, hi, tex->last_frame, (*it)->gpu, (*it)->gpu + (*it)->guest_size,
|
|
(*it)->last_written))
|
|
return true;
|
|
return false;
|
|
};
|
|
// (the transient targets: a frame's own, not sampled again, their storage drawn over by another since. 16 frames left
|
|
// ~500 MB of them in Hyrule Castle Town's market at 20 fps (59 768 x 432 RGBA16F, 135 384 x 384, 41 1536 x 864 at a
|
|
// time), VRAM at the driver's limit; the GPU's own use is checked below (last_value). RX_VK_DEAD_MIN_UNUSED=<frames>, 3)
|
|
static const uint64_t dead_min_unused =
|
|
getenv("RX_VK_DEAD_MIN_UNUSED") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_DEAD_MIN_UNUSED"))) : 3;
|
|
// (hookshot) a small texture the GPU drew in one frame that the game has finalized, with nothing drawn over its storage
|
|
// since: the game may make a texture over that storage again and sample it without drawing it. OoT's terrain does that with its
|
|
// tiles' 260 x 260 RG8 normal maps: drawn when a chunk comes in, finalized when it goes out of range, made again and sampled as
|
|
// they are when it comes back. On the console the texels are still in that memory; here they were only in the image, and one
|
|
// evicted came back empty (made again over storage the GPU rendered, not uploaded): streaks over the ground in square patches
|
|
// after a while of play (issues 107 and 386). Kept, up to RX_VK_KEEP_FINALIZED_MB (512 by default; 0:
|
|
// evicted as before), the least recently used beyond that going as before. (Not written back into the memory instead: the game
|
|
// may have put other things there since, and a write-back over scene data once crashed it)
|
|
static const uint64_t keep_finalized_max =
|
|
(uint64_t)(getenv("RX_VK_KEEP_FINALIZED_MB") ? std::max(0, atoi(getenv("RX_VK_KEEP_FINALIZED_MB"))) : 512) << 20;
|
|
const auto keep_finalized = [&](const Texture* tex) {
|
|
if (!keep_finalized_max || !tex->gpu_written || !tex->released || !tex->first_written ||
|
|
tex->first_written != tex->last_written || tex->bytes > (4ull << 20))
|
|
return false;
|
|
const auto g = gpu_owned.find(tex->gpu); // (its storage's range still the one the GPU drew it in)
|
|
if (g == gpu_owned.end() || g->second.first != tex->gpu + std::max<uint64_t>(tex->guest_size, 1)) return false;
|
|
return !drawn_over(tex) && !drawn_over_shifted(tex);
|
|
};
|
|
uint64_t finalized_kept = 0;
|
|
size_t finalized_kept_n = 0;
|
|
if (keep_finalized_max)
|
|
for (auto& [last, tex] : order)
|
|
if (keep_finalized(tex)) finalized_kept += tex->bytes, finalized_kept_n++;
|
|
if (finalized_kept && frame_no % 600 == 0) {
|
|
std::map<std::tuple<uint32_t, uint32_t, uint32_t, uint32_t>, std::pair<size_t, uint64_t>> kinds; // (w, h, layers, NVN format)
|
|
for (auto& [last, tex] : order)
|
|
if (keep_finalized(tex)) {
|
|
auto& k = kinds[{tex->width, tex->height, tex->layers, tex->nvn_format}];
|
|
k.first++, k.second += tex->bytes;
|
|
}
|
|
char line[512];
|
|
int n = 0;
|
|
for (const auto& [k, v] : kinds)
|
|
if (n < 440)
|
|
n += snprintf(line + n, sizeof line - n, " %ux%ux%u NVN %#x: %zu (%llu KB);", std::get<0>(k), std::get<1>(k),
|
|
std::get<2>(k), std::get<3>(k), v.first, (unsigned long long)(v.second >> 10));
|
|
logf("vk: finalized textures the GPU drew once, kept for being made again: %zu (%llu KB):%s", finalized_kept_n,
|
|
(unsigned long long)(finalized_kept >> 10), n ? line : "");
|
|
}
|
|
const uint64_t need_unused = sweep ? std::max<uint64_t>(120, min_unused) : min_unused;
|
|
const uint64_t need_dead = dead_targets ? std::min(need_unused, dead_min_unused) : need_unused;
|
|
static uint64_t n_dead, b_dead, dead_logged_at;
|
|
// (hookshot) the textures to go are chosen first and destroyed after; the write-backs among them go to the GPU in a few
|
|
// batches (flush_textures), not one submission and one wait each: a sweep of 500 at a scene load stalled for seconds
|
|
struct Evict { Texture* tex; bool flush, shifted; };
|
|
std::vector<Evict> evict;
|
|
uint64_t pending_bytes = 0; // (chosen below, their bytes not yet off total_bytes)
|
|
// (hookshot, RX_VK_COMPACT) a pass the compaction's hold asked for (evict_to) lets go of the dead render targets as a sweep
|
|
// does, all of them, and of live textures only for the excess left after those, and only of ones long unused: vram_compact.h's
|
|
// hold_met and hold_target. So it does not stop at the first texture past the excess: what the dead targets hold is counted first
|
|
const bool by_hold = evict_to != 0 && !sweep;
|
|
uint64_t hold_dead = 0, hold_live = 0; // (what the dead targets hold; the live textures chosen so far)
|
|
if (by_hold)
|
|
for (auto& [last, tex] : order) {
|
|
if (frame_no - last < need_dead) break;
|
|
if (tex->last_value > completed) continue;
|
|
const bool shifted = drawn_over_shifted(tex);
|
|
if (frame_no - last < need_unused && !shifted) continue;
|
|
if (shifted || drawn_over(tex)) hold_dead += tex->bytes + tex->pad;
|
|
}
|
|
// (Triforce, 11-12) What the game has let go goes first while VRAM is short. At an area's arrival the textures of the area
|
|
// left stayed until a rule asked for room, which was after the new area's had come in: those took new blocks, the old ones
|
|
// then left holes in theirs, and for seconds the driver counted both. A lab arrival in Hyrule Field from Kakariko: VRAM
|
|
// 10,993 to 11,249 MB of 10,468 to 10,663 allowed with 1.3 GB free inside the blocks, and each image made meanwhile took
|
|
// 93 to 124 ms (three frames of 97 to 134 ms). A texture uploaded from memory whose texture the game finalized
|
|
// RX_VK_LET_GO_AFTER presents ago (30) and that nothing has used since goes at the next pass while VRAM is over
|
|
// RX_VK_LET_GO_PCT% of what the driver allows (75; 0: off, as before), whatever its age: its room is there when the next
|
|
// area's textures come. One the game makes again is uploaded from memory again, as after any eviction.
|
|
// Only when the game has let go of much at once, RX_VK_LET_GO_MB and more (256: an area left behind): at the Market the
|
|
// game finalizes and makes again a few backdrop textures all the time, and with every one of those going at once the
|
|
// uploads there went from 1.4-1.7 GB to 3.6 GB in 95 seconds. (Not "or whenever VRAM is over 92%" either: a card that
|
|
// sits there for good, 5.6 GB allowed, had more frames over 40 ms with that than without, 559 against 369 in a tour.)
|
|
// And only where the driver allows 9 GB and more, unless RX_VK_LET_GO_PCT is set: at 5.6 GB allowed three tours with this
|
|
// and the blocks made ahead had 559-896 frames over 40 ms against 369-614 without (vk_vma.cpp)
|
|
static const bool let_go_set = getenv("RX_VK_LET_GO_PCT") != nullptr;
|
|
static const uint64_t let_go_pct = let_go_set ? (uint64_t)std::clamp(atoi(getenv("RX_VK_LET_GO_PCT")), 0, 100) : 75;
|
|
static const uint64_t let_go_after = getenv("RX_VK_LET_GO_AFTER") ? (uint64_t)std::max(1, atoi(getenv("RX_VK_LET_GO_AFTER"))) : 30;
|
|
static const uint64_t let_go_mass_min = (uint64_t)(getenv("RX_VK_LET_GO_MB") ? std::max(0, atoi(getenv("RX_VK_LET_GO_MB"))) : 256) << 20;
|
|
bool let_go_short = let_go_pct && known && used > allowed / 100 * let_go_pct && (let_go_set || allowed >= (9ull << 30));
|
|
if (let_go_short) {
|
|
uint64_t mass = 0;
|
|
for (auto& [last, tex] : order)
|
|
if (tex->let_go && !tex->gpu_written && frame_no - tex->let_go >= let_go_after) mass += tex->bytes + tex->pad;
|
|
let_go_short = mass >= let_go_mass_min;
|
|
}
|
|
static uint64_t let_go_n, let_go_bytes, let_go_logged_at;
|
|
for (auto& [last, tex] : order) {
|
|
// (total_bytes can have drifted below the textures' sum (destroy_later clamps it): the chosen ones' bytes past it mean the
|
|
// budget is met, not a wrapped difference that never is)
|
|
if ((!sweep && !by_hold && (pending_bytes >= total_bytes || total_bytes - pending_bytes <= budget / 8 * 7)) ||
|
|
frame_no - last < need_dead)
|
|
break;
|
|
const bool shifted = drawn_over_shifted(tex);
|
|
const bool gone_early = let_go_short && tex->let_go && !tex->gpu_written &&
|
|
frame_no - tex->let_go >= let_go_after;
|
|
if (frame_no - last < need_unused && !shifted && !gone_early) continue; // (younger: only a dead transient target goes)
|
|
if (tex->last_value > completed) continue;
|
|
// (nor what the GPU wrote and the game still has: the image is its only copy, and one made again over that
|
|
// storage started empty or from stale memory: a sky or course texture drawn once, garbage from then on.
|
|
// RX_VK_EVICT_RENDERED=1: as before)
|
|
static const bool evict_rendered = getenv("RX_VK_EVICT_RENDERED") != nullptr;
|
|
// (one unused for 3600 presents (a minute) goes: a course left behind; the game draws it again when it comes
|
|
// back to that course. Without it a course change kept 1.4 GB arrays and VRAM ran over)
|
|
// (600 presents near what the driver allows, at the budget's 97%: past it the driver pages VRAM to system memory.
|
|
// AMD's driver reports the use at most at what it allows, so "over it" never held there: keep_rendered.h.
|
|
// RX_VK_KEEP_RENDERED_OVER=1: only past what it allows, as before)
|
|
static const bool keep_over_only = getenv("RX_VK_KEEP_RENDERED_OVER") != nullptr;
|
|
const uint64_t keep_rendered = keep_rendered::presents(known, used, allowed, keep_over_only);
|
|
const bool overwritten = shifted || drawn_over(tex); // (above)
|
|
// (hookshot, RX_VK_COMPACT: the hold's pass, above. A live texture stays once the excess over its line is covered: the
|
|
// reserve's line for one long unused, the hold's own for one unused ten seconds, and what the other rules ask (the 90%
|
|
// rule, the budget's size, the driver's limit) for one unused as long as an eviction asks with the compaction off)
|
|
if (by_hold && !overwritten && !gone_early) {
|
|
const uint64_t to = vram_compact::hold_target(frame_no - last, need_unused, rule_to, hold_to, evict_to);
|
|
if (to == UINT64_MAX || vram_compact::hold_met(total_bytes, hold_live, hold_dead, to)) continue;
|
|
}
|
|
if (finalized_kept && keep_finalized(tex)) { // (above. Oldest first: beyond the cap the least recently used go)
|
|
if (finalized_kept <= keep_finalized_max) continue;
|
|
finalized_kept -= tex->bytes;
|
|
static int logged;
|
|
if (logged++ < 100)
|
|
logf("vk: kept finalized textures over %llu MB: #%u %#llx %ux%u NVN %#x goes (unused for %llu frames)",
|
|
(unsigned long long)(keep_finalized_max >> 20), tex->id, (unsigned long long)tex->gpu, tex->width,
|
|
tex->height, tex->nvn_format, (unsigned long long)(frame_no - last));
|
|
}
|
|
if (!overwritten && tex->gpu_written && !tex->released && !evict_rendered && frame_no - last < keep_rendered) continue;
|
|
if (sweep && !overwritten && !gone_early) continue;
|
|
if (gone_early) let_go_n++, let_go_bytes += tex->bytes + tex->pad;
|
|
// (hookshot) nor a live texture the GPU drew whose texels cannot be written back into its memory (ASTC kept as BC3, sparse,
|
|
// past the write-back's size, ...): its image is the only copy, and one made again over that storage is not uploaded (the
|
|
// memory counts as the GPU's), so it showed whatever its new memory held: the rainbow/garbled textures that spread over play
|
|
// time on 4-8 GB cards (issue 993)? Up to 256 MB of such textures each stay; bigger ones (a whole area left behind) go as
|
|
// before. On by default since 2026-11-09 (8 GB RTX 3070s still showed rainbow textures once VRAM filled, Discord): the VRAM it
|
|
// costs is bounded (256 MB a texture) and a wrong texture is worse than a fuller budget. RX_VK_EVICT_UNFLUSHABLE=1: evicted, as before
|
|
static const bool evict_unflushable = getenv("RX_VK_EVICT_UNFLUSHABLE") && atoi(getenv("RX_VK_EVICT_UNFLUSHABLE")) != 0;
|
|
if (!overwritten && tex->gpu_written && !tex->released && !evict_unflushable && tex->bytes < (256ull << 20) && !flushable(*tex)) {
|
|
static std::atomic<int> kept_logged{0};
|
|
if (kept_logged.fetch_add(1) < 40)
|
|
logf("vk: texture #%u %#llx %ux%u x%u NVN %#x the GPU drew kept, not evicted: its texels cannot be written back",
|
|
tex->id, (unsigned long long)tex->gpu, tex->width, tex->height, tex->layers, tex->nvn_format);
|
|
continue;
|
|
}
|
|
// (and a small one never: its texels exist nowhere else, and it is made once and kept by the game. The sky's
|
|
// 256 x 256 lookup table, unused while a menu was up, went and came back from memory as garbage: the rainbow
|
|
// sky. The big ones that may go are a course left behind)
|
|
// (a render target, drawn in the frames it was used in, goes: the dynamic resolution makes a new set of ~20 at
|
|
// each size, and the old ones are not used again)
|
|
const bool drawn_once = tex->last_written + 8 < tex->last_frame;
|
|
if (!overwritten && tex->gpu_written && !tex->released && !evict_rendered && tex->bytes < (64ull << 20) && drawn_once) continue;
|
|
// (nor a small one the GPU wrote in one frame alone: a table the game bakes once and samples now and then. The
|
|
// pause screen's colour grading, 8 x 8 x 8, its slices drawn at boot and sampled next when the game is paused,
|
|
// went after a minute unused, and the one made again over storage only the GPU had written stayed empty: the
|
|
// pause blur black. Not a finalized one: the transient targets the game makes and finalizes every frame are
|
|
// each "written in one frame", and kept so they added up, ~20 MB a minute)
|
|
const bool baked = tex->first_written && tex->first_written == tex->last_written && tex->bytes <= (4ull << 20);
|
|
if (!overwritten && tex->gpu_written && !tex->released && !evict_rendered && baked) continue;
|
|
// (a texture the GPU drew that still holds the game's content goes into the game's memory first: made again later, it is
|
|
// uploaded from there with what the GPU drew. Evicted without it, it came back empty or stale: a terrain chunk dark, a
|
|
// tiled pattern over the Deku Tree's planks and Hyrule Field's path, with a 16 GB card's textures at the 6 GB budget
|
|
// (issue 386). RX_VK_EVICT_NO_FLUSH=1: not written back, as before)
|
|
static const bool no_flush = getenv("RX_VK_EVICT_NO_FLUSH") != nullptr;
|
|
evict.push_back({tex, !no_flush && !overwritten && tex->gpu_written && !tex->released && flushable(*tex), shifted});
|
|
pending_bytes += tex->bytes + tex->pad;
|
|
if (by_hold && !overwritten) hold_live += tex->bytes + tex->pad;
|
|
}
|
|
{ // (the write-backs: batches of up to 64 textures or ~512 MB of staging memory, each one submission and one wait)
|
|
std::vector<Flush> batch;
|
|
uint64_t batch_bytes = 0;
|
|
const auto run = [&] {
|
|
if (batch.empty()) return;
|
|
flush_textures(batch);
|
|
for (const Flush& f : batch)
|
|
if (f.ok) {
|
|
static std::atomic<int64_t> flushed{0};
|
|
if (++flushed % 50 == 1) logf("vk: textures the GPU drew, written back into their memory before eviction: %lld so far", (long long)flushed.load());
|
|
}
|
|
batch.clear();
|
|
batch_bytes = 0;
|
|
};
|
|
for (const Evict& e : evict) {
|
|
if (!e.flush) continue;
|
|
batch.push_back(Flush{e.tex, e.tex->gpu, e.tex->gpu + e.tex->guest_size});
|
|
batch_bytes += e.tex->guest_size; // (what its staging buffer holds: the texels as the storage lays them out)
|
|
if (batch.size() >= 64 || batch_bytes >= (512ull << 20)) run();
|
|
}
|
|
run();
|
|
}
|
|
// (Triforce, texture detail: the ones a shader reads by texel position go now, whatever their age, and their keys are
|
|
// made whole from here on: the next draw that binds one makes it again from the game's memory, every level. A pointer
|
|
// is only compared here: its texture may have gone since it was asked)
|
|
if (!whole_pending.empty()) {
|
|
static uint64_t n_whole, logged_at;
|
|
for (Texture* want : whole_pending)
|
|
for (auto& kv : textures)
|
|
if (kv.second.get() == want && want->detail_skip && want->whole_asked) {
|
|
whole_keys.insert(kv.first);
|
|
if (std::none_of(evict.begin(), evict.end(), [&](const Evict& e) { return e.tex == want; }))
|
|
evict.push_back({want, false, false});
|
|
n_whole++;
|
|
break;
|
|
}
|
|
whole_pending.clear();
|
|
if (n_whole >= logged_at * 2 + 1) {
|
|
logged_at = n_whole;
|
|
logf("vk: texture detail: %llu textures kept whole so far: a shader reads them by texel position or asks "
|
|
"their size", (unsigned long long)n_whole);
|
|
}
|
|
}
|
|
for (const Evict& e : evict) { // (and each one's bookkeeping and its end, as before)
|
|
Texture* tex = e.tex;
|
|
if (e.shifted) n_dead++, b_dead += tex->bytes, shifted_now++;
|
|
watch_tex(tex, "evicted");
|
|
if (!tex->gpu_written && tex->uploaded && !remade_skip_old()) { // (above: made again, it is uploaded from memory)
|
|
std::lock_guard lk(from_memory_mu);
|
|
if (evicted_from_memory.size() > 50000) evicted_from_memory.clear(); // (a bound)
|
|
evicted_from_memory[evicted_key(tex->gpu, tex->width, tex->height, tex->nvn_format)] = tex->guest_size;
|
|
}
|
|
if (remade_log_on()) {
|
|
std::lock_guard lk(evicted_mu);
|
|
if (evicted_at.size() > 20000) evicted_at.clear(); // (a bound: only the recent ones matter)
|
|
evicted_at[evicted_key(tex->gpu, tex->width, tex->height, tex->nvn_format)] = {frame_no, tex->gpu_written};
|
|
remade.erase(tex);
|
|
}
|
|
destroy_later(*tex);
|
|
forget_gpu(tex);
|
|
gone.push_back(tex);
|
|
}
|
|
for (auto it = textures.begin(); it != textures.end() && !gone.empty();) {
|
|
auto g = std::find(gone.begin(), gone.end(), it->second.get());
|
|
if (g != gone.end()) {
|
|
gone.erase(g);
|
|
it = textures.erase(it);
|
|
} else {
|
|
++it;
|
|
}
|
|
}
|
|
if (n_dead && frame_no >= dead_logged_at + 600) { // (the dead transient targets let go so far, every 600 presents)
|
|
dead_logged_at = frame_no;
|
|
logf("vk: render targets drawn over from another start, let go: %llu so far (%llu MB)", (unsigned long long)n_dead,
|
|
(unsigned long long)(b_dead >> 20));
|
|
}
|
|
const size_t n = order.size() - textures.size();
|
|
if (n && !dead_only) logf("vk: %zu textures retired (%zu left, %llu MB)", n, textures.size(), (unsigned long long)(total_bytes >> 20));
|
|
if (let_go_n / 256 != let_go_logged_at) {
|
|
let_go_logged_at = let_go_n / 256;
|
|
logf("vk: textures the game had let go, gone before their age while VRAM is over %llu%% of what the driver allows: %llu so "
|
|
"far (%llu MB; RX_VK_LET_GO_PCT=0: off)", (unsigned long long)let_go_pct, (unsigned long long)let_go_n,
|
|
(unsigned long long)(let_go_bytes >> 20));
|
|
}
|
|
// (the next pass for the dead targets: by how fast this one found them coming)
|
|
if (dead_quick && dead_targets) {
|
|
if (shifted_now >= pass_presents) dead_sweep_next = frame_no + 1;
|
|
else if (shifted_now * 3 >= pass_presents) dead_sweep_next = frame_no + 3;
|
|
}
|
|
if (dead_only) {
|
|
quick_passes++;
|
|
quick_ns += (uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - pass_t0).count();
|
|
}
|
|
if (quick_passes && frame_no >= quick_logged_at + 600) {
|
|
quick_logged_at = frame_no;
|
|
logf("vk: passes for the dead render targets between the looks: %llu so far, %.2f ms each", (unsigned long long)quick_passes,
|
|
quick_ns / 1e6 / (double)quick_passes);
|
|
}
|
|
}
|
|
|
|
void texture_log_guest(const Texture* t) {
|
|
if (!t || !t->guest_size) return;
|
|
std::vector<uint8_t> b(65536);
|
|
char lv[512] = "";
|
|
int k = 0;
|
|
uint64_t all_nz = 0, all_unmapped = 0;
|
|
for (uint32_t l = 0; l < std::max(1u, t->d.levels) && l < 16; l++) {
|
|
const uint64_t lo = host->tex_level_offset(&t->d, l);
|
|
const uint64_t hi = l + 1 < std::max(1u, t->d.levels) ? host->tex_level_offset(&t->d, l + 1) : t->guest_size;
|
|
uint64_t nz = 0, unmapped = 0, pages = 0, empty_pages = 0;
|
|
for (uint64_t off = lo; off < hi; off += 65536) {
|
|
const uint64_t n = std::min<uint64_t>(65536, hi - off);
|
|
if (!host->gpu_read(t->gpu + off, b.data(), n)) { unmapped += n; continue; }
|
|
uint64_t page_nz = 0;
|
|
for (uint64_t i = 0; i < n; i++) page_nz += b[i] != 0;
|
|
nz += page_nz, pages++, empty_pages += !page_nz;
|
|
}
|
|
all_nz += nz, all_unmapped += unmapped;
|
|
if (k < 480)
|
|
k += snprintf(lv + k, sizeof lv - k, " L%u: %llu KB, %llu not zero, %llu of %llu mapped pages empty%s;", l,
|
|
(unsigned long long)((hi - lo) >> 10), (unsigned long long)nz, (unsigned long long)empty_pages,
|
|
(unsigned long long)pages, unmapped ? " (part unmapped)" : "");
|
|
}
|
|
const uint64_t g0 = host->gpu_guest(t->gpu);
|
|
logf("vk: dump guest memory of #%u %#llx %ux%u NVN %#x (guest %#llx%s): %llu bytes not zero, %llu KB unmapped;%s", t->id,
|
|
(unsigned long long)t->gpu, t->width, t->height, t->nvn_format, (unsigned long long)g0,
|
|
t->gpu >= (1ull << 40) ? ", a virtual pool's" : "", (unsigned long long)all_nz,
|
|
(unsigned long long)(all_unmapped >> 10), lv);
|
|
}
|
|
|
|
void texture_released(const nvn_tex* t) {
|
|
if (!t) return;
|
|
auto [a, b] = by_gpu.equal_range(t->gpu);
|
|
for (auto it = a; it != b; ++it) it->second->let_go = std::max<uint64_t>(frame_no, 1); // (texture_sweep: first to go while VRAM is short)
|
|
// (Triforce, a lab switch, off: RX_VK_RELEASE_OWN=1 finalizes only the image the released texture has, NX2navi's rule
|
|
// (there a release that finalized every image at the storage left terrain height maps drawn at height zero). Not
|
|
// the default: this renderer keeps an image for each size and format of a storage, and in the lab's first minute of
|
|
// play 32,767 images were spared this way, most of them at the start of one pool, render targets of other sizes
|
|
// that only such a release ever marks as let go. What that does to memory and to the write-backs before eviction
|
|
// has not been measured (the one run with it on was slow for another reason). The count of the others spared is
|
|
// logged as it doubles)
|
|
static const bool own_only = getenv("RX_VK_RELEASE_OWN") && atoi(getenv("RX_VK_RELEASE_OWN")) != 0;
|
|
const Texture* own = nullptr;
|
|
bool own_known = false;
|
|
if (own_only) {
|
|
const VkFormatInfo fi = vk_format(t->d.format);
|
|
int bw = 1, bh = 1;
|
|
const int bpb = fi.format != VK_FORMAT_UNDEFINED ? nvn_block_size(t->d.format, &bw, &bh) : 0;
|
|
if (bpb > 0) {
|
|
const auto it = textures.find(key_of(t, fi, bpb));
|
|
own_known = true, own = it == textures.end() ? nullptr : it->second.get();
|
|
}
|
|
}
|
|
for (auto it = a; it != b; ++it)
|
|
if (it->second->gpu_written && !it->second->released) {
|
|
if (own_known && it->second != own) {
|
|
static uint64_t spared, told_at;
|
|
if (++spared >= told_at * 2 + 1) {
|
|
told_at = spared;
|
|
logf("vk: a texture let go at %#llx (%ux%u NVN %#x): another texture's image there, #%u %ux%u NVN %#x, "
|
|
"which the GPU drew, stays as it is (%llu such so far)", (unsigned long long)t->gpu, t->d.width,
|
|
t->d.height, t->d.format, it->second->id, it->second->width, it->second->height,
|
|
it->second->nvn_format, (unsigned long long)spared);
|
|
}
|
|
continue;
|
|
}
|
|
it->second->released = true, it->second->release_gen = memory_generation();
|
|
watch_tex(it->second, "finalized");
|
|
static int logged; // (a big one: the course maps' kind)
|
|
if (it->second->bytes >= (32ull << 20) && logged++ < 40)
|
|
logf("vk: texture #%u %#llx %ux%u x%u NVN %#x (%llu MB, rendered or copied into) finalized by the game",
|
|
it->second->id, (unsigned long long)it->second->gpu, it->second->width, it->second->height,
|
|
it->second->layers, it->second->nvn_format, (unsigned long long)(it->second->bytes >> 20));
|
|
}
|
|
}
|
|
|
|
uint64_t texture_made() {
|
|
return st_made;
|
|
}
|
|
void texture_unswizzle_stats(int64_t* bytes, int64_t* ns) {
|
|
*bytes = st_unswizzle_bytes, *ns = st_unswizzle_ns;
|
|
}
|
|
size_t texture_count() {
|
|
return textures.size();
|
|
}
|
|
uint64_t texture_bytes() {
|
|
return total_bytes;
|
|
}
|
|
void texture_stats(int64_t* uploads, int64_t* astc_gpu, int64_t* bytes_uploaded) {
|
|
*uploads = st_uploads, *astc_gpu = st_astc, *bytes_uploaded = st_bytes;
|
|
}
|
|
|
|
bool texels_fit(const nvn_tex* t, const NVNcopyRegion* r, uint64_t row, uint64_t avail) {
|
|
if (!t || !r) return false;
|
|
int bw = 1, bh = 1;
|
|
const int bpb = nvn_block_size(t->d.format, &bw, &bh);
|
|
if (bpb <= 0) return true; // (texture_read_texels copies nothing)
|
|
const uint64_t w = (uint64_t)((std::max(0, r->width) + bw - 1) / bw), h = (uint64_t)((std::max(0, r->height) + bh - 1) / bh);
|
|
const uint64_t dd = (uint64_t)std::max(1, r->depth);
|
|
if (!w || !h) return true;
|
|
const uint64_t pitch = row ? row : w * bpb;
|
|
uint64_t image = 0, a = 0, b = 0, n = 0;
|
|
return !__builtin_mul_overflow(pitch, h, &image) && !__builtin_mul_overflow(dd - 1, image, &a) &&
|
|
!__builtin_mul_overflow(h - 1, pitch, &b) && !__builtin_add_overflow(a, b, &n) &&
|
|
!__builtin_add_overflow(n, w * bpb, &n) && n <= avail;
|
|
}
|
|
|
|
/// ReadTexels / WriteTexels: the CPU's copy of a region in guest memory (block linear, the Switch 2 GOB swizzle).
|
|
/// A write also stamps the pages, so the image is uploaded again; a texture the GPU wrote is read from guest memory
|
|
/// as it is (its newest texels are in the image: not yet read back).
|
|
void texture_read_texels(const nvn_tex* t, unsigned level, unsigned layer0, const NVNcopyRegion* r, uint8_t* data,
|
|
uint64_t row, uint64_t img, bool write) {
|
|
if (!t || !t->gpu || !r) return;
|
|
{ // (RX_VK_WATCH_RANGE: ReadTexels / WriteTexels of textures overlapping it, 400 lines)
|
|
static const std::pair<uint64_t, uint64_t> range = [] {
|
|
unsigned long long a = 0, b = 0;
|
|
if (const char* e = getenv("RX_VK_WATCH_RANGE")) sscanf(e, "%llx:%llx", &a, &b);
|
|
return std::pair<uint64_t, uint64_t>{a, b};
|
|
}();
|
|
static int logged;
|
|
if (range.second && t->gpu < range.second && t->gpu + std::max<uint64_t>(t->L.size, 1) > range.first && logged++ < 400)
|
|
logf("vk: watch range %s %#llx %ux%u NVN %#x level %u layer %u region (%d,%d %dx%d) mapped %s", write ? "WriteTexels" : "ReadTexels",
|
|
(unsigned long long)t->gpu, t->d.width, t->d.height, t->d.format, level, layer0, r->xoffset, r->yoffset,
|
|
r->width, r->height, host->gpu_guest(t->gpu + host->tex_level_offset(&t->d, level)) ? "(level start yes)" : "(level start no)");
|
|
}
|
|
int bw = 1, bh = 1;
|
|
const int bpb = nvn_block_size(t->d.format, &bw, &bh);
|
|
if (bpb <= 0) return;
|
|
const nvn_texdesc& d = t->d;
|
|
const uint32_t lw = (std::max(1u, d.width >> level) + bw - 1) / bw;
|
|
const uint32_t lh = d.target == NVN_TEXTURE_TARGET_1D || d.target == NVN_TEXTURE_TARGET_1D_ARRAY
|
|
? 1 : (std::max(1u, d.height >> level) + bh - 1) / bh;
|
|
const bool vol = d.target == NVN_TEXTURE_TARGET_3D;
|
|
const uint64_t base = t->gpu + host->tex_level_offset(&d, level);
|
|
unsigned th = 0, td = 0;
|
|
host->tex_level_tiles(&d, level, &th, &td);
|
|
const uint32_t x0 = (uint32_t)std::max(0, r->xoffset) / bw, y0 = (uint32_t)std::max(0, r->yoffset) / bh;
|
|
const uint32_t w = (uint32_t)(std::max(0, r->width) + bw - 1) / bw, h = (uint32_t)(std::max(0, r->height) + bh - 1) / bh;
|
|
const uint32_t z0 = (uint32_t)std::max(0, r->zoffset), dd = (uint32_t)std::max(1, r->depth);
|
|
const uint64_t pitch = row ? row : (uint64_t)w * bpb, image = img ? img : pitch * h;
|
|
const bool linear = nvn_tex_is_linear(&d);
|
|
const uint32_t stride = linear ? (d.stride ? d.stride : host->tex_linear_stride(&d)) : 0;
|
|
const uint64_t gobs_x = ((uint64_t)lw * bpb + 63) >> 6;
|
|
const uint64_t rows_y = (lh + (8u << th) - 1) >> (3 + th);
|
|
const uint64_t block = 512ull << (th + td);
|
|
for (uint32_t k = 0; k < dd; k++) {
|
|
const uint32_t z = vol ? z0 + k : 0;
|
|
const uint64_t layer_base = base + (vol ? 0 : (uint64_t)(layer0 + z0 + k) * t->L.layer_size);
|
|
for (uint32_t y = 0; y < h && y0 + y < lh; y++) {
|
|
uint8_t* line = data + k * image + y * pitch;
|
|
const uint32_t yy = y0 + y;
|
|
for (uint32_t xb = 0; xb < (uint64_t)w * bpb;) {
|
|
const uint32_t xbyte = x0 * bpb + xb;
|
|
uint64_t off;
|
|
uint32_t run;
|
|
if (linear) {
|
|
off = (uint64_t)yy * stride + xbyte;
|
|
run = w * bpb - xb;
|
|
} else { // (16 bytes of a GOB row are contiguous)
|
|
const uint64_t gob = ((xbyte & 63) >> 5) << 8 | ((yy & 7) >> 2) << 7 | ((xbyte & 31) >> 4) << 6 |
|
|
((yy >> 1) & 1) << 5 | (yy & 1) << 4 | (xbyte & 15);
|
|
off = (uint64_t)(z >> td) * gobs_x * rows_y * block + (uint64_t)(yy >> (3 + th)) * gobs_x * block +
|
|
(uint64_t)(xbyte >> 6) * block + ((uint64_t)(z & ((1u << td) - 1)) << (9 + th)) +
|
|
((uint64_t)((yy >> 3) & ((1u << th) - 1)) << 9) + gob;
|
|
run = std::min<uint32_t>(16 - (xbyte & 15), w * bpb - xb);
|
|
}
|
|
if (write) host->gpu_write(layer_base + off, line + xb, run); // (the shadows hear of the whole: guest_cpu_wrote below)
|
|
else host->gpu_read(layer_base + off, line + xb, run);
|
|
xb += run;
|
|
}
|
|
}
|
|
}
|
|
if (write) {
|
|
guest_cpu_wrote(t->gpu, t->L.size); // (the write-back diff's shadows: one report for the 16-byte runs above)
|
|
const uint64_t guest = host->gpu_guest(t->gpu);
|
|
if (guest) memory_stamp(guest, t->L.size);
|
|
auto [a, b] = by_gpu.equal_range(t->gpu);
|
|
for (auto it = a; it != b; ++it) it->second->gpu_written = false, it->second->checked_epoch = 0;
|
|
}
|
|
}
|
|
|
|
} // namespace mkwvk
|