mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-19 16:20:01 -04:00
What was merged from NX2navi (DLSS frame generation, AMD FSR 4 in Setup builds, texture detail by video memory, sprint, volume, frame caps, safer saves, the above-60 fixes), the ultrawide HUD at the screen edges, camera speed, movies without the AV1 Video Extension, the 5120x2880 + RenoDX HDR start fix, the renderer work of 11-14 and the first float changes of the lift.
434 lines
28 KiB
C++
434 lines
28 KiB
C++
// The Vulkan back end's shared state (gpu/vk).
|
|
#pragma once
|
|
|
|
#include <cstdint>
|
|
#include <array>
|
|
#include <atomic>
|
|
#include <mutex>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "vk_loader.h"
|
|
#include "vk_api.h"
|
|
#include "vk_vendor.h"
|
|
|
|
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
|
|
#include <intrin.h>
|
|
#define HOOKSHOT_RDTSC() ((uint64_t)__rdtsc())
|
|
#define HOOKSHOT_PAUSE() _mm_pause()
|
|
#elif defined(__x86_64__) || defined(__i386__)
|
|
#include <x86intrin.h>
|
|
#define HOOKSHOT_RDTSC() ((uint64_t)__rdtsc())
|
|
#define HOOKSHOT_PAUSE() _mm_pause()
|
|
#endif
|
|
|
|
typedef struct VmaAllocation_T* VmaAllocation; // (vk_mem_alloc.h's handle)
|
|
|
|
namespace mkwvk {
|
|
|
|
extern std::vector<std::string> gpu_device_names;
|
|
extern const MkwVkHost* host;
|
|
void logf(const char* fmt, ...);
|
|
void queue_flush_locked(); // (vk_replay.cpp, RX_VK_BATCH: held submits out; dev.present_mu held)
|
|
void queue_flush(); // (the same, taking dev.present_mu)
|
|
// (hookshot) held submits out, then vkDeviceWaitIdle with the present queue's lock taken too (dev.present_mu held): the device's
|
|
// every queue is synchronized by the host around it
|
|
void device_wait_idle_locked();
|
|
|
|
struct Device {
|
|
VkInstance instance = VK_NULL_HANDLE;
|
|
VkPhysicalDevice physical = VK_NULL_HANDLE;
|
|
VkPhysicalDeviceProperties props{};
|
|
VkPhysicalDeviceMemoryProperties memory{};
|
|
VkPhysicalDeviceFeatures features{};
|
|
VkDevice device = VK_NULL_HANDLE;
|
|
uint32_t queue_family = 0;
|
|
VkQueue queue = VK_NULL_HANDLE;
|
|
// extensions in use
|
|
bool mesh_shader = false, mesh_queries = false, external_memory_host = false, external_memory_fd = false, dma_buf_import = false, pipeline_library = false,
|
|
graphics_pipeline_library = false, custom_border_color = false, dynamic_state3 = false,
|
|
depth_clip_control = false, null_descriptor = false, index_uint8 = false, push_descriptor = false,
|
|
conditional_rendering = false, provoking_vertex = false, vertex_divisor = false, stencil_export = false, memory_budget = false,
|
|
checkpoints = false, // (RX_VK_CHECKPOINTS: VK_NV_device_diagnostic_checkpoints)
|
|
buffer_marker = false, // (RX_VK_CHECKPOINTS=1 without the NV extension: VK_AMD_buffer_marker, AMD and Intel)
|
|
device_fault = false, // (VK_EXT_device_fault: what a lost device faulted on)
|
|
sparse_images = false, // (sparse resident 2D images on the queue: the terrain's huge arrays)
|
|
sparse_buffers = false, // (sparse-bound buffers on the queue: a pool imported as several dma-bufs)
|
|
view_2d_of_3d = false, // (VK_EXT_image_2d_view_of_3d: a 3D texture a 2D sample instruction reads)
|
|
coop_matrix = false, // (VK_KHR_cooperative_matrix with its feature)
|
|
native_mma = false; // (HMMA on it: the cooperative matrix test passed, vk_coopmat.cpp)
|
|
bool eds3_polygon = false, eds3_clamp = false, eds3_blend = false, eds3_a2c = false;
|
|
bool swapchain_colorspace = false; // (hookshot) VK_EXT_swapchain_colorspace: HDR10 surfaces are listed
|
|
bool hdr = false; // (hookshot) the swapchain is HDR10 (ST 2084, Rec. 2020): framegen_hdr writes it
|
|
bool mesh_fallback = false; // (hookshot) no VK_EXT_mesh_shader: task and mesh programs as compute shaders (vk_shader.cpp)
|
|
bool task_mesh_fallback = false;
|
|
bool uses_mesh_fallback(bool has_task) const {
|
|
return mesh_fallback || (task_mesh_fallback && has_task);
|
|
}
|
|
VkPhysicalDeviceVulkan12Features vk12{};
|
|
VkPhysicalDeviceVulkan13Features vk13{};
|
|
VkShaderStageFlags sg32_stages = 0; // (stages that run at 32 lanes when asked: the size the translated programs assume)
|
|
uint32_t subgroup_size = 32, subgroup_max = 32; // (the lanes a stage not pinned runs at, and the most it can have)
|
|
VkDriverId driver_id{};
|
|
VendorFixes fixes{}; // (hookshot) the vendor and driver workarounds this device needs (vk_vendor.h; vk_main.cpp create_device)
|
|
uint32_t max_push_descriptors = 0;
|
|
// no window (HOS_HEADLESS): presents go to an image of their own, which RX_VK_SHOTS can write out
|
|
bool offscreen = false;
|
|
// the window
|
|
VkSurfaceKHR surface = VK_NULL_HANDLE;
|
|
VkSwapchainKHR swapchain = VK_NULL_HANDLE;
|
|
VkFormat swap_format = VK_FORMAT_UNDEFINED;
|
|
VkExtent2D swap_extent{};
|
|
std::vector<VkImage> swap_images;
|
|
std::vector<VkImageView> swap_views; // (hookshot) a view of each, for Dear ImGui's overlays (VK_NULL_HANDLE: none)
|
|
VkFormat swap_view_format = VK_FORMAT_UNDEFINED; // (their format: the UNORM one of an sRGB swapchain when it can be)
|
|
bool swap_mutable = false; // (VK_KHR_swapchain_mutable_format: an sRGB swapchain's images viewed as UNORM)
|
|
std::vector<VkSemaphore> swap_rendered; // present waits belong to acquired images, not frame slots
|
|
std::mutex present_mu; // (the swapchain: presents and resizes)
|
|
std::atomic<uint64_t> swap_resize{0}; // (a new window size, width << 32 | height: the next present makes the swapchain)
|
|
// (hookshot) a queue of a compute family for vkQueuePresentKHR alone (vk_main.cpp create_device; RX_VK_PRESENT_QUEUE=0 / 1).
|
|
// VK_NULL_HANDLE: presents go on `queue` under present_mu. Its own lock: taken after present_mu, never before it
|
|
VkQueue present_queue = VK_NULL_HANDLE;
|
|
std::mutex present_queue_mu;
|
|
uint32_t present_family = 0; // (present_queue's family, the swapchain's images shared with it)
|
|
};
|
|
extern Device dev;
|
|
|
|
/// A range of guest memory as the GPU sees it: a buffer (an imported pool), the offset in it, and its device address
|
|
struct BufferRef {
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
uint64_t offset = 0, size = 0;
|
|
VkDeviceAddress address = 0;
|
|
explicit operator bool() const {
|
|
return buffer != VK_NULL_HANDLE;
|
|
}
|
|
};
|
|
bool memory_init(); // vk_memory.cpp
|
|
void memory_cpu_owned(uint64_t guest, uint64_t size); // (hookshot: words the replay writes on the CPU, left out of write-backs)
|
|
// (hookshot) The replay's own CPU writes of guest memory, by GPU address: every one goes through guest_cpu_write (or, for a
|
|
// write made in pieces, host->gpu_write and then guest_cpu_wrote over the whole range), so that the write-back diff's shadows
|
|
// hear of it before this submission's write-back (vk_memory.cpp, "the write-back's diff"). tests/test_wb_diff_cpu_writes.py
|
|
int guest_cpu_write(uint64_t gpu, const void* src, uint64_t n);
|
|
void guest_cpu_wrote(uint64_t gpu, uint64_t n);
|
|
void memory_table_begin(); // (hookshot: a recording's page table changes held back for its upload commands)
|
|
bool memory_table_flush();
|
|
uint64_t memory_guest_pages(); // (hookshot: the device address of guest memory's page table, 0: none)
|
|
void memory_sync(); // (at each submit: pools the game made or let go)
|
|
extern std::atomic<int64_t> memsync_lock_wait_ns; // (hookshot, diagnostics: time memory_sync waited for its two locks; the slow-submit line)
|
|
extern std::atomic<int64_t> memsync_part_ns[3]; // (hookshot, diagnostics: submit_memory_work's memory_sync, memory_take_capture, memory_collect_written)
|
|
BufferRef gpu_buffer(uint64_t gpu, uint64_t size); // (empty when no pool holds the range)
|
|
/// A range as a binding of the replay sees it: the pool's VRAM mirror when it has one (vk_memory.cpp), else the
|
|
/// imported pool. written: the GPU may write it (copied back to guest memory at the submission's end).
|
|
BufferRef gpu_buffer_bind(uint64_t gpu, uint64_t size, bool written);
|
|
/// (Evil Hookshot) the binds that follow come from a compute program's storage buffers (1) or not (0): vk_memory.cpp's mirror trials
|
|
void memory_bind_from_compute(bool on);
|
|
BufferRef stream_copy(const void* src, uint64_t n); // vk_draw.cpp: bytes in the recording's stream, as a binding
|
|
/// Changes when a read binding remembered earlier must be looked up again (a pool left, a mirror was made, a new submission)
|
|
uint64_t gpu_bind_stamp();
|
|
/// Around each submission's recording: mirrors wanted since the last one are made (begin), what the GPU wrote into
|
|
/// mirrors goes back to guest memory (end, outside any pass, before the closing barrier)
|
|
void memory_begin_submission();
|
|
void memory_sync_mirrors_before_segments(); // (hookshot: vk_memory.cpp)
|
|
bool probe_on(); // (hookshot: vk_draw.cpp, RX_VK_PROBE)
|
|
bool probe_serial();
|
|
void probe_dump_all(const char* why);
|
|
void probe_log(uint64_t frame);
|
|
void memory_end_submission(VkCommandBuffer cmd);
|
|
/// The game made a memory pool (gpu address, bytes): imported in the background, before it is used
|
|
/// RX_VK_CHECKPOINTS: the draws the GPU reached, logged once the device is lost (vk_draw.cpp)
|
|
void checkpoints_report();
|
|
/// The device is lost: that report once, then the process ends (vk_replay.cpp)
|
|
[[noreturn]] void device_lost();
|
|
/// VK_AMD_buffer_marker's buffer, for RX_VK_CHECKPOINTS on a device without the NV checkpoints (vk_draw.cpp)
|
|
void buffer_markers_init();
|
|
void coopmat_init(); // vk_coopmat.cpp (once the device is made)
|
|
/// A marker of another kind of GPU work (m: mirror sync, b: write-back, u: texture upload, g: GPU unswizzle; key: its
|
|
/// pool or texture), RX_VK_CHECKPOINTS
|
|
void checkpoint_mark(VkCommandBuffer cmd, uint64_t key, char kind);
|
|
|
|
/// Parallel replay (vk_replay.cpp, RX_VK_PARALLEL): while segments of a submission are recorded on several threads,
|
|
/// what they share (textures and their uploads, programs, pipelines, samplers, pools and memory stamps) is used under
|
|
/// one lock; the per-draw caches are each thread's own
|
|
extern std::atomic<bool> par_active;
|
|
extern std::atomic<int64_t> par_lock_wait_ns, par_locks; // (contention: the time threads waited for it)
|
|
std::recursive_mutex& par_mutex();
|
|
int64_t par_now_ns();
|
|
struct ParLock {
|
|
bool on;
|
|
explicit ParLock(bool want = true) : on(want && par_active.load(std::memory_order_relaxed)) {
|
|
if (!on) return;
|
|
par_locks.fetch_add(1, std::memory_order_relaxed);
|
|
if (par_mutex().try_lock()) return;
|
|
const int64_t t0 = par_now_ns();
|
|
par_mutex().lock();
|
|
par_lock_wait_ns.fetch_add(par_now_ns() - t0, std::memory_order_relaxed);
|
|
}
|
|
~ParLock() {
|
|
if (on) par_mutex().unlock();
|
|
}
|
|
ParLock(const ParLock&) = delete;
|
|
ParLock& operator=(const ParLock&) = delete;
|
|
};
|
|
void memory_pool_created(uint64_t gpu, uint64_t size);
|
|
/// Whether guest memory in [lo, hi) is in an imported pool, live or retired (the GPU may read or write it)
|
|
int memory_guest_imported(uint64_t lo, uint64_t hi);
|
|
void memory_stats(int64_t* mirrors, uint64_t* mirror_mb, int64_t* sync_bytes, int64_t* sync_copies, int64_t* wb_bytes);
|
|
// Page write generations (vk_memory.cpp, after totk-pc-recomp's vk_memory.c): at each submit the guest pages the CPU
|
|
// wrote since the last one (the soft write watch) are stamped with a new generation; a texture uploaded at
|
|
// generation g is stale when a page of its storage carries a newer one. GPU writes into guest memory that textures
|
|
// may be read from (storage buffers, buffer copies) are stamped the same way.
|
|
void memory_collect_written();
|
|
/// RX_VK_CAPTURE (on with the mirrors; =0 off): at a submit, on the submitting thread, the pages the CPU wrote since the
|
|
/// last one, and a copy of them as they are then; the replay stamps them and fills the mirrors from that copy
|
|
bool memory_capture_submit(); // (true: a capture was queued for the replay to take)
|
|
/// The replay of the next submit: its capture (false: none, the pages are collected now as before)
|
|
bool memory_take_capture(uint64_t value, uint64_t completed);
|
|
bool memory_capture_now(uint64_t value, uint64_t completed); // (hookshot: made and taken at once)
|
|
/// The submission the replay records next (bindings note it: a pool's mirror is in use until it is done)
|
|
void memory_recording_value(uint64_t value, uint64_t completed);
|
|
/// RX_VK_TEX_CAPTURE (on with the capture; =0 off): the storage of a texture uploaded from guest memory joins the
|
|
/// capture (its pool up to 256 MB, else its 2 MB regions), and what the capture took of it is kept for the uploads
|
|
void memory_note_texture_upload(uint64_t guest, uint64_t size);
|
|
/// Whether [guest, guest + n) has pages kept as a capture took them
|
|
bool memory_has_shadow(uint64_t guest, uint64_t n);
|
|
/// A read of guest memory by the replay as the submission being recorded had it: the pages a kept capture holds from
|
|
/// there (the game writes several submits ahead of the replay), the rest from guest memory. False when unreadable
|
|
bool memory_read_as_submitted(uint64_t gpu, void* dst, uint64_t n);
|
|
uint64_t memory_capture_view(); // (bumped when a capture is published or any page is stamped)
|
|
uint32_t memory_page_stamp(uint64_t gpu); // (that address's page only)
|
|
uint64_t gpu_pool_end(uint64_t gpu); // (hookshot) where the memory pool holding that address ends, 0: in none (vk_memory.cpp)
|
|
uint64_t gpu_pool_start(uint64_t gpu); // (EvilOOT) and where it begins
|
|
bool create_buffer_cached(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation,
|
|
void** mapped, bool dedicated = false); // (vk_vma.cpp: cached host memory, mapped)
|
|
void flush_mapped(VmaAllocation allocation, uint64_t size);
|
|
bool host_cached(VmaAllocation allocation); // (its memory type has VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
|
|
void flush_mapped_range(VmaAllocation allocation, uint64_t offset, uint64_t size);
|
|
void invalidate_mapped(VmaAllocation allocation, uint64_t size); // (the CPU is to read what the GPU wrote: a no-op in coherent memory)
|
|
void memory_stamp_gpu(uint64_t guest, uint64_t size); // (a GPU write: memory_stamp, and the pages noted)
|
|
void memory_protect_gpu(uint64_t guest, uint64_t size); // (RX_VK_PROTECT_BARE=1: pages a shader writes through a table pointer)
|
|
/// (vk_replay.cpp) guest memory as the stream's place sees it: as submitted, the stream's uniform updates before it over it
|
|
bool ub_read(uint64_t addr, void* dst, uint64_t n);
|
|
/// [guest, guest + n) as of the submit being replayed: live itself, or scratch holding it with the kept pages over it
|
|
const uint8_t* memory_as_submitted(uint64_t guest, const uint8_t* live, uint64_t n, std::vector<uint8_t>& scratch);
|
|
void memory_drop_shadow(uint64_t guest, uint64_t size); // (a write the capture did not take: the copies go)
|
|
/// (EvilBoyce) the newest submission value whose write-back reaches into [guest, guest + size) (0: none ever); *ever set
|
|
/// when any write-back ever did
|
|
uint64_t memory_writeback_value(uint64_t guest, uint64_t size, bool* ever);
|
|
void memory_note_readback(uint64_t guest, uint64_t size); // (PR 430) a CPU read of GPU results: see memory_rb_refresh
|
|
/// (Triforce) around the write of an NVN event's word by a command buffer's signal: its write-back is not one the captures of
|
|
/// its page wait for (vk_memory.cpp). This thread's bindings while on
|
|
void memory_bind_event_write(bool on);
|
|
bool memory_rb_refresh(VkCommandBuffer up);
|
|
void memory_rb_recording_begun(); // (hookshot) at a recording's start, on its thread: the submission it joins (memory_rb_refresh)
|
|
void memory_rb_segments_begin(uint64_t base, uint64_t n); // (and a parallel replay's segments: their order, base on)
|
|
/// (hookshot) a range a program may write in the mirror that is not bound as written (not copied back): memory_rb_refresh
|
|
void memory_note_gpu_write(uint64_t gpu, uint64_t size);
|
|
/// (vk_replay.cpp) The GPU's progress on the recordings' timeline, and a wait for it to reach v
|
|
uint64_t vk_timeline_completed();
|
|
void vk_timeline_wait(uint64_t v);
|
|
uint32_t memory_generation();
|
|
uintptr_t memory_stamp_who(uint64_t guest, uint32_t* gen, uintptr_t* anchor); // (RX_VK_STAMP_SRC: who wrote that page last)
|
|
/// 1: a page of [guest, guest + size) was written after generation gen; 0: none was
|
|
bool memory_written_since(uint64_t guest, uint64_t size, uint32_t gen);
|
|
void memory_stamp(uint64_t guest, uint64_t size); // a write the watch does not see (GPU writes, host writes)
|
|
/// The page runs of [guest, guest + size) written after gen, as byte ranges from guest (appended to out)
|
|
void memory_written_runs(uint64_t guest, uint64_t size, uint32_t gen, std::vector<std::pair<uint64_t, uint64_t>>& out, bool tex = false);
|
|
uint64_t memory_pages_collected(); // (statistics)
|
|
|
|
/// An NVN format as Vulkan stores it (vk_formats.cpp): ASTC is decoded to RGBA8, block_w x block_h
|
|
/// Bytes per block and block size of an NVN format (ASTC blocks are 16 bytes)
|
|
int nvn_block_size(int format, int* bw, int* bh);
|
|
|
|
struct VkFormatInfo {
|
|
VkFormat format;
|
|
bool astc;
|
|
int block_w, block_h;
|
|
};
|
|
VkFormatInfo vk_format(int nvn_format);
|
|
bool is_depth_format(VkFormat f);
|
|
bool has_stencil(VkFormat f);
|
|
|
|
// device memory (vk_vma.cpp, Vulkan Memory Allocator)
|
|
bool vma_init(PFN_vkGetInstanceProcAddr gipa);
|
|
extern uint64_t test_card_ballast; // (vk_vma.cpp, RX_VK_TEST_CARD_MB: 0 outside that lab test)
|
|
void vma_blocks_ahead_check(); // (vk_vma.cpp: the blocks made ahead given back while VRAM is short; every 600 presents)
|
|
/// (textures: from the textures' own pool, the one image_compact_* moves allocations in: vk_texture.cpp's compaction)
|
|
bool create_image(const VkImageCreateInfo& ci, VkImage& image, VmaAllocation& allocation, bool dedicated = false, bool target = false,
|
|
bool textures = false);
|
|
// (hookshot) the textures' pool compacted: VMA's defragmentation, a pass at a time (vk_texture.cpp's compact_step)
|
|
struct ImageMove {
|
|
VmaAllocation src; // (the allocation to move: it stays the texture's, and points at the new place once the pass ends)
|
|
};
|
|
/// The free space inside the textures' pool's blocks, and what the blocks hold in all; false without the pool
|
|
bool textures_pool_usage(uint64_t* block_bytes, uint64_t* free_bytes);
|
|
bool target_floor_kept(); // (the render targets' pool keeps a floor of blocks for the session: RX_VK_TARGET_FLOOR_MB)
|
|
bool image_compact_begin(uint64_t max_bytes_a_pass, uint32_t max_moves_a_pass);
|
|
bool image_compact_active();
|
|
/// A pass begun: the moves VMA asks for. False: nothing left to move, the compaction is over (then *freed_bytes and
|
|
/// *freed_blocks say what went back to the driver in all)
|
|
bool image_compact_pass(std::vector<ImageMove>& moves, uint64_t* freed_bytes, uint32_t* freed_blocks);
|
|
/// The new image of move i bound at its new place; false: not bound (the caller destroys the image and ignores the move)
|
|
bool image_compact_bind(size_t i, VkImage image);
|
|
void image_compact_ignore(size_t i);
|
|
/// The pass ended (its old images destroyed by the caller, nothing of the pool freed since it began). False: that was
|
|
/// the last one; then *freed_bytes and *freed_blocks say what went back to the driver in all
|
|
bool image_compact_pass_end(uint64_t* freed_bytes, uint32_t* freed_blocks);
|
|
void destroy_image(VkImage image, VmaAllocation allocation);
|
|
/// What an allocation really takes (VkMemoryRequirements.size: padding, alignment, the driver's metadata), 0 for none
|
|
uint64_t allocation_size(VmaAllocation allocation);
|
|
/// (dedicated: its own memory object, not a share of a 64 MB block: for the 16 to 32 MB ones that fit two to a block and leave
|
|
/// the rest of it free; RX_VK_BLOCK_BUFFERS=1: as before)
|
|
bool create_buffer(uint64_t size, VkBufferUsageFlags usage, bool host_visible, VkBuffer& buffer,
|
|
VmaAllocation& allocation, void** mapped, bool dedicated = false);
|
|
void destroy_buffer(VkBuffer buffer, VmaAllocation allocation);
|
|
/// A buffer in VRAM only (within the driver's budget), or false
|
|
bool create_device_buffer(uint64_t size, VkBufferUsageFlags usage, VkBuffer& buffer, VmaAllocation& allocation, bool dedicated = false);
|
|
/// (Triforce) device memory for a piece of a sparse buffer, and its release (vk_vma.cpp)
|
|
bool vma_piece_alloc(uint64_t size, uint64_t alignment, uint32_t type_bits, VmaAllocation& allocation, VkDeviceMemory& memory,
|
|
uint64_t& offset);
|
|
void vma_piece_free(VmaAllocation allocation);
|
|
/// (Triforce) the mirrors' pieces made resident in this recording, bound before it is submitted (vk_memory.cpp)
|
|
void memory_mirror_binds_flush();
|
|
bool memory_mirror_bind_wait(VkSemaphoreSubmitInfo* wait); // (and the wait a queue submission makes for those binds)
|
|
/// (Triforce, vk_memory.cpp) The memory handling played on NVIDIA cards only (mirrors in pieces, compacted textures): on there,
|
|
/// and on trial on a card of another make until triforce_memtrial.txt says it passed or failed. tick: at the 600th presents;
|
|
/// device_lost: from device_lost()
|
|
bool memory_trial_on();
|
|
void memory_trial_tick();
|
|
void memory_trial_device_lost();
|
|
void memory_trial_orderly_end(); // (the menu's Restart and Exit: no verdict for a session ended on purpose)
|
|
/// (Triforce) Set while the start's own shader work runs (vk_main.cpp started_up): a fault in the driver then starts the game
|
|
/// again (vk_draw.cpp driver_fault_restart, Windows)
|
|
extern std::atomic<bool> startup_work_on;
|
|
/// Device-local memory this process uses and what the driver allows it now (VK_EXT_memory_budget), summed over the
|
|
/// device-local heaps; false when unknown
|
|
bool vram_usage(uint64_t* used, uint64_t* budget, uint64_t* block_free = nullptr);
|
|
/// vram_usage asked of the driver now (VMA asks again for a new frame index), for the status line; false when the driver
|
|
/// gives no budget (VK_EXT_memory_budget): used is then the game's own allocations
|
|
bool vram_usage_now(uint64_t* used, uint64_t* budget);
|
|
void vma_log_heaps();
|
|
void vma_log_image_blocks(); // (the image pools' blocks: what they hold, and their free ranges)
|
|
PFN_vkGetInstanceProcAddr instance_proc(); // vk_main.cpp
|
|
|
|
bool create_swapchain(uint32_t width, uint32_t height);
|
|
bool replay_start(); // vk_replay.cpp
|
|
bool precompile_wanted(); // vk_draw.cpp
|
|
void precompile_run(void (*progress)(size_t done, size_t total));
|
|
bool prepare_run(void (*progress)(size_t done, size_t total), bool background);
|
|
bool prepare_is_update();
|
|
void prepare_background();
|
|
void state_list_run(); // (an update: the state list's pipelines noted before precompile_run)
|
|
void predict_background(); // (Triforce: the predicted pipelines predict_run did not get to, built behind the game)
|
|
void frame_shown_note(int draws); // (Triforce, 11-14: a frame was shown with this many draws; the builder rests in play)
|
|
void predict_plan_ensure(); // (Triforce: its plan, made before the precompile when an older build prepared this install)
|
|
void predict_run(void (*progress)(size_t done, size_t total)); // (after a prepare_run that prepared: the notes' predictions)
|
|
/// (hookshot) The window was closed while the shaders were prepared: prepare_run and precompile_run take nothing more and return,
|
|
/// what is done kept (the program cache, the driver's) and not counted as a start that ended inside them
|
|
void prepare_stop();
|
|
bool prepare_stopped();
|
|
/// What prepare_run is doing, for the start-up screen: 0 the shaders translated, 1 the graphics driver's part of them
|
|
int prepare_pass();
|
|
/// Whether prepare_run has that second pass at all here (off unless the stage libraries are switched on): the screen's steps
|
|
bool prepare_compiles();
|
|
void fill_replay_table(MkwVkRenderer& t); // vk_replay.cpp
|
|
|
|
} // namespace mkwvk
|
|
|
|
// ---- a scoped timer per phase of the replay (logged per frame with the frame statistics) ----
|
|
namespace mkwvk {
|
|
enum ProfPhase { PROF_SUBMIT, PROF_PRESENT, PROF_DRAW, PROF_DESC, PROF_TEXGET, PROF_UPLOAD, PROF_PIPE, PROF_PROGRAM,
|
|
PROF_CMDS, PROF_DISPATCH, PROF_MEMSYNC, PROF_QSUBMIT, PROF_SLOTWAIT, PROF_COUNT };
|
|
extern int64_t prof_ns[PROF_COUNT];
|
|
/// Nanoseconds from a performance-counter reading. An integer number of nanoseconds per tick is a multiply, so a
|
|
/// reading past 2^53 does not lose its low bits the way the double divide does.
|
|
inline int64_t perf_ticks_to_ns(int64_t ticks, int64_t freq) {
|
|
if (freq > 0 && 1000000000LL % freq == 0)
|
|
return ticks * (1000000000LL / freq);
|
|
if (freq <= 0) return 0;
|
|
return (int64_t)((double)ticks * 1e9 / (double)freq);
|
|
}
|
|
// A cheap nanosecond clock for the per-phase replay timers. Ten QueryPerformanceCounter reads a draw were ~4% of
|
|
// the GPU thread (gpu/render/rx_internal.h). The hot path reads the invariant TSC; the counter calibrates the scale once.
|
|
inline int64_t prof_now() {
|
|
#if defined(HOOKSHOT_RDTSC)
|
|
struct Scale { double ns_per; bool ok; };
|
|
static const Scale scale = [] {
|
|
LARGE_INTEGER f, a, b;
|
|
QueryPerformanceFrequency(&f);
|
|
const uint64_t c0 = HOOKSHOT_RDTSC();
|
|
QueryPerformanceCounter(&a);
|
|
volatile int sink = 0;
|
|
for (int i = 0; i < 400000; i++) { HOOKSHOT_PAUSE(); sink += 1; }
|
|
const uint64_t c1 = HOOKSHOT_RDTSC();
|
|
QueryPerformanceCounter(&b);
|
|
const int64_t dns = perf_ticks_to_ns(b.QuadPart - a.QuadPart, f.QuadPart);
|
|
Scale s;
|
|
s.ok = c1 > c0 && dns > 0 && sink > 0;
|
|
s.ns_per = s.ok ? (double)dns / (double)(c1 - c0) : 0.0;
|
|
return s;
|
|
}();
|
|
if (scale.ok) return (int64_t)((double)HOOKSHOT_RDTSC() * scale.ns_per);
|
|
#endif
|
|
LARGE_INTEGER t;
|
|
QueryPerformanceCounter(&t);
|
|
static const int64_t freq = [] { LARGE_INTEGER f; QueryPerformanceFrequency(&f); return f.QuadPart; }();
|
|
return perf_ticks_to_ns(t.QuadPart, freq);
|
|
}
|
|
// (hookshot) The per-draw phases (PROF_HOT) run a few thousand times a frame, and each timer is two counter reads: ~25k
|
|
// reads a frame were ~2% of the replay thread. One draw in eight is timed, picked at random per thread, and counted eight
|
|
// times: the outermost PROF_HOT on the thread picks, and those nested in it follow, so a phase never reads more than the
|
|
// one around it. What is slow and rare inside (a program or pipeline made, a texture created or uploaded, a wait) is timed
|
|
// by PROF / PROF_EXACT and counted once as it is, sampled draw or not: the PROF_HOT scopes around it weigh only the rest.
|
|
// Every scope timed (prof_full): RX_VK_PROF_ALL=1, RX_VK_SLOW_LOG, and while RX_VK_BENCH replays its frame.
|
|
inline std::atomic<int> prof_full{getenv("RX_VK_PROF_ALL") || getenv("RX_VK_SLOW_LOG") ? 1 : 0};
|
|
struct ProfThread {
|
|
uint32_t rng; // (xorshift32, seeded at the thread's first pick)
|
|
int open, k; // (the PROF_HOT scopes open, and the outermost one's weight: 0 not timed)
|
|
int64_t exact; // (the nanoseconds PROF / PROF_EXACT timed on this thread so far)
|
|
};
|
|
inline thread_local ProfThread prof_t;
|
|
struct ProfScope {
|
|
int phase; // (-1: PROF_EXACT, no phase of its own)
|
|
int64_t t0, x0;
|
|
explicit ProfScope(int p) : phase(p), t0(prof_now()), x0(prof_t.exact) {}
|
|
~ProfScope() {
|
|
const int64_t dt = prof_now() - t0;
|
|
if (phase >= 0) prof_ns[phase] += dt;
|
|
prof_t.exact = x0 + dt; // (what was exact inside is part of dt: not counted twice)
|
|
}
|
|
};
|
|
/// A thread's first state: splitmix64 of a counter, so the threads' sequences are unrelated; never 0, xorshift's fixed point
|
|
inline uint32_t prof_seed() {
|
|
static std::atomic<uint64_t> threads{0};
|
|
uint64_t z = (threads.fetch_add(1, std::memory_order_relaxed) + 1) * 0x9E3779B97F4A7C15ull;
|
|
z = (z ^ z >> 30) * 0xBF58476D1CE4E5B9ull;
|
|
z = (z ^ z >> 27) * 0x94D049BB133111EBull;
|
|
return (uint32_t)((z ^ z >> 31) >> 32) | 1u;
|
|
}
|
|
inline int prof_pick(ProfThread& t) { // (the weight: 8 one time in eight, else 0; 1 when every scope is timed)
|
|
if (prof_full.load(std::memory_order_relaxed)) return 1;
|
|
if (!t.rng) t.rng = prof_seed();
|
|
t.rng ^= t.rng << 13, t.rng ^= t.rng >> 17, t.rng ^= t.rng << 5;
|
|
return t.rng >> 29 ? 0 : 8; // (the top bits: xorshift's low ones are its weakest)
|
|
}
|
|
struct ProfScopeHot {
|
|
int phase, k;
|
|
int64_t t0, x0;
|
|
explicit ProfScopeHot(int p) : phase(p) {
|
|
ProfThread& t = prof_t;
|
|
if (!t.open++) t.k = prof_pick(t);
|
|
k = t.k, x0 = t.exact, t0 = k ? prof_now() : 0;
|
|
}
|
|
~ProfScopeHot() {
|
|
ProfThread& t = prof_t;
|
|
const int64_t x = t.exact - x0; // (timed exactly inside: once, as it is)
|
|
if (k) prof_ns[phase] += (prof_now() - t0 - x) * k + x;
|
|
else if (x) prof_ns[phase] += x;
|
|
t.open--;
|
|
}
|
|
};
|
|
} // namespace mkwvk
|
|
#define PROF(phase) ::mkwvk::ProfScope prof_scope_##phase(::mkwvk::phase)
|
|
#define PROF_HOT(phase) ::mkwvk::ProfScopeHot prof_scope_##phase(::mkwvk::phase) // (the per-draw phases: sampled, above)
|
|
#define PROF_EXACT() ::mkwvk::ProfScope prof_scope_exact(-1) // (a slow, rare path inside them: timed exactly, above)
|