diff --git a/docs/building.md b/docs/building.md index 0946079d..a7a3bb74 100644 --- a/docs/building.md +++ b/docs/building.md @@ -282,7 +282,11 @@ Forest (1.1.1, RTX 4090) that costs about 1 ms a frame (copy-in 0.5 ms, write-ba after a frame's first submission covers the write-backs there (`runtime/vk_sync.c`, "the executed timeline"): the game rewrites what those passes wrote without a fence, and a write-back that landed after the rewrite left the GPU adding light onto its own last output, the whole scene brighter for a frame. `NX2NAVI_SHADOW=1` turns it on under Windows too. The Steam Deck's RADV has `VK_EXT_external_memory_host` as -well: the shadow copies are chosen for the measured cost above, not for a missing extension. +well: the shadow copies are chosen for the measured cost above, not for a missing extension. NVIDIA's own driver on Linux +imports instead (`NX2NAVI_SHADOW=1` brings the copies back): it imports guest memory without that cost, so there the +copies are only the slower path. In Kokiri Forest on 0.7.1 (RTX 4090, driver 615.78), imports with VRAM mirrors ran at +51.9 frames a second and the copies at 27.4; on that version the copies also flickered between two looks almost every +frame, which the wait for the write-backs under shadow storage (`vk_sync_exec_wait`) has since addressed. The device address table (`runtime/vk_gpa.c`) names a pool's shadow copy as it names an import, so shaders follow guest pointers into either. A shader that reads through the table resolves nothing, so the pool it reaches is not queued for diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 9780ab94..5a76c791 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -89,8 +89,8 @@ static void mesa_cache_beside_exe(void) { // beside the exe, which nothing else writes, with a limit of NV_CACHE_BYTES. The driver makes GLCache inside that // folder but not the folder, so it is made here; pointed at a folder that is missing or not writable it caches nothing // at all, so when that folder cannot be made or written neither variable is set and the driver keeps its own folder -// and limit. Without NVIDIA's kernel driver (no /proc/driver/nvidia/version, the test play.sh makes) the variables do -// nothing and the folder would stay empty, so nothing is done. +// and limit. Without NVIDIA's kernel driver (no /proc/driver/nvidia/version, the test the AppImage's build makes for +// DLSS) the variables do nothing and the folder would stay empty, so nothing is done. #define NV_CACHE_BYTES "8589934592" // 8 GiB static void nvidia_cache_beside_exe(void) { if (plat_fs_kind("/proc/driver/nvidia/version") != PLAT_FS_FILE) return; @@ -274,6 +274,7 @@ retry: } free(qp); rt_log("vk: using gpu %d: %s", best, g_vk.props.deviceName); + vk_caps_driver(g_vk.phys); // before vk_mem_class_log, which chooses the pools' storage by driver vk_mem_class_log(&g_vk.props, &g_vk.mem_props); return 1; } @@ -421,7 +422,6 @@ static int create_device(void) { // The driver fills the chain with what it supports; the same chain then enables all of it (robustBufferAccess // stays on: guest shaders index constant buffers freely and out-of-range reads must stay harmless). vkGetPhysicalDeviceFeatures2(g_vk.phys, &c->f2); - vk_caps_driver(g_vk.phys); vk_caps_features(&c->f2.features); if (gpl_link) { VkPhysicalDeviceGraphicsPipelineLibraryPropertiesEXT gp = { diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index d9653d29..0a1c1176 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -1152,7 +1152,8 @@ static int query_stamp(uint64_t lo, uint64_t hi, uint32_t g) { // copy-ins that overwrote a store through the table, per pool; the write-backs that moved bytes into a page the CPU // had written since its copy-in (cpu_wrote_since); the bytes both sides changed since their base copy (taken from the // CPU at a copy-in, from the GPU at a fence); what the copy-ins cost by kind. -// NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine. +// NX2NAVI_SHADOW=1/0 forces the mode on or off; the default is on for Linux and Wine, except for NVIDIA's own driver on +// Linux, where imports ran about twice as fast on 0.7.1 (mirror_mode). #define SB_PAGE 4096ull static PlatMutex shadow_lock = PLAT_MUTEX_INIT; // after pool_lock; before gen_lock static PoolRec **stouch; // shadow_lock: pools resolved since the last sync @@ -2003,7 +2004,7 @@ static PlatMutex mirror_stat_lock = PLAT_MUTEX_INIT; static volatile int64_t m_live_n, m_live_bytes, m_refused_n[MR_N], m_refused_bytes[MR_N]; static volatile int64_t m_flush_calls, m_flush_bytes, v_pages, v_skipped, v_mismatch; -static void mirror_mode(int discrete) { +static void mirror_mode(int discrete, int nvidia_own) { const char *e = cfg_value("NX2NAVI_MEMORY"); int mode = discrete ? 1 : 0; // 0 import, 1 mirror, 2 verify const char *why = discrete ? "auto, discrete gpu" : "auto, uma"; @@ -2033,7 +2034,13 @@ static void mirror_mode(int discrete) { #ifdef _WIN32 shadow_on = sh && *sh ? strcmp(sh, "0") != 0 : plat_is_wine(); #else - shadow_on = !sh || !*sh || strcmp(sh, "0"); + // NVIDIA's own driver imports guest memory without the cost shadow storage avoids on amdgpu (re-protecting imported + // pages), so there the copies are only the slower path: in Kokiri Forest on 0.7.1 (RTX 4090, driver 615.78) imports + // with VRAM mirrors ran at 51.9 frames a second and the copies at 27.4, and the copies hold about 5 GB. What a + // shader stores through the device address table into a pool with a shadow copy also does not come back to guest + // memory (vk_gpa.c). The pools stay imports unless NX2NAVI_SHADOW asks for the copies. The open driver for the same + // cards keeps the copies: it is not measured. The AppImage's play.sh has set the same for this driver since 0.6.0. + shadow_on = sh && *sh ? strcmp(sh, "0") != 0 : !nvidia_own; #endif if (shadow_on) rt_log("mem: shadow storage: CPU-visible pools are host-visible copies of guest memory, not imports%s", sh && *sh ? " (NX2NAVI_SHADOW)" : ""); @@ -3899,7 +3906,7 @@ void vk_mem_class_log(const VkPhysicalDeviceProperties *p, const VkPhysicalDevic f & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT ? " LAZY" : ""); } rt_log("%s", line); - mirror_mode(discrete); + mirror_mode(discrete, vk_caps.driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY); } // ---------------------------------------------------------------- upload batches and deferred destruction diff --git a/tools/make_appimage.py b/tools/make_appimage.py index 571dccd7..c2a2af22 100644 --- a/tools/make_appimage.py +++ b/tools/make_appimage.py @@ -153,14 +153,11 @@ else ev step dlss '"state":"skip"' fi -# play.sh: the environment the game wants on this machine, then the game +# play.sh: starts the game from its own folder. It sets no storage mode: the game leaves shadow storage off for +# NVIDIA's own driver itself (runtime/vk_memory.c mirror_mode), by the driver of the device it runs on. { echo '#!/bin/bash' echo 'cd "$(dirname "$(readlink -f "$0")")"' - if [ -e /proc/driver/nvidia/version ]; then - echo '# NVIDIA: the default shadow storage leaves the 3D world black (issue #72)' - echo 'export NX2NAVI_SHADOW="${NX2NAVI_SHADOW:-0}"' - fi echo 'exec ./ocarina "$@"' } > "$OUT/play.sh" chmod +x "$OUT/play.sh"