diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 725cde21..a9ef9ed1 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -288,6 +288,9 @@ # which game memory gets no copy in video memory (memory = mirror), which costs far more frame time than a texture # uploaded again. 0: textures never give way for video memory. # tex_vram_pressure = 80 +# Textures the game released that are kept as they are (1 to 16384); past this count only those that lose nothing +# when dropped go. Default: one per 2 MB of the video memory budget, 2048 to 8192. Read at start. +# tex_released_max = 4096 # Megabytes of this process's commit above which the log says so. Windows counts video memory against the commit; # with a discrete GPU the video memory in use is left out of the comparison, since it is not in RAM. Textures no longer # give way at this bound: dropping one lowers both sides of the comparison alike, so nothing they could do brought the @@ -421,6 +424,20 @@ # 3D render scale: the scene's size relative to the game resolution, game (5/6, the game's own 1600x900 at 1080p) or # a percentage (25 to 400, at most 4096 pixels a side); above 100 supersamples the scene (anti-aliasing). Read at start. # render_scale = game +# Discrete GPUs: a game memory pool whose copies to and from video memory each frame reach this percentage of its size +# is read and written by the GPU in system memory instead (0: every pool keeps its video memory copy). For GPUs and +# buses short of bandwidth; on an RX 580 the pools the game refills every frame cost about 8 ms a frame in copies. +# mirror_hot = 0 +# 2: the next Rebuild the Game compiles the game's code at -O2 instead of -O1 (faster game threads on a slower CPU, a +# build several times longer). Read by tools/make_game.py; the environment variable NX2NAVI_GUEST_OPT wins. +# guest_opt = 1 +# GPUs without mesh shaders (VK_EXT_mesh_shader: AMD before RDNA 2, NVIDIA before Turing): 1 runs the game's mesh +# and task shaders (grass and other foliage) as compute shaders and draws what they make; 0 skips those draws. The +# arena for their vertices and triangles is mesh_emu_mb MiB of video memory, with mesh_emu_tasks task records (3 KiB +# each) per submission; with trace = perf the log says how much a submission used. Read at start. +# mesh_emulation = 0 +# mesh_emu_mb = 256 +# mesh_emu_tasks = 16384 # Scaling to the window when it is larger than the game resolution: fsr (AMD FSR 1) or bilinear, and FSR's sharpening # from 0 (off) to 100. Changes while the game runs take effect at once. # upscaler = fsr diff --git a/runtime/cfg.c b/runtime/cfg.c index 6c1a781c..420c8945 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -62,6 +62,7 @@ static const char *const known[] = { "ONE_PER_CORE", "OPERATION_MODE", "OUTPUT_RES", "PAD_LAYOUT", "PACE_HZ", "PASS_CRUMBS", "PATCH_OUTPUT", "PATCH_RES_FIXED", "PATCH_SCENE", "PATCH_WIDE_ASPECT", "PATCH_WIDE_HUD", "PATCH_WIDE_MOVIE", "PATCH_WIDE_OUT", "PATCH_FREECAM", "PATCH_WIDE_SCENE", "PCSS", "PCSS_K", "PERF_OVERLAY", "PERF_SLOW_MS", "PIPELINE_CACHE", "PIPELINE_CACHE_MAX_MB", "PIPELINE_CACHE_PLAY", "PIPELINE_CACHE_PLAY_MB", "PIPE_BINARIES", "PIPE_BINARIES_MAX_MB", "PIPE_FEEDBACK", "PIPE_MEMO", "PIPE_PREBUILD", "PIPE_SEED", "PIPE_SKIP", "PIPE_SYNC", "PIPE_WAIT", "POWER_FULL", "TEX_SAME", "TEX_SAME_BIG", "TEX_FLAT3D", "TEX_KEEP_TAKEN", "TEX_STALE_KEPT", "PRESENT_HELD", "HELD_SYNC", "PREPARE_PIPELINES", "PREWARM_FULL", "PREWARM_WAIT_MS", "PREWARM_XLATE", "PROBE_MOVIE", "PROBE_MOVIE_SHOT", "PROFILE", "PROFILE_CHAINS", "PROFILE_DIR", "PROFILE_FRAMES", "PROFILE_REPLAY", "PROFILE_SLOW", "POOL_FAST", "POOL_GROUP", "POOL_GROUP_CHECK", "POOL_PRESNAP", "POOL_PRESNAP_RUNS", + "SUBMIT_FINISH_GPU", "SVC_WAITS", "SNAP_WIDEN", "SUBMIT_FINISH_RESULTS", "SUBMIT_FINISH_KINDS", "FOLIAGE", "MIRROR_STAGE", "FLUSH_SKIP_CLEAN", "MIRROR_HOT", "MESH_EMULATION", "MESH_EMU_MB", "MESH_EMU_TASKS", "MESH_EMU_BATCH", "GUEST_OPT", "LOG_TEX_NEW", "DUMP_KEY", "VEC_INDEX_FIX", "PASS_TIMES", "PROFILE_MAIN", "PROFILE_RENDER", "PROMPTS", "PUSH_SKIP", "PUSH_STAGE", "RDOC", "RDOC_DIR", "RDOC_DLL", "REALTIME", "REC_END", "REEXEC", "REFLEX", "RENDER_SCALE", "REPLAY_ASYNC", "REPLAY_PIPE", "REPLAY_RUNS", "REPLAY_WORKERS", "RTSS", "RTSS_ENUM", "SCENE_SHARE", "PRESNAP_PLACE", @@ -74,6 +75,7 @@ static const char *const known[] = { "SET_AT", "SWKBD_TEST", "TAA", "DBG_TEST", "DEBUG_MENU", "MENU_TEST_DEBUG", "FSR4", "FSR4_FG", "FG4_VERSION", "FG4_SWAPCHAIN", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "DEPTH_INIT_INLINE", "DEPTH_ALIAS_LOG", "TEX_DISUSE", "TEX_DISUSE_EVICT", "TEX_GEN_UNDO", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_RELEASED_MAX", "TEX_REUSE", "TEX_SPLIT_FILL", "TEX_COMMIT_FREE_MB", "TEX_PROC_COMMIT_MB", "TEX_EVICT", "TEX_COMMIT_EVICT", "MIRROR_EVICT", "VRAM_BUDGET_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "EVO_TRACE", "EVENT_AT_SYNC", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NAN_PRESERVE", "VK_NOOPT", "VK_NO_FLOAT64", "VK_PRESENT", "VK_RECONVERGE", "VK_NAN_BITS", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WATER_PROJECTIVE", "WAIT_WATCH_S", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WB_NO_REUP", "WB_MERGE_MAX", "WB_PROBE", "WB_SPLIT", "WB_LAZY", "WB_LAZY_CHUNK", "WB_LAZY_TOUCH", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", + "UPLOAD_BATCH_MB", "VRAM_HEAP_LIMIT", "VRAM_RESERVE_MB", "TEX_CLOBBER", "ZSTD_CHECK", "ZSTD_NATIVE", "BIND_REUSE", "DYN_DELTA", "STG_ARENA", "SUBMIT_BATCH", "SUBMIT_THREAD", "ASTC_GPU_DESWIZZLE", "ASTC_SCRATCH_KEEP", "ASTC_ARENA", "ASTC_BC1", "TEXUP_LOG_MS", "DYN_RES", "DYN_RES_TARGET", "DYN_RES_STEP", "TEXTURE_DETAIL", @@ -89,6 +91,12 @@ static const char *const known[] = { // The nx2navi.ini section a key cfg_set may write goes to when the file lacks it (nx2navi.ini.example's sections). static const struct { const char *key, *section; } sections[] = { + // the menu's Lama patches tab (ui/menu.cpp tab_lama) + {"TEX_RELEASED_MAX", "performance"}, {"MIRROR_HOT", "performance"}, {"FREEZE_IDLE", "performance"}, {"SUBMIT_FINISH_KINDS", "performance"}, {"FOLIAGE", "performance"}, + {"ZSTD_NATIVE", "performance"}, + {"VRAM_HEAP_LIMIT", "performance"}, {"VRAM_RESERVE_MB", "performance"}, {"UPLOAD_BATCH_MB", "performance"}, {"MESH_EMULATION", "performance"}, + {"MESH_EMU_MB", "performance"}, {"MESH_EMU_TASKS", "performance"}, {"MESH_EMU_BATCH", "performance"}, {"VEC_INDEX_FIX", "performance"}, + {"GUEST_OPT", "performance"}, {"PASS_TIMES", "diagnostics"}, {"DUMP_KEY", "diagnostics"}, {"FPS_LIMIT", "frame rate"}, {"FPS60", "frame rate"}, {"RTSS", "frame rate"}, {"REALTIME", "frame rate"}, {"CLOTH_REALTIME", "frame rate"}, {"HIGHFPS_FIX", "frame rate"}, {"PHYSICS_STEPS", "frame rate"}, {"PICKUP_COUNT", "frame rate"}, {"STEP_STEADY", "frame rate"}, @@ -138,6 +146,7 @@ static const struct { const char *key, *section; } sections[] = { {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"DEPTH_INIT_INLINE", "performance"}, {"DEPTH_ALIAS_LOG", "diagnostics"}, {"TEX_DISUSE", "performance"}, {"TEX_DISUSE_EVICT", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_SPLIT_FILL", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"WATER_PROJECTIVE", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_CACHE_FLUSH", "performance"}, {"SHADER_CARRY", "performance"}, {"PIPELINE_CACHE_PLAY", "performance"}, {"PIPELINE_CACHE_PLAY_MB", "performance"}, {"FFX_PIPE_CACHE", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, + {"TEX_CLOBBER", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_IDLE", "performance"}, {"SPLIT_IN_PASS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"VK_NAN_PRESERVE", "performance"}, {"VK_NAN_BITS", "performance"}, @@ -249,7 +258,7 @@ static const CfgMeta metas[] = { // offer's number, which this setting or the menu accepts), always (every room, the save still copied first), or a number. {"COOP_ACCEPT", "ask", NULL, LIVE}, // Advanced: scheduling - {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "0", "0|1", RESTART}, + {"ASYNC_PREEMPT", "1", "0|1|2", RESTART}, {"REPLAY_ASYNC", "1", "0|1", RESTART}, {"REPLAY_WORKERS", "3", "0..16", RESTART}, {"REPLAY_PIPE", "0", "0..4", RESTART}, {"FREE_CORES", "0", "0|1", RESTART}, {"TIMER_1MS", "1", "0|1", RESTART}, // dynamic resolution (dynres.c): the scene follows the GPU time toward DYN_RES_TARGET milliseconds @@ -302,9 +311,22 @@ static const CfgMeta metas[] = { // all|: the compute pipelines the boot keeps, those dispatched most recently (vk_pipeline.c, compute-used.bin) {"COMPUTE_KEEP", "512", NULL, RESTART}, {"SHADER_CARRY", "1", "0|1", RESTART}, // gfx_shader.c "Carry": a previous translator's translations until translated again + // the menu's Lama patches tab (ui/menu.cpp tab_lama): the settings of the RX 580 patches + {"TEX_RELEASED_MAX", "", NULL, RESTART}, // vk_texture.c released_max: "" by the video memory budget, else 1..16384 + {"MIRROR_HOT", "25", "0..1000", RESTART}, {"FREEZE_IDLE", "1800", "0..100000", RESTART}, + {"SUBMIT_FINISH_KINDS", "31", "0..31", RESTART}, // vk_sync.c res_mask + {"FOLIAGE", "full", "full|reduced|off", RESTART}, // vk_exec_draw.c foliage_skip + {"VRAM_HEAP_LIMIT", "auto", NULL, RESTART}, // vk_device.c vram_heap_limits: auto, 0 or MiB + {"VRAM_RESERVE_MB", "512", "0..4096", RESTART}, // vk_vma_budget.c: video memory kept free of VMA's allocations + {"UPLOAD_BATCH_MB", "256", NULL, RESTART}, // vk_memory.c: MiB of staging before a batch goes early, 0 no limit + {"MESH_EMULATION", "1", "0|1", RESTART}, {"MESH_EMU_MB", "256", "32..2048", RESTART}, + {"MESH_EMU_TASKS", "16384", "256..262144", RESTART}, {"MESH_EMU_BATCH", "1", "0|1", RESTART}, {"VEC_INDEX_FIX", "0", "0|auto|1", RESTART}, + {"GUEST_OPT", "2", "1|2", RESTART}, // tools/make_game.py: the next Rebuild the Game + {"PASS_TIMES", "0", "0|1", RESTART}, {"DUMP_KEY", "", NULL, RESTART}, // vk_crumbs.c, vk_dump.c // textures and memory {"ASTC", "auto", "auto|gpu|native|cpu", RESTART}, {"MEMORY", "auto", "auto|import|mirror|verify", RESTART}, {"TEX_ALIAS", "1", "0|1", RESTART}, {"DEPTH_INIT_INLINE", "1", "0|1", RESTART}, {"DEPTH_ALIAS_LOG", "0", "0|1", RESTART}, {"TEX_DISUSE", "1", "0|1|2", RESTART}, {"TEX_DISUSE_EVICT", "0", "0|1", RESTART}, {"TEX_GEN_UNDO", "1", "0|1", RESTART}, {"TEX_REUSE", "1", "0|1", RESTART}, {"TEX_SPLIT_FILL", "1", "0|1|2", RESTART}, {"TEX_VRAM_PRESSURE", "80", "0..100", RESTART}, {"ASTC_BC", "", "0|1", RESTART}, {"ZSTD_NATIVE", "1", "0|1", RESTART}, {"ZSTD_CHECK", "0", "0|1", RESTART}, + {"TEX_CLOBBER", "1", "0|1", RESTART}, {"STG_ARENA", "1", "0|1", RESTART}, {"ASTC_GPU_DESWIZZLE", "1", "0|1", RESTART}, {"ASTC_SCRATCH_KEEP", "1", "0|1", RESTART}, {"ASTC_ARENA", "1", "0|1", RESTART}, // vk_astc.c scratch_get, vk_texture.c astc_arena // render fixes diff --git a/runtime/gfx_present.c b/runtime/gfx_present.c index b0d87802..86aa44b8 100644 --- a/runtime/gfx_present.c +++ b/runtime/gfx_present.c @@ -140,6 +140,41 @@ static VkImageView g_sc_view[MAX_IMAGES]; // colour attachment views for the m static VkSemaphore g_render_done[MAX_IMAGES]; static VkCommandPool g_pool; static VkCommandBuffer g_cb[FRAMES]; + +// The GPU timeline (NX2NAVI_TRACE=perf, vk.h vk_gtl_note): a timestamp pair per present slot around the present's command +// buffer, read once the slot's fence is waited for again. Its gap before it shows a wait for the acquired image (the +// submission waits on the acquire semaphore) as idle time with work queued before presents. +static VkQueryPool pts_pool; +static int pts_state; // 0 not set up, 1 on, -1 off +static uint8_t pts_on[FRAMES]; // the slot's pair was written by its last present +static uint64_t pts_sub[FRAMES]; // when that present went to the queue + +static void pts_begin(VkCtx *c, VkCommandBuffer cb, unsigned slot) { + pts_on[slot] = 0; + if (!(rt_trace & RT_TRACE_PERF) || pts_state < 0) return; + if (!pts_state) { + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * FRAMES}; + pts_state = c->props.limits.timestampComputeAndGraphics && + vkCreateQueryPool(c->device, &qi, NULL, &pts_pool) == VK_SUCCESS ? 1 : -1; + if (pts_state < 0) return; + } + vkCmdResetQueryPool(cb, pts_pool, 2 * slot, 2); + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pts_pool, 2 * slot); + pts_on[slot] = 1; +} + +static void pts_end(VkCommandBuffer cb, unsigned slot) { + if (pts_on[slot]) vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, pts_pool, 2 * slot + 1); +} + +static void pts_read(VkCtx *c, unsigned slot) { // the slot's fence was waited for: its last present is done + if (!pts_on[slot]) return; + pts_on[slot] = 0; + uint64_t t[2]; + if (vkGetQueryPoolResults(c->device, pts_pool, 2 * slot, 2, sizeof t, t, sizeof t[0], VK_QUERY_RESULT_64_BIT) == VK_SUCCESS) + vk_gtl_note(GTL_PRESENT, pts_sub[slot], t[0], t[1]); +} static VkFence g_fence[FRAMES]; static VkSemaphore g_acquire[FRAMES]; static unsigned g_frame; @@ -907,8 +942,18 @@ static VkImage hudless_record(VkCommandBuffer cb, unsigned slot, int w, int h, c VkImageView pv; uint32_t pw, ph; VkFormat pf; - if (!(vk_fg_active() || ng_running()) || !vk_dlaa_fg_hudless(&pic, &pv, &pw, &ph, &pf) || (int)pw != w || (int)ph != h) + if (!(vk_fg_active() || ng_running()) || !vk_dlaa_fg_hudless(&pic, &pv, &pw, &ph, &pf) || !pw || !ph || w <= 0 || h <= 0) return VK_NULL_HANDLE; + // The upscaler's output can be smaller than the presented picture: the scene is 75% of it in handheld mode + // (widescreen.c, 1440x810 in a 1920x1080 picture) and the game's last passes scale it up with the HUD. Taken only + // at the same size, as before, that frame had no HUD-less picture and was not generated: 5 of 3600 frames generated + // in handheld mode on an RX 580 (2026-11-11). The crop is taken at the same place of the smaller picture. + int prect[4] = {rect[0], rect[1], rect[2], rect[3]}; + if ((int)pw != w || (int)ph != h) { + prect[0] = (int)((int64_t)rect[0] * pw / w), prect[1] = (int)((int64_t)rect[1] * ph / h); + prect[2] = (int)((int64_t)rect[2] * pw / w), prect[3] = (int)((int64_t)rect[3] * ph / h); + if (prect[2] <= 0 || prect[3] <= 0) return VK_NULL_HANDLE; + } VkCtx *c = vk_ctx(); if (g_hl && (g_hl_w != g_sc_ext.width || g_hl_h != g_sc_ext.height || g_hl_fmt != g_sc_fmt)) { vk_queue_lock(), vkDeviceWaitIdle(c->device), vk_queue_unlock(); @@ -945,7 +990,7 @@ static VkImage hudless_record(VkCommandBuffer cb, unsigned slot, int w, int h, c } // the steps' per-slot views of their source: a slot of its own (the presented picture's views stay valid for this // command buffer); with HDR the PQ encode as for the presented float picture - put_picture(cb, slot + FRAMES, pic, pf, w, h, rect, g_hl, to, general); + put_picture(cb, slot + FRAMES, pic, pf, (int)pw, (int)ph, prect, g_hl, to, general); sync_barrier(cb, g_hl, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); @@ -1373,6 +1418,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w rt_log("gfx: present fence wait timed out"); goto out; } + pts_read(c, slot); // the GPU timeline (vk_gtl_note): this slot's last present is done uint32_t idx = 0; // NGX frame generation: the frame is composed into ng.bb and the presenter shows it (ng_show); no acquire here const int ngp = ng_ready(); @@ -1457,6 +1503,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(cb, &bi); + pts_begin(c, cb, slot); uint64_t crumb = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); vk_crumb(cb, VK_CRUMB_PRESENT, crumb); VkImage dst = ngp ? ng.bb.img : g_sc_img[idx]; @@ -1549,6 +1596,7 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w const int via_ng = ng_count >= 0; if (!via_ng) sync_barrier(cb, dst, last, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, last_stage, last_write, VK_PIPELINE_STAGE_2_NONE, 0); vk_crumb(cb, VK_CRUMB_PRESENT_END, crumb); + pts_end(cb, slot); vkEndCommandBuffer(cb); VkSemaphoreSubmitInfo wait = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = g_acquire[slot], @@ -1572,8 +1620,10 @@ static void present_image(VkImage src, VkImageLayout layout, VkFormat fmt, int w if (ng_n > 2) ng_wait_signaled(ng_n - 2); } vk_queue_lock(); + pts_sub[slot] = plat_now_ns(); r = vkQueueSubmit2(c->gfx_queue, 1, &si, g_fence[slot]); vk_queue_unlock(); + if (r != VK_SUCCESS) pts_on[slot] = 0; if (r == VK_SUCCESS && via_ng) { // the presenter shows the generated frames, then this one ng.n = ng_n; ng_hand(idx, ng_count, ng_set, ng_n); diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 77be1d9f..7fd3bdf5 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1175,6 +1175,185 @@ out: free(arrayed), free(need), free(bind), free(org), free(query); } +// Every load of one component of a uniform block vector at a computed component index (OpAccessChain into a Uniform +// block whose last index, into a vector, is not a constant: the translator's c[n][i >> 2][i & 3] for an LDC with a +// register offset) becomes a load of the whole vector, its components extracted with constant indices and the one +// wanted picked with OpSelect. AMD's Windows drivers can read the wrong component through such an access chain, the bug +// Ryujinx works around with HasVectorIndexingBug (VectorComponentSelect). The volumetric fog's per-volume +// extinction reads its colour this way. It did not change the fog's teal blotches on an RX 580 (23.10.01.14), so it is +// off by default. NX2NAVI_VEC_INDEX_FIX: 1 everywhere, auto on AMD's own drivers, unset or 0 off. In memory only. +int vk_caps_vec_index_bug(void); // vk_caps.c + +enum { VX_TYPE_BOOL = 20, VX_TYPE_INT = 21, VX_TYPE_FLOAT = 22, VX_TYPE_VECTOR = 23, VX_TYPE_ARRAY = 28, + VX_TYPE_RTARRAY = 29, VX_TYPE_STRUCT = 30, VX_TYPE_POINTER = 32, VX_CONSTANT = 43, VX_FUNCTION = 54, + VX_VARIABLE = 59, VX_LOAD = 61, VX_ACCESS_CHAIN = 65, VX_IN_BOUNDS_ACCESS_CHAIN = 66, VX_COMPOSITE_EXTRACT = 81, + VX_SELECT = 169, VX_IEQUAL = 170, VX_SC_UNIFORM = 2 }; + +static void patch_vec_index(GfxShader *s) { + static int on = -1; + static volatile int32_t n_logged; + if (on < 0) { + const char *e = cfg_get("NX2NAVI_VEC_INDEX_FIX"); + on = e && *e ? (strcmp(e, "auto") ? *e != '0' : vk_caps_vec_index_bug()) : 0; + if (on) rt_log("gfx_shader: uniform vector components at computed indices are read whole and selected " + "(NX2NAVI_VEC_INDEX_FIX=0 turns it off)"); + } + uint32_t *w = s->spirv; + uint64_t n = s->words; + if (!on || n < 5) return; + uint32_t bound = w[3]; + if (bound > (1u << 22)) return; + uint64_t *def = calloc(bound, sizeof *def); // word index of the instruction defining each id + uint32_t *ptr_v = calloc(bound, sizeof *ptr_v); // per vector type: its Uniform pointer type, if any + uint32_t *target = calloc(bound, sizeof *target); // per access chain id to rewrite: the vector access chain's id + uint32_t *acv = calloc(bound, sizeof *acv); // per access chain id to rewrite: the vector type + if (!def || !ptr_v || !target || !acv) goto out; + uint64_t first_fn = 0; + uint32_t bool_t = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) goto out; + if (op == VX_FUNCTION && !first_fn) first_fn = i; + uint32_t rid = 0; + switch (op) { + // no result id (or none at word 2): debug, annotations, mode setting, stores, barriers, control flow + case 3: case 4: case 5: case 6: case 8: case 10: case 14: case 15: case 16: case 17: case 56: case 62: case 63: + case 71: case 72: case 73: case 74: case 75: case 99: case 224: case 225: case 228: case 246: case 247: case 248: + case 249: case 250: case 251: case 252: case 253: case 254: case 255: case 317: case 330: case 331: case 332: + case 4416: case 5294: case 5295: case 5632: case 5633: break; + case 7: case 11: // OpString, OpExtInstImport: result at word 1 + case VX_TYPE_BOOL: case VX_TYPE_INT: case VX_TYPE_FLOAT: case VX_TYPE_VECTOR: case VX_TYPE_ARRAY: + case VX_TYPE_RTARRAY: case VX_TYPE_STRUCT: case VX_TYPE_POINTER: rid = len >= 2 ? w[i + 1] : 0; break; + default: + if (op >= 19 && op <= 39) rid = len >= 2 ? w[i + 1] : 0; // the other OpType* + else if (len >= 3) rid = w[i + 2]; + } + if (op == VX_TYPE_BOOL && !bool_t) bool_t = w[i + 1]; + if (op == VX_TYPE_POINTER && len == 4 && w[i + 2] == VX_SC_UNIFORM && w[i + 3] < bound && !ptr_v[w[i + 3]]) + ptr_v[w[i + 3]] = w[i + 1]; + if (rid && rid < bound && !def[rid]) def[rid] = i; + i += len; + } + if (!first_fn) goto out; +#define VX_OP(id) ((id) < bound && def[id] ? w[def[id]] & 0xffff : 0) + // the access chains to rewrite and what they need + int found = 0; + uint32_t next = bound; + uint32_t need_ptr[64], need_ptr_v[64], nneed = 0; // new Uniform pointer types: id, vector type + uint32_t cint[8][4] = {{0}}, cint_t[8] = {0}, ncint = 0; // constants 0..3 per index type + for (uint64_t i = first_fn; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if ((op == VX_ACCESS_CHAIN || op == VX_IN_BOUNDS_ACCESS_CHAIN) && len >= 6) { + uint32_t rt = w[i + 1], id = w[i + 2], base = w[i + 3], nidx = len - 4, last = w[i + len - 1]; + uint32_t vt = 0; + if (VX_OP(rt) == VX_TYPE_POINTER && w[def[rt] + 2] == VX_SC_UNIFORM && VX_OP(base) == VX_VARIABLE && + VX_OP(last) != VX_CONSTANT && VX_OP(w[def[base] + 1]) == VX_TYPE_POINTER) { + uint32_t t = w[def[w[def[base] + 1]] + 3]; // the variable's pointee + int ok = 1; + for (uint32_t k = 0; k + 1 < nidx && ok; k++) { + uint32_t top = VX_OP(t), ix = w[i + 4 + k]; + if (top == VX_TYPE_STRUCT) { + if (VX_OP(ix) != VX_CONSTANT) { ok = 0; break; } + uint32_t m = w[def[ix] + 3], slen = w[def[t]] >> 16; + if (2 + m >= slen) { ok = 0; break; } + t = w[def[t] + 2 + m]; + } else if (top == VX_TYPE_ARRAY || top == VX_TYPE_RTARRAY) { + t = w[def[t] + 2]; + } else ok = 0; + } + uint32_t sc = w[def[rt] + 3]; + if (ok && VX_OP(t) == VX_TYPE_VECTOR && w[def[t] + 2] == sc && w[def[t] + 3] >= 2 && w[def[t] + 3] <= 4 && + def[last] && (w[def[last]] >> 16) >= 3) { + uint32_t it = w[def[last] + 1]; // the index's type + if (VX_OP(it) == VX_TYPE_INT) vt = t; + if (vt && !ptr_v[vt]) { + if (nneed == 64) vt = 0; + else need_ptr[nneed] = next++, need_ptr_v[nneed++] = vt, ptr_v[vt] = need_ptr[nneed - 1]; + } + if (vt) { + uint32_t c = 0; + while (c < ncint && cint_t[c] != it) c++; + if (c == ncint) { + if (ncint == 8) vt = 0; + else cint_t[ncint++] = it; + } + } + } + } + if (vt && id < bound) target[id] = next++, acv[id] = vt, found++; + } + i += len; + } + if (!found) goto out; + // constants 0..3 of each index type: existing ones where the module has them + for (uint64_t i = 5; i < first_fn;) { + uint32_t len = w[i] >> 16; + if ((w[i] & 0xffff) == VX_CONSTANT && len == 4 && w[i + 3] < 4) + for (uint32_t c = 0; c < ncint; c++) + if (cint_t[c] == w[i + 1] && !cint[c][w[i + 3]]) cint[c][w[i + 3]] = w[i + 2]; + i += len; + } + uint32_t newc[8][4] = {{0}}; + for (uint32_t c = 0; c < ncint; c++) + for (int v = 0; v < 4; v++) + if (!cint[c][v]) cint[c][v] = newc[c][v] = next++; + uint32_t new_bool = bool_t ? 0 : next++; + if (!bool_t) bool_t = new_bool; + // count the loads to size the output: each becomes 1 + 4 + 3 + 3 instructions of up to 5 words + uint64_t loads = 0; + for (uint64_t i = first_fn; i < n;) { + uint32_t len = w[i] >> 16; + if ((w[i] & 0xffff) == VX_LOAD && len >= 4 && w[i + 3] < bound && target[w[i + 3]]) loads++; + i += len; + } + uint32_t *o = malloc((n + 64 * 4 + 32 * 4 + 4 + (uint64_t)found * 16 + loads * 64) * sizeof *o); + if (!o) goto out; + uint64_t k = 0; + for (uint64_t i = 0; i < 5; i++) o[k++] = w[i]; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, len = w[i] >> 16; + if (i == first_fn) { // the new declarations, last in the global section + for (uint32_t j = 0; j < nneed; j++) + o[k++] = 4u << 16 | VX_TYPE_POINTER, o[k++] = need_ptr[j], o[k++] = VX_SC_UNIFORM, o[k++] = need_ptr_v[j]; + if (new_bool) o[k++] = 2u << 16 | VX_TYPE_BOOL, o[k++] = new_bool; + for (uint32_t c = 0; c < ncint; c++) + for (uint32_t v = 0; v < 4; v++) + if (newc[c][v]) o[k++] = 4u << 16 | VX_CONSTANT, o[k++] = cint_t[c], o[k++] = newc[c][v], o[k++] = v; + } + for (uint32_t j = 0; j < len; j++) o[k++] = w[i + j]; + if ((op == VX_ACCESS_CHAIN || op == VX_IN_BOUNDS_ACCESS_CHAIN) && len >= 6 && w[i + 2] < bound && target[w[i + 2]]) { + // the same chain without its last index: a pointer to the vector + o[k++] = (len - 1) << 16 | op, o[k++] = ptr_v[acv[w[i + 2]]], o[k++] = target[w[i + 2]]; + for (uint32_t j = 3; j < len - 1; j++) o[k++] = w[i + j]; + } else if (op == VX_LOAD && len >= 4 && w[i + 3] < bound && target[w[i + 3]]) { + k -= len; // replaced + uint32_t st = w[i + 1], res = w[i + 2], ac = w[i + 3], vt = acv[ac], nc = w[def[vt] + 3]; + uint32_t idx = w[def[ac] + (w[def[ac]] >> 16) - 1], it = w[def[idx] + 1], c = 0; + while (cint_t[c] != it) c++; + uint32_t vec = next++, e[4], sel = 0; + o[k++] = 4u << 16 | VX_LOAD, o[k++] = vt, o[k++] = vec, o[k++] = target[ac]; + for (uint32_t m = 0; m < nc; m++) + e[m] = next++, o[k++] = 5u << 16 | VX_COMPOSITE_EXTRACT, o[k++] = st, o[k++] = e[m], o[k++] = vec, o[k++] = m; + sel = e[0]; + for (uint32_t m = 1; m < nc; m++) { + uint32_t eq = next++, r = m + 1 == nc ? res : next++; + o[k++] = 5u << 16 | VX_IEQUAL, o[k++] = bool_t, o[k++] = eq, o[k++] = idx, o[k++] = cint[c][m]; + o[k++] = 6u << 16 | VX_SELECT, o[k++] = st, o[k++] = r, o[k++] = eq, o[k++] = e[m], o[k++] = sel; + sel = r; + } + } + i += len; + } + o[3] = next; + free(s->spirv); + s->spirv = o, s->words = k; + if (plat_inc32(&n_logged) <= 4) + rt_log("gfx_shader: %d uniform vector component reads at computed indices now select the component", found); +#undef VX_OP +out: + free(def), free(ptr_v), free(target), free(acv); +} + static void reflect_stage(GfxShader *s) { const char *js = s->reflection; char stage[8]; @@ -1197,6 +1376,7 @@ static void reflect(GfxShader *s) { reflect_spirv(s); reflect_exact(s); patch_txd_transpose(s); + patch_vec_index(s); } // ---- use masks (see the cache key at the top) diff --git a/runtime/host.h b/runtime/host.h index 63fa2b9a..0638fdcb 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -247,6 +247,7 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o int mem_lazy_armed(uint64_t a, uint64_t n); // Copies the ranges src[i] = {address, bytes} of guest memory whose pages read without a fault to dst[i], ok[i] set. int mem_copy_readable(uint8_t *const *dst, const uint64_t (*src)[2], int n, uint8_t *ok); +int mem_lazy_touches(uint64_t a, uint64_t n); // 0: the CPU can read [a, a + n) without a fault now void mem_lazy_window(int open); void mem_lazy_release(uint64_t a, uint64_t n); void mem_lazy_report(uint64_t frame, uint64_t skipped); diff --git a/runtime/input.c b/runtime/input.c index e1547b95..01d5b28e 100644 --- a/runtime/input.c +++ b/runtime/input.c @@ -533,6 +533,50 @@ static void pad_primary(SDL_Gamepad *p, const char *why) { pad_east_a ? "labelled A" : "not labelled A"); } +// Joysticks SDL has no gamepad mapping for are not gamepads to it, and this file opens gamepads only. Known ones get a +// mapping built from their own GUID (so it matches whatever bus, version and name checksum SDL put in it); any other is +// logged once with its name, ids and GUID. +// - hekate's USB HID Joy-Con mode (Nyx "USB Tools" > "Joy-Con mode", bdk/usb/usb_descriptors.c, usb_gadget_hid.c): +// "Nyx Joy-Con", VID 0x11EC, PID 0xA7E1, a plain HID joystick: X, Y (left stick), Z, Rz (right stick) from 0 to 255 +// with up at 0, a hat for the D-pad, and buttons 1-12 B, A, Y, X (south, east, west, north), L, R, ZL, ZR, Minus, +// Plus, left and right stick clicks. Capture and Home are not sent. +static const struct { uint16_t vid, pid; const char *name, *map; } known_joysticks[] = { + {0x11EC, 0xA7E1, "hekate Joy-Con (USB HID)", + "a:b0,b:b1,x:b2,y:b3,leftshoulder:b4,rightshoulder:b5,lefttrigger:b6,righttrigger:b7,back:b8,start:b9,leftstick:b10," + "rightstick:b11,dpup:h0.1,dpright:h0.2,dpdown:h0.4,dpleft:h0.8,leftx:a0,lefty:a1,rightx:a2,righty:a3"}, +}; + +static void map_joysticks(void) { + static SDL_JoystickID seen[32]; + static int nseen; + int n = 0; + SDL_JoystickID *ids = SDL_GetJoysticks(&n); + for (int i = 0; ids && i < n; i++) { + int done = 0; + for (int k = 0; k < nseen && !done; k++) done = seen[k] == ids[i]; + if (done || SDL_IsGamepad(ids[i])) continue; + if (nseen < (int)(sizeof seen / sizeof *seen)) seen[nseen++] = ids[i]; + uint16_t vid = SDL_GetJoystickVendorForID(ids[i]), pid = SDL_GetJoystickProductForID(ids[i]); + char guid[64]; + SDL_GUIDToString(SDL_GetJoystickGUIDForID(ids[i]), guid, (int)sizeof guid); + const char *jname = SDL_GetJoystickNameForID(ids[i]); + int mapped = 0; + for (size_t k = 0; k < sizeof known_joysticks / sizeof *known_joysticks; k++) { + if (known_joysticks[k].vid != vid || known_joysticks[k].pid != pid) continue; + char m[512]; + snprintf(m, sizeof m, "%s,%s,%s", guid, known_joysticks[k].name, known_joysticks[k].map); + int r = SDL_AddGamepadMapping(m); + rt_log("input: joystick \"%s\" (%04x:%04x, GUID %s) mapped as a gamepad (%s)%s%s", jname ? jname : "?", vid, pid, + guid, known_joysticks[k].name, r < 0 ? ": mapping refused, " : "", r < 0 ? SDL_GetError() : ""); + mapped = 1; + } + if (!mapped) + rt_log("input: joystick \"%s\" (%04x:%04x, GUID %s) has no gamepad mapping in SDL: not used (an SDL mapping " + "in the SDL_GAMECONTROLLERCONFIG environment variable adds one)", jname ? jname : "?", vid, pid, guid); + } + SDL_free(ids); +} + // Under lock: closes the gamepads that went away and opens the ones that came; the primary falls back to the first // open one when it left. Logs each change. // The game is always told one controller is connected, so a disconnect read as nothing pressed and play went on @@ -540,6 +584,7 @@ static void pad_primary(SDL_Gamepad *p, const char *why) { // its place, the menu opens: the player sees it, and it takes the keyboard and mouse until it is closed. static void scan_pads(void) { int had = pad != NULL; + map_joysticks(); for (int i = 0; i < npads;) { if (SDL_GamepadConnected(pads[i])) { i++; continue; } rt_log("input: gamepad \"%s\" disconnected", SDL_GetGamepadName(pads[i]) ? SDL_GetGamepadName(pads[i]) : "?"); diff --git a/runtime/kernel.c b/runtime/kernel.c index 003e76e0..990813de 100644 --- a/runtime/kernel.c +++ b/runtime/kernel.c @@ -2372,7 +2372,66 @@ HOST_OVERRIDE(KERN_SYM_GET_CURRENT_FIBER, kern_fiber_current); // NX2NAVI_SVC_SLOW_MS (default 0, off): logs each kernel service that blocks longer than that, with the service number. // A stalled frame shows up as time in the replay; this says whether the guest was waiting in a service, and which one, // instead of leaving it to a sampler to find. +// NX2NAVI_SVC_WAITS=1 (diagnosis, default 0): the guest main thread's time in each kernel service, per service and the +// guest call chain that made it (the lr and up to three callers from the guest's frame records), logged every 10 s as +// milliseconds per second with the chain named (ldr_describe: the SDK's symbols). What the game's main thread waits for +// between frames: standing in Hyrule Field on an RX 580 it spent about a fifth of its time blocked in one service call. +#define SW_N 256 +#define SW_DEPTH 4 +static struct { uint64_t key, lr[SW_DEPTH], ns, n; uint32_t imm; } svw[SW_N]; +static uint64_t svw_t0; + +static void svc_wait_note(const Cpu *c, uint32_t imm, uint64_t ns) { + uint64_t lr[SW_DEPTH] = {c->x[30]}, fp = c->x[29], key = imm * 0x9E3779B97F4A7C15ull ^ c->x[30]; + for (int d = 1; d < SW_DEPTH && fp && !(fp & 7) && mem_readable(fp, 16); d++) { + uint64_t next = *(const volatile uint64_t *)(uintptr_t)fp, ret = *(const volatile uint64_t *)(uintptr_t)(fp + 8); + lr[d] = ret, key = (key ^ ret) * 0x100000001B3ull; + if (next <= fp) break; + fp = next; + } + if (!key) key = 1; + for (uint32_t k = 0; k < 32; k++) { + typeof(svw[0]) *e = &svw[(key + k) % SW_N]; + if (e->key && e->key != key) continue; + if (!e->key) e->key = key, e->imm = imm, memcpy(e->lr, lr, sizeof lr); + e->ns += ns, e->n++; + break; + } + uint64_t now = kern_now_ns(); + if (!svw_t0) svw_t0 = now; + if (now - svw_t0 < 10000000000ull) return; + double secs = (now - svw_t0) / 1e9; + svw_t0 = now; + for (int top = 0; top < 6; top++) { + int best = -1; + for (int i = 0; i < SW_N; i++) + if (svw[i].key && svw[i].ns && (best < 0 || svw[i].ns > svw[best].ns)) best = i; + if (best < 0) break; + char d[SW_DEPTH][96]; + for (int k = 0; k < SW_DEPTH; k++) { + if (svw[best].lr[k]) ldr_describe(svw[best].lr[k], d[k], sizeof d[k]); + else d[k][0] = 0; + } + rt_log("kernel: main thread in service %#x: %.1f ms/s over %llu calls (%.2f ms each) from %s < %s < %s < %s", + svw[best].imm, svw[best].ns / 1e6 / secs, (unsigned long long)svw[best].n, svw[best].ns / 1e6 / (double)svw[best].n, + d[0], d[1], d[2], d[3]); + svw[best].ns = 0; // listed: out of this round + } + memset(svw, 0, sizeof svw); +} + int kern_svc(Cpu *c, uint32_t imm) { + static int waits = -1; + if (waits < 0) { // no settings row: cfg_bool only on a set value (an unset key is fatal there) + const char *v = cfg_get("NX2NAVI_SVC_WAITS"); + waits = v && *v && cfg_bool("NX2NAVI_SVC_WAITS"); + } + if (waits && plat_thread_id() == prof_main_tid) { + uint64_t w0 = kern_now_ns(); + int r = kern_svc_inner(c, imm); + svc_wait_note(c, imm, kern_now_ns() - w0); + return r; + } static double lim = -1; if (lim < 0) lim = strtod(cfg_value("NX2NAVI_SVC_SLOW_MS"), NULL); if (lim <= 0) return kern_svc_inner(c, imm); diff --git a/runtime/main.c b/runtime/main.c index 9e376322..a6e67e07 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -291,6 +291,9 @@ static void crash_handler(const PlatCrash *x) { if (foreign_hook(x->module)) rt_logf(" %s is not part of the game: a recording, overlay or capture program put it into the game. Close that " "program or turn off its Vulkan capture, then start the game again\n", x->module); + // what the thread was doing (a pipeline build names its shaders): the stderr note crash_veh writes misses the log + const char *note = plat_crash_context_get(); + if (note) rt_logf(" the thread was %s\n", note); uint64_t f = x->module ? 0 : lifted_function(rip); if (f) rt_logf(" in lifted function f_%llx = %s\n", (unsigned long long)f, ldr_describe(f, d, sizeof d)); // abort and invalid parameter: where the host code called it; a fault in another module: the host calls that led diff --git a/runtime/memory.c b/runtime/memory.c index fa010e77..6ce3cf29 100644 --- a/runtime/memory.c +++ b/runtime/memory.c @@ -106,6 +106,12 @@ static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by m static int frz_async = 1; // the setting, read in mem_init typedef struct { uint8_t n; uint16_t run[16][2]; } FrzRuns; // pages of the region, [first, end) static FrzRuns *frz_runs[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock (written at a freeze): NULL or n 0 = not kept +// vm_lock, a leaf lock held around every write-watch query (written_pages) and around the page calls of a freeze and a +// thaw: on Windows each kind takes the process's address-space lock, and with both running at once a game thread's +// thaw waited about 16 ms per page call behind the replay's queries, 100-300 ms for a thaw of several runs (a frame of +// that length about every second standing in Hyrule Field on an RX 580). Taken in turns here, a thaw waits for one +// query at most. +static PlatMutex vm_lock = PLAT_MUTEX_INIT; static volatile int64_t frz_async_n; // thaws done outside the lock, since the start static volatile int64_t frz_protect_fails; // VirtualProtect failures of those thaws, since the start @@ -168,9 +174,16 @@ static void unprobe_locked(uint64_t r) { // has the part's writes. Only a fault of the replay thread itself, a wait of 20 ms, or a release by a thaw goes ahead // inside the window (lz_gaps counts the faults): that part's writes to the range then reach guest memory with the // range's next write-back. +// The list of armed segments has a lock of its own, lz_lock, held only for a few steps over the list and never across a +// system call or a fetch: the replay asks it at every draw that writes a mirrored pool (mem_lazy_skip) and at the end of +// every part (mem_lazy_armed, mem_lazy_window). Under frz_lock it waited for freezes, thaws and arms on other threads, +// whose page queries and protection changes take that lock for tens to hundreds of microseconds: 8% of the replay +// thread's samples standing in Hyrule Field on an RX 580 (0.7.1, lazy write-back on by default). A change of the list +// holds both locks (frz_lock first); a read holds either. typedef struct { uint64_t lo, hi, h0, h1; int64_t delta; uint8_t stale; } LzSeg; // pages [lo, hi); buffer offset = address + delta -static LzSeg *lzs; // frz_lock -static int lz_n, lz_cap; // frz_lock +static PlatMutex lz_lock = PLAT_MUTEX_INIT; // inside frz_lock, never the other way round +static LzSeg *lzs; // changed under frz_lock and lz_lock, read under either +static int lz_n, lz_cap; // the same static volatile uint8_t lzr[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // a segment reaches into the region static uint32_t lz_hot[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: plat_now_ms() | 1 until which nothing is armed here static uint8_t lz_back[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: seconds of the region's last hold @@ -222,7 +235,7 @@ static struct { uint64_t addr, lo, hi, out; uint32_t tid; uint8_t write, stale, static int lz_nnote; // frz_lock: since the last report: faults by a read, by a write, other releases, fetches, their bytes and time static uint64_t lz_faults[2], lz_other, lz_fetches, lz_fetch_bytes, lz_fetch_ns, lz_gaps; -static volatile uint32_t lz_win_tid; // the replay thread's id while its part is between check and hand-over; 0 else +static volatile uint32_t lz_win_tid; // lz_lock: the replay thread's id while its part is between check and hand-over // For a fatal error's summary line (mem_lazy_note): written under frz_lock and read without it, since the thread that // fails may be the one inside a fetch. static volatile uint64_t lz_fetch_total; // fetches since the start @@ -239,11 +252,28 @@ static void lazy_flags_locked(uint64_t lo, uint64_t hi) { } } +// frz_lock and lz_lock held: segment i leaves the list, with the stale mark mem_lazy_skip may have set until now. A +// later mem_lazy_skip no longer finds it and writes its range back as ever. +static LzSeg lazy_take_locked(int i) { + LzSeg s = lzs[i]; + lzs[i] = lzs[--lz_n]; + return s; +} + +static void lazy_gone_locked(LzSeg s, int why); + // frz_lock held exclusively. Segment i goes: its bytes come down first when the mirror's are newer, then its pages // are read-write again. why: 0 a read fault, 1 a write fault (both hold the regions back), else a thaw or the mirror // going away. static void lazy_drop_locked(int i, int why) { - LzSeg s = lzs[i]; + plat_mutex_lock(&lz_lock); + LzSeg s = lazy_take_locked(i); + plat_mutex_unlock(&lz_lock); + lazy_gone_locked(s, why); +} + +// frz_lock held exclusively: the pages of s, out of the list already, get their bytes and their access back. +static void lazy_gone_locked(LzSeg s, int why) { if (s.stale && lz_fetch) { uint64_t t0 = plat_now_ns(); __atomic_store_n(&lz_fetch_now, s.hi - s.lo, __ATOMIC_RELAXED); @@ -258,7 +288,6 @@ static void lazy_drop_locked(int i, int why) { if (plat_mem_state(a, s.hi, &b) == PLAT_PROT_NONE && !plat_mem_protect(a, b - a, PLAT_PROT_RW)) rt_fatal("memory: giving back the lazily written pages %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a)); - lzs[i] = lzs[--lz_n]; lazy_flags_locked(s.lo, s.hi); if (why > 1) { lz_other++; return; } lz_faults[why]++; @@ -428,6 +457,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t } uint64_t slo = x, shi = e; uint8_t stale = 0; + plat_mutex_lock(&lz_lock); for (int j = 0; j < lz_n; j++) // one segment for what touches in the same buffer if (lzs[j].h0 == h0 && lzs[j].delta == delta && lzs[j].lo <= shi && lzs[j].hi >= slo) { if (lzs[j].lo < slo) slo = lzs[j].lo; @@ -437,6 +467,7 @@ uint64_t mem_lazy_arm(uint64_t a, uint64_t n, uint64_t h0, uint64_t h1, uint64_t j = -1; } lazy_push_locked((LzSeg){slo, shi, h0, h1, delta, stale}); + plat_mutex_unlock(&lz_lock); __atomic_store_n(&lzr[r], 1, __ATOMIC_RELEASE); armed += e - x; } @@ -455,13 +486,31 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o int any = 0, k = 0; for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = lzr[r]; if (!any) return 0; - plat_mutex_lock(&frz_lock); - for (int i = lz_n; i-- > 0;) - if (lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta)) { - lzs[i].stale = 0; - lz_foreign++; - lazy_drop_locked(i, 4); + // A segment armed for another buffer goes back under frz_lock (lazy_gone_locked), with arms held off until the + // rest is marked, as before lz_lock; that is rare, and the replay's per-draw check holds lz_lock only. + int foreign = 0; + plat_mutex_lock(&lz_lock); + for (int i = 0; i < lz_n && !foreign; i++) + foreign = lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta); + if (foreign) { + plat_mutex_unlock(&lz_lock); + plat_mutex_lock(&frz_lock); + for (;;) { + plat_mutex_lock(&lz_lock); + int i = lz_n; + while (i-- > 0 && !(lzs[i].lo < a + n && lzs[i].hi > a && (lzs[i].h0 != h0 || lzs[i].delta != delta))) {} + LzSeg s; + if (i >= 0) { + lzs[i].stale = 0; + lz_foreign++; + s = lazy_take_locked(i); + } + plat_mutex_unlock(&lz_lock); + if (i < 0) break; + lazy_gone_locked(s, 4); } + plat_mutex_lock(&lz_lock); + } for (int i = 0; i < lz_n && k < max; i++) { uint64_t lo = lzs[i].lo > a ? lzs[i].lo : a, hi = lzs[i].hi < a + n ? lzs[i].hi : a + n; if (lo >= hi) continue; @@ -470,16 +519,33 @@ int mem_lazy_skip(uint64_t a, uint64_t n, uint64_t h0, int64_t delta, uint64_t o for (; at > 0 && out[at - 1][0] > lo; at--) out[at][0] = out[at - 1][0], out[at][1] = out[at - 1][1]; out[at][0] = lo, out[at][1] = hi; } - plat_mutex_unlock(&frz_lock); + plat_mutex_unlock(&lz_lock); + if (foreign) plat_mutex_unlock(&frz_lock); return k; } +// 0 when no page of [a, a + n) can be inaccessible right now (armed by the lazy write-back or probed), so the CPU can +// read it without a fault; 1 when one may be. +int mem_lazy_touches(uint64_t a, uint64_t n) { + if (!n) return 0; + if (a >= RT_ADDRESS_LIMIT || a + n > RT_ADDRESS_LIMIT || __atomic_load_n(&prb_on, __ATOMIC_RELAXED)) return 1; + if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; + int any = 0; + for (uint64_t r = a >> FRZ_SHIFT; r <= (a + n - 1) >> FRZ_SHIFT && !any; r++) any = __atomic_load_n(&lzr[r], __ATOMIC_ACQUIRE); + if (!any) return 0; + int hit = 0; + plat_mutex_lock(&lz_lock); + for (int i = 0; i < lz_n && !hit; i++) hit = lzs[i].lo < a + n && lzs[i].hi > a; + plat_mutex_unlock(&lz_lock); + return hit; +} + // [a, a + n) is armed as a whole still. int mem_lazy_armed(uint64_t a, uint64_t n) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return 0; - plat_mutex_lock(&frz_lock); + plat_mutex_lock(&lz_lock); int ok = lazy_covered_locked(a, a + n); - plat_mutex_unlock(&frz_lock); + plat_mutex_unlock(&lz_lock); return ok; } @@ -530,16 +596,17 @@ int mem_lazy_enable(void (*fetch)(uint64_t h0, uint64_t h1, uint64_t off, uint64 return 1; } -// Opens (under frz_lock, so no release is half done) or closes the window of the replay's part. +// Opens (under lz_lock: a fault's segment is out of the list before it, or the fault sees it) or closes the window of +// the replay's part. void mem_lazy_window(int open) { if (!__atomic_load_n(&lz_on, __ATOMIC_RELAXED)) return; if (!open) { __atomic_store_n(&lz_win_tid, 0, __ATOMIC_RELEASE); return; } - plat_mutex_lock(&frz_lock); + plat_mutex_lock(&lz_lock); // a fault takes its segment out before this or sees the window __atomic_store_n(&lz_win_tid, plat_thread_id(), __ATOMIC_RELEASE); - plat_mutex_unlock(&frz_lock); + plat_mutex_unlock(&lz_lock); } static int lazy_fault(uint64_t a, int write) { @@ -547,14 +614,17 @@ static int lazy_fault(uint64_t a, int write) { uint32_t me = plat_thread_id(); for (int tries = 0;; tries++) { plat_mutex_lock(&frz_lock); + plat_mutex_lock(&lz_lock); // the window's test and the segment's leaving the list in one step int i = 0; while (i < lz_n && !(a >= lzs[i].lo && a < lzs[i].hi)) i++; if (i == lz_n) { + plat_mutex_unlock(&lz_lock); plat_mutex_unlock(&frz_lock); return 0; } uint32_t w = __atomic_load_n(&lz_win_tid, __ATOMIC_ACQUIRE); if (w && w != me && tries < 400) { // up to 20 ms: after the hand-over the fetch has that part's writes + plat_mutex_unlock(&lz_lock); plat_mutex_unlock(&frz_lock); plat_wait_ns(50000); continue; @@ -575,11 +645,14 @@ static int lazy_fault(uint64_t a, int write) { } } if (reader) lz_heavy_ms[r] = now | 1; - if (lz_chunk && !reader) { + if (lz_chunk && !reader) { // the piece leaves the list and the rest goes back into it under lz_lock out = lazy_piece_locked(i, a, write != 0); + plat_mutex_unlock(&lz_lock); if (k >= 0) lz_note[k].hold = lazy_hot_locked(a >> LZ_CHUNK_SHIFT, 1)->back; - } else { - lazy_drop_locked(i, write != 0); + } else { // the segment leaves the list under lz_lock; its pages come back without it + LzSeg s = lazy_take_locked(i); + plat_mutex_unlock(&lz_lock); + lazy_gone_locked(s, write != 0); if (k >= 0) lz_note[k].hold = lz_back[r]; } if (k >= 0) lz_note[k].out = out; @@ -683,7 +756,15 @@ void mem_lazy_report(uint64_t frame, uint64_t skipped) { // was read-write at the freeze, and mapping changes thaw or wait before they run, so these are all ours). strict: under // frz_lock, a failure ends the process; outside it (a thaw in flight, FRZ_THAWING) the pages are what the freeze left, // and a failure is only counted. +static void thaw_pages_vm(uint64_t r, int strict); + static void thaw_pages(uint64_t r, int strict) { + plat_mutex_lock(&vm_lock); + thaw_pages_vm(r, strict); + plat_mutex_unlock(&vm_lock); +} + +static void thaw_pages_vm(uint64_t r, int strict) { // vm_lock held uint64_t base = r << FRZ_SHIFT; const FrzRuns *k = frz_runs[r]; if (k && k->n) { @@ -879,19 +960,22 @@ void mem_thaw(uint64_t a, uint64_t n) { // reserved holes (heap pages given back) stay as they are, and thaw_locked finds the read-only runs again. static int freeze_locked(uint64_t r) { uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, run[16][2]; - int n = 0; - for (uint64_t x = a, y; x < end; x = y) { + int n = 0, ok = 1; + plat_mutex_lock(&vm_lock); + for (uint64_t x = a, y; x < end && ok; x = y) { int st = plat_mem_state(x, end, &y); if (st >= PLAT_PROT_NONE) { - if (st != PLAT_PROT_RW || n == 16) return 0; - run[n][0] = x, run[n][1] = y, n++; - } else if (st != PLAT_MEM_RESERVED) return 0; + if (st != PLAT_PROT_RW || n == 16) ok = 0; + else run[n][0] = x, run[n][1] = y, n++; + } else if (st != PLAT_MEM_RESERVED) ok = 0; } - for (int i = 0; i < n; i++) + for (int i = 0; ok && i < n; i++) if (!plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_READ)) { while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW); - return 0; + ok = 0; } + plat_mutex_unlock(&vm_lock); + if (!ok) return 0; // the runs, for a thaw without VirtualQuery (thaw_pages); kept per region from its first freeze on if (n && !frz_runs[r]) frz_runs[r] = calloc(1, sizeof *frz_runs[r]); if (frz_runs[r]) { @@ -1192,7 +1276,10 @@ int mem_start_note(int probe) { static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, int reset) { if (!max || !size) return 0; if (!watched(addr, size)) return UINT64_MAX; - return plat_mem_written(addr, size, pages, max, reset); + plat_mutex_lock(&vm_lock); + uint64_t n = plat_mem_written(addr, size, pages, max, reset); + plat_mutex_unlock(&vm_lock); + return n; } uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) { diff --git a/runtime/mesh_emu_spirv.c b/runtime/mesh_emu_spirv.c new file mode 100644 index 00000000..114d8292 --- /dev/null +++ b/runtime/mesh_emu_spirv.c @@ -0,0 +1,1097 @@ +// Mesh shader emulation: SPIR-V rewriting (mesh_emu_spirv.h). The input is what tools/xlate/MeshSpirv.cs makes: the +// translator's compute main, called from an entry function that copies its shared-memory stage buffer into the mesh +// outputs after SetMeshOutputsEXT (mesh), or into the task payload before EmitMeshTasksEXT (task). +// +// Mesh -> GLCompute: the output arrays and the task payload move to Workgroup memory; SetMeshOutputsEXT is dropped (its +// counts are kept); at the entry function's end thread 0 reserves vertex slots and indices in the arena with atomics, +// and every thread copies vertices and triangles out (indices made absolute vertex slots, relative to the vertex area). +// With a task stage mesh workgroup (x, y) is mesh group x of task record (task base + y): a prologue returns when x is +// past the record's count, loads the payload, and the module's WorkgroupId reads become (x, 0, 0). +// Task -> GLCompute: the payload moves to Workgroup memory; EmitMeshTasksEXT becomes a store of the payload and the +// mesh group count into the task record, and workgroup 0 writes the mesh dispatch {max_mesh, task workgroups, 1} +// into the draw record. +#include "mesh_emu_spirv.h" + +#include +#include +#include + +enum { + OpSourceContinued = 2, OpSource = 3, OpSourceExtension = 4, OpName = 5, OpMemberName = 6, OpString = 7, OpLine = 8, + OpExtension = 10, OpExtInstImport = 11, OpExtInst = 12, OpMemoryModel = 14, OpEntryPoint = 15, OpExecutionMode = 16, + OpCapability = 17, OpTypeVoid = 19, OpTypeBool = 20, OpTypeInt = 21, OpTypeFloat = 22, OpTypeVector = 23, + OpTypeArray = 28, OpTypeRuntimeArray = 29, OpTypeStruct = 30, OpTypePointer = 32, OpTypeFunction = 33, + OpConstant = 43, OpConstantComposite = 44, OpFunction = 54, OpFunctionEnd = 56, OpVariable = 59, OpLoad = 61, + OpStore = 62, OpAccessChain = 65, OpInBoundsAccessChain = 66, OpDecorate = 71, OpMemberDecorate = 72, + OpCompositeConstruct = 80, OpCompositeExtract = 81, OpUConvert = 113, OpConvertUToPtr = 120, OpBitcast = 124, + OpIAdd = 128, OpIMul = 132, OpLogicalAnd = 167, OpSelect = 169, OpIEqual = 170, OpINotEqual = 171, + OpULessThan = 176, OpULessThanEqual = 178, OpControlBarrier = 224, OpAtomicIAdd = 234, OpAtomicUMax = 239, + OpSelectionMerge = 247, OpLabel = 248, OpBranch = 249, OpBranchConditional = 250, OpReturn = 253, + OpNoLine = 317, OpModuleProcessed = 330, OpExecutionModeId = 331, OpConstantNull = 46, OpCopyObject = 83, + OpEmitMeshTasksEXT = 5294, OpSetMeshOutputsEXT = 5295, +}; +enum { ScInput = 1, ScOutput = 3, ScWorkgroup = 4, ScPrivate = 6, ScFunction = 7, ScPushConstant = 9, + ScPhysicalStorageBuffer = 5349, ScTaskPayload = 5402 }; +enum { DecBlock = 2, DecArrayStride = 6, DecBuiltIn = 11, DecLocation = 30, DecOffset = 35, DecPerPrimitive = 5271 }; +enum { BiPosition = 0, BiPointSize = 1, BiPrimPoint = 5294, BiVertexIndex = 42, BiNumWorkgroups = 24, BiWorkgroupId = 26, BiLocalInvocationIndex = 29, + BiPrimTri = 5296 }; +enum { CapShader = 1, CapInt64 = 11, CapMeshShading = 5283, CapPSBAddresses = 5347 }; +enum { ModelVertex = 0, ModelGLCompute = 5, ModelTask = 5364, ModelMesh = 5365 }; +enum { ModeLocalSize = 17, ModeOutputVertices = 26, ModeOutputPoints = 27, ModeOutputLines = 5269, + ModeOutputPrimitives = 5270, ModeOutputTriangles = 5298 }; +enum { GlslUMin = 38 }; +enum { MaxIns = 4096 }; + +typedef struct { uint32_t *w; size_t n, cap; int bad; } Buf; + +static void put(Buf *b, uint32_t x) { + if (b->n == b->cap) { + size_t c = b->cap ? b->cap * 2 : 1024; + uint32_t *w = realloc(b->w, c * sizeof *w); + if (!w) { b->bad = 1; return; } + b->w = w, b->cap = c; + } + b->w[b->n++] = x; +} +static void putn(Buf *b, const uint32_t *w, size_t n) { for (size_t i = 0; i < n; i++) put(b, w[i]); } +static void op(Buf *b, uint32_t opc, int n, ...) { + va_list ap; + va_start(ap, n); + put(b, (uint32_t)(n + 1) << 16 | opc); + for (int i = 0; i < n; i++) put(b, va_arg(ap, uint32_t)); + va_end(ap); +} +static void op_str(Buf *b, uint32_t opc, const uint32_t *pre, int npre, const char *s, const uint32_t *post, int npost) { + size_t sl = strlen(s) + 1, sw = (sl + 3) / 4; + put(b, (uint32_t)(1 + npre + sw + npost) << 16 | opc); + for (int i = 0; i < npre; i++) put(b, pre[i]); + for (size_t i = 0; i < sw; i++) { + uint32_t x = 0; + for (size_t k = 0; k < 4 && i * 4 + k < sl; k++) x |= (uint32_t)(uint8_t)s[i * 4 + k] << (8 * k); + put(b, x); + } + for (int i = 0; i < npost; i++) put(b, post[i]); +} + +// ---- the module +typedef struct { + const uint32_t *w; + size_t n; + uint32_t bound; + uint32_t t_void, t_bool, t_u32, t_u64, t_f32, t_v3u, t_v4u, t_v4f; + uint32_t glsl, ep_fn, model, wgid, lii, nwg; + int has_int64, has_psb, has_psb_ext; + struct { uint32_t id, sc, to; } ptr[4096]; + int nptr; + struct { uint32_t id, type, sc; } var[2048]; + int nvar; + struct { uint32_t id, type, val; } cst[8192]; + int ncst; +} Mod; + +static uint32_t find_ptr(const Mod *m, uint32_t sc, uint32_t to) { + for (int i = 0; i < m->nptr; i++) + if (m->ptr[i].sc == sc && m->ptr[i].to == to) return m->ptr[i].id; + return 0; +} +static uint32_t ptr_to(const Mod *m, uint32_t id) { + for (int i = 0; i < m->nptr; i++) + if (m->ptr[i].id == id) return m->ptr[i].to; + return 0; +} +static const uint32_t *find_type(const Mod *m, uint32_t id) { + for (size_t i = 5; i < m->n;) { + uint32_t o = m->w[i] & 0xffff, len = m->w[i] >> 16; + if (!len || o == OpFunction) break; + if ((o == OpTypeArray || o == OpTypeRuntimeArray || o == OpTypeVector || o == OpTypeStruct) && m->w[i + 1] == id) + return m->w + i; + i += len; + } + return NULL; +} +static int const_u32(const Mod *m, uint32_t id, uint32_t *v) { + for (int i = 0; i < m->ncst; i++) + if (m->cst[i].id == id && m->cst[i].type == m->t_u32) return *v = m->cst[i].val, 1; + return 0; +} +static int is_debug(uint32_t o) { + return o == OpSourceContinued || o == OpSource || o == OpSourceExtension || o == OpName || o == OpMemberName || + o == OpString || o == OpModuleProcessed; +} +static int is_annot(uint32_t o) { return o == OpDecorate || o == OpMemberDecorate || o == 73 || o == 74 || o == 75 || o == 332 || o == 5632 || o == 5633; } + +static int parse(Mod *m, const uint32_t *w, size_t n, const char **why) { + m->w = w, m->n = n; + if (n < 5 || w[0] != 0x07230203) return *why = "not SPIR-V", 0; + m->bound = w[3]; + for (size_t i = 5; i < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + if (!len || i + len > n) return *why = "malformed", 0; + if (len > MaxIns) return *why = "an instruction too long", 0; + const uint32_t *a = w + i + 1; + if (o == OpFunction) break; + switch (o) { + case OpCapability: + if (a[0] == CapInt64) m->has_int64 = 1; + if (a[0] == CapPSBAddresses) m->has_psb = 1; + break; + case OpExtension: if (!strcmp((const char *)a, "SPV_KHR_physical_storage_buffer")) m->has_psb_ext = 1; break; + case OpExtInstImport: if (!strcmp((const char *)(a + 1), "GLSL.std.450")) m->glsl = a[0]; break; + case OpEntryPoint: + if (m->ep_fn) return *why = "several entry points", 0; + m->model = a[0], m->ep_fn = a[1]; + break; + case OpTypeVoid: if (!m->t_void) m->t_void = a[0]; break; + case OpTypeBool: if (!m->t_bool) m->t_bool = a[0]; break; + case OpTypeInt: + if (a[1] == 32 && !a[2] && !m->t_u32) m->t_u32 = a[0]; + if (a[1] == 64 && !a[2] && !m->t_u64) m->t_u64 = a[0]; + break; + case OpTypeFloat: if (a[1] == 32 && !m->t_f32) m->t_f32 = a[0]; break; + case OpTypeVector: + if (a[1] == m->t_u32 && a[2] == 3 && !m->t_v3u) m->t_v3u = a[0]; + if (a[1] == m->t_u32 && a[2] == 4 && !m->t_v4u) m->t_v4u = a[0]; + if (a[1] == m->t_f32 && m->t_f32 && a[2] == 4 && !m->t_v4f) m->t_v4f = a[0]; + break; + case OpTypePointer: + if (m->nptr == 4096) return *why = "too many pointer types", 0; + m->ptr[m->nptr].id = a[0], m->ptr[m->nptr].sc = a[1], m->ptr[m->nptr++].to = a[2]; + break; + case OpVariable: + if (m->nvar == 2048) return *why = "too many variables", 0; + m->var[m->nvar].id = a[1], m->var[m->nvar].type = a[0], m->var[m->nvar++].sc = a[2]; + break; + case OpConstant: + if (len == 4 && m->ncst < 8192) m->cst[m->ncst].id = a[1], m->cst[m->ncst].type = a[0], m->cst[m->ncst++].val = a[2]; + break; + case OpDecorate: + if (len >= 4 && a[1] == DecBuiltIn) { + if (a[2] == BiWorkgroupId) m->wgid = a[0]; + if (a[2] == BiLocalInvocationIndex) m->lii = a[0]; + if (a[2] == BiNumWorkgroups) m->nwg = a[0]; + } + break; + } + i += len; + } + if (!m->ep_fn || (m->model != ModelMesh && m->model != ModelTask)) return *why = "not a task or mesh module", 0; + if (!m->t_u32 || !m->t_void || !m->glsl || !m->lii || !m->wgid) return *why = "missing basic types or built-ins", 0; + return 1; +} + +// ---- emission over new ids +typedef struct { + Mod *m; + Buf *types, *annot, *code; + uint32_t p_u32_psb, p_v4u_psb, p_words, p_slots, p_pc_u64, p_pc_u32, pc_var, c64_16; + struct { uint32_t id, val; } cc[256]; + int ncc; +} Em; + +static uint32_t nid(Mod *m) { return m->bound++; } +static uint32_t cu(Em *e, uint32_t v) { + for (int i = 0; i < e->m->ncst; i++) + if (e->m->cst[i].type == e->m->t_u32 && e->m->cst[i].val == v) return e->m->cst[i].id; + for (int i = 0; i < e->ncc; i++) + if (e->cc[i].val == v) return e->cc[i].id; + uint32_t id = nid(e->m); + op(e->types, OpConstant, 3, e->m->t_u32, id, v); + if (e->ncc < 256) e->cc[e->ncc].id = id, e->cc[e->ncc++].val = v; + return id; +} +static uint32_t E1(Em *e, uint32_t o, uint32_t t, uint32_t a) { uint32_t r = nid(e->m); op(e->code, o, 3, t, r, a); return r; } +static uint32_t E2(Em *e, uint32_t o, uint32_t t, uint32_t a, uint32_t b) { uint32_t r = nid(e->m); op(e->code, o, 4, t, r, a, b); return r; } +static uint32_t E3(Em *e, uint32_t o, uint32_t t, uint32_t a, uint32_t b, uint32_t c) { uint32_t r = nid(e->m); op(e->code, o, 5, t, r, a, b, c); return r; } +static uint32_t add(Em *e, uint32_t a, uint32_t b) { return E2(e, OpIAdd, e->m->t_u32, a, b); } +static uint32_t mul(Em *e, uint32_t a, uint32_t b) { return E2(e, OpIMul, e->m->t_u32, a, b); } +static uint32_t ult(Em *e, uint32_t a, uint32_t b) { return E2(e, OpULessThan, e->m->t_bool, a, b); } +static uint32_t ld(Em *e, uint32_t t, uint32_t p) { return E1(e, OpLoad, t, p); } +static uint32_t pc_u64(Em *e) { return ld(e, e->m->t_u64, E2(e, OpAccessChain, e->p_pc_u64, e->pc_var, cu(e, 0))); } +static uint32_t pc_u32(Em *e, uint32_t k) { return ld(e, e->m->t_u32, E2(e, OpAccessChain, e->p_pc_u32, e->pc_var, cu(e, k))); } +static uint32_t words_base(Em *e) { return E1(e, OpConvertUToPtr, e->p_words, pc_u64(e)); } +static uint32_t word_ptr(Em *e, uint32_t base, uint32_t idx) { return E3(e, OpAccessChain, e->p_u32_psb, base, cu(e, 0), idx); } +static uint32_t word_load(Em *e, uint32_t base, uint32_t idx) { + uint32_t p = word_ptr(e, base, idx), r = nid(e->m); + op(e->code, OpLoad, 5, e->m->t_u32, r, p, 2u, 4u); // Aligned 4 + return r; +} +static void word_store(Em *e, uint32_t base, uint32_t idx, uint32_t v) { op(e->code, OpStore, 4, word_ptr(e, base, idx), v, 2u, 4u); } +static uint32_t atomic(Em *e, uint32_t o, uint32_t base, uint32_t idx, uint32_t v) { + uint32_t p = word_ptr(e, base, idx), r = nid(e->m); + op(e->code, o, 6, e->m->t_u32, r, p, cu(e, 1), cu(e, 0), v); // Device scope, relaxed + return r; +} +static void barrier(Em *e) { op(e->code, OpControlBarrier, 3, cu(e, 2), cu(e, 2), cu(e, 264)); } // Workgroup, AcqRel|WorkgroupMemory +static void if_begin(Em *e, uint32_t cond, uint32_t *merge) { + uint32_t t = nid(e->m); + *merge = nid(e->m); + op(e->code, OpSelectionMerge, 2, *merge, 0u); + op(e->code, OpBranchConditional, 3, cond, t, *merge); + op(e->code, OpLabel, 1, t); +} +static void if_end(Em *e, uint32_t merge) { + op(e->code, OpBranch, 1, merge); + op(e->code, OpLabel, 1, merge); +} + +typedef struct { uint32_t var, len, role, loc, slot; } OutVar; // role 0 position, 1 generic, 2 indices, 3 point size + +// The mesh epilogue (replaces the entry function's return) +static void mesh_epilogue(Em *e, const MeshEmuInfo *info, const OutVar *ov, int nov, uint32_t vcount, uint32_t pcount, + uint32_t sh_vb, uint32_t sh_ib, uint32_t p_wg_v4f, uint32_t p_wg_v3u, uint32_t p_wg_f32, + uint32_t p_wg_u32) { + Mod *m = e->m; + uint32_t S = info->slots, T = info->threads, K = info->prim; + barrier(e); + uint32_t lid = ld(e, m->t_u32, m->lii), base = words_base(e); + uint32_t isz = mul(e, pcount, cu(e, K)), vsz = mul(e, vcount, cu(e, S)); + // header word 6: mesh workgroups that reached the output copy; word 7: the primitives they declared (diagnosis) + { + uint32_t l0 = E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)); + atomic(e, OpAtomicIAdd, base, cu(e, 6), E3(e, OpSelect, m->t_u32, l0, cu(e, 1), cu(e, 0))); + atomic(e, OpAtomicIAdd, base, cu(e, 7), E3(e, OpSelect, m->t_u32, l0, pcount, cu(e, 0))); + } + uint32_t mz; + if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); + { + // indices from the draw's own range (reserved by the allocation shader), vertex slots from the arena's + uint32_t drw = pc_u32(e, 1), vcap = pc_u32(e, 5); + uint32_t icap = word_load(e, base, add(e, drw, cu(e, MESH_EMU_R_ICAP))); + uint32_t ib = atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_ICUR)), isz); + uint32_t vb = atomic(e, OpAtomicIAdd, base, cu(e, 0), vsz); + uint32_t iok = E2(e, OpULessThanEqual, m->t_bool, add(e, ib, isz), icap); + ib = add(e, word_load(e, base, add(e, drw, cu(e, 2))), ib); // relative to the area, as firstIndex + uint32_t vok = E2(e, OpULessThanEqual, m->t_bool, add(e, vb, vsz), vcap); + uint32_t mi; + if_begin(e, iok, &mi); + atomic(e, OpAtomicIAdd, base, drw, isz); // the draw record's index count + if_end(e, mi); + uint32_t all = E2(e, OpLogicalAnd, m->t_bool, iok, vok); + atomic(e, OpAtomicIAdd, base, cu(e, 3), E3(e, OpSelect, m->t_u32, all, cu(e, 0), cu(e, 1))); + op(e->code, OpStore, 2, sh_vb, E3(e, OpSelect, m->t_u32, vok, vb, cu(e, 0xffffffffu))); + op(e->code, OpStore, 2, sh_ib, E3(e, OpSelect, m->t_u32, iok, ib, cu(e, 0xffffffffu))); + } + if_end(e, mz); + barrier(e); + uint32_t VB = ld(e, m->t_u32, sh_vb), IB = ld(e, m->t_u32, sh_ib); + uint32_t m3; + if_begin(e, E2(e, OpINotEqual, m->t_bool, IB, cu(e, 0xffffffffu)), &m3); + { + uint32_t vbok = E2(e, OpINotEqual, m->t_bool, VB, cu(e, 0xffffffffu)); + uint32_t m4; + if_begin(e, vbok, &m4); + { + uint32_t vw = pc_u32(e, 4); + uint32_t a64 = E2(e, OpIAdd, m->t_u64, pc_u64(e), E2(e, OpIMul, m->t_u64, E1(e, OpUConvert, m->t_u64, vw), e->c64_16)); + uint32_t sbase = E1(e, OpConvertUToPtr, e->p_slots, a64); + for (uint32_t r = 0; r * T < info->max_v; r++) { + uint32_t v = add(e, lid, cu(e, r * T)), mv; + if_begin(e, ult(e, v, vcount), &mv); + uint32_t s0 = add(e, VB, mul(e, v, cu(e, S))); + for (int k = 0; k < nov; k++) { + if (ov[k].role == 2) continue; + uint32_t bits; + if (ov[k].role == 3) { // the point size, in x + uint32_t ps = ld(e, m->t_f32, E2(e, OpAccessChain, p_wg_f32, ov[k].var, v)); + uint32_t pb = E1(e, OpBitcast, m->t_u32, ps); + bits = nid(m); + op(e->code, OpCompositeConstruct, 6, m->t_v4u, bits, pb, cu(e, 0), cu(e, 0), cu(e, 0)); + } else { + uint32_t src = ld(e, m->t_v4f, E2(e, OpAccessChain, p_wg_v4f, ov[k].var, v)); + bits = E1(e, OpBitcast, m->t_v4u, src); + } + uint32_t dst = E3(e, OpAccessChain, e->p_v4u_psb, sbase, cu(e, 0), add(e, s0, cu(e, ov[k].slot))); + op(e->code, OpStore, 4, dst, bits, 2u, 16u); + } + if_end(e, mv); + } + } + if_end(e, m4); + uint32_t iw = pc_u32(e, 2), idxv = 0; + for (int k = 0; k < nov; k++) + if (ov[k].role == 2) idxv = ov[k].var; + for (uint32_t r = 0; r * T < info->max_p; r++) { + uint32_t p = add(e, lid, cu(e, r * T)), mp; + if_begin(e, ult(e, p, pcount), &mp); + uint32_t tri = K == 3 ? ld(e, m->t_v3u, E2(e, OpAccessChain, p_wg_v3u, idxv, p)) + : ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, idxv, p)); + uint32_t at = add(e, add(e, iw, IB), mul(e, p, cu(e, K))); + for (uint32_t c = 0; c < K; c++) { + uint32_t li = K == 3 ? E2(e, OpCompositeExtract, m->t_u32, tri, c) : tri; + uint32_t val = E3(e, OpSelect, m->t_u32, vbok, add(e, VB, mul(e, li, cu(e, S))), cu(e, 0)); + word_store(e, base, add(e, at, cu(e, c)), val); + } + if_end(e, mp); + } + } + if_end(e, m3); + op(e->code, OpReturn, 0); +} + +// The task epilogue (replaces EmitMeshTasksEXT x y z) +static void task_epilogue(Em *e, const MeshEmuInfo *info, uint32_t payload, uint32_t p_wg_u32, uint32_t nwg, + uint32_t ex, uint32_t ey, uint32_t ez) { + Mod *m = e->m; + uint32_t T = info->threads; + uint32_t cnt = mul(e, ex, mul(e, ey, ez)); + uint32_t nc = E3(e, OpSelect, m->t_u32, ult(e, cnt, cu(e, info->max_mesh)), cnt, cu(e, info->max_mesh)); + barrier(e); + uint32_t lid = ld(e, m->t_u32, m->lii), wg = ld(e, m->t_v3u, m->wgid), nw = ld(e, m->t_v3u, nwg); + uint32_t wx = E2(e, OpCompositeExtract, m->t_u32, wg, 0u), wy = E2(e, OpCompositeExtract, m->t_u32, wg, 1u), + wz = E2(e, OpCompositeExtract, m->t_u32, wg, 2u); + uint32_t nx = E2(e, OpCompositeExtract, m->t_u32, nw, 0u), ny = E2(e, OpCompositeExtract, m->t_u32, nw, 1u), + nz = E2(e, OpCompositeExtract, m->t_u32, nw, 2u); + uint32_t nxy = mul(e, nx, ny); + uint32_t t = add(e, wx, add(e, mul(e, wy, nx), mul(e, wz, nxy))); + uint32_t base = words_base(e), drw = pc_u32(e, 1); + uint32_t tb = word_load(e, base, add(e, drw, cu(e, 8))); + uint32_t trecs = pc_u32(e, 6), slot = add(e, tb, t), m1; + uint32_t mz; + if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &mz); + atomic(e, OpAtomicUMax, base, cu(e, 2), add(e, slot, cu(e, 1))); // records asked for, past the capacity too + if_end(e, mz); + if_begin(e, ult(e, slot, trecs), &m1); + { + uint32_t recw = add(e, cu(e, MESH_EMU_TOFF / 4), mul(e, slot, cu(e, MESH_EMU_TWORDS))); + for (uint32_t r = 0; r * T < info->payload_words; r++) { // the last round's threads past the payload idle + uint32_t k = add(e, lid, cu(e, r * T)), mk = 0; + if ((r + 1) * T > info->payload_words) if_begin(e, ult(e, k, cu(e, info->payload_words)), &mk); + uint32_t v = ld(e, m->t_u32, E2(e, OpAccessChain, p_wg_u32, payload, k)); + word_store(e, base, add(e, add(e, recw, cu(e, 1)), k), v); + if (mk) if_end(e, mk); + } + uint32_t m2; + if_begin(e, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), &m2); + word_store(e, base, recw, nc); + atomic(e, OpAtomicIAdd, base, add(e, drw, cu(e, MESH_EMU_R_GROUPS)), nc); // for the index range + if_end(e, m2); + } + if_end(e, m1); + uint32_t first = E2(e, OpLogicalAnd, m->t_bool, E2(e, OpIEqual, m->t_bool, lid, cu(e, 0)), + E2(e, OpIEqual, m->t_bool, t, cu(e, 0))); + uint32_t m3; + if_begin(e, first, &m3); + { + uint32_t total = mul(e, nxy, nz); + // task records past the area are not written: their mesh groups are not dispatched + uint32_t room = E3(e, OpSelect, m->t_u32, ult(e, tb, trecs), E2(e, 130 /* OpISub */, m->t_u32, trecs, tb), cu(e, 0)); + uint32_t ty = E3(e, OpSelect, m->t_u32, ult(e, total, room), total, room); + word_store(e, base, add(e, drw, cu(e, 5)), cu(e, info->max_mesh)); + word_store(e, base, add(e, drw, cu(e, 6)), ty); + word_store(e, base, add(e, drw, cu(e, 7)), cu(e, 1)); + } + if_end(e, m3); + op(e->code, OpReturn, 0); +} + +uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why) { + static const char *dummy; + if (!why) why = &dummy; + *why = NULL; + Mod *m = calloc(1, sizeof *m); + uint32_t *ins = malloc(MaxIns * 4), *ret = NULL; + Buf caps = {0}, exts = {0}, imps = {0}, modes = {0}, debug = {0}, annot = {0}, types = {0}, ntypes = {0}, nannot = {0}, + code = {0}, out = {0}; + if (!m || !ins) { *why = "out of memory"; goto done; } + if (!parse(m, w, n, why)) goto done; + memset(info, 0, sizeof *info); + int mesh = m->model == ModelMesh; + info->task = !mesh; + for (size_t i = 5; i < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + const uint32_t *a = w + i + 1; + if (o == OpFunction) break; + if (o == OpExecutionMode && a[0] == m->ep_fn) { + if (a[1] == ModeLocalSize) info->threads = a[2] * a[3] * a[4]; + if (a[1] == ModeOutputVertices) info->max_v = a[2]; + if (a[1] == ModeOutputPrimitives) info->max_p = a[2]; + if (a[1] == ModeOutputTriangles) info->prim = 3; + if (a[1] == ModeOutputLines) info->prim = 2; + if (a[1] == ModeOutputPoints) info->prim = 1; + } + i += len; + } + if (!info->threads) { *why = "no LocalSize"; goto done; } + if (mesh && info->prim != 3 && info->prim != 1) { *why = "only triangle and point meshes are emulated"; goto done; } + if (!m->t_f32 || (mesh && !m->t_v4f)) { *why = "no float types"; goto done; } + + // output variables and the payload + OutVar ov[40]; + int nov = 0; + uint32_t payload = 0, payload_len = 0; + for (int i = 0; i < m->nvar; i++) { + uint32_t sc = m->var[i].sc; + if (sc != ScOutput && sc != ScTaskPayload) continue; + const uint32_t *arr = find_type(m, ptr_to(m, m->var[i].type)); + uint32_t len = 0; + if (!arr || (arr[0] & 0xffff) != OpTypeArray || !const_u32(m, arr[3], &len)) { *why = "an output or payload is not a sized array"; goto done; } + if (sc == ScTaskPayload) { + if (arr[2] != m->t_u32) { *why = "a payload that is not uint[]"; goto done; } + payload = m->var[i].id, payload_len = len; + continue; + } + if (nov == 40) { *why = "too many outputs"; goto done; } + ov[nov] = (OutVar){m->var[i].id, len, 9, 0, 0}; + for (size_t k = 5; k < n;) { + uint32_t o = w[k] & 0xffff, l2 = w[k] >> 16; + const uint32_t *a = w + k + 1; + if (o == OpFunction) break; + if (o == OpDecorate && a[0] == m->var[i].id) { + if (a[1] == DecBuiltIn && a[2] == BiPosition) ov[nov].role = 0; + else if (a[1] == DecBuiltIn && (a[2] == BiPrimTri || a[2] == BiPrimPoint)) ov[nov].role = 2; + else if (a[1] == DecBuiltIn && a[2] == BiPointSize) ov[nov].role = 3; + else if (a[1] == DecLocation) ov[nov].role = 1, ov[nov].loc = a[2]; + else if (a[1] == DecBuiltIn) { *why = "an unsupported mesh output built-in"; goto done; } + else if (a[1] == DecPerPrimitive) { *why = "per-primitive outputs are not emulated"; goto done; } + } + k += l2; + } + if (ov[nov].role == 9) { *why = "an output without a role"; goto done; } + if (ov[nov].role < 2 && arr[2] != m->t_v4f) { *why = "a vertex output that is not a vec4"; goto done; } + if (ov[nov].role == 3 && arr[2] != m->t_f32) { *why = "a point size that is not a float"; goto done; } + if (ov[nov].role == 2 && arr[2] != (info->prim == 3 ? m->t_v3u : m->t_u32)) { *why = "indices of an unexpected type"; goto done; } + nov++; + } + info->payload_words = payload_len; + if (payload_len > MESH_EMU_TWORDS - 1) { *why = "payload too large"; goto done; } + if (mesh) { + uint32_t locs = 0; + int have_pos = 0, have_idx = 0; + for (int i = 0; i < nov; i++) { + if (ov[i].role == 1) { + if (ov[i].loc >= 32 || locs >> ov[i].loc & 1) { *why = "bad generic location"; goto done; } + locs |= 1u << ov[i].loc; + } + have_pos |= ov[i].role == 0, have_idx |= ov[i].role == 2; + } + if (!have_pos || !have_idx) { *why = "no position or no indices"; goto done; } + for (uint32_t l = 0; l < 32; l++) + if (locs >> l & 1) info->generic_loc[info->n_generic++] = (uint8_t)l; + info->slots = 1 + info->n_generic; + for (int i = 0; i < nov; i++) + if (ov[i].role == 3) info->psize_slot = info->slots, ov[i].slot = info->slots++; + for (int i = 0; i < nov; i++) { + if (ov[i].role == 0) ov[i].slot = 0; + if (ov[i].role == 1) + for (uint32_t g = 0; g < info->n_generic; g++) + if (info->generic_loc[g] == ov[i].loc) ov[i].slot = 1 + g; + } + } + uint32_t emit_x = 0, emit_y = 0, emit_z = 0; + if (!mesh) { + size_t i = 5; + while (i < n && (w[i] & 0xffff) != OpFunction) i += w[i] >> 16; + for (; i < n; i += w[i] >> 16) + if ((w[i] & 0xffff) == OpEmitMeshTasksEXT) emit_x = w[i + 1], emit_y = w[i + 2], emit_z = w[i + 3]; + if (!emit_x || !payload) { *why = "no EmitMeshTasksEXT or no payload"; goto done; } + info->max_mesh = 32; + i = 5; + while (i < n && (w[i] & 0xffff) != OpFunction) i += w[i] >> 16; + for (; i < n; i += w[i] >> 16) { + uint32_t v; + if ((w[i] & 0xffff) == OpExtInst && (w[i] >> 16) == 7 && w[i + 2] == emit_x && w[i + 3] == m->glsl && + w[i + 4] == GlslUMin && const_u32(m, w[i + 6], &v) && v && v <= 1024) + info->max_mesh = v; + } + } + + // ---- new types, constants and variables + Em e = {.m = m, .types = &ntypes, .annot = &nannot, .code = &code}; + if (!m->t_bool) m->t_bool = nid(m), op(&ntypes, OpTypeBool, 1, m->t_bool); + if (!m->t_u64) m->t_u64 = nid(m), op(&ntypes, OpTypeInt, 3, m->t_u64, 64u, 0u); + if (!m->t_v3u) m->t_v3u = nid(m), op(&ntypes, OpTypeVector, 3, m->t_v3u, m->t_u32, 3u); + if (!m->t_v4u) m->t_v4u = nid(m), op(&ntypes, OpTypeVector, 3, m->t_v4u, m->t_u32, 4u); + uint32_t rt_u = nid(m), s_words = nid(m), rt_v4 = nid(m), s_slots = nid(m), pc_struct = nid(m); + op(&ntypes, OpTypeRuntimeArray, 2, rt_u, m->t_u32); + op(&ntypes, OpTypeStruct, 2, s_words, rt_u); + op(&ntypes, OpTypeRuntimeArray, 2, rt_v4, m->t_v4u); + op(&ntypes, OpTypeStruct, 2, s_slots, rt_v4); + op(&ntypes, OpTypeStruct, 8, pc_struct, m->t_u64, m->t_u32, m->t_u32, m->t_u32, m->t_u32, m->t_u32, m->t_u32); + e.p_words = nid(m), op(&ntypes, OpTypePointer, 3, e.p_words, (uint32_t)ScPhysicalStorageBuffer, s_words); + e.p_slots = nid(m), op(&ntypes, OpTypePointer, 3, e.p_slots, (uint32_t)ScPhysicalStorageBuffer, s_slots); +#define NEED_PTR(DST, SC, TO) do { DST = find_ptr(m, SC, TO); if (!DST) { DST = nid(m); op(&ntypes, OpTypePointer, 3, DST, (uint32_t)(SC), TO); \ + m->ptr[m->nptr].id = DST, m->ptr[m->nptr].sc = SC, m->ptr[m->nptr++].to = TO; } } while (0) + NEED_PTR(e.p_u32_psb, ScPhysicalStorageBuffer, m->t_u32); + NEED_PTR(e.p_v4u_psb, ScPhysicalStorageBuffer, m->t_v4u); + uint32_t p_pc = nid(m); + op(&ntypes, OpTypePointer, 3, p_pc, (uint32_t)ScPushConstant, pc_struct); + NEED_PTR(e.p_pc_u64, ScPushConstant, m->t_u64); + NEED_PTR(e.p_pc_u32, ScPushConstant, m->t_u32); + e.pc_var = nid(m), op(&ntypes, OpVariable, 3, p_pc, e.pc_var, (uint32_t)ScPushConstant); + e.c64_16 = nid(m), op(&ntypes, OpConstant, 4, m->t_u64, e.c64_16, 16u, 0u); + op(&nannot, OpDecorate, 3, rt_u, (uint32_t)DecArrayStride, 4u); + op(&nannot, OpDecorate, 2, s_words, (uint32_t)DecBlock); + op(&nannot, OpMemberDecorate, 4, s_words, 0u, (uint32_t)DecOffset, 0u); + op(&nannot, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u); + op(&nannot, OpDecorate, 2, s_slots, (uint32_t)DecBlock); + op(&nannot, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u); + op(&nannot, OpDecorate, 2, pc_struct, (uint32_t)DecBlock); + for (uint32_t k = 0; k < 7; k++) op(&nannot, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u); + + // Output and payload pointers become Workgroup pointers: in place when no such pointer exists yet, else mapped + struct { uint32_t from, to; } pmap[512]; + int npmap = 0; + for (int i = 0; i < m->nptr; i++) { + if (m->ptr[i].sc != ScOutput && m->ptr[i].sc != ScTaskPayload) continue; + uint32_t have = find_ptr(m, ScWorkgroup, m->ptr[i].to); + if (have) { + if (npmap == 512) { *why = "too many pointer types"; goto done; } + pmap[npmap].from = m->ptr[i].id, pmap[npmap++].to = have; + m->ptr[i].sc = 0xffffffffu; // dropped + } else { + m->ptr[i].sc = ScWorkgroup; // rewritten in place below + } + } + uint32_t p_wg_u32, p_wg_v4f = 0, p_wg_v3u = 0; + NEED_PTR(p_wg_u32, ScWorkgroup, m->t_u32); + uint32_t p_wg_f32 = 0; + if (mesh) { NEED_PTR(p_wg_v4f, ScWorkgroup, m->t_v4f); NEED_PTR(p_wg_v3u, ScWorkgroup, m->t_v3u); NEED_PTR(p_wg_f32, ScWorkgroup, m->t_f32); } + uint32_t sh_vb = 0, sh_ib = 0, wgemu = 0, p_priv_v3 = 0, p_priv_u32 = 0, p_in_u32 = find_ptr(m, ScInput, m->t_u32); + if (mesh) { + sh_vb = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_vb, (uint32_t)ScWorkgroup); + sh_ib = nid(m), op(&ntypes, OpVariable, 3, p_wg_u32, sh_ib, (uint32_t)ScWorkgroup); + } + int remap_wg = mesh && payload; + if (remap_wg) { + NEED_PTR(p_priv_v3, ScPrivate, m->t_v3u); + NEED_PTR(p_priv_u32, ScPrivate, m->t_u32); + wgemu = nid(m), op(&ntypes, OpVariable, 3, p_priv_v3, wgemu, (uint32_t)ScPrivate); + } + uint32_t nwg = m->nwg; + if (!mesh && !nwg) { + uint32_t p_in_v3; + NEED_PTR(p_in_v3, ScInput, m->t_v3u); + nwg = nid(m), op(&ntypes, OpVariable, 3, p_in_v3, nwg, (uint32_t)ScInput); + op(&nannot, OpDecorate, 3, nwg, (uint32_t)DecBuiltIn, (uint32_t)BiNumWorkgroups); + } +#undef NEED_PTR + + // ---- one pass over the module + uint32_t moved[48]; + int nmoved = 0; + for (int i = 0; i < nov; i++) moved[nmoved++] = ov[i].var; + if (payload) moved[nmoved++] = payload; + uint32_t vcount = 0, pcount = 0; + int in_fn = 0, in_ep = 0, ep_started = 0, ep_done = 0; + for (size_t i = 5; i < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + memcpy(ins, w + i, len * 4); + const uint32_t *a = w + i + 1; + if (o == OpFunction) in_fn = 1, in_ep = a[1] == m->ep_fn; + if (!in_fn) { + if (o == OpCapability) { if (a[0] != CapMeshShading) putn(&caps, ins, len); } + else if (o == OpExtension) { if (strcmp((const char *)a, "SPV_EXT_mesh_shader")) putn(&exts, ins, len); } + else if (o == OpExtInstImport) putn(&imps, ins, len); + else if (o == OpMemoryModel || o == OpEntryPoint) {} + else if (o == OpExecutionMode || o == OpExecutionModeId) { + if (!(o == OpExecutionMode && (a[1] == ModeOutputVertices || a[1] == ModeOutputPrimitives || + a[1] == ModeOutputTriangles || a[1] == ModeOutputLines || a[1] == ModeOutputPoints))) + putn(&modes, ins, len); + } else if (is_debug(o)) putn(&debug, ins, len); + else if (is_annot(o)) { + int drop = 0; + for (int k = 0; k < nmoved; k++) drop |= o == OpDecorate && a[0] == moved[k]; + if (!drop) putn(&annot, ins, len); + } else { // types, constants, global variables + int drop = 0; + if (o == OpTypePointer) { + for (int k = 0; k < npmap; k++) drop |= pmap[k].from == a[0]; + if (a[1] == ScOutput || a[1] == ScTaskPayload) ins[2] = ScWorkgroup; + } + if (o == OpVariable && (a[2] == ScOutput || a[2] == ScTaskPayload)) ins[3] = ScWorkgroup; + for (uint32_t k = 1; k < len; k++) { + int idpos = o == OpTypeFunction || o == OpTypeStruct || o == OpTypeArray || o == OpTypeRuntimeArray || + (o == OpTypePointer && k == 3) || k == 1; + if (o == OpTypePointer && k == 1) idpos = 0; + for (int q = 0; idpos && q < npmap; q++) + if (ins[k] == pmap[q].from) ins[k] = pmap[q].to; + } + if (!drop) putn(&types, ins, len); + } + i += len; + continue; + } + // function code + if (len >= 2) + for (int q = 0; q < npmap; q++) + if (ins[1] == pmap[q].from) ins[1] = pmap[q].to; + if (remap_wg && len >= 4 && (o == OpLoad || o == OpAccessChain || o == OpInBoundsAccessChain) && ins[3] == m->wgid) { + ins[3] = wgemu; + if (o != OpLoad) { + if (!p_in_u32 || ins[1] != p_in_u32) { *why = "an unexpected WorkgroupId access"; goto done; } + ins[1] = p_priv_u32; + } + } else if (remap_wg) { + for (uint32_t k = 1; k < len; k++) { // id operands only: literals of these instructions may equal the id + int lit = (o == OpExtInst && k == 4) || (o == 81 && k >= 4) || (o == 82 && k >= 5) || (o == 79 && k >= 5) || + (o == OpLoad && k >= 4) || (o == OpStore && k >= 3) || o == 251 /* OpSwitch */ || o == OpSelectionMerge || + o == 246 /* OpLoopMerge */; + if (!lit && ins[k] == m->wgid) { *why = "an unexpected WorkgroupId use"; goto done; } + } + } + if (o == OpFunctionEnd) in_fn = 0, in_ep = 0; + if (in_ep && o == OpLabel && !ep_started) { + putn(&code, ins, len); + size_t j = i + len; // function variables stay first + while (j < n && (w[j] & 0xffff) == OpVariable) putn(&code, w + j, w[j] >> 16), j += w[j] >> 16; + ep_started = 1; + i = j; + if (remap_wg) { + uint32_t lid = ld(&e, m->t_u32, m->lii), wg = ld(&e, m->t_v3u, m->wgid); + uint32_t x = E2(&e, OpCompositeExtract, m->t_u32, wg, 0u), y = E2(&e, OpCompositeExtract, m->t_u32, wg, 1u); + uint32_t base = words_base(&e), drw = pc_u32(&e, 1); + uint32_t tb = word_load(&e, base, add(&e, drw, cu(&e, 8))); + uint32_t slot = add(&e, tb, y), ok = ult(&e, slot, pc_u32(&e, 6)); + uint32_t slotc = E3(&e, OpSelect, m->t_u32, ok, slot, cu(&e, 0)); + uint32_t recw = add(&e, cu(&e, MESH_EMU_TOFF / 4), mul(&e, slotc, cu(&e, MESH_EMU_TWORDS))); + uint32_t cnt = word_load(&e, base, recw); + uint32_t act = E2(&e, OpLogicalAnd, m->t_bool, ok, ult(&e, x, cnt)); + // header words 4 and 5: mesh workgroups started and past this check (diagnosis: vk_dump mesh.txt) + uint32_t l0 = E2(&e, OpIEqual, m->t_bool, lid, cu(&e, 0)); + atomic(&e, OpAtomicIAdd, base, cu(&e, 4), E3(&e, OpSelect, m->t_u32, l0, cu(&e, 1), cu(&e, 0))); + atomic(&e, OpAtomicIAdd, base, cu(&e, 5), + E3(&e, OpSelect, m->t_u32, E2(&e, OpLogicalAnd, m->t_bool, l0, act), cu(&e, 1), cu(&e, 0))); + uint32_t cont = nid(m), retl = nid(m); + op(&code, OpSelectionMerge, 2, cont, 0u); + op(&code, OpBranchConditional, 3, act, cont, retl); + op(&code, OpLabel, 1, retl); + op(&code, OpReturn, 0); + op(&code, OpLabel, 1, cont); + uint32_t v3 = nid(m); + op(&code, OpCompositeConstruct, 5, m->t_v3u, v3, x, cu(&e, 0), cu(&e, 0)); + op(&code, OpStore, 2, wgemu, v3); + for (uint32_t r = 0; r * info->threads < payload_len; r++) { // as task_epilogue's copy + uint32_t k = add(&e, lid, cu(&e, r * info->threads)), mk = 0; + if ((r + 1) * info->threads > payload_len) if_begin(&e, ult(&e, k, cu(&e, payload_len)), &mk); + uint32_t v = word_load(&e, base, add(&e, add(&e, recw, cu(&e, 1)), k)); + op(&code, OpStore, 2, E2(&e, OpAccessChain, p_wg_u32, payload, k), v); + if (mk) if_end(&e, mk); + } + barrier(&e); + } + continue; + } + if (in_ep && o == OpSetMeshOutputsEXT) { + vcount = a[0], pcount = a[1]; + i += len; + continue; + } + if (in_ep && mesh && o == OpReturn) { + if (!vcount || ep_done) { *why = "no SetMeshOutputsEXT before the return, or several returns"; goto done; } + mesh_epilogue(&e, info, ov, nov, vcount, pcount, sh_vb, sh_ib, p_wg_v4f, p_wg_v3u, p_wg_f32, p_wg_u32); + ep_done = 1; + i += len; + continue; + } + if (in_ep && !mesh && o == OpEmitMeshTasksEXT) { + task_epilogue(&e, info, payload, p_wg_u32, nwg, emit_x, emit_y, emit_z); + ep_done = 1; + i += len; + continue; + } + putn(&code, ins, len); + i += len; + } + if (!ep_done) { *why = "the entry function's end was not found"; goto done; } + + // ---- assemble + put(&out, 0x07230203); + put(&out, w[1] < 0x00010500 ? 0x00010500 : w[1]); // SPIR-V 1.5: physical storage buffers are core + put(&out, w[2]); + put(&out, m->bound); + put(&out, 0); + putn(&out, caps.w, caps.n); + if (!m->has_int64) op(&out, OpCapability, 1, (uint32_t)CapInt64); + if (!m->has_psb) op(&out, OpCapability, 1, (uint32_t)CapPSBAddresses); + putn(&out, exts.w, exts.n); + putn(&out, imps.w, imps.n); + op(&out, OpMemoryModel, 2, 5348u, 1u); // PhysicalStorageBuffer64, GLSL450 + { // entry point: every global variable + Buf ep = {0}; + uint32_t pre[2] = {ModelGLCompute, m->ep_fn}; + uint32_t ifc[2048 + 16]; + int ni = 0; + for (int i = 0; i < m->nvar && ni < 2048; i++) { + int dropped = 0; + if (!dropped && m->var[i].sc != ScFunction) ifc[ni++] = m->var[i].id; + } + // the new ones: scan ntypes for OpVariable + for (size_t i = 0; i < ntypes.n;) { + uint32_t o = ntypes.w[i] & 0xffff, len = ntypes.w[i] >> 16; + if (o == OpVariable && ni < 2048 + 16) ifc[ni++] = ntypes.w[i + 2]; + i += len; + } + op_str(&ep, OpEntryPoint, pre, 2, "main", ifc, ni); + putn(&out, ep.w, ep.n); + free(ep.w); + } + putn(&out, modes.w, modes.n); + putn(&out, debug.w, debug.n); + putn(&out, annot.w, annot.n); + putn(&out, nannot.w, nannot.n); + putn(&out, types.w, types.n); + putn(&out, ntypes.w, ntypes.n); + putn(&out, code.w, code.n); + out.w[3] = m->bound; + if (caps.bad || exts.bad || imps.bad || modes.bad || debug.bad || annot.bad || types.bad || ntypes.bad || + nannot.bad || code.bad || out.bad) { *why = "out of memory"; goto done; } + ret = out.w, *out_n = out.n, out.w = NULL; +done: + free(caps.w), free(exts.w), free(imps.w), free(modes.w), free(debug.w), free(annot.w), free(types.w), free(ntypes.w), + free(nannot.w), free(code.w), free(out.w); + free(m), free(ins); + return ret; +} + +int mesh_emu_prim(const uint32_t *w, size_t n) { + uint32_t ep = 0; + for (size_t i = 5; i + 1 < n;) { + uint32_t o = w[i] & 0xffff, len = w[i] >> 16; + if (!len || o == OpFunction) break; + if (o == OpEntryPoint && len >= 3) ep = w[i + 1] == ModelMesh || w[i + 1] == ModelTask; + if (o == OpExecutionMode && len >= 3 && ep) { + if (w[i + 2] == ModeOutputTriangles) return 3; + if (w[i + 2] == ModeOutputLines) return 2; + if (w[i + 2] == ModeOutputPoints) return 1; + } + i += len; + } + return 0; +} + +// ---- the vertex shader of an emulated mesh pipeline +// The fragment shader's input variables as declared: Location, Component, scalar type and width, Flat. 0 when one +// is not a plain 32-bit scalar or vector (the caller then goes by the reflection's per-location summary). +typedef struct { uint32_t loc, comp, type, comps, flat; } FsIn; +static int fs_inputs(const uint32_t *w, size_t n, FsIn *o, int max) { + if (!w || n < 5 || w[0] != 0x07230203) return 0; + uint32_t bound = w[3]; + if (bound > (1u << 22)) return 0; + uint32_t *loc = calloc(bound, 4), *comp = calloc(bound, 4), *ty = calloc(bound, 4), *cn = calloc(bound, 4), *pt = calloc(bound, 4); + uint8_t *flat = calloc(bound, 1), *bi = calloc(bound, 1), *hl = calloc(bound, 1); + int k = 0, ok = loc && comp && ty && cn && pt && flat && bi && hl; + for (size_t i = 5; ok && i < n;) { + uint32_t o2 = w[i] & 0xffff, len = w[i] >> 16; + const uint32_t *a = w + i + 1; + if (!len || i + len > n || o2 == OpFunction) break; + if (o2 == OpDecorate && len >= 3 && a[0] < bound) { + if (a[1] == DecLocation && len >= 4) loc[a[0]] = a[2], hl[a[0]] = 1; + if (a[1] == 31 && len >= 4) comp[a[0]] = a[2]; // Component + if (a[1] == 14) flat[a[0]] = 1; // Flat + if (a[1] == DecBuiltIn) bi[a[0]] = 1; + } + if (o2 == OpTypeFloat && len >= 3 && a[0] < bound && a[1] == 32) ty[a[0]] = 1, cn[a[0]] = 1; + if (o2 == OpTypeInt && len >= 4 && a[0] < bound && a[1] == 32) ty[a[0]] = a[2] ? 2 : 3, cn[a[0]] = 1; + if (o2 == OpTypeVector && len >= 4 && a[0] < bound && a[1] < bound && ty[a[1]]) ty[a[0]] = ty[a[1]], cn[a[0]] = a[2]; + if (o2 == OpTypePointer && len >= 4 && a[0] < bound && a[1] == ScInput) pt[a[0]] = a[2] + 1; + if (o2 == OpVariable && len >= 4 && a[2] == ScInput && a[1] < bound && a[0] < bound && !bi[a[1]] && hl[a[1]]) { + uint32_t p = pt[a[0]] ? pt[a[0]] - 1 : 0; + if (!p || p >= bound || !ty[p] || cn[p] > 4 || loc[a[1]] >= 32 || k == max) { k = 0, ok = 0; break; } + o[k++] = (FsIn){loc[a[1]], comp[a[1]], ty[p] - 1, cn[p], flat[a[1]]}; + } + i += len; + } + free(loc), free(comp), free(ty), free(cn), free(pt), free(flat), free(bi), free(hl); + return ok ? k : 0; +} + +uint32_t *mesh_emu_vs(const MeshEmuInfo *mi, uint32_t need, const uint8_t types[32], const uint8_t comps[32], + const uint32_t *fsw, size_t fsn, size_t *out_n) { + Buf b = {0}, t = {0}, an = {0}, c = {0}; + FsIn in[64]; + int nin = fs_inputs(fsw, fsn, in, 64); + if (!nin) // the per-location summary: one vector per location + for (uint32_t l = 0; l < 32; l++) + if (need >> l & 1) + in[nin++] = (FsIn){l, 0, types[l] <= 2 ? types[l] : 0, comps[l] >= 1 && comps[l] <= 4 ? comps[l] : 4, types[l] == 1 || types[l] == 2}; + uint32_t id = 1; +#define N() (id++) + uint32_t glsl = N(), fn = N(), t_void = N(), t_fn = N(), t_u32 = N(), t_i32 = N(), t_f32 = N(), t_u64 = N(), + t_v4u = N(), t_v4f = N(), t_v4i = N(), rt_v4 = N(), s_slots = N(), p_slots = N(), p_v4u_psb = N(), + pc_struct = N(), p_pc = N(), pc_var = N(), p_pc_u64 = N(), p_pc_u32 = N(), p_in_i32 = N(), + vidx = N(), p_out_v4f = N(), pos = N(), c0 = N(), c4 = N(), c16 = N(), p_out_f32 = N(), psz = N(); + (void)glsl; + uint32_t vecs[3][5] = {{0}}; // [type][comps] vector ids (comps 2..4), [type][1] the scalar + vecs[0][1] = t_f32, vecs[1][1] = t_i32, vecs[2][1] = t_u32, vecs[0][4] = t_v4f, vecs[1][4] = t_v4i, vecs[2][4] = t_v4u; + op(&t, OpTypeVoid, 1, t_void); + op(&t, OpTypeFunction, 2, t_fn, t_void); + op(&t, OpTypeInt, 3, t_u32, 32u, 0u); + op(&t, OpTypeInt, 3, t_i32, 32u, 1u); + op(&t, OpTypeFloat, 2, t_f32, 32u); + op(&t, OpTypeInt, 3, t_u64, 64u, 0u); + op(&t, OpTypeVector, 3, t_v4u, t_u32, 4u); + op(&t, OpTypeVector, 3, t_v4f, t_f32, 4u); + op(&t, OpTypeVector, 3, t_v4i, t_i32, 4u); + for (int ty = 0; ty < 3; ty++) + for (int k = 2; k < 4; k++) { + vecs[ty][k] = N(); + op(&t, OpTypeVector, 3, vecs[ty][k], vecs[ty][1], (uint32_t)k); + } + op(&t, OpTypeRuntimeArray, 2, rt_v4, t_v4u); + op(&t, OpTypeStruct, 2, s_slots, rt_v4); + op(&t, OpTypePointer, 3, p_slots, (uint32_t)ScPhysicalStorageBuffer, s_slots); + op(&t, OpTypePointer, 3, p_v4u_psb, (uint32_t)ScPhysicalStorageBuffer, t_v4u); + op(&t, OpTypeStruct, 8, pc_struct, t_u64, t_u32, t_u32, t_u32, t_u32, t_u32, t_u32); + op(&t, OpTypePointer, 3, p_pc, (uint32_t)ScPushConstant, pc_struct); + op(&t, OpTypePointer, 3, p_pc_u64, (uint32_t)ScPushConstant, t_u64); + op(&t, OpTypePointer, 3, p_pc_u32, (uint32_t)ScPushConstant, t_u32); + op(&t, OpTypePointer, 3, p_in_i32, (uint32_t)ScInput, t_i32); + op(&t, OpTypePointer, 3, p_out_v4f, (uint32_t)ScOutput, t_v4f); + op(&t, OpConstant, 3, t_u32, c0, 0u); + op(&t, OpConstant, 3, t_u32, c4, 4u); + op(&t, OpConstant, 4, t_u64, c16, 16u, 0u); + op(&t, OpVariable, 3, p_pc, pc_var, (uint32_t)ScPushConstant); + op(&t, OpVariable, 3, p_in_i32, vidx, (uint32_t)ScInput); + op(&t, OpVariable, 3, p_out_v4f, pos, (uint32_t)ScOutput); + if (mi->psize_slot) { + op(&t, OpTypePointer, 3, p_out_f32, (uint32_t)ScOutput, t_f32); + op(&t, OpVariable, 3, p_out_f32, psz, (uint32_t)ScOutput); + op(&an, OpDecorate, 3, psz, (uint32_t)DecBuiltIn, (uint32_t)BiPointSize); + } + op(&an, OpDecorate, 3, rt_v4, (uint32_t)DecArrayStride, 16u); + op(&an, OpDecorate, 2, s_slots, (uint32_t)DecBlock); + op(&an, OpMemberDecorate, 4, s_slots, 0u, (uint32_t)DecOffset, 0u); + op(&an, OpDecorate, 2, pc_struct, (uint32_t)DecBlock); + for (uint32_t k = 0; k < 7; k++) op(&an, OpMemberDecorate, 4, pc_struct, k, (uint32_t)DecOffset, k ? 4 + 4 * k : 0u); + op(&an, OpDecorate, 3, vidx, (uint32_t)DecBuiltIn, (uint32_t)BiVertexIndex); + op(&an, OpDecorate, 3, pos, (uint32_t)DecBuiltIn, (uint32_t)BiPosition); + uint32_t ifc[80], ni = 0; + ifc[ni++] = pc_var, ifc[ni++] = vidx, ifc[ni++] = pos; + if (mi->psize_slot) ifc[ni++] = psz; + // outputs: one per fragment input, same Location, Component and type + uint32_t outv[64] = {0}, outt[64] = {0}; + for (int q = 0; q < nin; q++) { + int ty = in[q].type <= 2 ? (int)in[q].type : 0, k = in[q].comps >= 1 && in[q].comps <= 4 ? (int)in[q].comps : 4; + if (in[q].comp + (uint32_t)k > 4) k = (int)(4 - in[q].comp); + in[q].type = (uint32_t)ty, in[q].comps = (uint32_t)k; + outt[q] = vecs[ty][k]; + uint32_t p = N(), v = N(); + op(&t, OpTypePointer, 3, p, (uint32_t)ScOutput, outt[q]); + op(&t, OpVariable, 3, p, v, (uint32_t)ScOutput); + op(&an, OpDecorate, 3, v, (uint32_t)DecLocation, in[q].loc); + if (in[q].comp) op(&an, OpDecorate, 3, v, 31u, in[q].comp); // Component + if (in[q].flat || ty) op(&an, OpDecorate, 2, v, 14u); // Flat + outv[q] = v; + ifc[ni++] = v; + } + // code: slot = vertex area + VertexIndex; each output from slot + its index + uint32_t lab = N(); + op(&c, OpFunction, 4, t_void, fn, 0u, t_fn); + op(&c, OpLabel, 1, lab); + uint32_t e64 = N(), e64p = N(), vw = N(), vwp = N(), vi = N(), viu = N(), s = N(), s64 = N(), off = N(), a = N(), base = N(); + op(&c, OpAccessChain, 4, p_pc_u64, e64p, pc_var, c0); + op(&c, OpLoad, 3, t_u64, e64, e64p); + op(&c, OpAccessChain, 4, p_pc_u32, vwp, pc_var, c4); + op(&c, OpLoad, 3, t_u32, vw, vwp); + op(&c, OpLoad, 3, t_i32, vi, vidx); + op(&c, OpBitcast, 3, t_u32, viu, vi); + op(&c, OpIAdd, 4, t_u32, s, vw, viu); + op(&c, OpUConvert, 3, t_u64, s64, s); + op(&c, OpIMul, 4, t_u64, off, s64, c16); + op(&c, OpIAdd, 4, t_u64, a, e64, off); + op(&c, OpConvertUToPtr, 3, p_slots, base, a); + if (mi->psize_slot) { + uint32_t ci = N(), p = N(), v = N(), x = N(), f = N(); + op(&t, OpConstant, 3, t_u32, ci, mi->psize_slot); + op(&c, OpAccessChain, 5, p_v4u_psb, p, base, c0, ci); + op(&c, OpLoad, 5, t_v4u, v, p, 2u, 16u); + op(&c, OpCompositeExtract, 4, t_u32, x, v, 0u); + op(&c, OpBitcast, 3, t_f32, f, x); + op(&c, OpStore, 2, psz, f); + } + uint32_t slot_of[33]; + for (int k = 0; k < 33; k++) slot_of[k] = 0xffffffffu; + slot_of[32] = 0; // position + for (uint32_t g = 0; g < mi->n_generic; g++) slot_of[mi->generic_loc[g]] = 1 + g; + for (int q = -1; q < nin; q++) { // -1: the position + uint32_t l = q < 0 ? 32 : in[q].loc; + uint32_t val; + if (slot_of[l] == 0xffffffffu) { + val = 0; + } else { + uint32_t ci = N(), p = N(); + val = N(); + op(&t, OpConstant, 3, t_u32, ci, slot_of[l]); + op(&c, OpAccessChain, 5, p_v4u_psb, p, base, c0, ci); + op(&c, OpLoad, 5, t_v4u, val, p, 2u, 16u); + } + if (q < 0) { + uint32_t f = N(); + op(&c, OpBitcast, 3, t_v4f, f, val); + op(&c, OpStore, 2, pos, f); + continue; + } + int ty = (int)in[q].type, k = (int)in[q].comps; + uint32_t v4t = vecs[ty][4], r, c0i = in[q].comp; + if (!val) { // not written by the mesh stage: zero + uint32_t z = N(); + r = N(); + op(&t, OpConstantNull, 2, outt[q], z); + op(&c, OpCopyObject, 3, outt[q], r, z); + } else { + uint32_t cv = val; + if (ty != 2) cv = N(), op(&c, OpBitcast, 3, v4t, cv, val); + if (k == 4) r = cv; + else if (k == 1) r = N(), op(&c, OpCompositeExtract, 4, outt[q], r, cv, c0i); + else { + r = N(); + put(&c, (uint32_t)(5 + k) << 16 | 79u); // OpVectorShuffle: components comp..comp+k-1 + put(&c, outt[q]), put(&c, r), put(&c, cv), put(&c, cv); + for (int z = 0; z < k; z++) put(&c, c0i + (uint32_t)z); + } + } + op(&c, OpStore, 2, outv[q], r); + } + op(&c, OpReturn, 0); + op(&c, OpFunctionEnd, 0); + // assemble + put(&b, 0x07230203), put(&b, 0x00010500), put(&b, 0), put(&b, id), put(&b, 0); + op(&b, OpCapability, 1, (uint32_t)CapShader); + op(&b, OpCapability, 1, (uint32_t)CapInt64); + op(&b, OpCapability, 1, (uint32_t)CapPSBAddresses); + op(&b, OpMemoryModel, 2, 5348u, 1u); + uint32_t pre[2] = {ModelVertex, fn}; + op_str(&b, OpEntryPoint, pre, 2, "main", ifc, (int)ni); + putn(&b, an.w, an.n); + putn(&b, t.w, t.n); + putn(&b, c.w, c.n); + b.w[3] = id; + int bad = b.bad || t.bad || an.bad || c.bad; + free(t.w), free(an.w), free(c.w); + if (bad) { free(b.w); return NULL; } + *out_n = b.n; + return b.w; +#undef N +} + +// The allocation shader (mesh_emu_spirv.h): glslangValidator -V -g0 --target-env vulkan1.2, spirv-opt -O --strip-debug, +// spirv-val clean. Its source: +// #version 460 +// #extension GL_EXT_buffer_reference : require +// #extension GL_EXT_buffer_reference_uvec2 : require +// layout(local_size_x = 1) in; +// layout(buffer_reference, std430, buffer_reference_align = 4) buffer Words { uint w[]; }; +// layout(push_constant) uniform PC { uvec2 addr; uint first; uint n; uint mode; uint icap; uint trecs; uint pad; } pc; +// void main() { +// Words a = Words(pc.addr); +// for (uint d = 0u; d < pc.n; d++) { +// uint r = 64u + ((pc.first + d) % 4096u) * 16u; +// bool task = a.w[r + 14u] != 0u; +// if (pc.mode == 0u && task) { +// uint tb = a.w[2], k = a.w[r + 10u]; +// a.w[r + 8u] = tb; +// a.w[2] = k > pc.trecs ? pc.trecs + tb : tb + k; +// } else if ((pc.mode == 0u) != task) { +// uint g = pc.mode == 0u ? a.w[r + 10u] : a.w[r + 9u], per = a.w[r + 13u], ib = a.w[1]; +// uint ni = per != 0u && g > pc.icap / per ? pc.icap + 1u : g * per; +// a.w[r + 2u] = ib; +// a.w[r + 12u] = ib >= pc.icap ? 0u : min(ni, pc.icap - ib); +// a.w[1] = min(ib + ni, 0x7fffffffu); +// } +// } +// } +static const uint32_t alloc_words[] = { + 0x07230203, 0x00010500, 0x0008000b, 0x000000df, 0x00000000, 0x00020011, 0x00000001, 0x00020011, + 0x000014e3, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, + 0x000014e4, 0x00000001, 0x0006000f, 0x00000005, 0x00000004, 0x6e69616d, 0x00000000, 0x0000000f, + 0x00060010, 0x00000004, 0x00000011, 0x00000001, 0x00000001, 0x00000001, 0x00040047, 0x00000008, + 0x00000006, 0x00000004, 0x00030047, 0x00000009, 0x00000002, 0x00050048, 0x00000009, 0x00000000, + 0x00000023, 0x00000000, 0x00030047, 0x0000000d, 0x00000002, 0x00050048, 0x0000000d, 0x00000000, + 0x00000023, 0x00000000, 0x00050048, 0x0000000d, 0x00000001, 0x00000023, 0x00000008, 0x00050048, + 0x0000000d, 0x00000002, 0x00000023, 0x0000000c, 0x00050048, 0x0000000d, 0x00000003, 0x00000023, + 0x00000010, 0x00050048, 0x0000000d, 0x00000004, 0x00000023, 0x00000014, 0x00050048, 0x0000000d, + 0x00000005, 0x00000023, 0x00000018, 0x00050048, 0x0000000d, 0x00000006, 0x00000023, 0x0000001c, + 0x00040047, 0x000000cb, 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, + 0x00000002, 0x00030027, 0x00000006, 0x000014e5, 0x00040015, 0x00000007, 0x00000020, 0x00000000, + 0x0003001d, 0x00000008, 0x00000007, 0x0003001e, 0x00000009, 0x00000008, 0x00040020, 0x00000006, + 0x000014e5, 0x00000009, 0x00040017, 0x0000000c, 0x00000007, 0x00000002, 0x0009001e, 0x0000000d, + 0x0000000c, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00040020, + 0x0000000e, 0x00000009, 0x0000000d, 0x0004003b, 0x0000000e, 0x0000000f, 0x00000009, 0x00040015, + 0x00000010, 0x00000020, 0x00000001, 0x0004002b, 0x00000010, 0x00000011, 0x00000000, 0x00040020, + 0x00000012, 0x00000009, 0x0000000c, 0x0004002b, 0x00000007, 0x00000018, 0x00000000, 0x0004002b, + 0x00000010, 0x0000001f, 0x00000002, 0x00040020, 0x00000020, 0x00000009, 0x00000007, 0x00020014, + 0x00000023, 0x0004002b, 0x00000010, 0x00000027, 0x00000001, 0x0004002b, 0x00000007, 0x0000002c, + 0x00001000, 0x0004002b, 0x00000007, 0x0000002e, 0x00000010, 0x00040020, 0x00000037, 0x000014e5, + 0x00000007, 0x0004002b, 0x00000010, 0x0000003b, 0x00000003, 0x0004002b, 0x00000010, 0x00000056, + 0x00000005, 0x0004002b, 0x00000010, 0x00000094, 0x00000004, 0x0004002b, 0x00000007, 0x000000a0, + 0x00000001, 0x0004002b, 0x00000007, 0x000000c5, 0x7fffffff, 0x00040017, 0x000000ca, 0x00000007, + 0x00000003, 0x0006002c, 0x000000ca, 0x000000cb, 0x000000a0, 0x000000a0, 0x000000a0, 0x0004002b, + 0x00000007, 0x000000d8, 0x0000004e, 0x0004002b, 0x00000007, 0x000000d9, 0x00000049, 0x0004002b, + 0x00000007, 0x000000da, 0x0000004a, 0x0004002b, 0x00000007, 0x000000db, 0x0000004d, 0x0004002b, + 0x00000007, 0x000000dc, 0x00000042, 0x0004002b, 0x00000007, 0x000000dd, 0x0000004c, 0x0004002b, + 0x00000007, 0x000000de, 0x00000048, 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, + 0x000200f8, 0x00000005, 0x00050041, 0x00000012, 0x00000013, 0x0000000f, 0x00000011, 0x0004003d, + 0x0000000c, 0x00000014, 0x00000013, 0x0004007c, 0x00000006, 0x00000015, 0x00000014, 0x000200f9, + 0x00000019, 0x000200f8, 0x00000019, 0x000700f5, 0x00000007, 0x000000cc, 0x00000018, 0x00000005, + 0x000000c9, 0x0000001c, 0x00050041, 0x00000020, 0x00000021, 0x0000000f, 0x0000001f, 0x0004003d, + 0x00000007, 0x00000022, 0x00000021, 0x000500b0, 0x00000023, 0x00000024, 0x000000cc, 0x00000022, + 0x000400f6, 0x0000001b, 0x0000001c, 0x00000000, 0x000400fa, 0x00000024, 0x0000001a, 0x0000001b, + 0x000200f8, 0x0000001a, 0x00050041, 0x00000020, 0x00000028, 0x0000000f, 0x00000027, 0x0004003d, + 0x00000007, 0x00000029, 0x00000028, 0x00050080, 0x00000007, 0x0000002b, 0x00000029, 0x000000cc, + 0x00050089, 0x00000007, 0x0000002d, 0x0000002b, 0x0000002c, 0x00050084, 0x00000007, 0x0000002f, + 0x0000002d, 0x0000002e, 0x00050080, 0x00000007, 0x00000036, 0x0000002f, 0x000000d8, 0x00060041, + 0x00000037, 0x00000038, 0x00000015, 0x00000011, 0x00000036, 0x0006003d, 0x00000007, 0x00000039, + 0x00000038, 0x00000002, 0x00000004, 0x000500ab, 0x00000023, 0x0000003a, 0x00000039, 0x00000018, + 0x00050041, 0x00000020, 0x0000003c, 0x0000000f, 0x0000003b, 0x0004003d, 0x00000007, 0x0000003d, + 0x0000003c, 0x000500aa, 0x00000023, 0x0000003e, 0x0000003d, 0x00000018, 0x000500a7, 0x00000023, + 0x00000040, 0x0000003e, 0x0000003a, 0x000300f7, 0x00000042, 0x00000000, 0x000400fa, 0x00000040, + 0x00000041, 0x00000067, 0x000200f8, 0x00000041, 0x00060041, 0x00000037, 0x00000045, 0x00000015, + 0x00000011, 0x0000001f, 0x0006003d, 0x00000007, 0x00000046, 0x00000045, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x0000004b, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, 0x0000004c, + 0x00000015, 0x00000011, 0x0000004b, 0x0006003d, 0x00000007, 0x0000004d, 0x0000004c, 0x00000002, + 0x00000004, 0x00050080, 0x00000007, 0x00000051, 0x0000002f, 0x000000de, 0x00060041, 0x00000037, + 0x00000053, 0x00000015, 0x00000011, 0x00000051, 0x0005003e, 0x00000053, 0x00000046, 0x00000002, + 0x00000004, 0x00050041, 0x00000020, 0x00000057, 0x0000000f, 0x00000056, 0x0004003d, 0x00000007, + 0x00000058, 0x00000057, 0x000500ac, 0x00000023, 0x00000059, 0x0000004d, 0x00000058, 0x000300f7, + 0x0000005c, 0x00000000, 0x000400fa, 0x00000059, 0x0000005b, 0x00000061, 0x000200f8, 0x0000005b, + 0x00050080, 0x00000007, 0x00000060, 0x00000058, 0x00000046, 0x000200f9, 0x0000005c, 0x000200f8, + 0x00000061, 0x00050080, 0x00000007, 0x00000064, 0x00000046, 0x0000004d, 0x000200f9, 0x0000005c, + 0x000200f8, 0x0000005c, 0x000700f5, 0x00000007, 0x000000d0, 0x00000060, 0x0000005b, 0x00000064, + 0x00000061, 0x0005003e, 0x00000045, 0x000000d0, 0x00000002, 0x00000004, 0x000200f9, 0x00000042, + 0x000200f8, 0x00000067, 0x000500a5, 0x00000023, 0x0000006c, 0x0000003e, 0x0000003a, 0x000300f7, + 0x0000006e, 0x00000000, 0x000400fa, 0x0000006c, 0x0000006d, 0x0000006e, 0x000200f8, 0x0000006d, + 0x000300f7, 0x00000075, 0x00000000, 0x000400fa, 0x0000003e, 0x00000074, 0x0000007b, 0x000200f8, + 0x00000074, 0x00050080, 0x00000007, 0x00000078, 0x0000002f, 0x000000da, 0x00060041, 0x00000037, + 0x00000079, 0x00000015, 0x00000011, 0x00000078, 0x0006003d, 0x00000007, 0x0000007a, 0x00000079, + 0x00000002, 0x00000004, 0x000200f9, 0x00000075, 0x000200f8, 0x0000007b, 0x00050080, 0x00000007, + 0x0000007f, 0x0000002f, 0x000000d9, 0x00060041, 0x00000037, 0x00000080, 0x00000015, 0x00000011, + 0x0000007f, 0x0006003d, 0x00000007, 0x00000081, 0x00000080, 0x00000002, 0x00000004, 0x000200f9, + 0x00000075, 0x000200f8, 0x00000075, 0x000700f5, 0x00000007, 0x000000cd, 0x0000007a, 0x00000074, + 0x00000081, 0x0000007b, 0x00050080, 0x00000007, 0x00000087, 0x0000002f, 0x000000db, 0x00060041, + 0x00000037, 0x00000088, 0x00000015, 0x00000011, 0x00000087, 0x0006003d, 0x00000007, 0x00000089, + 0x00000088, 0x00000002, 0x00000004, 0x00060041, 0x00000037, 0x0000008c, 0x00000015, 0x00000011, + 0x00000027, 0x0006003d, 0x00000007, 0x0000008d, 0x0000008c, 0x00000002, 0x00000004, 0x000500ab, + 0x00000023, 0x00000090, 0x00000089, 0x00000018, 0x000300f7, 0x00000092, 0x00000000, 0x000400fa, + 0x00000090, 0x00000091, 0x00000092, 0x000200f8, 0x00000091, 0x00050041, 0x00000020, 0x00000095, + 0x0000000f, 0x00000094, 0x0004003d, 0x00000007, 0x00000096, 0x00000095, 0x00050086, 0x00000007, + 0x00000098, 0x00000096, 0x00000089, 0x000500ac, 0x00000023, 0x00000099, 0x000000cd, 0x00000098, + 0x000200f9, 0x00000092, 0x000200f8, 0x00000092, 0x000700f5, 0x00000023, 0x0000009a, 0x00000090, + 0x00000075, 0x00000099, 0x00000091, 0x000300f7, 0x0000009d, 0x00000000, 0x000400fa, 0x0000009a, + 0x0000009c, 0x000000a2, 0x000200f8, 0x0000009c, 0x00050041, 0x00000020, 0x0000009e, 0x0000000f, + 0x00000094, 0x0004003d, 0x00000007, 0x0000009f, 0x0000009e, 0x00050080, 0x00000007, 0x000000a1, + 0x0000009f, 0x000000a0, 0x000200f9, 0x0000009d, 0x000200f8, 0x000000a2, 0x00050084, 0x00000007, + 0x000000a5, 0x000000cd, 0x00000089, 0x000200f9, 0x0000009d, 0x000200f8, 0x0000009d, 0x000700f5, + 0x00000007, 0x000000ce, 0x000000a1, 0x0000009c, 0x000000a5, 0x000000a2, 0x00050080, 0x00000007, + 0x000000aa, 0x0000002f, 0x000000dc, 0x00060041, 0x00000037, 0x000000ac, 0x00000015, 0x00000011, + 0x000000aa, 0x0005003e, 0x000000ac, 0x0000008d, 0x00000002, 0x00000004, 0x00050080, 0x00000007, + 0x000000b0, 0x0000002f, 0x000000dd, 0x00050041, 0x00000020, 0x000000b2, 0x0000000f, 0x00000094, + 0x0004003d, 0x00000007, 0x000000b3, 0x000000b2, 0x000500ae, 0x00000023, 0x000000b4, 0x0000008d, + 0x000000b3, 0x000300f7, 0x000000b7, 0x00000000, 0x000400fa, 0x000000b4, 0x000000b6, 0x000000b8, + 0x000200f8, 0x000000b6, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b8, 0x00050082, 0x00000007, + 0x000000bd, 0x000000b3, 0x0000008d, 0x0007000c, 0x00000007, 0x000000be, 0x00000001, 0x00000026, + 0x000000ce, 0x000000bd, 0x000200f9, 0x000000b7, 0x000200f8, 0x000000b7, 0x000700f5, 0x00000007, + 0x000000cf, 0x00000018, 0x000000b6, 0x000000be, 0x000000b8, 0x00060041, 0x00000037, 0x000000c0, + 0x00000015, 0x00000011, 0x000000b0, 0x0005003e, 0x000000c0, 0x000000cf, 0x00000002, 0x00000004, + 0x00050080, 0x00000007, 0x000000c4, 0x0000008d, 0x000000ce, 0x0007000c, 0x00000007, 0x000000c6, + 0x00000001, 0x00000026, 0x000000c4, 0x000000c5, 0x0005003e, 0x0000008c, 0x000000c6, 0x00000002, + 0x00000004, 0x000200f9, 0x0000006e, 0x000200f8, 0x0000006e, 0x000200f9, 0x00000042, 0x000200f8, + 0x00000042, 0x000200f9, 0x0000001c, 0x000200f8, 0x0000001c, 0x00050080, 0x00000007, 0x000000c9, + 0x000000cc, 0x00000027, 0x000200f9, 0x00000019, 0x000200f8, 0x0000001b, 0x000100fd, 0x00010038, +}; +const uint32_t *mesh_emu_alloc_spirv(size_t *n) { + *n = sizeof alloc_words / sizeof *alloc_words; + return alloc_words; +} +_Static_assert(MESH_EMU_DOFF / 4 == 64 && MESH_EMU_DRECS == 4096 && MESH_EMU_DWORDS == 16 && MESH_EMU_R_GROUPS == 9 && + MESH_EMU_R_FIRST == 10 && MESH_EMU_R_ICAP == 12 && MESH_EMU_R_PER == 13 && MESH_EMU_R_TASK == 14, + "the allocation shader's constants"); diff --git a/runtime/mesh_emu_spirv.h b/runtime/mesh_emu_spirv.h new file mode 100644 index 00000000..0584e3ae --- /dev/null +++ b/runtime/mesh_emu_spirv.h @@ -0,0 +1,74 @@ +#pragma once +// Mesh shader emulation for GPUs without VK_EXT_mesh_shader (NX2NAVI_MESH_EMULATION, vk_pipeline.c / vk_exec_draw.c): +// the translator's TaskEXT and MeshEXT modules (tools/xlate/MeshSpirv.cs) become GLCompute modules that write their +// vertices, triangles and task payloads into one GPU buffer (the arena), and a generated vertex shader reads the +// vertices back for the game's fragment shader. Plain C over SPIR-V words; no Vulkan types. +#include +#include + +// The arena: one buffer, its address in the push constants of every emulation stage. +// words 0-3: vertex slots used, indices reserved, task records reserved, overflows (the replay zeroes them per +// submission) +// MESH_EMU_DOFF: draw records, MESH_EMU_DWORDS words each: DrawIndexedIndirect {indexCount, instanceCount, +// firstIndex, vertexOffset, firstInstance}, mesh dispatch {x, y, z}, task base, then the MESH_EMU_R_* +// words; the replay fills them, the allocation shader (mesh_emu_alloc_spirv) reserves each draw's +// task records and index range, so that the stages of many draws can run together: a draw's +// indices are one range (its DrawIndexedIndirect) whatever the order its mesh workgroups run in +// MESH_EMU_TOFF: task records, MESH_EMU_TWORDS words each: mesh workgroup count, then the payload +// index area (uint32 indices), vertex area (16-byte slots: position, then each generic output) at offsets the +// replay passes in the push constants. +#define MESH_EMU_DOFF 256u +#define MESH_EMU_DWORDS 16u +#define MESH_EMU_DRECS 4096u +#define MESH_EMU_TOFF (MESH_EMU_DOFF + MESH_EMU_DRECS * MESH_EMU_DWORDS * 4u) +#define MESH_EMU_R_GROUPS 9u // mesh workgroups with output: the task stage adds the counts it emits +#define MESH_EMU_R_FIRST 10u // the first stage's workgroups (the replay: the count, or a copy of the indirect one) +#define MESH_EMU_R_ICUR 11u // indices taken from the draw's range (the mesh stage's atomics) +#define MESH_EMU_R_ICAP 12u // the draw's range: indices reserved (fit in the area) +#define MESH_EMU_R_PER 13u // indices one mesh workgroup may emit (OutputPrimitives x vertices per primitive) +#define MESH_EMU_R_TASK 14u // 1 with a task stage +#define MESH_EMU_TWORDS 780u // 1 + payloads of up to 776 words (3072 bytes and the 32 the + // translator adds for the payload header since 0.6.0), padded +// task records: as many as the push constants say (the replay sizes them, NX2NAVI_MESH_EMU_TASKS); the index area +// follows them +// Push constants of every emulation stage (compute and the vertex shader), 32 bytes: +// u64 arena address; uint draw record (word index); uint index area (word index); uint index capacity; +// uint vertex area (16-byte slot index); uint vertex capacity (slots); uint task record capacity +// Header words 0-2 count what the stages asked for, beyond the capacities too (the replay logs the most per +// submission); word 3 counts mesh workgroups that did not fit. +#define MESH_EMU_PC_BYTES 32u + +// The allocation shader (GLCompute, one invocation): for the n draw records from ring index first, mode 0 reserves the +// task records of draws with a task stage (MESH_EMU_R_FIRST of them, from word 2) and the index ranges of draws +// without one (MESH_EMU_R_FIRST x MESH_EMU_R_PER, from word 1); mode 1, after the task stages, the index ranges of +// draws with one (MESH_EMU_R_GROUPS x MESH_EMU_R_PER). Push constants, 32 bytes: u64 arena address; uint first, n, +// mode, index capacity, task record capacity, unused. Static words, *n of them. +const uint32_t *mesh_emu_alloc_spirv(size_t *n); + +typedef struct MeshEmuInfo { + int task; // 1 TaskEXT, 0 MeshEXT + uint32_t threads; // LocalSize x + uint32_t max_v, max_p; // mesh: OutputVertices, OutputPrimitivesEXT + uint32_t prim; // mesh: 3 triangles, 2 lines, 1 points (triangles and points are emulated) + uint32_t psize_slot; // mesh: the slot holding the point size (x), 0 without one + uint32_t payload_words; // the task payload's size (mesh: what it reads, 0 without a task stage) + uint32_t max_mesh; // task: the most mesh workgroups one task workgroup launches (dispatch width) + uint32_t slots; // mesh: 16-byte slots per vertex: position and each generic output + uint32_t n_generic; // mesh: generic outputs, in slot order 1..n + uint8_t generic_loc[32]; // their locations +} MeshEmuInfo; + +// The compute module for a TaskEXT or MeshEXT module (malloc'd, *out_n words); NULL with why set when the module is not +// one the converter knows (it then stays unemulated: its draws are skipped as before). +uint32_t *mesh_emu_compute(const uint32_t *w, size_t n, MeshEmuInfo *info, size_t *out_n, const char **why); + +// The vertex shader of an emulated mesh pipeline: position and the generic outputs from the vertex slots, as types[l] +// (0 float, 1 signed, 2 unsigned) with comps[l] components for every location l in need (the fragment stage's +// inputs); a location the mesh stage does not write is zero. malloc'd, *out_n words. +// A task or mesh module's output primitive: 3 triangles, 2 lines, 1 points, 0 not a mesh module. +int mesh_emu_prim(const uint32_t *w, size_t n); + +// fsw: the fragment shader's SPIR-V; its input variables are matched one for one (Location, Component, type, Flat), +// else (NULL, or a declaration the parser does not take) one vector per location of need. +uint32_t *mesh_emu_vs(const MeshEmuInfo *m, uint32_t need, const uint8_t types[32], const uint8_t comps[32], + const uint32_t *fsw, size_t fsn, size_t *out_n); diff --git a/runtime/nvn_api_core.c b/runtime/nvn_api_core.c index 2f443c3c..3945b419 100644 --- a/runtime/nvn_api_core.c +++ b/runtime/nvn_api_core.c @@ -750,6 +750,9 @@ NVN_API(nvnQueueSubmitCommands) { // point). Copies of the data at submit made it worse, a mix of two frames' versions. // NX2NAVI_SUBMIT_FINISH=: bit i waits after the i-th submission of a frame (default 0x1); a plain 1 waits // after every submission, 0 never. + // Since the staged mirror uploads (vk_memory.c) the copies of guest memory a submission reads are taken while the + // replay records it, and the wait ends once the replay has submitted it (vk_sync_recorded_wait); the GPU's wait + // below remains for a submission whose GPU reads guest memory itself. // The wait (exec_wait, vk_sync.c vk_sync_exec_wait) ends when the GPU has run the submission's passes and the // results the game reads once it goes on are in its memory, the last frame's occlusion counts among them; the // write-backs and the completion thread's other side effects are not waited for. NX2NAVI_WB_SPLIT=0 makes it diff --git a/runtime/ov_movie2.c b/runtime/ov_movie2.c index 7766869f..4b89087c 100644 --- a/runtime/ov_movie2.c +++ b/runtime/ov_movie2.c @@ -289,20 +289,64 @@ static void stripe_run(void *arg) { to_nv12(t); } } +// The NSTRIPES - 1 helper threads are made once and kept. Made per call (two calls per picture, 30-60 pictures a +// second) they were about 300 new threads a second for as long as a movie played, and every DLL in the process (the +// GPU driver's, overlays and audio hooks) sets itself up for each new thread: a cost per thread, and with an AMD driver +// on an RX 580 the process commit grew by about 300 MB a second during the title movie until the system ran out +// (FATAL "movie: out of memory for a 1920x1080 frame" at 45.7 GB). One caller at a time (call_lock); a worker that +// could not be started leaves its stripe to the caller. +static struct { + PlatMutex call_lock; // one run_stripes at a time (several decoders may exist) + PlatMutex lock; // the fields below + PlatCond go, done; + Stripe job[NSTRIPES]; + uint64_t gen; // bumped per batch + int pending; // helper stripes of the batch not finished + int nworkers; // started helpers (stripes 1..nworkers); -1 not tried yet +} SP = {PLAT_MUTEX_INIT, PLAT_MUTEX_INIT, PLAT_COND_INIT, PLAT_COND_INIT, {{0}}, 0, 0, -1}; + +static void stripe_worker(void *arg) { + int i = (int)(intptr_t)arg; + uint64_t seen = 0; + plat_mutex_lock(&SP.lock); + for (;;) { + while (SP.gen == seen) plat_cond_wait(&SP.go, &SP.lock); + seen = SP.gen; + Stripe t = SP.job[i]; + plat_mutex_unlock(&SP.lock); + stripe_run(&t); + plat_mutex_lock(&SP.lock); + if (!--SP.pending) plat_cond_broadcast(&SP.done); + } +} + static void run_stripes(Stripe base) { - Stripe st[NSTRIPES]; - PlatThread *th[NSTRIPES] = {0}; + plat_mutex_lock(&SP.call_lock); + if (SP.nworkers < 0) { + SP.nworkers = 0; + for (int i = 1; i < NSTRIPES; i++, SP.nworkers++) + if (!plat_thread_spawn(stripe_worker, (void *)(intptr_t)i, "movie convert", 0)) break; + if (SP.nworkers < NSTRIPES - 1) rt_log("movie: %d of %d convert threads started", SP.nworkers, NSTRIPES - 1); + } int rows = ((base.h + NSTRIPES - 1) / NSTRIPES + 1) & ~1; // even: chroma rows split at the same places + Stripe st[NSTRIPES]; for (int i = 0; i < NSTRIPES; i++) { st[i] = base; st[i].y0 = i * rows < base.h ? i * rows : base.h; st[i].y1 = (i + 1) * rows < base.h ? (i + 1) * rows : base.h; - if (i) th[i] = plat_thread_new(stripe_run, &st[i], "movie convert"); - if (i && !th[i]) stripe_run(&st[i]); } + plat_mutex_lock(&SP.lock); + memcpy(SP.job, st, sizeof st); + SP.pending = SP.nworkers; + SP.gen++; + plat_cond_broadcast(&SP.go); + plat_mutex_unlock(&SP.lock); stripe_run(&st[0]); - for (int i = 1; i < NSTRIPES; i++) - if (th[i]) plat_thread_join(th[i]); + for (int i = SP.nworkers + 1; i < NSTRIPES; i++) stripe_run(&st[i]); // helpers that could not start + plat_mutex_lock(&SP.lock); + while (SP.pending) plat_cond_wait(&SP.done, &SP.lock); + plat_mutex_unlock(&SP.lock); + plat_mutex_unlock(&SP.call_lock); } // Out of memory for a decoded frame: the frame is dropped (the game shows the previous one a little longer) and diff --git a/runtime/platform/linux.c b/runtime/platform/linux.c index da703894..8c531f28 100644 --- a/runtime/platform/linux.c +++ b/runtime/platform/linux.c @@ -1294,6 +1294,7 @@ void plat_mem_access_hook(int (*fn)(uint64_t addr, int write, uint64_t ip)) { void plat_crash_context(const char *s) { (void)s; } // the Windows crash notes only int plat_crash_filter_reassert(char *who, size_t n) { (void)who, (void)n; return 0; } // Windows only +const char *plat_crash_context_get(void) { return NULL; } // plat_jmp_save / plat_jmp_go (platform.h): leaf code that does not move rsp before it leaves, as on Windows. _Static_assert(sizeof(PlatJmp) == 72 && __builtin_offsetof(PlatJmp, mxcsr) == 64, "plat_jmp_save layout"); diff --git a/runtime/platform/platform.h b/runtime/platform/platform.h index 0722cf08..64c0b056 100644 --- a/runtime/platform/platform.h +++ b/runtime/platform/platform.h @@ -296,6 +296,7 @@ void plat_crash_context(const char *s); // Puts the crash hook's last-chance exception filter back in place. 1 when a DLL loaded later had set its own in its // place (who: that filter's module and offset), 0 when it was still there. Windows only; 0 elsewhere. int plat_crash_filter_reassert(char *who, size_t n); +const char *plat_crash_context_get(void); // the calling thread's note, for the crash report (NULL: none) int plat_mem_read(uint64_t addr, void *out, size_t n); // copies n bytes, 0 instead of faulting on unmapped memory int plat_mem_readable(uint64_t addr, uint64_t n); // every page of [addr, addr + n) can be read; no fault void *plat_aligned_alloc(size_t size, size_t align); // free with plat_aligned_free; NULL on failure diff --git a/runtime/platform/win32.c b/runtime/platform/win32.c index fa742a84..cfaa5c9e 100644 --- a/runtime/platform/win32.c +++ b/runtime/platform/win32.c @@ -908,6 +908,7 @@ static void (*crash_fn)(const PlatCrash *c); static const char *image_name(const char *base); static __thread const char *crash_ctx; void plat_crash_context(const char *s) { crash_ctx = s; } +const char *plat_crash_context_get(void) { return crash_ctx; } static void raw_note(const char *tag, DWORD code, uintptr_t rip, uintptr_t addr) { char b[256]; void *base = NULL; diff --git a/runtime/prof_syms.c b/runtime/prof_syms.c new file mode 100644 index 00000000..dbf098df --- /dev/null +++ b/runtime/prof_syms.c @@ -0,0 +1,112 @@ +// Names for a sampling profile's exe addresses from the exe's own COFF symbol table (runtime/prof_win.c). A MinGW link +// keeps the table in the file (PointerToSymbolTable), so a profile names its functions without the link map: for each +// address the profile lists, `sym
+`. Plain file reads, no Windows headers, so the parser +// builds and is tested anywhere. +#include +#include +#include +#include + +#include "prof_syms.h" + +typedef struct { uint32_t rva; uint32_t str; char sname[8]; } CSym; // str: string-table offset, 0 short name + +static int cmp_sym(const void *a, const void *b) { + uint32_t x = ((const CSym *)a)->rva, y = ((const CSym *)b)->rva; + return x < y ? -1 : x > y; +} + +static int cmp_u64(const void *a, const void *b) { + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + return x < y ? -1 : x > y; +} + +static uint32_t rd32(const uint8_t *p) { return p[0] | p[1] << 8 | p[2] << 16 | (uint32_t)p[3] << 24; } +static uint16_t rd16(const uint8_t *p) { return (uint16_t)(p[0] | p[1] << 8); } + +int prof_syms_write(FILE *out, const char *exe_path, uint64_t base, const uint64_t *addr, size_t n) { + FILE *f = fopen(exe_path, "rb"); + if (!f) return -1; + uint8_t h[0x400]; + int named = -1; + CSym *syms = NULL; + char *strtab = NULL; + uint8_t *raw = NULL; + uint64_t *a = NULL; + if (fread(h, 1, 0x40, f) != 0x40 || h[0] != 'M' || h[1] != 'Z') goto done; + uint32_t pe = rd32(h + 0x3c); + if (fseek(f, pe, SEEK_SET) || fread(h, 1, 24, f) != 24 || memcmp(h, "PE\0\0", 4)) goto done; + uint16_t nsec = rd16(h + 6), opt = rd16(h + 20); + uint32_t symptr = rd32(h + 12), nsym = rd32(h + 16); + if (!symptr || !nsym || nsec > 96) goto done; + uint32_t sec_va[96], sec_exec[96]; + if (fseek(f, pe + 24 + opt, SEEK_SET)) goto done; + for (int i = 0; i < nsec; i++) { + if (fread(h, 1, 40, f) != 40) goto done; + sec_va[i] = rd32(h + 12); + sec_exec[i] = (rd32(h + 36) & 0x20000020) != 0; // CNT_CODE or MEM_EXECUTE + } + // the string table follows the symbols: its first 4 bytes are its size + uint8_t szb[4]; + if (fseek(f, (long)symptr + (long)nsym * 18, SEEK_SET) || fread(szb, 1, 4, f) != 4) goto done; + uint32_t strsz = rd32(szb); + strtab = malloc(strsz + 1u); + if (!strtab) goto done; + memcpy(strtab, szb, 4); + if (strsz > 4 && fread(strtab + 4, 1, strsz - 4, f) != strsz - 4) goto done; + strtab[strsz] = 0; + // the symbols in chunks; functions in executable sections + size_t ns = 0, cs = 1 << 16; + syms = malloc(cs * sizeof *syms); + enum { CH = 1 << 16 }; + raw = malloc(CH * 18); + if (!syms || !raw || fseek(f, symptr, SEEK_SET)) goto done; + uint32_t skip = 0; + for (uint32_t done_n = 0; done_n < nsym;) { + uint32_t k = nsym - done_n < CH ? nsym - done_n : CH; + if (fread(raw, 18, k, f) != k) goto done; + for (uint32_t i = 0; i < k; i++) { + const uint8_t *s = raw + i * 18; + if (skip) { skip--; continue; } // auxiliary records + skip = s[17]; + int16_t sec = (int16_t)rd16(s + 12); + uint8_t cls = s[16]; + if (sec <= 0 || sec > nsec || !sec_exec[sec - 1] || (cls != 2 && cls != 3)) continue; + CSym c = {sec_va[sec - 1] + rd32(s + 8), 0, {0}}; + if (rd32(s)) memcpy(c.sname, s, 8); + else c.str = rd32(s + 4); + const char *nm = c.str ? (c.str < strsz ? strtab + c.str : "") : c.sname; + if (nm[0] == '.' || !nm[0]) continue; // section symbols + if (ns == cs) { + CSym *g = realloc(syms, (cs *= 2) * sizeof *syms); + if (!g) goto done; + syms = g; + } + syms[ns++] = c; + } + done_n += k; + } + qsort(syms, ns, sizeof *syms, cmp_sym); + a = malloc((n ? n : 1) * sizeof *a); + if (!a) goto done; + memcpy(a, addr, n * sizeof *a); + qsort(a, n, sizeof *a, cmp_u64); + named = 0; + size_t j = 0; + for (size_t i = 0; i < n; i++) { + if (i && a[i] == a[i - 1]) continue; + if (a[i] < base || a[i] - base >= 0xffffffffull) continue; + uint32_t rva = (uint32_t)(a[i] - base); + while (j < ns && syms[j].rva <= rva) j++; // a is sorted, so j only grows + if (!j) continue; + const CSym *c = &syms[j - 1]; + char sn[9] = {0}; + const char *nm = c->str ? (c->str < strsz ? strtab + c->str : "?") : (memcpy(sn, c->sname, 8), sn); + fprintf(out, "sym %llx %s+0x%x\n", (unsigned long long)a[i], nm, rva - c->rva); + named++; + } +done: + free(a), free(raw), free(syms), free(strtab); + fclose(f); + return named; +} diff --git a/runtime/prof_syms.h b/runtime/prof_syms.h new file mode 100644 index 00000000..c583e8f4 --- /dev/null +++ b/runtime/prof_syms.h @@ -0,0 +1,9 @@ +#pragma once +#include +#include +#include + +// Writes `sym
+0x` for each of the n addresses (any order, repeats allowed) that falls in +// the image loaded at base, from the COFF symbol table of the file at exe_path. Returns the lines written, -1 when the +// file has no symbol table or cannot be read. +int prof_syms_write(FILE *out, const char *exe_path, uint64_t base, const uint64_t *addr, size_t n); diff --git a/runtime/prof_win.c b/runtime/prof_win.c index 95470d9c..a8df4008 100644 --- a/runtime/prof_win.c +++ b/runtime/prof_win.c @@ -29,6 +29,7 @@ #include "host.h" #include "perfmon.h" #include "prof.h" +#include "prof_syms.h" #define render_tid prof_render_tid #define main_tid prof_main_tid @@ -355,7 +356,21 @@ void prof_sampler_write(void) { fputc('\n', f); } if (stacks_dropped) fprintf(f, "stacks_dropped %llu\n", (unsigned long long)stacks_dropped); + // the exe's own symbol table names the addresses above, so the profile needs no link map + size_t na = 0, ca = (size_t)cnt + 1; + for (int i = 0; i < ns && i < PROF_STACKS; i++) ca += stacks[i].depth; + uint64_t *addrs = malloc(ca * sizeof *addrs); + char exe[MAX_PATH]; + int named = -1; + if (addrs && GetModuleFileNameA(NULL, exe, sizeof exe)) { + for (int i = 0; i < cnt && i < PROF_TOP; i++) addrs[na++] = table[i].rip; + for (int i = 0; i < ns && i < PROF_STACKS; i++) + for (uint32_t d = 0; d < stacks[i].depth; d++) addrs[na++] = stacks[i].rip[d]; + named = prof_syms_write(f, exe, (uint64_t)(uintptr_t)GetModuleHandleW(NULL), addrs, na); + } + free(addrs); fclose(f); + if (named < 0) rt_log("prof: no symbol table in the exe: the addresses stay unnamed"); rt_log("prof: %llu samples, %d Rips -> %s", (unsigned long long)total, cnt, out_path); } diff --git a/runtime/ui/menu.cpp b/runtime/ui/menu.cpp index e318b28e..7b40d8d7 100644 --- a/runtime/ui/menu.cpp +++ b/runtime/ui/menu.cpp @@ -83,6 +83,7 @@ int vk_fg4_failed(void); // ... it failed in this run, FSR 3.1's int vk_fg4_active(void); // ... its swapchain is in place const char *vk_fg4_name(void); // ... the version the library runs here ("3.1.6", "4.0.2") int vk_texture_detail_auto(void); // vk_texture.c: the texture detail auto picks on this card, 0 full to 2 low +int vk_mesh_shader_native(void); // vk_device.c: the GPU has mesh shaders (the mesh emulation does nothing) double frame_step_limit_hz(void); int frame_limit_adjustable(void); int rtss_limit_state(void); @@ -504,7 +505,7 @@ static int fonts_up(float scale) { // what the title face shows in the CJK languages: the menu's title and tabs (Simplified Chinese menu text) and // the boot screen's title (the game's language) static const UiTextId titles[] = {UI_MENU_TITLE, UI_MENU_TAB_SCREEN, UI_MENU_TAB_GAME, UI_MENU_TAB_SOUND, - UI_MENU_TAB_CONTROLS, UI_MENU_TAB_COOP, UI_BOOT_TITLE}; + UI_MENU_TAB_CONTROLS, UI_MENU_TAB_LAMA, UI_MENU_TAB_COOP, UI_BOOT_TITLE}; for (int i = 0; i < N; i++) { if (!data[i]) continue; ImFontGlyphRangesBuilder b; @@ -572,8 +573,8 @@ extern "C" void menu_vk_release(void) { // NX2NAVI_ prefix optional) puts the cursor on that setting's row and scrolls it into view when the row is next drawn // (opt_focus_key), so a page's shot shows a row below the first screenful; it does not switch tabs. COOP_PAGE, COOP_JOIN // and COOP_ACCEPT_OFFER drive the Co-op tab (ui/coop_page.h coop_test_set). -// Ocarina of Time: the options this port has and co-op (ui/coop_page.cpp) -enum { TAB_SCREEN, TAB_GAME, TAB_SOUND, TAB_CONTROLS, TAB_COOP, TAB_COUNT }; +// Ocarina of Time: the options this port has, co-op (ui/coop_page.cpp) and the Lama patches tab (the RX 580 patches) +enum { TAB_SCREEN, TAB_GAME, TAB_SOUND, TAB_CONTROLS, TAB_LAMA, TAB_COOP, TAB_COUNT }; // co-op last: tabs_shown enum { TEST_PAGE_FRAMES = 30, TEST_SETS = 16, TEST_OPENS = 8 }; struct TestSet { uint64_t frame; char key[64], value[64]; }; static struct { @@ -1623,8 +1624,81 @@ static void tab_coop(float scale) { coop_after_page(scale); } +// ---- the Lama patches tab: the settings the RX 580 patches add (apply_rx580_fix), each read where its patch reads it +static void tab_lama(float scale) { + opt_begin("##lama", scale); + opt_text(UI_MENU_LAMA_INTRO); + opt_group(UI_MENU_LAMA_MEMORY); + { // vk_texture.c sweep_released + static const char *const values[] = {"", "512", "1024", "2048", "4096", "8192"}; + const char *names[] = {menu_text(UI_MENU_AUTO), "512", "1024", "2048", "4096", "8192"}; + opt_choice_list("NX2NAVI_TEX_RELEASED_MAX", UI_MENU_LAMA_RELEASED, values, names, 6, UI_MENU_LAMA_RELEASED_DESC); + } + opt_slider("NX2NAVI_MIRROR_HOT", UI_MENU_LAMA_MIRROR_HOT, 0, 100, UI_MENU_LAMA_MIRROR_HOT_DESC, 5, "%.0f %%"); + { // vk_device.c vram_heap_limits + static const char *const values[] = {"auto", "0"}; + const char *names[] = {menu_text(UI_MENU_AUTO), menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_VRAM_HEAP_LIMIT", UI_MENU_LAMA_HEAP_LIMIT, values, names, 2, UI_MENU_LAMA_HEAP_LIMIT_DESC); + } + { // vk_vma_budget.c reserve_bytes + static const char *const values[] = {"512", "256", "768", "1024", "0"}; + const char *names[] = {"512 MB", "256 MB", "768 MB", "1024 MB", menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_VRAM_RESERVE_MB", UI_MENU_LAMA_RESERVE, values, names, 5, UI_MENU_LAMA_RESERVE_DESC); + } + { // vk_memory.c upload_batch_cap + static const char *const values[] = {"128", "256", "512", "1024", "0"}; + const char *names[] = {"128 MB", "256 MB", "512 MB", "1024 MB", menu_text(UI_MENU_LAMA_NO_LIMIT)}; + opt_choice_list("NX2NAVI_UPLOAD_BATCH_MB", UI_MENU_LAMA_UPLOAD_BATCH, values, names, 5, UI_MENU_LAMA_UPLOAD_BATCH_DESC); + } + opt_toggle("NX2NAVI_ASTC_SCRATCH_KEEP", UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC); + { // vk_memory.c freeze_pass: submissions a region stays unwritten before it is frozen + static const char *const values[] = {"1800", "1200", "600", "60", "0"}; + const char *names[] = {"1800", "1200", "600", "60", menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_FREEZE_IDLE", UI_MENU_LAMA_FREEZE, values, names, 5, UI_MENU_LAMA_FREEZE_DESC); + } + { // vk_sync.c res_mask: the result kinds the submit wait counts (31 all, 30 all but events) + static const char *const values[] = {"31", "30"}; + const char *names[] = {menu_text(UI_MENU_LAMA_RESULTS_ALL), menu_text(UI_MENU_LAMA_RESULTS_NO_EVENTS)}; + opt_choice_list("NX2NAVI_SUBMIT_FINISH_KINDS", UI_MENU_LAMA_RESULTS, values, names, 2, UI_MENU_LAMA_RESULTS_DESC); + } + opt_toggle("NX2NAVI_ZSTD_NATIVE", UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC); // ov_zstd.c trusted + opt_group(UI_MENU_LAMA_RENDER); + opt_toggle("NX2NAVI_REPLAY_ASYNC", UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC); // vk_backend.c + const bool native = vk_mesh_shader_native() != 0; // vk_exec_draw.c: the emulation only where the GPU lacks them + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); + opt_toggle("NX2NAVI_MESH_EMULATION", UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC); + { // vk_exec_draw.c foliage_skip + static const char *const values[] = {"full", "reduced", "off"}; + const char *names[] = {menu_text(UI_MENU_LAMA_FOLIAGE_FULL), menu_text(UI_MENU_LAMA_FOLIAGE_REDUCED), menu_text(UI_MENU_OFF)}; + opt_choice_list("NX2NAVI_FOLIAGE", UI_MENU_LAMA_FOLIAGE, values, names, 3, UI_MENU_LAMA_FOLIAGE_DESC); + } + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); + opt_slider("NX2NAVI_MESH_EMU_MB", UI_MENU_LAMA_MESH_MB, 64, 2048, UI_MENU_LAMA_MESH_MB_DESC, 64, "%.0f MB"); + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); + opt_slider("NX2NAVI_MESH_EMU_TASKS", UI_MENU_LAMA_MESH_TASKS, 4096, 65536, UI_MENU_LAMA_MESH_TASKS_DESC, 4096, "%.0f"); + if (native) opt_disable_next(UI_MENU_LAMA_MESH_NATIVE); + opt_toggle("NX2NAVI_MESH_EMU_BATCH", UI_MENU_LAMA_MESH_BATCH, UI_MENU_LAMA_MESH_BATCH_DESC); // vk_exec_draw.c + opt_toggle("NX2NAVI_TEX_CLOBBER", UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC); + static const UiTextId vec[] = {UI_MENU_OFF, UI_MENU_LAMA_VEC_AUTO, UI_MENU_ON}; // gfx_shader.c patch_vec_index + opt_choice("NX2NAVI_VEC_INDEX_FIX", UI_MENU_LAMA_VEC_INDEX, vec, 3, UI_MENU_LAMA_VEC_INDEX_DESC); + opt_group(UI_MENU_LAMA_BUILD); + // tools/make_game.py reads it from nx2navi.ini at the next Rebuild the Game (guest_opt_ini) + opt_toggle_value("NX2NAVI_GUEST_OPT", UI_MENU_LAMA_GUEST_OPT, UI_MENU_LAMA_GUEST_OPT_DESC, "2", "1", + menu_text(UI_MENU_LAMA_GUEST_OPT_NOTE)); + opt_group(UI_MENU_LAMA_DIAG); + opt_toggle("NX2NAVI_PASS_TIMES", UI_MENU_LAMA_PASS_TIMES, UI_MENU_LAMA_PASS_TIMES_DESC); // vk_crumbs.c +#ifdef _WIN32 + { // vk_dump.c: the keys it knows + static const char *const values[] = {"", "pause", "scrolllock", "insert", "f12"}; + const char *names[] = {menu_text(UI_MENU_OFF), "Pause", "Scroll Lock", "Insert", "F12"}; + opt_choice_list("NX2NAVI_DUMP_KEY", UI_MENU_LAMA_DUMP_KEY, values, names, 5, UI_MENU_LAMA_DUMP_KEY_DESC); + } +#endif + opt_end(); +} + static const UiTextId kTabNames[TAB_COUNT] = {UI_MENU_TAB_SCREEN, UI_MENU_TAB_GAME, UI_MENU_TAB_SOUND, - UI_MENU_TAB_CONTROLS, UI_MENU_TAB_COOP}; + UI_MENU_TAB_CONTROLS, UI_MENU_TAB_LAMA, UI_MENU_TAB_COOP}; static int g_tab; // the tab shown // The tabs the menu shows: the Co-op tab is a beta, shown only with coop_beta = 1 in nx2navi.ini (NX2NAVI_COOP_BETA), @@ -1751,6 +1825,7 @@ static void layout_menu(uint32_t w, uint32_t h, float scale) { case TAB_SOUND: tab_sound(scale); break; case TAB_CONTROLS: tab_controls(scale); break; case TAB_COOP: tab_coop(scale); break; + case TAB_LAMA: tab_lama(scale); break; } ImGui::EndTabItem(); } diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 9f7402bc..30f95186 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -89,6 +89,93 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_TEX_DETAIL_DESC] = "Sharpness of large surface textures. Reduced and Low leave out their largest versions, so that an area fits the video memory of 6 and 8 GB cards; only surfaces seen up close look softer. Automatic picks by the card's video memory.", [UI_MENU_TEX_DETAIL_AUTO] = "Automatic: %s", [UI_MENU_TEX_DETAIL_FULL] = "Full", [UI_MENU_TEX_DETAIL_REDUCED] = "Reduced", [UI_MENU_TEX_DETAIL_LOW] = "Low", + [UI_MENU_TAB_LAMA] = "Lama patches", + [UI_MENU_LAMA_INTRO] = "Settings of the RX 580 patches (apply_rx580_fix). Most take effect after a restart.", + [UI_MENU_LAMA_MEMORY] = "Video memory and textures", + [UI_MENU_LAMA_RENDER] = "Rendering", + [UI_MENU_LAMA_BUILD] = "Build", + [UI_MENU_LAMA_DIAG] = "Diagnostics", + [UI_MENU_LAMA_RELEASED] = "Released textures kept", + [UI_MENU_LAMA_RELEASED_DESC] = "How many textures the game has released are kept as they are; past it only those that " + "lose nothing when dropped go. Fewer leaves more video memory for the rest; more " + "saves making them again; too few lets some blink black for a moment. Auto: one per 2 MB " + "of the video memory budget, 2048 to 8192.", + [UI_MENU_LAMA_CLOBBER] = "Fix overwritten textures", + [UI_MENU_LAMA_CLOBBER_DESC] = "A texture the game reads after another image was drawn over its memory gets memory of " + "its own. Fixes the teal splotches in Kokiri Forest's fog.", + [UI_MENU_LAMA_MIRROR_HOT] = "Pools copied every frame stay in system memory", + [UI_MENU_LAMA_MIRROR_HOT_DESC] = "A game memory pool whose copies to and from video memory each frame reach this " + "percentage of its size is read by the GPU from system memory instead, if the " + "CPU does not write it (a pool it refills each frame stays in video memory: read " + "in place, the GPU saw the next frame's data). For GPUs and buses short of " + "bandwidth; 25 suits an RX 580. 0: off.", + [UI_MENU_LAMA_RESERVE] = "Video memory kept free", + [UI_MENU_LAMA_RESERVE_DESC] = "Room kept free in each video memory heap for what FidelityFX (FSR 3.1, frame generation) and the " + "driver allocate on their own: a heap with less than this left takes no new allocation while " + "another heap can. Off: video memory can fill up with frame generation on, and the game ends.", + [UI_MENU_LAMA_HEAP_LIMIT] = "Video memory heap limit", + [UI_MENU_LAMA_HEAP_LIMIT_DESC] = "Auto: on GPUs with a second video memory heap (an RX 580 shows 6 GB and 2 GB), " + "each heap is filled only to its budget before the next one is used, so the " + "driver does not run out. Off: no limit.", + [UI_MENU_LAMA_UPLOAD_BATCH] = "Upload batch limit", + [UI_MENU_LAMA_UPLOAD_BATCH_DESC] = "Texture uploads waiting in staging memory are sent to the GPU early once they " + "hold this much, so an area load cannot run video memory out.", + [UI_MENU_LAMA_NO_LIMIT] = "No limit", + [UI_MENU_LAMA_ASTC_KEEP] = "Keep the ASTC decode buffer", + [UI_MENU_LAMA_ASTC_KEEP_DESC] = "The buffer that GPU-decodes compressed textures is kept between upload batches " + "instead of being allocated again for each one.", + [UI_MENU_LAMA_MESH_EMU] = "Mesh shader emulation", + [UI_MENU_LAMA_MESH_EMU_DESC] = "Draws grass, moss and other foliage on GPUs without mesh shaders (AMD RX 400/500 " + "and Vega, GTX 10-series and older), with compute shaders. GPUs with mesh shaders " + "ignore it.", + [UI_MENU_LAMA_MESH_NATIVE] = "this GPU has mesh shaders", + [UI_MENU_LAMA_MESH_MB] = "Mesh emulation memory", + [UI_MENU_LAMA_MESH_MB_DESC] = "Video memory for the emulated meshes' vertices and indices, in MB.", + [UI_MENU_LAMA_MESH_BATCH] = "Mesh emulation before the pass", + [UI_MENU_LAMA_MESH_BATCH_DESC] = "The emulated mesh stages of a render pass run before the pass in a command buffer " + "of their own, instead of splitting the pass at every emulated draw. Off: the " + "split, as before.", + [UI_MENU_LAMA_MESH_TASKS] = "Mesh emulation task records", + [UI_MENU_LAMA_MESH_TASKS_DESC] = "Task records per submission for the emulated meshes. Raise it when the log reports " + "that a submission asked for more.", + [UI_MENU_LAMA_VEC_INDEX] = "Vector index fix", + [UI_MENU_LAMA_VEC_INDEX_DESC] = "Shaders read uniform vector components at computed indices whole and select the " + "component, for drivers that get such reads wrong. Auto: on AMD's own drivers.", + [UI_MENU_LAMA_VEC_AUTO] = "Auto (AMD)", + [UI_MENU_LAMA_GUEST_OPT] = "Optimised game code (-O2)", + [UI_MENU_LAMA_GUEST_OPT_DESC] = "The next Rebuild the Game in the launcher compiles the game's code at -O2 instead " + "of -O1: the game's own threads run faster on a slower CPU. The build takes " + "several times longer.", + [UI_MENU_LAMA_GUEST_OPT_NOTE] = "at the next rebuild", + [UI_MENU_LAMA_FREEZE] = "Freeze idle memory after", + [UI_MENU_LAMA_FREEZE_DESC] = "Submissions a memory region stays unwritten before it is made read-only, so the write " + "tracking can skip it. 1800 (about 15 s): few thaws and small tracking queries. 60: " + "the game thaws regions all the time and every thaw stalls a frame. Off: no stalls, but " + "the tracking scans all memory every frame.", + [UI_MENU_LAMA_FOLIAGE] = "Foliage", + [UI_MENU_LAMA_FOLIAGE_DESC] = "Full: all of it. Reduced: no small grass or moss on the ground; ferns, leaves and " + "other plants stay. Off: no foliage. Terrain, rocks and buildings always stay.", + [UI_MENU_LAMA_FOLIAGE_FULL] = "Full", + [UI_MENU_LAMA_FOLIAGE_REDUCED] = "Reduced", + [UI_MENU_LAMA_RESULTS] = "Wait for GPU results", + [UI_MENU_LAMA_RESULTS_DESC] = "After a frame's first submission the game waits until the GPU wrote the results it " + "reads back. All: events, counter and occlusion reports, write-backs. Without events: " + "a shorter wait; on an RX 580 nothing flickered.", + [UI_MENU_LAMA_RESULTS_ALL] = "All", + [UI_MENU_LAMA_RESULTS_NO_EVENTS] = "Without events", + [UI_MENU_LAMA_ZSTD] = "Native data decompression", + [UI_MENU_LAMA_ZSTD_DESC] = "Decompresses the game's zstd data with the PC's libzstd instead of the game's own " + "decoder translated from the console. Lighter streaming while you move.", + [UI_MENU_LAMA_ASYNC] = "Asynchronous replay", + [UI_MENU_LAMA_ASYNC_DESC] = "Translates the game's GPU commands on a thread of their own instead of the game's, so " + "the game prepares the next frame meanwhile.", + [UI_MENU_LAMA_PASS_TIMES] = "Pass times in the log", + [UI_MENU_LAMA_PASS_TIMES_DESC] = "GPU time of every render pass and dispatch goes to the log, with the largest " + "viewport each kind of pass draws with. Costs a little frame time.", + [UI_MENU_LAMA_DUMP_KEY] = "Frame dump key", + [UI_MENU_LAMA_DUMP_KEY_DESC] = "Each press of this key in the game window dumps the next frame (images and the " + "frame's passes) to the dump folder, for a glitch caught while it is on screen. " + "Windows only.", [UI_MENU_SHARPNESS] = "FSR sharpness", [UI_MENU_SHARPNESS_DESC] = "How sharp FSR makes the picture. 0 turns it off.", [UI_MENU_SHARPNESS_NEEDS_FSR] = "only with AMD FSR 1 or FSR 3.1", diff --git a/runtime/ui_text.h b/runtime/ui_text.h index 9d098b1f..6793d7be 100644 --- a/runtime/ui_text.h +++ b/runtime/ui_text.h @@ -332,6 +332,26 @@ typedef enum { UI_MENU_FRAMEGEN_MULT_UPTO, // printf: "%d", the highest multiplier this GPU and driver allow UI_MENU_FRAMEGEN_MULT_OLD_DLL, // nx2navi_dlssg.dll is older than multi frame generation UI_MENU_FRAMEGEN_UNAVAILABLE, UI_MENU_FRAMEGEN_OFF_FILES, + // the Lama patches tab (ui/menu.cpp tab_lama): the settings of the RX 580 patches, English only + UI_MENU_TAB_LAMA, UI_MENU_LAMA_INTRO, UI_MENU_LAMA_MEMORY, UI_MENU_LAMA_RENDER, UI_MENU_LAMA_BUILD, + UI_MENU_LAMA_DIAG, + UI_MENU_LAMA_FREEZE, UI_MENU_LAMA_FREEZE_DESC, UI_MENU_LAMA_ZSTD, UI_MENU_LAMA_ZSTD_DESC, + UI_MENU_LAMA_FOLIAGE, UI_MENU_LAMA_FOLIAGE_DESC, UI_MENU_LAMA_FOLIAGE_FULL, UI_MENU_LAMA_FOLIAGE_REDUCED, + UI_MENU_LAMA_RESULTS, UI_MENU_LAMA_RESULTS_DESC, UI_MENU_LAMA_RESULTS_ALL, UI_MENU_LAMA_RESULTS_NO_EVENTS, + UI_MENU_LAMA_ASYNC, UI_MENU_LAMA_ASYNC_DESC, + UI_MENU_LAMA_RELEASED, UI_MENU_LAMA_RELEASED_DESC, + UI_MENU_LAMA_CLOBBER, UI_MENU_LAMA_CLOBBER_DESC, + UI_MENU_LAMA_MIRROR_HOT, UI_MENU_LAMA_MIRROR_HOT_DESC, + UI_MENU_LAMA_HEAP_LIMIT, UI_MENU_LAMA_HEAP_LIMIT_DESC, UI_MENU_LAMA_RESERVE, UI_MENU_LAMA_RESERVE_DESC, + UI_MENU_LAMA_UPLOAD_BATCH, UI_MENU_LAMA_UPLOAD_BATCH_DESC, UI_MENU_LAMA_NO_LIMIT, + UI_MENU_LAMA_ASTC_KEEP, UI_MENU_LAMA_ASTC_KEEP_DESC, + UI_MENU_LAMA_MESH_EMU, UI_MENU_LAMA_MESH_EMU_DESC, UI_MENU_LAMA_MESH_NATIVE, + UI_MENU_LAMA_MESH_MB, UI_MENU_LAMA_MESH_MB_DESC, UI_MENU_LAMA_MESH_TASKS, UI_MENU_LAMA_MESH_TASKS_DESC, + UI_MENU_LAMA_MESH_BATCH, UI_MENU_LAMA_MESH_BATCH_DESC, + UI_MENU_LAMA_VEC_INDEX, UI_MENU_LAMA_VEC_INDEX_DESC, UI_MENU_LAMA_VEC_AUTO, + UI_MENU_LAMA_GUEST_OPT, UI_MENU_LAMA_GUEST_OPT_DESC, UI_MENU_LAMA_GUEST_OPT_NOTE, + UI_MENU_LAMA_PASS_TIMES, UI_MENU_LAMA_PASS_TIMES_DESC, + UI_MENU_LAMA_DUMP_KEY, UI_MENU_LAMA_DUMP_KEY_DESC, UI_TEXT_COUNT } UiTextId; diff --git a/runtime/vk.h b/runtime/vk.h index 0babc740..2e8d04d5 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -6,6 +6,20 @@ #pragma clang diagnostic ignored "-Wnullability-completeness" #include "vk_mem_alloc.h" #pragma clang diagnostic pop +// Every VMA allocation goes by the device-local heaps' live budgets (vk_vma_budget.c): a heap within the reserve of its +// budget takes nothing new while another heap can. +VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, + VmaAllocation *a, VmaAllocationInfo *info); +VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info); +VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, + VmaAllocation *a, VmaAllocationInfo *info); +uint64_t vk_vma_moved(void); // allocations placed outside a full heap so far +#ifndef VK_VMA_BUDGET_IMPL +#define vmaCreateImage vk_vma_create_image +#define vmaCreateBuffer vk_vma_create_buffer +#define vmaAllocateMemory vk_vma_allocate_memory +#endif #include "nvn_obj.h" // pool_t, tex_t, view_t, samp_par, NVNcopyRegion (memory and textures block) #include "gfx.h" // GfxShader, GfxShaderRes, GfxShaderAnswers (pipelines block) @@ -128,6 +142,7 @@ typedef struct { uint32_t step; // index of the pass, or of the dispatch, within its submission, from 0 uint32_t w, h, d; // render area and layers; a dispatch: group counts (0 0 0: indirect) uint32_t draws; // draws recorded in the pass; UINT32_MAX until its end is recorded + uint32_t vw, vh; // the largest viewport a draw of the pass set (dynamic resolution draws in part) uint64_t fence; // the submission: the timeline value it signals VkFormat color[8], depth; // attachment formats (VK_FORMAT_UNDEFINED: none) const GfxShader *first[2], *last[2];// vertex and fragment shader of the first and last draw; a dispatch: first[0] @@ -145,6 +160,12 @@ enum { VK_GAP_MESH_RING, VK_GAP_BC_BACK, VK_GAP_N }; void vk_gpu_passes_part(VkCommandBuffer cb, int end); +// Result points (NX2NAVI_GPU_PASSES): the submission whose parts follow (vk_sync.c), a result just recorded (VK_RES_* +// bits, vk_exec_note_results), a submission the game's wait after a frame's first submission waited for. +void vk_gpu_passes_fence(uint64_t fence); +void vk_gpu_passes_result(int kinds); +void vk_gpu_passes_waited(uint64_t need); +int vk_sync_result_mask(void); // NX2NAVI_SUBMIT_FINISH_KINDS as read void vk_gpu_gap_note(int kind, uint64_t units); // Pass marks wanted and possible (NX2NAVI_PASS_CRUMBS and the extensions): set by vk_crumbs_init. extern int vk_pass_marks_on; @@ -346,6 +367,15 @@ void vk_mem_mirror_begin(void); int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence); int vk_mem_mirror_pending(void); void vk_mem_mirror_prologue(VkCommandBuffer cb); +// Staged mirror uploads (vk_memory.c): the GPU of the submission being recorded reads guest memory itself (noted by +// vk_mem_note_live); vk_mem_take_live returns that and starts the next submission's note. +void vk_mem_note_live(void); +int vk_mem_take_live(void); +// The submission being recorded hands results to the CPU at its completion (vk_exec_xfer.c): vk_sync_recorded_wait waits +// for the newest such submission up to the one it waits for. +enum { VK_RES_EVENT = 1, VK_RES_COUNTER = 2, VK_RES_OQ = 4, VK_RES_WRITE = 8, VK_RES_WRITEBACK = 16, VK_RES_KINDS = 5 }; +void vk_exec_note_results(int kind); // VK_RES_* +int vk_exec_take_results(void); // the kinds noted since the last take void vk_mem_lazy_part_end(void); // lazy write-back: a part ends (before its prologue), and it is handed over void vk_mem_lazy_handed(void); void vk_mem_lazy_decide(void); // once the driver is known (after vk_caps_driver): on or off, with one log line @@ -483,6 +513,7 @@ int vk_astc_time_get(uint32_t bw, uint32_t bh, uint64_t *texels, uint64_t *calls void vk_astc_time_summary(void); // logs the totals void vk_astc_release(void); // vk_device.c, before the device goes: frees the decoder's objects void vk_mem_note_submit(uint64_t timeline_value); // vk_sync.c: after each submit that signals ctx->timeline with this value +void vk_mem_oom_report(const char *what, VkResult r); // logs heaps and upload batches after an out-of-memory failure void vk_mem_collect(void); // frees what has completed typedef struct { uint64_t imported_pools, physical_pools, virtual_pools, import_failures, upload_batches; } VkMemStats; void vk_mem_stats(VkMemStats *s); @@ -612,7 +643,8 @@ void vk_pool_marks_read(const uint32_t *marks, uint32_t n); // vk_texture.c: s // set and brings it up to date. vk_pool_takes counts the takes whose sync left no slot behind (vk_exec_draw.c pool_sync); // once it has moved on by more than VK_POOL_SETS since a pool entry changed, every set holds the change (vk_texture.c // retire_sweep). It may count fewer takes than there were, which only delays that. -#define VK_POOL_SETS 4 +#define VK_POOL_SETS 8 // 4 before: with the game no longer held for the GPU after a frame's first submission, the + // replay waited for a set's GPU work at 15% of the switches uint64_t vk_pool_takes(void); // Texture lock held: 1 while the texture pool refresh looks its slots up (vk_exec_draw.c pool_sync_entries), 0 after. // Those lookups leave out a texture evicted by disuse, and are not counted as the texture being used. @@ -726,6 +758,7 @@ uint64_t vk_sync_reserve(void); // asynchronous replay: the value the int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns); // 1 when done; 0 on timeout or a value never submitted // The executed timeline (vk_sync.c): a submission's own commands are through the GPU, its write-backs maybe not yet. int vk_sync_exec_wait(uint64_t value, uint64_t timeout_ns); +int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)); int vk_sync_wb_split(void); // NX2NAVI_WB_SPLIT: the write-backs go into a part of their own // NX2NAVI_TRACE=perf (vk_sync.c "the executed timeline"): what a wait on that timeline left behind since the last call, // whether the completion thread's callback running now belongs to a value whose waiter went on already, and the @@ -761,6 +794,21 @@ VkCommandBuffer vk_sync_append_later(int n, int *at); void vk_sync_append_fill(int at, const VkCommandBuffer *cbs, int n); VkCommandBuffer vk_sync_worker_begin(int w); void vk_sync_worker_end(VkCommandBuffer cb); +// The part's pre-pass command buffer: submitted right before the part being recorded (after its mirror prologue), begun +// with a full barrier at the first call of a part; VK_NULL_HANDLE when it cannot be made. Replay thread, sub_lock held. +VkCommandBuffer vk_sync_pre(void); +// vk_exec_draw.c: records into the pre-pass command buffer, as it ends, the stages of the emulated mesh draws queued +// for it (NX2NAVI_MESH_EMU_BATCH). A weak vk_sync.c version does nothing. +void vk_exec_mesh_flush(VkCommandBuffer cb); +// Ends the part being recorded without handing it to the queue (it goes with the next part that does, in order) and +// returns the command buffer the replay continues in; VK_NULL_HANDLE when no more parts can wait. Replay thread. +VkCommandBuffer vk_sync_cut(void); +// The GPU timeline (NX2NAVI_TRACE=perf, vk_sync.c): a timed piece of GPU work of kind GTL_*, handed to the queue at +// sub_ns (plat_now_ns), its timestamps start and end (ticks of the graphics queue). vk_gtl_log: the gaps of the pieces +// noted since the last call, over frames frames, in the log. +enum { GTL_PART, GTL_MIRROR, GTL_PRE, GTL_UPLOAD, GTL_PRESENT, GTL_KINDS }; +void vk_gtl_note(int kind, uint64_t sub_ns, uint64_t start_tick, uint64_t end_tick); +void vk_gtl_log(uint64_t frame, int frames); extern int vk_sub_in_frame; // replay: the submission being recorded is the frame's n-th (0 the first; vk_backend.c) // Runs fn(arg) on the completion thread once fence is done on the GPU (event writes, counter reports, sync state). // Callbacks run in the order they were added, after those of earlier values; a value already done runs at once there. @@ -841,6 +889,12 @@ typedef struct { // from the start of the set's descriptors. VK_NULL_HANDLE: the set is written with VkWriteDescriptorSet structs. VkDescriptorUpdateTemplate tpl[2]; int pool; // a stage indexes the texture or sampler pool: bind set 2 (the pools) + // NX2NAVI_MESH_EMULATION: an emulated mesh pipeline (a generated vertex shader and the fragment stage; the task and + // mesh stages are compute pipelines of the same layout, vk_pipeline_mesh_emu), and set 0's compute-side template. + int emu; + VkDescriptorUpdateTemplate emu_tpl_c; + int8_t gpa; // vk_exec_draw.c gpa_reader: 0 not known yet, 1 no binding of the device + // address table, 2 one (its draws read guest memory as the GPU runs) } VkPipeInfo; // One descriptor of the packed array a descriptor update template reads (VkPipeInfo.tpl), 24 bytes for every type; the // bytes a type leaves unused are zero, so equal descriptors compare equal byte for byte. @@ -877,6 +931,10 @@ typedef struct { } VkPrewarmProgress; void vk_pipeline_prewarm_progress(VkPrewarmProgress *p); // any thread, while vk_pipeline_prewarm runs or after VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info); +// NX2NAVI_MESH_EMULATION: the compute pipelines of an emulated mesh pipeline (built with it); 0 when it has none. +struct MeshEmuInfo; +int vk_pipeline_mesh_emu(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, VkPipeline *mesh, + VkPipeline *task, struct MeshEmuInfo *mi); // Records every dynamic state that VkPipeState determines, after binding a graphics pipeline: cull mode, front face, // topology, depth and stencil tests and ops, depth bias enable, rasterizer discard, vertex input, and, where the GPU // has them dynamic, depth clamp, polygon mode, blend enables, equations, write masks and alpha to coverage. The replay diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 49f1fcd6..c30edca5 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -303,9 +303,14 @@ int vk_replay_async(void) { } // With the replay on its own thread a submission's value covers its replay too, and the wait stays the fence's. -static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { +static int vk_be_gpu_wait(uint64_t value, uint64_t timeout_ns) { return vk_replay_async() ? vk_sync_fence_wait(value, timeout_ns) : vk_sync_exec_wait(value, timeout_ns); } +// The submit wait (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): the GPU's only where the submission reads guest memory on the +// GPU (vk_sync_recorded_wait). +static int vk_be_exec_wait(uint64_t value, uint64_t timeout_ns) { + return vk_sync_recorded_wait(value, timeout_ns, vk_be_gpu_wait); +} // The front end's view (nvn_api_core.c, nvn_api_cmd.c): replay is asynchronous only on this back end. int gfx_replay_async(void) { return gfx_backend() == &gfx_vulkan_backend && vk_replay_async(); } diff --git a/runtime/vk_caps.c b/runtime/vk_caps.c index bcd0bde2..5945b135 100644 --- a/runtime/vk_caps.c +++ b/runtime/vk_caps.c @@ -41,6 +41,12 @@ void vk_caps_features(VkPhysicalDeviceFeatures *f) { vk_caps.float64 = f->shaderFloat64; } +static int amd_proprietary; // AMD's own Windows driver (any branch): gfx_shader.c patch_vec_index + +// gfx_shader.c patch_vec_index: the driver reads uniform vector components at computed indices wrongly (Ryujinx's +// HasVectorIndexingBug, which it sets for AMD's Windows drivers). +int vk_caps_vec_index_bug(void) { return amd_proprietary; } + void vk_caps_driver(VkPhysicalDevice p) { VkPhysicalDeviceDriverProperties drv = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; VkPhysicalDeviceProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, .pNext = &drv}; @@ -51,6 +57,7 @@ void vk_caps_driver(VkPhysicalDevice p) { // "26.10.1 (AMD proprietary shader compiler)" where the RDNA 3 and 4 driver names LLPC. vk_caps.amd_sc = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY && strstr(drv.driverInfo, "shader compiler"); vk_caps.amd_llpc = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY && !vk_caps.amd_sc; + amd_proprietary = drv.driverID == VK_DRIVER_ID_AMD_PROPRIETARY; vk_caps.mesa = drv.driverID == VK_DRIVER_ID_MESA_RADV || drv.driverID == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA || drv.driverID == VK_DRIVER_ID_MESA_NVK; } diff --git a/runtime/vk_crumbs.c b/runtime/vk_crumbs.c index eef9ebaa..110355aa 100644 --- a/runtime/vk_crumbs.c +++ b/runtime/vk_crumbs.c @@ -15,8 +15,14 @@ #include "cfg.h" #include "host.h" +#include "nvn_api.h" #include "vk.h" +// vk_pass_mark_cmd also runs on the recorder thread (vk_rec.c R_PassMark), where vk.h's macro would turn the timestamp +// into a new packet for the part being recorded: a second producer on the ring, which corrupts it ("vk_rec: unknown +// packet" with NX2NAVI_PASS_TIMES=1 on NVIDIA, where the recorder is on). The command goes straight into cb. +#undef vkCmdWriteTimestamp + static PFN_vkCmdWriteBufferMarkerAMD write_marker; static VkBuffer buf; static VmaAllocation alloc; @@ -27,6 +33,11 @@ int vk_pass_marks_on; static int pm_amd; // marks into words 2 and 3 static PFN_vkCmdSetCheckpointNV set_checkpoint; static void pass_marks_init(VkCtx *c); +static void pass_times_init(VkCtx *c); +static void pass_times_poll(void); +static VkQueryPool pt_pool; // pass times (below): two timestamps per ring slot +static volatile uint8_t pt_timed[VK_PASS_MARKS]; // the slot's mark writes timestamps; cleared once they are read + static void buffer_init(VkCtx *c) { write_marker = (PFN_vkCmdWriteBufferMarkerAMD)vk_dpa(c->device, "vkCmdWriteBufferMarkerAMD"); @@ -49,11 +60,14 @@ static void buffer_init(VkCtx *c) { void vk_crumbs_init(VkCtx *c) { if (c->has_buffer_marker) buffer_init(c); pass_marks_init(c); + pass_times_init(c); } void vk_crumbs_release(void) { VkCtx *c = vk_ctx(); if (buf && c) vmaDestroyBuffer(c->vma, buf, alloc); + if (pt_pool && c) vkDestroyQueryPool(c->device, pt_pool, NULL); + pt_pool = VK_NULL_HANDLE; buf = VK_NULL_HANDLE, alloc = NULL, words = NULL, write_marker = NULL; vk_pass_marks_on = pm_amd = 0, set_checkpoint = NULL; } @@ -120,7 +134,11 @@ static void pass_marks_init(VkCtx *c) { : "no VK_AMD_buffer_marker; NX2NAVI_PASS_CRUMBS=1 asks for NVIDIA checkpoints"); } + void vk_pass_mark_cmd(VkCommandBuffer cb, uint32_t id, int end) { + if (pt_pool && pt_timed[id & (VK_PASS_MARKS - 1)] == 1) + vkCmdWriteTimestamp(cb, end ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, pt_pool, + (id & (VK_PASS_MARKS - 1)) * 2 + (end != 0)); if (pm_amd) write_marker(cb, end ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, buf, end ? 12 : 8, id); @@ -179,6 +197,40 @@ static uint64_t gp_shader_hash(const GfxShader *s) { static int kind_is_pass(const VkPassMark *m) { return m->kind == VK_MARK_PASS; } +// Result points (gpu_passes): where in a submission's GPU time the results the CPU waits for are done. The replay +// notes a result (vk_exec_note_results: an event, a counter or occlusion report, a GPU write of unmirrored memory, a +// mirror write-back) as it records it; the next timestamp the capture writes after it (a pass's or dispatch's start or +// end, a part's end) is when the GPU is done with it at the latest. The report compares that, for the submissions the +// game's wait after a frame's first submission waited for (vk_sync_recorded_wait), with the submission's end: how much +// earlier the wait could end if it ended at the results instead of at the whole submission's completion. +#define GP_RES 16384 +typedef struct { uint64_t fence; uint32_t q; uint8_t kind; } GpRes; // q: query in gp_pool, UINT32_MAX until written +static GpRes gp_res[GP_RES]; +static uint32_t gp_nres, gp_res_open; +static uint64_t gp_fence, gp_part_fence[GP_SLOTS]; +#define GP_WAITED 256 +static uint64_t gp_waited[GP_WAITED]; +static uint32_t gp_nwaited; // atomic: the game's threads add + +static void gp_resolve(uint32_t q) { + for (uint32_t i = gp_res_open; i < gp_nres; i++) gp_res[i].q = q; + gp_res_open = gp_nres; +} + +void vk_gpu_passes_fence(uint64_t fence) { gp_fence = fence; } + +void vk_gpu_passes_result(int kinds) { + if (!gp_capture || gp_part == UINT32_MAX) return; + for (int b = 0; b < VK_RES_KINDS && gp_nres < GP_RES; b++) + if (kinds >> b & 1) gp_res[gp_nres++] = (GpRes){gp_fence, UINT32_MAX, (uint8_t)b}; +} + +void vk_gpu_passes_waited(uint64_t need) { + if (!gp_capture) return; + uint32_t i = __atomic_fetch_add(&gp_nwaited, 1, __ATOMIC_RELAXED); + if (i < GP_WAITED) __atomic_store_n(&gp_waited[i], need, __ATOMIC_RELEASE); +} + int vk_gpu_passes_capturing(void) { return gp_capture; } static void gp_gap_close(uint32_t to) { @@ -198,12 +250,14 @@ void vk_gpu_passes_part(VkCommandBuffer cb, int end) { if (!gp_capture || gp_n >= GP_SLOTS) return; gp_labels[gp_n] = (GpLabel){.kind = GP_KIND_PART}; vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, gp_n * 2); + gp_part_fence[gp_n] = gp_fence; gp_part = gp_n++; if (!gp_in_mark) gp_gap_from = gp_part * 2; return; } if (gp_part == UINT32_MAX) return; vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, gp_part * 2 + 1); + gp_resolve(gp_part * 2 + 1); if (!gp_in_mark) gp_gap_close(gp_part * 2 + 1); gp_labels[gp_part].used = 1; gp_part = UINT32_MAX, gp_gap_from = UINT32_MAX; @@ -215,6 +269,7 @@ static void gp_begin(VkCommandBuffer cb, VkPassMark *m) { if (gp_part != UINT32_MAX) gp_gap_close(gp_n * 2); // the gap before this mark ends at its start gp_labels[gp_n].used = 0; vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, gp_n * 2); + gp_resolve(gp_n * 2); if (gp_prims && kind_is_pass(m)) vkCmdBeginQuery(cb, gp_prims, gp_n, 0); if (gp_stats && kind_is_pass(m)) vkCmdBeginQuery(cb, gp_stats, gp_n, 0); m->gp = ++gp_n; @@ -226,6 +281,7 @@ static void gp_end(VkCommandBuffer cb, const VkPassMark *m) { if (gp_prims && kind_is_pass(m)) vkCmdEndQuery(cb, gp_prims, i); if (gp_stats && kind_is_pass(m)) vkCmdEndQuery(cb, gp_stats, i); vkCmdWriteTimestamp2(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, gp_pool, i * 2 + 1); + gp_resolve(i * 2 + 1); GpLabel *l = &gp_labels[i]; l->kind = m->kind, l->w = m->w, l->h = m->h, l->d = m->d, l->draws = m->draws == UINT32_MAX ? 0 : m->draws; l->layer = m->layer; @@ -292,6 +348,66 @@ static void gp_report_gaps(double tick_ms, double parts_ms, uint32_t parts) { gp_kind_names[b], (double)gp_kind_n[b] / GP_FRAMES, gp_kind_units[b] / 1e6 / GP_FRAMES); } +// The result points of the timed frames (vk_gpu_passes_result), for the submissions the game's wait waited for. +static void gp_report_results(double tick_ms) { + VkCtx *c = vk_ctx(); + static const char *const names[VK_RES_KINDS] = {"events", "counter reports", "occlusion reports", + "unmirrored writes", "mirror write-backs"}; + uint32_t nw = __atomic_load_n(&gp_nwaited, __ATOMIC_ACQUIRE); + if (nw > GP_WAITED) nw = GP_WAITED; + uint64_t seen[GP_WAITED]; + uint32_t ns = 0, subs = 0; + double span = 0, last[VK_RES_KINDS] = {0}, cnt[VK_RES_KINDS] = {0}, mask_end = 0, all_end = 0; + int mask = vk_sync_result_mask(); + for (uint32_t w = 0; w < nw; w++) { + uint64_t f = __atomic_load_n(&gp_waited[w], __ATOMIC_ACQUIRE); + int dup = !f; + for (uint32_t k = 0; k < ns && !dup; k++) dup = seen[k] == f; + if (dup) continue; + seen[ns++] = f; + uint64_t t0 = UINT64_MAX, t1 = 0, t; + for (uint32_t i = 0; i < gp_n; i++) { // the submission's parts: its first start, its last end + if (gp_labels[i].kind != GP_KIND_PART || !gp_labels[i].used || gp_part_fence[i] != f) continue; + if (gp_tick(c, i * 2, &t) && t < t0) t0 = t; + if (gp_tick(c, i * 2 + 1, &t) && t > t1) t1 = t; + } + if (t0 == UINT64_MAX || t1 <= t0) continue; + uint64_t lk[VK_RES_KINDS] = {0}; + int any = 0; + for (uint32_t i = 0; i < gp_nres; i++) + if (gp_res[i].fence == f && gp_res[i].q != UINT32_MAX && gp_tick(c, gp_res[i].q, &t) && t >= t0) { + if (t > lk[gp_res[i].kind]) lk[gp_res[i].kind] = t; + any = 1; + } + if (!any) continue; + subs++; + span += (double)(t1 - t0) * tick_ms; + uint64_t me = t0, ae = t0; + for (int b = 0; b < VK_RES_KINDS; b++) { + if (!lk[b]) continue; + last[b] += (double)(lk[b] - t0) * tick_ms, cnt[b]++; + if (lk[b] > ae) ae = lk[b]; + if (mask >> b & 1 && lk[b] > me) me = lk[b]; + } + mask_end += (double)(me - t0) * tick_ms, all_end += (double)(ae - t0) * tick_ms; + } + if (!subs) { + rt_log("gpu passes: result points: none of the %u waited-for submissions was timed with its results", ns); + return; + } + char list[400]; + int o = 0; + for (int b = 0; b < VK_RES_KINDS && o < (int)sizeof list - 80; b++) + if (cnt[b]) + o += snprintf(list + o, sizeof list - (size_t)o, "%s%s %.2f ms (%.0f of %u)", o ? ", " : "", names[b], + last[b] / cnt[b], cnt[b], subs); + list[o] = 0; + rt_log("gpu passes: result points: %u submissions the game's wait waited for, %.2f ms of GPU time each from their " + "start to their end; their last results done at: %s", subs, span / subs, list); + rt_log("gpu passes: result points: the wait could end %.2f ms before the submission's end (the kinds it counts, " + "%#x), %.2f ms with every kind", (span - mask_end) / subs, mask, (span - all_end) / subs); +} + typedef struct { GpLabel l; double ms; uint32_t n; uint64_t draws, prims, frags; } GpSum; static int gp_sum_cmp(const void *a, const void *b) { @@ -364,6 +480,7 @@ static void gp_report(uint64_t frame) { (double)sum / GP_FRAMES, top, list); } gp_report_gaps(tick_ms, parts_ms, parts); + gp_report_results(tick_ms); for (int s = 0; s < n && s < GP_TOP; s++) { const GpSum *e = &sums[s]; if (e->l.kind == VK_MARK_DISPATCH) @@ -435,6 +552,8 @@ void vk_gpu_passes_frame(uint64_t frame) { if (gp_stats) vkResetQueryPool(c->device, gp_stats, 0, GP_SLOTS); gp_ngaps = 0, gp_gap_from = UINT32_MAX, gp_gap_kinds = 0, gp_part = UINT32_MAX; memset(gp_kind_n, 0, sizeof gp_kind_n), memset(gp_kind_units, 0, sizeof gp_kind_units); + gp_nres = gp_res_open = 0; + __atomic_store_n(&gp_nwaited, 0, __ATOMIC_RELEASE); gp_n = 0, gp_start = frame, gp_capture = 1, gp_phase = 1; } else if (gp_phase == 1) { if (frame < gp_start + GP_FRAMES) return; @@ -447,10 +566,13 @@ void vk_gpu_passes_frame(uint64_t frame) { VkPassMark *vk_pass_mark_begin(VkCommandBuffer cb, int kind, uint64_t fence) { if (!vk_pass_marks_on || !cb) return NULL; + if (pt_pool) pass_times_poll(); uint32_t id = ++seq; if (!id) id = ++seq; // 0 means no mark VkPassMark *m = &ring[id & (VK_PASS_MARKS - 1)]; m->id = id, m->kind = (uint8_t)kind, m->fence = fence, m->draws = UINT32_MAX; + // timed unless the slot's previous mark is still waiting to be read (its queries are not reset yet) + if (pt_pool) pt_timed[id & (VK_PASS_MARKS - 1)] = !pt_timed[id & (VK_PASS_MARKS - 1)] ? 1 : 2; rec_pass_mark(cb, id, 0); gp_begin(cb, m); gp_in_mark = 1; @@ -591,3 +713,148 @@ void vk_pass_marks_report(void) { } nv_report(); } + +// ---- pass times (NX2NAVI_PASS_TIMES=1): GPU time per render pass and dispatch, for GPUs no vendor profiler supports +// (RGP covers RDNA only, so an RX 580 has none, and RenderDoc cannot capture this program). Every pass mark also +// writes a timestamp at its two points (TOP_OF_PIPE before the pass, BOTTOM_OF_PIPE after it), into the query slot +// pair of its ring slot. pass_times_poll, on the replay thread at each new mark, takes in the marks whose submission +// has finished, resets their slots on the host (hostQueryReset) and sums their time per kind of pass: its size, +// attachment formats and its first fragment (or compute) shader. Every 600 presented frames the log lists the passes +// with the most GPU time per frame. Passes overlap on the GPU, so the times are each pass's span, not its exclusive +// share; their order is what matters. A slot whose mark was not read yet when the ring comes round is not timed +// (pt_timed 2) for that turn. +#define PT_SUMS 1024 +#define PT_REPORT_FRAMES 600 +#define PT_TOP 15 +typedef struct { + uint64_t key; + uint8_t kind, ncolor; + uint32_t w, h, d; + VkFormat c0, depth; + unsigned long long sid; + double us; + uint64_t n, draws; + uint32_t vw, vh; // the largest viewport its draws set +} PtSum; +static PtSum pt_sum[PT_SUMS]; +static uint32_t pt_read; // the newest mark id taken in +static double pt_ns_per_tick, pt_total_us; +static uint64_t pt_mask, pt_frame0, pt_marks, pt_lost; + +static void pass_times_init(VkCtx *c) { + const char *e = cfg_get("NX2NAVI_PASS_TIMES"); + if (!e || !*e || !strcmp(e, "0")) return; + if (!vk_pass_marks_on) { + rt_log("vk: NX2NAVI_PASS_TIMES: pass marks are off (NX2NAVI_PASS_CRUMBS=0 or no marker extension): no pass times"); + return; + } + VkPhysicalDeviceVulkan12Features f12 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES}; + VkPhysicalDeviceFeatures2 f2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, .pNext = &f12}; + vkGetPhysicalDeviceFeatures2(c->phys, &f2); + uint32_t nq = 0; + vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, NULL); + VkQueueFamilyProperties qp[16]; + if (nq > 16) nq = 16; + vkGetPhysicalDeviceQueueFamilyProperties(c->phys, &nq, qp); + uint32_t bits = c->gfx_family < nq ? qp[c->gfx_family].timestampValidBits : 0; + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * VK_PASS_MARKS}; + if (!f12.hostQueryReset || !bits || c->props.limits.timestampPeriod <= 0 || + vkCreateQueryPool(c->device, &qi, NULL, &pt_pool) != VK_SUCCESS) { + rt_log("vk: NX2NAVI_PASS_TIMES: no hostQueryReset or no timestamps on the queue: no pass times"); + pt_pool = VK_NULL_HANDLE; + return; + } + vkResetQueryPool(c->device, pt_pool, 0, 2 * VK_PASS_MARKS); + pt_ns_per_tick = c->props.limits.timestampPeriod; + pt_mask = bits >= 64 ? ~0ull : (1ull << bits) - 1; + pt_read = seq; + pt_frame0 = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + rt_log("vk: pass times on (NX2NAVI_PASS_TIMES): GPU time per render pass and dispatch, the top %d every %d frames", + PT_TOP, PT_REPORT_FRAMES); +} + +static void pt_add(const VkPassMark *m, double us) { + unsigned long long sid = shader_id(m->kind == VK_MARK_DISPATCH ? m->first[0] : m->first[1]); + uint64_t key = sid * 0x9E3779B97F4A7C15ull ^ (uint64_t)m->kind << 56 ^ (uint64_t)m->w << 32 ^ (uint64_t)m->h << 16 ^ + (uint64_t)m->ncolor << 8 ^ (uint64_t)m->color[0] * 0x100000001b3ull ^ (uint64_t)m->depth << 40; + if (!key) key = 1; + for (uint32_t i = (uint32_t)(key % PT_SUMS), k = 0; k < PT_SUMS; k++, i = (i + 1) % PT_SUMS) { + PtSum *e = &pt_sum[i]; + if (e->key && e->key != key) continue; + if (!e->key) + *e = (PtSum){key, m->kind, m->ncolor, m->w, m->h, m->d, m->ncolor ? m->color[0] : VK_FORMAT_UNDEFINED, + m->depth, sid, 0, 0, 0, 0, 0}; + if (m->vw > e->vw) e->vw = m->vw; + if (m->vh > e->vh) e->vh = m->vh; + e->us += us, e->n++, e->draws += m->draws == UINT32_MAX ? 0 : m->draws; + return; + } + pt_lost++; // table full: counted in the total only +} + +static int pt_cmp(const void *a, const void *b) { + double x = ((const PtSum *)a)->us, y = ((const PtSum *)b)->us; + return x < y ? 1 : x > y ? -1 : 0; +} + +static void pt_report(uint64_t frames) { + static PtSum top[PT_SUMS]; + int n = 0; + for (int i = 0; i < PT_SUMS; i++) + if (pt_sum[i].key) top[n++] = pt_sum[i]; + qsort(top, (size_t)n, sizeof *top, pt_cmp); + double f = frames ? (double)frames : 1; + rt_log("vk: pass times over %llu frames: %.2f ms of pass spans per frame in %.0f passes and dispatches (%d kinds%s)", + (unsigned long long)frames, pt_total_us / f / 1e3, pt_marks / f, n, pt_lost ? ", table full" : ""); + for (int i = 0; i < n && i < PT_TOP; i++) { + const PtSum *e = &top[i]; + if (e->kind == VK_MARK_DISPATCH) + rt_log("vk: %5.2f ms/frame %5.1f%% dispatch %ux%ux%u groups, cs %016llx: %.1f per frame, %.3f ms each", + e->us / f / 1e3, pt_total_us > 0 ? e->us * 100 / pt_total_us : 0, e->w, e->h, e->d, e->sid, e->n / f, + e->us / (double)e->n / 1e3); + else + rt_log("vk: %5.2f ms/frame %5.1f%% pass %ux%u x%u, %u colour (first fmt %d), depth fmt %d, fs %016llx: %.1f per " + "frame, %.3f ms and %.0f draws each, viewport up to %ux%u", e->us / f / 1e3, + pt_total_us > 0 ? e->us * 100 / pt_total_us : 0, e->w, e->h, e->d, e->ncolor, (int)e->c0, (int)e->depth, + e->sid, e->n / f, e->us / (double)e->n / 1e3, (double)e->draws / (double)e->n, e->vw, e->vh); + } + memset(pt_sum, 0, sizeof pt_sum); + pt_total_us = 0, pt_marks = 0, pt_lost = 0; +} + +// Replay thread (vk_pass_mark_begin). +static void pass_times_poll(void) { + VkCtx *c = vk_ctx(); + uint64_t done = vk_sync_fence_done(); + while (pt_read != seq) { + uint32_t id = pt_read + 1; + if (!id) { pt_read = id; continue; } + uint32_t slot = id & (VK_PASS_MARKS - 1); + const VkPassMark *m = &ring[slot]; + if (m->id == id) { + if (!m->fence || m->fence > done) break; // its submission is still being recorded or running + // the submission finished: both timestamps are in, or one was never written (a pass whose end was not + // recorded), which leaves that mark out + if (pt_timed[slot] == 1 && m->draws != UINT32_MAX) { + uint64_t r[4] = {0}; + VkResult q = vkGetQueryPoolResults(c->device, pt_pool, slot * 2, 2, sizeof r, r, 16, + VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT); + if ((q == VK_SUCCESS || q == VK_NOT_READY) && r[1] && r[3]) { + double us = (double)((r[2] - r[0]) & pt_mask) * pt_ns_per_tick / 1e3; + if (us < 1e6) pt_add(m, us), pt_total_us += us, pt_marks++; + } + } + if (pt_timed[slot]) { + vkResetQueryPool(c->device, pt_pool, slot * 2, 2); + pt_timed[slot] = 0; + } + } + pt_read = id; + } + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (now - pt_frame0 >= PT_REPORT_FRAMES) { + pt_report(now - pt_frame0); + pt_frame0 = now; + } +} diff --git a/runtime/vk_device.c b/runtime/vk_device.c index 9effdb76..0d509e7a 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -21,6 +21,10 @@ PFN_vkGetDeviceProcAddr vk_dpa = vkGetDeviceProcAddr; static uint32_t g_gfx_queues = 1; // queues the graphics family offers (pick_device) +#define QFAM_MAX 16 +static uint32_t g_nqfam; // the device's queue families (pick_device): their flags and queue counts +static VkQueueFlags g_qfam_flags[QFAM_MAX]; +static uint32_t g_qfam_count[QFAM_MAX]; static VkCtx g_vk; static int g_ready; @@ -296,6 +300,8 @@ retry: vkGetPhysicalDeviceQueueFamilyProperties(g_vk.phys, &nq, qp); g_vk.gfx_family = find_gfx_family(g_vk.phys, qp, nq); g_gfx_queues = qp[g_vk.gfx_family].queueCount; + g_nqfam = nq < QFAM_MAX ? nq : QFAM_MAX; + for (uint32_t i = 0; i < g_nqfam; i++) g_qfam_flags[i] = qp[i].queueFlags, g_qfam_count[i] = qp[i].queueCount; g_vk.xfer_family = g_vk.gfx_family; for (uint32_t i = 0; i < nq; i++) // a family that does transfer only (async DMA engine) if ((qp[i].queueFlags & VK_QUEUE_TRANSFER_BIT) && !(qp[i].queueFlags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT))) { @@ -308,6 +314,53 @@ retry: vk_mem_class_log(&g_vk.props, &g_vk.mem_props); return 1; } +// Heap limits for VMA where the GPU has more than one device-local heap. An RX 580 (8 GiB, AMD 23.10) shows a 6 GiB +// heap and a 2 GiB host-visible one; Windows granted the game 5.1 GiB of the first. VMA, asked for device-local memory, +// takes the first heap whatever its budget, and a save load put 5.6 GiB there while the second held 69 MiB: the next +// submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. With a limit VMA counts a heap full at it and places what follows in +// the next memory type that allows it (the other device-local heap first), for every allocation: textures, render +// targets, VRAM mirrors, pools. The limit is each device-local heap's budget when the device is made (VK_EXT_memory_budget, +// what other programs leave) less a margin for memory VMA does not see (swapchain, FidelityFX): a tenth, at least +// 256 MiB. NX2NAVI_VRAM_HEAP_LIMIT: auto (default), 0 (no limits), or a number of MiB for the largest device-local heap. +static int vram_heap_limits(VkDeviceSize *limit) { + const char *e = cfg_get("NX2NAVI_VRAM_HEAP_LIMIT"); + long mb = e && *e && strcmp(e, "auto") ? atol(e) : -1; + if (!mb) return 0; + VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, + .pNext = g_vk.has_memory_budget ? &b : NULL}; + vkGetPhysicalDeviceMemoryProperties2(g_vk.phys, &p2); + const VkPhysicalDeviceMemoryProperties *mp = &p2.memoryProperties; + int nlocal = 0; + uint32_t largest = 0; + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if (mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) { + if (!nlocal++ || mp->memoryHeaps[h].size > mp->memoryHeaps[largest].size) largest = h; + } + if (mb < 0 && (nlocal < 2 || !g_vk.has_memory_budget)) return 0; // auto: only where another heap can take the rest + for (uint32_t h = 0; h < VK_MAX_MEMORY_HEAPS; h++) limit[h] = VK_WHOLE_SIZE; + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) { + if (!(mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)) continue; + VkDeviceSize lim; + if (mb > 0) { + if (h != largest) continue; + lim = (VkDeviceSize)mb << 20; + } else { + VkDeviceSize budget = b.heapBudget[h] && b.heapBudget[h] < mp->memoryHeaps[h].size ? b.heapBudget[h] + : mp->memoryHeaps[h].size; + VkDeviceSize margin = budget / 10 > (256ull << 20) ? budget / 10 : (256ull << 20); + if (budget <= margin * 2) continue; // too small to limit + lim = budget - margin; + } + if (lim >= mp->memoryHeaps[h].size) continue; + limit[h] = lim; + rt_log("vk: memory heap %u (%llu MiB, budget %llu MiB): the allocator uses at most %llu MiB, then the next heap " + "(NX2NAVI_VRAM_HEAP_LIMIT)", h, (unsigned long long)(mp->memoryHeaps[h].size >> 20), + (unsigned long long)(b.heapBudget[h] >> 20), (unsigned long long)(lim >> 20)); + } + return 1; +} + typedef struct { VkPhysicalDeviceFeatures2 f2; @@ -492,9 +545,13 @@ static int create_device(void) { rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "the extension is there, its task or mesh feature is not"); } // vkGetDeviceFaultInfoEXT needs the feature, not just the extension { // mesh_fallback: auto (on a device without mesh shaders), 1 (on every device, for testing) or 0 - const char *mf = cfg_value("NX2NAVI_MESH_FALLBACK"); - g_vk.mesh_fallback = !strcmp(mf, "1") || (strcmp(mf, "0") && !g_vk.has_mesh_shader); - if (g_vk.mesh_fallback) + // without mesh shaders the mesh emulation (NX2NAVI_MESH_EMULATION, on unless 0) draws them; mesh_emulation = 0 + // leaves them to this fallback + const char *mf = cfg_value("NX2NAVI_MESH_FALLBACK"), *me = cfg_value("NX2NAVI_MESH_EMULATION"); + int emu = !g_vk.has_mesh_shader && strcmp(me, "0"); + g_vk.mesh_fallback = !emu && (!strcmp(mf, "1") || (strcmp(mf, "0") && !g_vk.has_mesh_shader)); + if (emu) rt_log("vk: mesh shaders: emulated with compute (mesh_emulation = 1); mesh_fallback is not used"); + else if (g_vk.mesh_fallback) rt_log("vk: mesh fallback: task and mesh programs run as compute passes and are drawn from their buffers (%s)", g_vk.has_mesh_shader ? "mesh_fallback = 1" : "this device has no mesh shaders"); else if (!g_vk.has_mesh_shader) @@ -515,18 +572,44 @@ static int create_device(void) { has_fbl && c->fbl.attachmentFeedbackLoopLayout, has_lr && c->lr.dynamicRenderingLocalRead); } - // Frame generation (vk_fg.cpp) presents on a second queue of the graphics family and acquires on a third - static const float prio[3] = {1.0f, 1.0f, 1.0f}; + // Frame generation (vk_fg.cpp) presents on a second queue of the graphics family and acquires on a third. AMD's + // swapchain needs a present queue nothing else submits to (FidelityFX: the engine must not use it, or it can + // deadlock); the image acquire queue needs no capability. Where the graphics family has one queue (AMD's Windows + // driver, an RX 580), both come from another family: a queue of a family that can present for the present, and + // a second queue of it, or of any other family, for the acquire (the present queue itself when there is none). + // Without them vk_fg_create_swapchain gave up without a word: frame generation set in the menu did nothing. + static const float prio[4] = {1.0f, 1.0f, 1.0f, 1.0f}; uint32_t nq_gfx = vk_fg_queue_count() < g_gfx_queues ? vk_fg_queue_count() : g_gfx_queues; - VkDeviceQueueCreateInfo qi[2] = { - {.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, .queueFamilyIndex = g_vk.gfx_family, .queueCount = nq_gfx, - .pQueuePriorities = prio}}; - uint32_t nqi = 1; - if (g_vk.xfer_family != g_vk.gfx_family) { - qi[1] = (VkDeviceQueueCreateInfo){.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, - .queueFamilyIndex = g_vk.xfer_family, .queueCount = 1, .pQueuePriorities = prio}; - nqi = 2; + uint32_t used[QFAM_MAX] = {0}; + if (g_vk.gfx_family < QFAM_MAX) used[g_vk.gfx_family] = nq_gfx; + if (g_vk.xfer_family != g_vk.gfx_family && g_vk.xfer_family < QFAM_MAX) used[g_vk.xfer_family] = 1; + uint32_t fg_pf = UINT32_MAX, fg_pi = 0, fg_af = UINT32_MAX, fg_ai = 0; + if (nq_gfx >= 2) { + fg_pf = fg_af = g_vk.gfx_family, fg_pi = 1, fg_ai = nq_gfx - 1; + } else { + for (int pass = 0; pass < 2 && fg_pf == UINT32_MAX; pass++) // compute families first, then the others + for (uint32_t f = 0; f < g_nqfam && fg_pf == UINT32_MAX; f++) { + if (f == g_vk.gfx_family || used[f] >= g_qfam_count[f]) continue; + if (!(g_qfam_flags[f] & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT))) continue; + if (!pass != !!(g_qfam_flags[f] & VK_QUEUE_COMPUTE_BIT)) continue; + if (!gfx_window_vk_presentable(g_vk.instance, g_vk.phys, f)) continue; + fg_pf = f, fg_pi = used[f]++; + } + if (fg_pf != UINT32_MAX) { + if (used[fg_pf] < g_qfam_count[fg_pf] && used[fg_pf] < 4) fg_af = fg_pf, fg_ai = used[fg_pf]++; + for (uint32_t f = 0; f < g_nqfam && fg_af == UINT32_MAX; f++) + if (f != g_vk.gfx_family && used[f] < g_qfam_count[f] && used[f] < 4) fg_af = f, fg_ai = used[f]++; + if (fg_af == UINT32_MAX) fg_af = fg_pf, fg_ai = fg_pi; + } } + VkDeviceQueueCreateInfo qi[QFAM_MAX]; + uint32_t nqi = 0; + qi[nqi++] = (VkDeviceQueueCreateInfo){.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, + .queueFamilyIndex = g_vk.gfx_family, .queueCount = nq_gfx, .pQueuePriorities = prio}; + for (uint32_t f = 0; f < g_nqfam; f++) + if (f != g_vk.gfx_family && used[f]) + qi[nqi++] = (VkDeviceQueueCreateInfo){.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, + .queueFamilyIndex = f, .queueCount = used[f], .pQueuePriorities = prio}; VkDeviceCreateInfo di = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, .pNext = &c->f2, .queueCreateInfoCount = nqi, .pQueueCreateInfos = qi, .enabledExtensionCount = n_ext, .ppEnabledExtensionNames = ext}; VkResult r = vk_dlssg_create_device(g_vk.instance, g_vk.phys, &di, &g_vk.device); @@ -539,11 +622,19 @@ static int create_device(void) { rt_log("vk: device entry points come through Streamline's interposer (DLSS Frame Generation)"); } vkGetDeviceQueue(g_vk.device, g_vk.gfx_family, 0, &g_vk.gfx_queue); - if (nq_gfx >= 2) { + if (fg_pf != UINT32_MAX) { VkQueue present, acquire; - vkGetDeviceQueue(g_vk.device, g_vk.gfx_family, 1, &present); - vkGetDeviceQueue(g_vk.device, g_vk.gfx_family, nq_gfx - 1, &acquire); - vk_fg_queues(present, acquire); + vkGetDeviceQueue(g_vk.device, fg_pf, fg_pi, &present); + vkGetDeviceQueue(g_vk.device, fg_af, fg_ai, &acquire); + vk_fg_queues(present, fg_pf, acquire, fg_af); + if (fg_pf != g_vk.gfx_family) + rt_log("vk: frame generation queues: the graphics family %u has %u queue%s; presents on family %u queue %u, " + "image acquires on family %u queue %u", g_vk.gfx_family, g_gfx_queues, g_gfx_queues == 1 ? "" : "s", + fg_pf, fg_pi, fg_af, fg_ai); + } else { + rt_log("vk: frame generation queues: none (the graphics family %u has %u queue%s and no other family with a free " + "queue can present): AMD FSR 3.1 frame generation cannot run", g_vk.gfx_family, g_gfx_queues, + g_gfx_queues == 1 ? "" : "s"); } if (g_vk.xfer_family != g_vk.gfx_family) vkGetDeviceQueue(g_vk.device, g_vk.xfer_family, 0, &g_vk.xfer_queue); else { g_vk.xfer_queue = g_vk.gfx_queue; g_vk.xfer_shared = 1; } @@ -562,6 +653,8 @@ static int create_device(void) { (g_vk.has_memory_budget ? VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT : 0), .preferredLargeHeapBlockSize = vk_vma_block_size(), .pDeviceMemoryCallbacks = vk_headroom_callbacks()}; + VkDeviceSize heap_limit[VK_MAX_MEMORY_HEAPS]; + if (vram_heap_limits(heap_limit)) ai.pHeapSizeLimit = heap_limit; if ((r = vmaCreateAllocator(&ai, &g_vk.vma)) != VK_SUCCESS) { rt_log("vk: vmaCreateAllocator failed (%d)", (int)r); return 0; } vk_mem_ballast(g_vk.vma); @@ -767,3 +860,6 @@ int probe_vkdevice(void) { printf("probe-vkdevice: %s (validation errors %ld)\n", g_errors_total ? "FAILED" : "passed", g_errors_total); return g_errors_total ? 1 : 0; } + +// The GPU has mesh shaders: the mesh emulation (NX2NAVI_MESH_EMULATION) does nothing there (the menu's Lama patches tab). +int vk_mesh_shader_native(void) { return g_vk.has_mesh_shader; } diff --git a/runtime/vk_dump.c b/runtime/vk_dump.c index db18c1a3..b43263ad 100644 --- a/runtime/vk_dump.c +++ b/runtime/vk_dump.c @@ -26,6 +26,9 @@ // (the terrain decals draw one pass each into the G-buffer and would use up both caps). // NX2NAVI_DUMP_SHADERS=,...: copy images only for passes using those shader hashes; metadata covers the whole frame. // NX2NAVI_DUMP_RAW=1: also save each copied image's native texel bytes as .bin, for numerical analysis. +// NX2NAVI_DUMP_KEY= (Windows): each press of that key in the game window dumps the next frame to +// NX2NAVI_DUMP_DIR/f. Keys: pause (also 1), scrolllock, insert, home, end, pageup, pagedown, f2-f7, f9, f12 (F1, F8, +// F10 and F11 are the menu, the flight recorder, the overlay and fullscreen). The game still gets a key it has bound. #include #include #include @@ -44,6 +47,9 @@ uint64_t vk_exec_cs_storage_addr(const VkExec *ex, const GfxShaderRes *r); // uint64_t vk_exec_storage_addr(const VkExec *ex, int st, const GfxShaderRes *r); // vk_exec_draw.c int vk_exec_raw_9e5(VkImage img, VkFormat f); // vk_exec_draw.c #include "platform/platform.h" +#ifdef _WIN32 +#include +#endif #define MAX_PASSES 400 #define PNG_CAP (2ull << 30) @@ -59,6 +65,8 @@ static char base_dir[400]; // NX2NAVI_DUMP_DIR; with NX2NAVI_DUMP_WH static uint64_t white_last; // NX2NAVI_DUMP_WHITE: the frame dumped last (0: none), checked 3 presents on static int white_tries, white_caught; static uint64_t dump_after; // NX2NAVI_DUMP_AFTER +static int dump_key; // NX2NAVI_DUMP_KEY: the Windows virtual-key code, 0 off +static const char *dump_key_name; static FILE *passes_f, *images_f; static int seq, finished, copied; // copied: passes and dispatches of the dump frame whose images were copied static int skip_runs, dump_ubo; // NX2NAVI_DUMP_SKIP_RUNS, NX2NAVI_DUMP_UBO (vec4 per block, 0 for none) @@ -1185,10 +1193,21 @@ void vk_dump_init(void) { key_mode = e && !strcmp(e, "key"); const char *df = cfg_get("NX2NAVI_DUMP_FILE"); if (df && *df) snprintf(dump_file, sizeof dump_file, "%s", df); + // NX2NAVI_DUMP_KEY: a key of ours, also outside NX2NAVI_DUMP=key + { + static const struct { const char *n; int vk; } keys[] = { + {"1", 0x13}, {"pause", 0x13}, {"scrolllock", 0x91}, {"insert", 0x2d}, {"home", 0x24}, {"end", 0x23}, + {"pageup", 0x21}, {"pagedown", 0x22}, {"f2", 0x71}, {"f3", 0x72}, {"f4", 0x73}, {"f5", 0x74}, {"f6", 0x75}, + {"f7", 0x76}, {"f9", 0x78}, {"f12", 0x7b}}; + const char *k = cfg_get("NX2NAVI_DUMP_KEY"); + for (size_t i = 0; k && *k && i < sizeof keys / sizeof *keys; i++) + if (!strcasecmp(k, keys[i].n)) dump_key = keys[i].vk, dump_key_name = i ? keys[i].n : "pause"; + if (k && *k && strcmp(k, "0") && !dump_key) rt_log("vk_dump: NX2NAVI_DUMP_KEY=%s: not a key this knows (pause, scrolllock, insert, home, end, pageup, pagedown, f2-f7, f9, f12)", k); + } if (dump_white || dump_after) { dump_frame = UINT64_MAX; vk_thumbs_enable(16); - } else if (movie_k || key_mode || dump_file[0]) { + } else if (movie_k || key_mode || dump_file[0] || dump_key) { dump_frame = UINT64_MAX; } else if (!e || !*e || !(dump_frame = strtoull(e, NULL, 10))) return; const char *lf = cfg_get("NX2NAVI_DUMP_LOG"); @@ -1233,10 +1252,27 @@ void vk_dump_init(void) { if (!bd_w || !bd_h) bd_stride = 0; snprintf(path, sizeof path, "%s/bufs.txt", dir); // a dump named as the run goes opens it with its folder (open_files); a named frame's folder exists already - if (bd_size && !dump_white && !movie_k && !key_mode && !dump_file[0] && !(bufs_f = fopen(path, "w"))) bd_size = 0; + if (bd_size && !dump_white && !movie_k && !key_mode && !dump_file[0] && !dump_key && !(bufs_f = fopen(path, "w"))) bd_size = 0; } vk_dump_armed = 1; - rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); + if (dump_key) rt_log("vk_dump: %s dumps the next frame to %s/f", dump_key_name, base_dir); + else rt_log("vk_dump: frame %llu goes to %s", (unsigned long long)dump_frame, dir); +} + +// NX2NAVI_DUMP_KEY: its key went down since the last present (the game window has the focus). +static int key_pressed(void) { +#ifdef _WIN32 + static int was; + HWND fg = GetForegroundWindow(); + DWORD pid = 0; + if (fg) GetWindowThreadProcessId(fg, &pid); + int down = pid == GetCurrentProcessId() && (GetAsyncKeyState(dump_key) & 0x8000) != 0; + int pressed = down && !was; + was = down; + return pressed; +#else + return 0; +#endif } // The thumbnail of frame f is blank white left of the overlay and above the dialogue box: -1 not available, 0 no, 1 yes. @@ -1306,6 +1342,13 @@ void vk_dump_present(uint64_t frame) { open_frame(frame + 1); rt_log("vk_dump: %s found: frame %llu goes to %s", dump_file, (unsigned long long)dump_frame, dir); } + if (dump_key && dump_frame == UINT64_MAX && key_pressed()) { // NX2NAVI_DUMP_KEY: our key names the next frame + file_more = 0; + snprintf(dir, sizeof dir, "%s/f%llu", base_dir, (unsigned long long)(frame + 1)); + open_files(); // before the frame is named: its passes write to them + dump_frame = frame + 1; + rt_log("vk_dump: %s: frame %llu goes to %s", dump_key_name, (unsigned long long)dump_frame, dir); + } if (frame != dump_frame) return; plat_mutex_lock(&rec_lock); finished = 1; @@ -1322,7 +1365,7 @@ void vk_dump_present(uint64_t frame) { passes_f = NULL; rt_log("vk_dump: frame %llu: %d passes and dispatches, about %llu MB of PNGs in %s", (unsigned long long)frame, seq, (unsigned long long)(png_bytes >> 20), dir); - if (dump_white || movie_k || key_mode || dump_file[0]) { // ready for another dump (vk_dump_present checks a white one's thumbnail 3 presents on) + if (dump_white || movie_k || key_mode || dump_file[0] || dump_key) { // ready for another dump (vk_dump_present checks a white one's thumbnail 3 presents on) if (dump_white) white_last = frame; plat_mutex_lock(&rec_lock); seq = copied = 0, last_targets = 0, pass_skip = NULL, png_bytes = 0; diff --git a/runtime/vk_exec.c b/runtime/vk_exec.c index 8e8371b2..bd9d6785 100644 --- a/runtime/vk_exec.c +++ b/runtime/vk_exec.c @@ -860,6 +860,7 @@ void vk_exec_pass_begin(VkExec *ex) { m->step = ex->mark_passes++; m->w = ex->pass_w, m->h = ex->pass_h, m->d = ex->pass_layers; m->layer = ex->depth.image ? ex->depth.base_layer : 0; + m->vw = m->vh = 0; m->ncolor = (uint8_t)(ex->pass_ncolor < 8 ? ex->pass_ncolor : 8); for (int i = 0; i < m->ncolor; i++) m->color[i] = ex->pass_color[i]; m->depth = ex->pass_depth ? ex->pass_depth : ex->pass_stencil; @@ -1119,7 +1120,19 @@ static void set_rt(VkExec *ex, const p_set_rt *r) { // command, it was 4% of the replay in Kokiri Forest. static int split_n = -1, split_idle, split_in_pass, split_draws, split_at, part_start; static uint64_t split_ns, part_t0; -static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits; +static uint64_t n_parts, n_subs, n_idle_parts, n_pass_splits, n_cuts; + +int vk_exec_split_now(VkExec *ex) { + int in_pass = ex->in_pass; + if (in_pass) vk_exec_pass_end(ex); + VkCommandBuffer cb = vk_sync_cut(); // not handed to the queue: the texture pool set stays (vk_exec_pool_split) + if (!cb) return 0; + ex->cb = cb; + ex->dirty = VK_EXEC_DIRTY_ALL; + vk_exec_draw_new_cb(); + n_cuts++; + return 1; +} static int split_wanted(void) { if (split_draws >= split_at) return 1; @@ -2090,9 +2103,10 @@ static uint32_t group_peek(const VkExec *ex, const op_hdr *h, const op_hdr **end int begun = 0; const p_set_rt *rt = &ex->rt; *why = GE_LIMIT; - // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared) and while - // NX2NAVI_GPU_PASSES copies their workgroup counts (one list) - int mesh_serial = vk_ctx()->mesh_fallback || vk_gpu_passes_capturing(); + // mesh draws stay on the submitting thread with the mesh fallback (its buffers are shared), with the mesh emulation + // (no VK_EXT_mesh_shader: its arena, queue and the part's pre-pass command buffer are the submitting thread's) and + // while NX2NAVI_GPU_PASSES copies their workgroup counts (one list) + int mesh_serial = vk_ctx()->mesh_fallback || !vk_ctx()->has_mesh_shader || vk_gpu_passes_capturing(); for (uint32_t ops = 0; ops < GROUP_PEEK; ops++) { if (h->size < sizeof *h || (h->size & 7) || h->op >= OP_COUNT_) { *why = GE_RUN; break; } // the walk reports it const void *a = nvn_op_payload(h); @@ -2587,10 +2601,10 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t if (n) n_subs++; if ((rt_trace & RT_TRACE_PERF) && n_subs >= 240) { rt_log("perf: replay command buffer parts per submission %.2f (%.2f first parts sent while the GPU was idle, %.2f " - "splits inside a pass); event order: held %llu, resumed %llu (%llu without their signal), switches %llu, " + "splits inside a pass, %.2f cuts for emulated mesh stages); event order: held %llu, resumed %llu (%llu without their signal), switches %llu, " "held now %d; waits a CPU signal met only after the submission's copies were taken: %llu", (double)(n_subs + n_parts) / (double)n_subs, (double)n_idle_parts / (double)n_subs, - (double)n_pass_splits / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed, + (double)n_pass_splits / (double)n_subs, (double)n_cuts / (double)n_subs, (unsigned long long)evo_held, (unsigned long long)evo_resumed, (unsigned long long)evo_forced, (unsigned long long)evo_switches, evo_pending, (unsigned long long)vk_exec_event_late_take()); uint64_t xw, xe, xl, xm, le, lc, lo; @@ -2682,7 +2696,7 @@ int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t } RW.groups = RW.group_draws = RW.chunks = RW.serial_draws = RW.wait_ns = 0; evo_held = evo_resumed = evo_forced = evo_switches = 0; - n_subs = n_parts = n_idle_parts = n_pass_splits = 0; + n_subs = n_parts = n_idle_parts = n_pass_splits = n_cuts = 0; } rec_spent_ns += kern_now_ns() - in_ns; if (rec_spent_ns >= REC_CLAIM_NS) rec_claim(); diff --git a/runtime/vk_exec.h b/runtime/vk_exec.h index 773986b2..795bd3fd 100644 --- a/runtime/vk_exec.h +++ b/runtime/vk_exec.h @@ -127,6 +127,8 @@ typedef struct VkExec { const void *fb_fs; // feedback_scan's memo: fragment shader, texture bind serial, pass serial uint32_t fb_serial, fb_pass, fb_sampled; uint32_t tex_serial, pass_serial; // bumped by every texture or image bind and pass begin + uint32_t pre_pass; // the pass (pass_serial) the part's pre-pass command buffer serves (vk_sync_pre; + // vk_exec_draw.c mesh emulation); a pass begins after it with another serial } VkExec; // Begins dynamic rendering with the current targets unless a pass is open (a full memory barrier precedes it). Call @@ -142,6 +144,9 @@ void vk_exec_barrier(VkExec *ex); void vk_exec_barrier_dispatch(VkExec *ex); // NX2NAVI_TRACE=perf, every 600 frames: the barriers per frame by kind. void vk_exec_barrier_log(uint64_t frame); +// Ends the open pass and the part without handing it to the queue (vk_sync_cut): the next draw begins the pass again +// at the start of a new command buffer, which the part's pre-pass command buffer precedes. 0 when no more can wait. +int vk_exec_split_now(VkExec *ex); // vk_texture.c (NX2NAVI_TEX_ALIAS): commands that must run at the replay's current position. vk_exec_inline_begin returns // the command buffer being recorded after a full barrier (an open pass is ended; *reopen says so), or VK_NULL_HANDLE // when this thread is not recording a replay. vk_exec_inline_end begins the pass again when it was open. diff --git a/runtime/vk_exec_draw.c b/runtime/vk_exec_draw.c index 5e8de0c3..d87d6a3c 100644 --- a/runtime/vk_exec_draw.c +++ b/runtime/vk_exec_draw.c @@ -34,6 +34,7 @@ #include "vk_mesh_fb.h" #include "platform/platform.h" #include "vk_dlaa.h" +#include "mesh_emu_spirv.h" #define MAX_BINDINGS 128 #define MAX_DESC 1024 @@ -154,11 +155,11 @@ typedef struct { VkBuffer sup_buf; VkDeviceSize sup_off; int sup_valid; - uint8_t c0[3][VK_C0_BYTES]; // last c0 bank per stage (vertex, fragment, compute) and where it lives - VkBuffer c0_buf[3]; - VkDeviceSize c0_off[3]; - int c0_valid[3]; - struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; uint8_t push[1024]; } c0_in[3]; // what vk_c0_fill read for c0[k] + uint8_t c0[5][VK_C0_BYTES]; // last c0 bank per stage (vertex, fragment, compute, task, mesh) and where it lives + VkBuffer c0_buf[5]; + VkDeviceSize c0_off[5]; + int c0_valid[5]; + struct { p_bind_buf ssbo[16], ubo[16]; uint32_t dim[3], binst; uint8_t push[1024]; } c0_in[5]; // what vk_c0_fill read for c0[k] VkPipelineLayout ds_layout; // graphics layout the sets below were last pushed or bound with VkDescriptorSet ds_bound[2]; // allocated set bound per set number (0: none or pushed since) VkDescriptorSet ds_last[2]; // last allocated and written set per set number, contents in DS @@ -1279,7 +1280,11 @@ static uint32_t nvn2_cs_tables_at(const uint8_t *ctl); static VkDescriptorBufferInfo c0_info(VkExec *ex, int st) { DrawCtx *const cx = ex->dc; - int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : 2; + // One bank per stage: the inputs below do not say which stage they are for, and the stages lay the storage table out + // at different offsets (c0_ssbo_base). Task and mesh shared the compute entry: the foliage's task and mesh stages + // bind the same storage buffers, so the task stage was handed the mesh stage's bank (its table at 0x770), read + // zero addresses at 0x670, culled every instance and launched no mesh workgroups. + int k = st == NVN_STAGE_VS ? 0 : st == NVN_STAGE_FS ? 1 : st == NVN_STAGE_TASK ? 3 : st == NVN_STAGE_MESH ? 4 : 2; // vk_c0_fill reads the storage slots, and the uniform slots and dispatch size for compute or the base instance for // graphics: equal inputs give the same bank, so neither the fill nor the 2 KiB compare is needed. // A vertex or fragment bank depends on the storage slots and the base instance alone (the rest of its inputs record @@ -1543,6 +1548,13 @@ static VkDescriptorBufferInfo ssbo_info(VkExec *ex, int st, const GfxShaderRes * // submission as their draws: either way this ran for the guest submission. if (r->cbuf_slot == VK_GPA_CB_SLOT) { vk_mem_shadow_table_read(); + // What the table names is mostly guest memory itself (pools without a mirror), which the GPU reads as it runs: + // the submission's submit wait is the GPU's (vk_mem_take_live). Without this the staged mirror uploads + // (NX2NAVI_MIRROR_STAGE) ended the wait once the replay had submitted, and the game wrote the next frame's + // instances while the GPU still drew from them: the small grass and moss, which reach their instances through + // the table, flickered and slid over the ground as the camera turned (RX 580, Kokiri Forest, 2026-11-11). + // Not every bind comes here (the bind skip, the kept descriptors): gpa_reader notes it at each bind. + vk_mem_note_live(); VkDescriptorBufferInfo t = vk_gpa_info(); return t.buffer ? t : (VkDescriptorBufferInfo){Z.zbuf, 0, VK_WHOLE_SIZE}; } @@ -2066,13 +2078,15 @@ static VkDescriptorSetLayout pool_layout_make(void) { static void pool_marks_make(void) { VkCtx *c = vk_ctx(); - VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)VK_POOL_TEX * 4, + // a uint past the slots: the spare that a fragment shader's marks of accesses it did not reach go to + // (vk_pipeline.c spirv_pool_mark); it is never read back + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = (VkDeviceSize)(VK_POOL_TEX + 1) * 4, .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT}; VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; if (vmaCreateBuffer(c->vma, &bi, &ai, &PL.marks, &PL.marks_alloc, NULL) != VK_SUCCESS) rt_fatal("vk: the texture pool marks buffer failed"); - bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + bi.size = (VkDeviceSize)VK_POOL_TEX * 4, bi.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; VmaAllocationCreateInfo hi = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; for (int i = 0; i < MARKS_RB; i++) { @@ -2221,6 +2235,22 @@ static uint64_t pool_create_ns(void) { return (uint64_t)(pool_loading && ns ? load_ns : ns); } static uint32_t pool_defer[VK_POOL_TEX]; // per slot: 1 + the low bits of the frame a refresh left it in +// The slots a refresh left (pool_refill failed: the creation budget or the frame's time was spent), so that the next +// refresh of the same submission looks up these and the changed slots (vk_texture_dirty_slots_locked) instead of +// walking the whole table again. While an area loads, slots are left at every refresh, and each refresh walked the +// table: about 67,000 slot reads and 1.65 ms of the replay per submission in Kokiri Forest on an RX 580 (2026-11-11), +// with six submissions a frame. pool_left_all: the list overflowed (or nothing tracks it yet): walk the table. +static uint32_t pool_left[VK_POOL_TEX], pool_nleft; +static uint8_t pool_is_left[VK_POOL_TEX]; +static int pool_left_all = 1; +static void pool_left_add(uint32_t id) { + if (id >= VK_POOL_TEX || pool_is_left[id]) return; + pool_is_left[id] = 1, pool_left[pool_nleft++] = id; +} +static void pool_left_clear(void) { + for (uint32_t i = 0; i < pool_nleft; i++) pool_is_left[pool_left[i]] = 0; + pool_nleft = 0; +} static VkWriteDescriptorSet pool_w[POOL_WBATCH]; static VkDescriptorImageInfo pool_ii[POOL_WBATCH]; static uint32_t pool_nw; @@ -2672,7 +2702,7 @@ static void pool_dirty_slot(int id, void *arg) { // vk_texture_dirty_slots_loc if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++; else PL.n_by_addr++; if (pool_refill(d->ex, (uint32_t)id, d->used, t, d->spent)) PL.tok[id] = t; - else d->retry++; + else d->retry++, pool_left_add((uint32_t)id); } // Every entry as the slots, kinds and samplers are now (pool_sync_entries has nothing to write). @@ -2774,7 +2804,7 @@ static void pool_sync_entries(VkExec *ex) { if (fresh || gen != PL.gen || ep != PL.epoch || PL.retry) { vk_texture_create_budget_locked(spent ? 0 : pool_create_budget()); PL.n_frame_stops += (uint64_t)spent; - // Only a slot change since the last sync (no new kinds, no new submission, nothing left over): the slots that + // Only a slot change since the last sync (no new kinds, no new submission): the slots that // changed (vk_texture_dirty_slots_locked), not the whole table. With NX2NAVI_POOL_FAST (default on) a new // submission too: its freshness check goes through the entries the slots resolved to and the slots that // resolved to none (vk_texture_pool_fresh_locked, pool_unres), not every slot. @@ -2784,8 +2814,32 @@ static void pool_sync_entries(VkExec *ex) { int grp = pool_group(); if (grp && !PL.gmode) vk_texture_groups_unstamp_locked(); // members were refilled one by one meanwhile PL.gmode = grp; - int walked = !fresh && (!newep || pool_fast()) && !PL.retry && PL.dirty_pos && - vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd); + // With slots left by an earlier refresh (the creation budget or the frame's time spent): those (pool_left) + // first, then the changed slots, instead of the walk over every slot. While an area loads, slots are left at + // every refresh, and each one walked the table: about 67,000 slot reads and 1.65 ms of the replay per + // submission in Kokiri Forest on an RX 580 (0.7.1, 2026-11-11), six submissions a frame. pool_left_all: no + // list kept yet (until the first walk): walk. + int quick = !fresh && (!newep || pool_fast()) && (!PL.retry || !pool_left_all) && PL.dirty_pos; + if (quick && PL.retry) { + uint32_t k = 0; + for (uint32_t i = 0; i < pool_nleft; i++) { + uint32_t id = pool_left[i]; + uint64_t t = (int)id < n ? vk_texture_slot_token((int)id) : 0; + int left = 0; + if (t != PL.tok[id]) { + PL.n_refill++; + if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++; + else PL.n_by_addr++; + if (pool_refill(ex, id, used, t, spent)) PL.tok[id] = t; + else left = 1; + } + if (left) pool_left[k++] = id; + else pool_is_left[id] = 0; + } + pool_nleft = k; + PL.n_slots += (uint64_t)k; + } + int walked = quick && vk_texture_dirty_slots_locked(&PL.dirty_pos, pool_dirty_slot, grp ? pool_dirty_group : NULL, &pd); if (walked) { int at = newep ? 0 : 1; PL.n_walk[at]++, PL.walk_visit[at] += PL.n_visit - v0, PL.walk_refill[at] += PL.n_refill - f0; @@ -2819,7 +2873,7 @@ static void pool_sync_entries(VkExec *ex) { PL.unres_ns += kern_now_ns() - tu; PL.n_ep++, PL.ep_ns += kern_now_ns() - te; } - if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(); + if (!walked) PL.dirty_pos = vk_texture_dirty_pos_locked(), pool_left_clear(), pool_left_all = 0; uint64_t tw = walked ? 0 : kern_now_ns(); for (int id = 0; id < n && !walked; id++) { // slots past n were never registered: stand-ins since the first sync uint64_t t = vk_texture_slot_token(id); @@ -2828,7 +2882,7 @@ static void pool_sync_entries(VkExec *ex) { if (t >> 32 != PL.tok[id] >> 32) PL.n_by_reg++; else PL.n_by_addr++; if (pool_refill(ex, (uint32_t)id, used, t, spent)) PL.tok[id] = t; - else retry++; + else retry++, pool_left_add((uint32_t)id); } else if (t && fresh) { pool_fill(ex, (uint32_t)id, fresh, t); } else if (t && ep != PL.epoch) { @@ -2840,10 +2894,11 @@ static void pool_sync_entries(VkExec *ex) { PL.n_slots += (uint64_t)n, PL.n_full++, PL.full_ns += kern_now_ns() - tw; vk_texture_groups_unstamp_locked(); // the walk over every slot refilled members one by one } + retry = (int)pool_nleft; // every slot left, by this refresh or an earlier one not filled since vk_texture_create_budget_locked(-1); } PL.retry = retry != 0; - // retry counts the slots this sync left; a sync with slots left walks the whole table, so that is all of them + // retry counts every slot left (pool_left: this sync's and the earlier ones' not filled since) int backlog = pool_load_backlog(); if (!pool_loading && backlog > 0 && retry >= backlog) { pool_load_made = (uint64_t)retry, pool_load_frame0 = PL.frame; @@ -3498,10 +3553,35 @@ static void write_set(const VkPipeInfo *info, uint32_t s, VkDescriptorSet set, c // graphics push equal to the last one at its set number, with no other layout or bind between, is not recorded. // The descriptors go into one packed array d, a VkDescSlot per descriptor in binding order, which a set's update // template reads directly (VkPipeInfo.tpl); write structs are filled only when a set has no template. +static _Thread_local const VkDescSlot *emu_d; // set 0's descriptors of the last bind_resources build (mesh emulation) +static uint32_t emu_d_bytes; +// A pipeline that binds the device address table reads what it names, mostly guest memory itself (pools without a +// mirror), as the GPU runs: the submission's submit wait is the GPU's (vk_mem_take_live). Without this the staged mirror +// uploads (NX2NAVI_MIRROR_STAGE) ended the wait once the replay had submitted, and the game wrote the next frame's +// instances while the GPU still drew from them: the small grass and moss, which reach their instances through the +// table, flickered and slid over the ground as the camera turned (RX 580, Kokiri Forest, 2026-11-11). Noted at every +// bind, before the bind skip and the kept descriptors (bind_sig, pb_take), which leave ssbo_info out: noted only there, +// it was missed once those had warmed up, and the flicker came back after a minute of play. +// the pipeline's binding list holds a table binding +static void gpa_reader(const VkPipeInfo *info) { + int8_t g = __atomic_load_n(&info->gpa, __ATOMIC_RELAXED); // the replay workers bind too + if (!g) { + g = 1; + for (uint32_t i = 0; i < info->n_bindings && g == 1; i++) { + const VkPipeBinding *b = &info->bindings[i]; + if (b->kind == GFX_RES_SSBO && b->res && b->res->cbuf_slot == VK_GPA_CB_SLOT) g = 2; + } + __atomic_store_n(&((VkPipeInfo *)info)->gpa, g, __ATOMIC_RELAXED); + } + if (g == 2) vk_mem_note_live(); +} + static void bind_resources(VkExec *ex, const VkPipeInfo *info) { + gpa_reader(info); DrawCtx *const cx = ex->dc; VkWriteDescriptorSet *w = cx->bind_w; VkDescSlot *d = cx->bind_d; + emu_d = d; VkBufferView *bv = cx->bind_bv; if (info->n_bindings > MAX_BINDINGS) rt_fatal("vk: %u descriptor bindings exceed the replay's %d", info->n_bindings, MAX_BINDINGS); bind_stats(info); @@ -3643,6 +3723,7 @@ static void bind_resources(VkExec *ex, const VkPipeInfo *info) { // An empty set 1 (a fragment stage without resources: depth and shadow draws) starts past the last binding, which // the loop never reaches: d0[1] stayed 0 and set 0 was pushed with 0 bytes, so the driver read stale descriptors if (n_sets == 2 && lo[1] == hi[1]) d0[1] = nd; + emu_d_bytes = (uint32_t)((n_sets > 1 ? d0[1] : nd) * sizeof *d); // Under the validation layer, a new graphics layout binds set 1 before set 0 is pushed or bound. The layer (SDK // 1.4.363, PipelineLayoutCompatDef::operator==) takes any two INDEPENDENT_SETS layouts as compatible, so // vkCmdPushDescriptorSetKHR keeps the push set it made for an earlier layout and drops writes to bindings that @@ -3763,6 +3844,10 @@ static VkViewport base_viewport(VkExec *ex) { static VkViewport draw_viewport(VkExec *ex) { VkViewport v = base_viewport(ex); + if (ex->mark) { // NX2NAVI_PASS_TIMES: how much of the pass the draws cover + if ((uint32_t)v.width > ex->mark->vw) ex->mark->vw = (uint32_t)v.width; + if ((uint32_t)-v.height > ex->mark->vh) ex->mark->vh = (uint32_t)-v.height; + } vk_dlaa_viewport(ex, &v); // the scene's draws move by the DLAA jitter return v; } @@ -4203,6 +4288,7 @@ static void prog_lead(const prog_t *p, int mesh) { (unsigned long long)PLD.n[m][4], (unsigned long long)PLD.n[m][5]); } +static const VkPipeInfo *PD_info; // the pipeline of the last draw prepare_draw_ex prepared // mesh: a mesh draw (op_draw_mesh): the program's mesh and task stages stand in for the vertex stage, and nothing // vertex-related (vertex buffers, motion vector bookkeeping of vertex shaders) is bound or noted. 2: a mesh draw of // the mesh fallback (mfb_draw): the vertex stage is DrawCtx.mfb_cur_vs, in an ordinary pipeline without vertex input. @@ -4405,7 +4491,9 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int delta = cfg_bool("NX2NAVI_DYN_DELTA"); if (!delta) rt_log("vk_exec: every dynamic state recorded after each pipeline bind (NX2NAVI_DYN_DELTA=0)"); } - if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb, + // an emulated mesh draw (NX2NAVI_MESH_EMULATION) binds a vertex pipeline: its topology, primitive restart and + // vertex input are dynamic and are set as for any vertex draw + if (delta) vk_exec_frame.dyn_calls += (uint64_t)vk_pipeline_set_dynamic_delta(ex->cb, &s, &cx->B.dyn, mesh && !fb && !info->emu, &vk_exec_frame.dyn_left); else vk_pipeline_set_dynamic(ex->cb, &s); ex->recorded++; @@ -4437,6 +4525,7 @@ static int prepare_draw_ex(VkExec *ex, int prim, int32_t count, int restart, int vk_exec_oq_draw(ex); DRS.issued[mesh]++; draw_stats_tick(); + PD_info = info; return 1; } @@ -4693,6 +4782,7 @@ static VkWriteDescriptorSet mo_write(const VkPipeBinding *b, const VkDescSlot *d // would hold the same); a pushed set is pushed by mo_emit. 0 for a pipeline with storage buffers, images or texel // buffers (not drawn again). static int mo_desc(VkExec *ex, const VkPipeInfo *info, MoDraw *q, const VkDescriptorImageInfo *pose_depth) { + gpa_reader(info); if (info->n_bindings > MAX_BINDINGS || info->pool) return 0; uint32_t total = 0; for (uint32_t i = 0; i < info->n_bindings; i++) total += info->bindings[i].count ? info->bindings[i].count : 1; @@ -5745,7 +5835,37 @@ static void mfb_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind) { ex->recorded++; } +static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args); +// The primitive an emulated draw's mesh stage outputs: point meshes draw point lists, the rest triangles. +static int mesh_emu_draw_prim(VkExec *ex) { + static struct { const GfxShader *s; int prim; } memo[256]; + int present = 0, pending = 0; + const GfxShader *ms = ex->prog ? shader_for(ex, NVN_STAGE_MESH, &present, &pending, 0) : NULL; + if (!ms) return NVN_DRAW_PRIMITIVE_TRIANGLES; + unsigned h = (unsigned)((uintptr_t)ms >> 4) & 255; + if (memo[h].s != ms) { + uint64_t n = 0; + const uint32_t *w = gfx_shader_spirv(ms, &n); + memo[h].s = ms, memo[h].prim = w && mesh_emu_prim(w, (size_t)n) == 1 ? NVN_DRAW_PRIMITIVE_POINTS : NVN_DRAW_PRIMITIVE_TRIANGLES; + } + return memo[h].prim; +} +static int mesh_emu_on(void) { + static int on = -1; + if (on < 0) { + on = !vk_ctx()->has_mesh_shader && rt_knob("MESH_EMULATION", "1"); + if (on) rt_log("vk: mesh shaders emulated with compute (NX2NAVI_MESH_EMULATION)"); + } + return on; +} + static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { + PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); + if (!fn && mesh_emu_on()) { + if (d->first) warn_once(W_MESH, "an emulated mesh draw from workgroup %u: drawn from workgroup 0", d->first); + if (d->count && rt_knob("MESH", "1")) mesh_emu_draw(ex, d->count, NULL, NULL); + return; + } static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); // NX2NAVI_MESH=0: mesh draws are skipped (diagnosis) if (!d->count || !on) return; @@ -5754,8 +5874,7 @@ static void op_draw_mesh(VkExec *ex, const p_draw_mesh *d) { mfb_draw(ex, d->count, NULL); return; } - PFN_vkCmdDrawMeshTasksEXT fn = draw_mesh_fn(); - if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + if (!fn) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader (mesh_emulation = 1 emulates them): skipped"); ex->dc->c0_binst = 0; if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; if (d->first) warn_once(W_MESH, "a mesh draw from workgroup %u: drawn from workgroup 0", d->first); @@ -5798,7 +5917,7 @@ uint32_t vk_exec_mesh_counts(const uint32_t **out) { *out = mesh_count_map; return mesh_count_map ? mesh_count_n : 0; } -static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { +static void mesh_count_note(VkExec *ex, VkCommandBuffer cb, const VkMemSpan *src) { uint32_t cap = (uint32_t)vk_gpu_passes_capturing(); if (!cap) { mesh_count_was = 0; return; } if (!mesh_count_was) mesh_count_was = 1, mesh_count_n = 0; @@ -5813,8 +5932,8 @@ static void mesh_count_note(VkExec *ex, const VkMemSpan *src) { } if (mesh_count_n >= MESH_COUNTS) return; VkBufferCopy cp = {src->offset, 4ull * mesh_count_n++, 4}; - vkCmdCopyBuffer(ex->cb, src->buffer, mesh_count_buf, 1, &cp); - ex->recorded++; + vkCmdCopyBuffer(cb, src->buffer, mesh_count_buf, 1, &cp); + if (cb == ex->cb) ex->recorded++; } // NX2NAVI_MESH_AHEAD (default 1; 0 = each draw by itself, "ab" switches with the A/B blocks): the copy of a record @@ -5851,7 +5970,11 @@ static int mesh_ahead_on(void) { return RT_KNOB_ON(mode); } -static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { +// A ring entry {count from src, 1, 1}, filled in cb; ex->cb: after a full barrier, which ends the pass (the next draw +// opens it again) and orders the culling's writes before the copy. Another cb (the pre-pass one of the mesh +// emulation) begins with one. d: the draw it is for, which may take an entry filled ahead (MESH_AHEAD, ex->cb only); +// NULL: none (the mesh emulation's). +static int mesh_ring_entry(VkExec *ex, VkCommandBuffer cb, const p_draw_mesh_ind *d, const VkMemSpan *src, VkDeviceSize *at) { DrawCtx *const cx = ex->dc; if ((rt_trace & RT_TRACE_PERF) && (mesh_ahead_taken + mesh_ahead_own) % 20000 == 19999) { char stops[160]; @@ -5860,7 +5983,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan "their pass; the runs ended: %s", (unsigned long long)(mesh_ahead_taken + mesh_ahead_own + 1), (unsigned long long)mesh_ahead_taken, stops); } - if (cx->ma_i < cx->ma_n) { + if (d && cb == ex->cb && cx->ma_i < cx->ma_n) { if (cx->ma[cx->ma_i].d == d && cx->ma[cx->ma_i].ind == d->ind && cx->ma_brk == vk_exec_brk_now()) { *at = cx->ma[cx->ma_i++].at; __atomic_add_fetch(&mesh_ahead_taken, 1, __ATOMIC_RELAXED); @@ -5871,14 +5994,18 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan __atomic_add_fetch(&mesh_ahead_own, 1, __ATOMIC_RELAXED); if (!__atomic_load_n(&mesh_ring, __ATOMIC_ACQUIRE) && (vk_exec_in_chunk() || !mesh_ring_make())) return 0; *at = 12ull * (__atomic_fetch_add(&mesh_ring_next, 1, __ATOMIC_RELAXED) % MESH_RING); - vk_exec_barrier(ex); // ends the pass (the next draw opens it again) and orders the culling's writes before the copy - vk_gpu_gap_note(VK_GAP_MESH_RING, 12); - vkCmdFillBuffer(ex->cb, mesh_ring, *at + 4, 8, 1); + int own = cb == ex->cb; + if (own) { + vk_exec_barrier(ex); + cb = ex->cb; + vk_gpu_gap_note(VK_GAP_MESH_RING, 12); + } + vkCmdFillBuffer(cb, mesh_ring, *at + 4, 8, 1); VkBufferCopy cp = {src->offset, *at, 4}; - vkCmdCopyBuffer(ex->cb, src->buffer, mesh_ring, 1, &cp); - mesh_count_note(ex, src); + vkCmdCopyBuffer(cb, src->buffer, mesh_ring, 1, &cp); + mesh_count_note(ex, cb, src); cx->ma_n = cx->ma_i = 0; - if (mesh_ahead_on()) { + if (d && own && mesh_ahead_on()) { const void *run[MESH_AHEAD]; int n = vk_exec_mesh_run(d, run, MESH_AHEAD); for (int i = 0; i < n; i++) { @@ -5891,7 +6018,7 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan vkCmdFillBuffer(ex->cb, mesh_ring, fa + 4, 8, 1); VkBufferCopy fc = {ip.offset, fa, 4}; vkCmdCopyBuffer(ex->cb, ip.buffer, mesh_ring, 1, &fc); - mesh_count_note(ex, &ip); + mesh_count_note(ex, ex->cb, &ip); ex->recorded += 2; cx->ma[cx->ma_n].d = f, cx->ma[cx->ma_n].ind = f->ind, cx->ma[cx->ma_n].at = fa; cx->ma_n++; @@ -5903,26 +6030,53 @@ static int mesh_ring_entry(VkExec *ex, const p_draw_mesh_ind *d, const VkMemSpan .dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, .dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT}; VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; - vkCmdPipelineBarrier2(ex->cb, &di); - ex->recorded += 3; + vkCmdPipelineBarrier2(cb, &di); + if (own) ex->recorded += 3; return 1; } +// NX2NAVI_FOLIAGE (full, reduced, off; default full; the Lama patches tab): Ocarina draws its foliage with +// DrawMeshTasksIndirect, the instances its compute passes culled on the GPU. With a task stage: the small grass and moss +// that carpet the ground; without one: ferns, leaves, clover and the other plants. Its other mesh draws (DrawMeshTasks +// with a count: terrain, rocks, buildings, doors) are not foliage. reduced skips the indirect draws with a task stage, +// off every indirect one, in every pass (their shadows too). In a frame dump of Kokiri Forest on an RX 580 +// (f8118, 2026-11-10) the grass and moss were one draw with task stage 2ddf7b0bf74e5a05; the plants nine without. +static int foliage_skip(VkExec *ex) { + static int mode = -1; + if (mode < 0) { + const char *v = cfg_get("NX2NAVI_FOLIAGE"); + mode = v && !strcmp(v, "off") ? 2 : v && !strcmp(v, "reduced") ? 1 : 0; + rt_log("vk: foliage %s (NX2NAVI_FOLIAGE)", mode == 2 ? "off" : mode ? "reduced: no small grass or moss" : "full"); + } + if (!mode) return 0; + if (mode == 2) return 1; + const uint8_t *ctl, *code; + uint32_t len; + return ex->prog && find_stage(ex->dc, ex->prog, NVN_STAGE_TASK, &ctl, &code, &len); +} + // DrawMeshTasksIndirect: one draw from its ring entry (mesh_ring_entry); x2-x4 of the call vary like leftovers, so they // are not read as a count or stride. static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { PFN_vkCmdDrawMeshTasksIndirectEXT fn = draw_mesh_ind_fn(); - int fallback = vk_ctx()->mesh_fallback; - if (!fn && !fallback) return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader: skipped"); + int emu = !fn && mesh_emu_on(), fallback = !emu && vk_ctx()->mesh_fallback; + if (!fn && !emu && !fallback) + return warn_once(W_MESH, "mesh draws need VK_EXT_mesh_shader (mesh_emulation = 1 emulates them): skipped"); static int on = -1; if (on < 0) on = rt_knob("MESH", "1"); - if (!on) return; + if (!on || foliage_skip(ex)) return; VkMemSpan ip; if (!vk_mem_resolve(d->ind, 12, &ip) || ip.offset % 4) return warn_once(W_INDIRECT, "indirect mesh draw arguments at %#llx are not backed or not aligned: skipped", (unsigned long long)d->ind); ex->dc->c0_binst = 0; VK_MEM_USE(ip); + if (emu) { // emulated: a ring entry {count, 1, 1}, made where the stages run, is the first stage's dispatch + mesh_emu_draw(ex, 0, NULL, &ip); + vk_mem_count(ip.cls, VK_MUSE_IND, 4); + VK_MEM_BOUND(ip); + return; + } if (fallback) { mfb_draw(ex, 0, &ip); vk_mem_count(ip.cls, VK_MUSE_IND, 4); @@ -5930,7 +6084,7 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { return; } VkDeviceSize at; - if (!mesh_ring_entry(ex, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + if (!mesh_ring_entry(ex, ex->cb, d, &ip, &at)) return warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); if (!prepare_draw_ex(ex, NVN_DRAW_PRIMITIVE_TRIANGLES, 0x7fffffff, 0, 1)) return; vk_exec_frame.draws++; static uint32_t n_logged; @@ -5948,6 +6102,356 @@ static void op_draw_mesh_ind(VkExec *ex, const p_draw_mesh_ind *d) { VK_MEM_BOUND(ip); } +// ---- NX2NAVI_MESH_EMULATION: mesh draws on GPUs without VK_EXT_mesh_shader +// Each mesh draw (prepared like one, prepare_draw_ex, for its layout, descriptors and graphics state) runs its task +// and mesh stages as compute pipelines (vk_pipeline_mesh_emu) outside the pass: they write vertices and triangles into +// the arena (mesh_emu_spirv.h) and the draw record's DrawIndexedIndirect, which the generated vertex shader and the +// game's fragment shader then draw. The arena's counters are zeroed at the first emulated draw of each submission; +// the draw records go round a ring of MESH_EMU_DRECS. NX2NAVI_MESH_EMU_MB sizes the arena (default 256 MiB), +// NX2NAVI_MESH_EMU_TASKS its task records per submission (default 16384, 3 KiB each). Before the counters start over +// they are copied to a readback ring; the perf log reports the most a submission asked for against the capacities. +#define ME_RB 64 +static struct { + VkBuffer buf, rb; + VmaAllocation alloc, rb_alloc; + uint32_t *rb_map; + uint64_t addr, resets; + uint32_t iw, icap, vw, vcap, trecs, epoch, rec; + int failed; + uint64_t draws, skipped; + uint32_t max[4]; + VkPipelineLayout alayout; // the allocation shader's (push constants only) + VkPipeline apipe; +} ME; + +// The allocation shader's pipeline (mesh_emu_spirv.h mesh_emu_alloc_spirv); 0 when it cannot be made. +static int me_alloc_pipe(void) { + VkDevice dev = vk_ctx()->device; + VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 32}; + VkPipelineLayoutCreateInfo li = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .pushConstantRangeCount = 1, + .pPushConstantRanges = &pcr}; + if (vkCreatePipelineLayout(dev, &li, NULL, &ME.alayout) != VK_SUCCESS) return ME.alayout = VK_NULL_HANDLE, 0; + size_t n = 0; + const uint32_t *w = mesh_emu_alloc_spirv(&n); + VkShaderModuleCreateInfo mi = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = n * 4, .pCode = w}; + VkShaderModule m; + if (vkCreateShaderModule(dev, &mi, NULL, &m) != VK_SUCCESS) return 0; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = ME.alayout}; + VkResult r = vkCreateComputePipelines(dev, VK_NULL_HANDLE, 1, &ci, NULL, &ME.apipe); + vkDestroyShaderModule(dev, m, NULL); + if (r != VK_SUCCESS) return ME.apipe = VK_NULL_HANDLE, 0; + return 1; +} + +static int me_arena(void) { + if (ME.buf) return 1; + if (ME.failed) return 0; + const char *v = cfg_get("NX2NAVI_MESH_EMU_MB"), *t = cfg_get("NX2NAVI_MESH_EMU_TASKS"); + int mb = v && atoi(v) >= 32 ? atoi(v) : 256; + if (mb > 2048) mb = 2048; + ME.trecs = t && atoi(t) >= 256 ? (uint32_t)atoi(t) : 16384; + if (ME.trecs > 262144) ME.trecs = 262144; + uint64_t tbytes = (uint64_t)MESH_EMU_TOFF + (uint64_t)ME.trecs * MESH_EMU_TWORDS * 4; + if (tbytes + (32ull << 20) > ((uint64_t)mb << 20)) mb = (int)((tbytes >> 20) + 64); // room for vertices and indices + VkDeviceSize size = (VkDeviceSize)mb << 20; + VkBufferCreateInfo bi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = size, + .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT}; + VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; + if (!me_alloc_pipe()) { + ME.failed = 1; + rt_log("vk: mesh emulation: the allocation shader's pipeline could not be created: mesh draws skipped"); + return 0; + } + if (vmaCreateBuffer(vk_ctx()->vma, &bi, &ai, &ME.buf, &ME.alloc, NULL) != VK_SUCCESS) { + ME.buf = VK_NULL_HANDLE, ME.failed = 1; + rt_log("vk: mesh emulation: the %d MiB arena could not be created: mesh draws skipped", mb); + return 0; + } + VkBufferDeviceAddressInfo dai = {.sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, .buffer = ME.buf}; + ME.addr = vkGetBufferDeviceAddress(vk_ctx()->device, &dai); + uint64_t ioff = (tbytes + 4095) & ~4095ull, ibytes = ((size - ioff) / 4) & ~4095ull; + uint64_t voff = ioff + ibytes; + ME.iw = (uint32_t)(ioff / 4), ME.icap = (uint32_t)(ibytes / 4), ME.vw = (uint32_t)(voff / 16), ME.vcap = (uint32_t)((size - voff) / 16); + rt_log("vk: mesh emulation arena: %d MiB, %u task records, %u indices, %u vertex slots (16 bytes)", mb, ME.trecs, + ME.icap, ME.vcap); + VkBufferCreateInfo rbi = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = ME_RB * 16, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT}; + VmaAllocationCreateInfo rai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST, + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT}; + VmaAllocationInfo rinfo; + if (vmaCreateBuffer(vk_ctx()->vma, &rbi, &rai, &ME.rb, &ME.rb_alloc, &rinfo) == VK_SUCCESS) ME.rb_map = rinfo.pMappedData; + else ME.rb = VK_NULL_HANDLE; + return 1; +} + +// The access masks also carry the generic MEMORY_WRITE / MEMORY_READ bits. SHADER_STORAGE_READ/WRITE exist only in +// synchronization2 (bits above 31), and AMD's driver for Polaris (23.10.01, RX 580) did not act on them: the mesh +// dispatch that reads the {groups, task workgroups, 1} the task stage had just written read zeros and launched no +// workgroup (mesh.txt: "mesh groups started 0" with the record holding 32 x 12 x 1), so every draw with a task stage +// (Kokiri's moss foliage, Link's door) drew nothing. Mesh draws without a task stage drew, because their record is +// read by the draw after a command buffer boundary. The general replay barrier (vk_exec.c barrier_to) uses those bits. +static void me_barrier(VkCommandBuffer cb, VkPipelineStageFlags2 ss, VkAccessFlags2 sa, VkPipelineStageFlags2 ds, VkAccessFlags2 da) { + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = ss, + .srcAccessMask = sa | VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = ds, + .dstAccessMask = da | VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); +} + +// The stages of the emulated draws whose records were filled, waiting to be recorded (me_flush): the draws of a pass +// with NX2NAVI_MESH_EMU_BATCH, which run in the part's pre-pass command buffer when it ends (vk_exec_mesh_flush, from +// vk_sync.c), or the one draw that runs in the part. Recorded by stage, with one barrier between stages for all the +// draws: the fills, the allocation (task records; index ranges of draws without a task stage), every task stage, the +// allocation of the other index ranges, every mesh stage. Each draw's indices are a range of its own and its task +// records too (the allocation shader), so the draws' stages need nothing of each other's. One by one, as before +// 2026-11-11, each draw took three barriers, and its task base and first index were copies of the arena's counters +// after the draw before it: an RX 580 spent 5.5 ms of a frame's GPU time on 69 draws' stages (Kokiri Forest), +// most of it waiting at the barriers with the GPU nearly idle. +typedef struct { + VkPipeline tp, mp; + VkPipelineLayout layout; + VkDescriptorUpdateTemplate tpl_c; + VkDescriptorSet set0, pset; + int push; + uint32_t count, rec, pc[8]; + VkBuffer ib; // the first stage's indirect dispatch (VK_NULL_HANDLE: count) + VkDeviceSize io; + size_t doff; + uint32_t dbytes; +} MeQ; +#define MEQ_MAX 256 +static struct { + MeQ q[MEQ_MAX]; + int n, tasks; + uint8_t *d; // the push descriptors' data of the queued draws + size_t dn, dcap; +} MEQ; + +static void me_bind(VkCommandBuffer cb, const MeQ *q) { + if (q->push) rec_push_tpl(vk_pipeline_push_tpl_fn(), cb, q->tpl_c, q->layout, 0, MEQ.d + q->doff, q->dbytes); + else vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 0, 1, &q->set0, 0, NULL); + if (q->pset) vkCmdBindDescriptorSets(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->layout, 2, 1, &q->pset, 0, NULL); + vkCmdPushConstants(cb, q->layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, q->pc); +} + +static void me_first(VkCommandBuffer cb, const MeQ *q) { + if (q->ib) vkCmdDispatchIndirect(cb, q->ib, q->io); + else vkCmdDispatch(cb, q->count, 1, 1); +} + +static void me_alloc(VkCommandBuffer cb, uint32_t mode) { + uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), MEQ.q[0].rec, (uint32_t)MEQ.n, mode, ME.icap, ME.trecs, 0}; + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, ME.apipe); + vkCmdPushConstants(cb, ME.alayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof pc, pc); + vkCmdDispatch(cb, 1, 1, 1); +} + +// The queued draws' stages into cb (the pre-pass command buffer, or the part for a draw that runs there); returns the +// commands recorded. +static int me_flush(VkCommandBuffer cb) { + if (!MEQ.n || !cb) return 0; + const VkAccessFlags2 rw = VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT; + const VkPipelineStageFlags2 cs = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT; + int k = 0; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 0), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + if (MEQ.tasks) { // the task stages wrote the mesh dispatch {max mesh groups, task workgroups, 1} into the draw record + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + if (!q->tp) continue; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->tp); + me_first(cb, q), k += 4; + } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + me_alloc(cb, 1), k += 4; + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, cs | VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + rw | VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT); + } + for (int i = 0; i < MEQ.n; i++) { + const MeQ *q = &MEQ.q[i]; + me_bind(cb, q); + vkCmdBindPipeline(cb, VK_PIPELINE_BIND_POINT_COMPUTE, q->mp); + if (q->tp) vkCmdDispatchIndirect(cb, ME.buf, MESH_EMU_DOFF + (VkDeviceSize)q->rec * MESH_EMU_DWORDS * 4 + 20); + else me_first(cb, q); + k += 4; + } + me_barrier(cb, cs, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT, + VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT | VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | + cs | VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_2_INDEX_READ_BIT | rw | VK_ACCESS_2_TRANSFER_READ_BIT); + MEQ.n = MEQ.tasks = 0, MEQ.dn = 0; + return k + 4; +} + +// vk.h: the pre-pass command buffer ends (vk_sync.c pre_end): the stages of the draws queued for it go in first. +void vk_exec_mesh_flush(VkCommandBuffer cb) { me_flush(cb); } + +// count: the first stage's workgroups, or ind: a {count, 1, 1} record to dispatch from +// args: an indirect draw's arguments in game memory (count, 1, 1 copied into a ring entry, which ind then is) +static int mesh_emu_draw(VkExec *ex, uint32_t count, const VkMemSpan *ind, const VkMemSpan *args) { + if (!me_arena()) return 0; + ex->dc->c0_binst = 0; + int prim = mesh_emu_draw_prim(ex); + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1)) return 0; + const VkPipeInfo *info = PD_info; + const GfxShader *ms = ex->dc->B.st.vs, *ts = ex->dc->B.st.ts; + VkPipeline mp = VK_NULL_HANDLE, tp = VK_NULL_HANDLE; + MeshEmuInfo mi; + if (!info || !info->emu || !ex->dc->B.st_valid || !vk_pipeline_mesh_emu(ms, ts, info->layout, &mp, &tp, &mi) || !!ts != !!tp || + (info->push && !info->emu_tpl_c) || (!info->push && !ex->dc->B.ds_bound[0])) { + static int logged; + if (logged++ < 8) + rt_log("vk: mesh emulation: a draw of program %llu skipped: %s", ex->prog ? (unsigned long long)ex->prog->serial : 0ull, + !info || !info->emu ? "its pipeline is not an emulated one" + : !mp ? "its stages are not emulated (see vk_pipeline: mesh emulation)" + : "its descriptors cannot be bound for compute"); + ME.skipped++; + return 0; + } + VkPipelineLayout layout = info->layout; + int push = info->push, pool = info->pool; + VkDescriptorUpdateTemplate tpl_c = info->emu_tpl_c; + VkDescriptorSet set0 = ex->dc->B.ds_bound[0]; + // NX2NAVI_MESH_EMU_BATCH (default on): the stages run in the part's pre-pass command buffer (vk_sync_pre), which + // the queue runs before the part, and the pass is not split for them. At the first emulated draw of a pass the part + // is cut there (vk_sync_cut: ended, but handed to the queue only with the next part, so the texture pool set stays; + // a split there took a new set, and each took 2.3 ms of the replay to refresh), so that the pass begins the next + // part and only what came before it runs before the stages. Split, a + // pass of 20 emulated draws was 20 passes, each emulated draw a pass end and begin with a full barrier, three more + // barriers and the targets stored and loaded again: on an RX 580 6 to 7 ms of a frame's GPU time went to the gaps + // between passes (gpu_passes, Kokiri Forest, 2026-11-10). + static int batch = -1; + if (batch < 0) { + batch = rt_knob("MESH_EMU_BATCH", "1"); + rt_log("vk: mesh emulation: the stages of a pass's emulated draws %s (NX2NAVI_MESH_EMU_BATCH=%d)", + batch ? "run before the pass, in a command buffer of their own" : "split the pass", batch); + } + VkCommandBuffer pre = VK_NULL_HANDLE; + if (batch && ex->in_pass) { + if (ex->pre_pass != ex->pass_serial && vk_exec_split_now(ex)) { + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { // the pass again, in the new part + ME.skipped++; + return 0; + } + ex->pre_pass = ex->pass_serial; + set0 = ex->dc->B.ds_bound[0]; + } + if (ex->pre_pass == ex->pass_serial) pre = vk_sync_pre(); + } + if (!pre) vk_exec_barrier(ex); // ends the pass; what the game wrote before (its culling) is visible to the stages + if (!pre && MEQ.n) me_flush(vk_sync_pre()); // the draws queued for the pre-pass command buffer go before this one + VkCommandBuffer cb = pre ? pre : ex->cb; + VkMemSpan ring = {0}; + if (args) { + if (!mesh_ring_entry(ex, cb, NULL, args, &ring.offset)) { + warn_once(W_MESH, "the indirect mesh ring could not be created: skipped"); + ME.skipped++; + return 0; + } + ring.buffer = mesh_ring, ind = ˚ + cb = pre ? pre : ex->cb; + } + uint32_t ep = vk_texture_epoch(); + if (ep != ME.epoch) { // the first emulated draw of this submission: the counters start over + ME.epoch = ep; + if (ME.rb && ME.rb_map) { // the last submission's counters to the readback ring; one 32 resets back is read + if (ME.resets >= 32) { + vmaInvalidateAllocation(vk_ctx()->vma, ME.rb_alloc, 0, VK_WHOLE_SIZE); + const uint32_t *h = ME.rb_map + 4 * ((ME.resets - 32) % ME_RB); + for (int k = 0; k < 4; k++) + if (h[k] > ME.max[k]) ME.max[k] = h[k]; + } + VkBufferCopy rc = {0, 16 * (ME.resets % ME_RB), 16}; + vkCmdCopyBuffer(cb, ME.buf, ME.rb, 1, &rc); + ME.resets++; + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT); + } + vkCmdFillBuffer(cb, ME.buf, 0, 32, 0); // the counters, and the diagnosis words 4-7 (mesh_emu_spirv.c) + me_barrier(cb, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT, + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); + } + uint32_t rec = ME.rec++ % MESH_EMU_DRECS; + VkDeviceSize ro = MESH_EMU_DOFF + (VkDeviceSize)rec * MESH_EMU_DWORDS * 4; + if (MEQ.n && (MEQ.n == MEQ_MAX || (MEQ.q[0].rec + (uint32_t)MEQ.n) % MESH_EMU_DRECS != rec)) + me_flush(vk_sync_pre()); // full, or not the next record (the allocation shader takes a run of them) + // the record: DrawIndexedIndirect {0, 1, 0, 0, 0}, mesh dispatch, task base, then the words the allocation shader + // reads (mesh_emu_spirv.h MESH_EMU_R_*); an indirect first stage's count is copied from its arguments + vkCmdFillBuffer(cb, ME.buf, ro, 4, 0); + vkCmdFillBuffer(cb, ME.buf, ro + 4, 4, 1); + vkCmdFillBuffer(cb, ME.buf, ro + 8, (MESH_EMU_R_FIRST - 2) * 4, 0); + if (ind) { + const VkMemSpan *src = args ? args : ind; + VkBufferCopy cp = {src->offset, ro + MESH_EMU_R_FIRST * 4, 4}; + vkCmdCopyBuffer(cb, src->buffer, ME.buf, 1, &cp); + } else vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_FIRST * 4, 4, count); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_ICUR * 4, 8, 0); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_PER * 4, 4, mi.max_p * (mi.prim ? mi.prim : 3)); + vkCmdFillBuffer(cb, ME.buf, ro + MESH_EMU_R_TASK * 4, 8, tp ? 1 : 0); // and the unused last word + uint32_t pc[8] = {(uint32_t)ME.addr, (uint32_t)(ME.addr >> 32), (uint32_t)(ro / 4), ME.iw, ME.icap, ME.vw, ME.vcap, ME.trecs}; + MeQ *q = &MEQ.q[MEQ.n]; + *q = (MeQ){.tp = tp, .mp = mp, .layout = layout, .tpl_c = tpl_c, .set0 = set0, .pset = pool ? PL.set : VK_NULL_HANDLE, + .push = push, .count = count, .rec = rec, .ib = ind ? ind->buffer : VK_NULL_HANDLE, .io = ind ? ind->offset : 0}; + memcpy(q->pc, pc, sizeof pc); + if (push) { + if (MEQ.dn + emu_d_bytes > MEQ.dcap) { + size_t c = MEQ.dcap ? MEQ.dcap * 2 : 65536; + while (c < MEQ.dn + emu_d_bytes) c *= 2; + uint8_t *d = realloc(MEQ.d, c); + if (!d) { + ME.skipped++; + return 0; + } + MEQ.d = d, MEQ.dcap = c; + } + memcpy(MEQ.d + MEQ.dn, emu_d, emu_d_bytes); + q->doff = MEQ.dn, q->dbytes = emu_d_bytes, MEQ.dn += emu_d_bytes; + } + MEQ.n++, MEQ.tasks += !!tp; + if (!pre) ex->recorded += 3 + me_flush(cb); // the one draw's stages, here + if (!pre) { + // the draw: prepared again (the pass reopens, every binding is recorded again) + vk_exec_draw_new_cb(); + ex->dirty = VK_EXEC_DIRTY_ALL; + if (!prepare_draw_ex(ex, prim, 0x7fffffff, 0, 1) || PD_info != info) { + ME.skipped++; + return 0; + } + } + cb = ex->cb; + vkCmdBindIndexBuffer(cb, ME.buf, (VkDeviceSize)ME.iw * 4, VK_INDEX_TYPE_UINT32); + vkCmdPushConstants(cb, layout, VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES, pc); + vkCmdDrawIndexedIndirect(cb, ME.buf, ro, 1, 20); + ex->recorded += 3; + vk_exec_frame.draws++; + // the index buffer and push constants are the emulation's: the next draw binds everything again + vk_exec_draw_new_cb(); + ex->dirty = VK_EXEC_DIRTY_ALL; + ME.draws++; + static uint64_t log_frame; + uint64_t f = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if ((rt_trace & RT_TRACE_PERF) && f / 600 != log_frame / 600) { + log_frame = f; + rt_log("perf: frame %llu: mesh emulation: %llu draws, %llu skipped (not emulated or not ready); most in one submission: " + "%u of %u vertex slots, %u of %u indices, %u of %u task records, %u mesh workgroups that did not fit", + (unsigned long long)f, (unsigned long long)ME.draws, (unsigned long long)ME.skipped, ME.max[0], ME.vcap, + ME.max[1], ME.icap, ME.max[2], ME.trecs, ME.max[3]); + ME.draws = ME.skipped = 0; + memset(ME.max, 0, sizeof ME.max); + } + return 1; +} + static void bind_index(VkExec *ex, const void *data, uint64_t bytes, VkIndexType type) { VkBuffer b; VkDeviceSize o; diff --git a/runtime/vk_exec_xfer.c b/runtime/vk_exec_xfer.c index 5e4c83f8..ef379279 100644 --- a/runtime/vk_exec_xfer.c +++ b/runtime/vk_exec_xfer.c @@ -62,6 +62,17 @@ static uint32_t max1(uint32_t v) { return v ? v : 1; } typedef struct { uint64_t addr, size; int back; } Range; // back: written into a VRAM mirror (vk_mem_mirrored) +// What the GPU work of the submission being recorded hands to the CPU at its completion without a fault to fetch it: +// event values, counter and occlusion reports, GPU writes of memory no VRAM mirror holds (vk_sync_recorded_wait waits +// for the newest such submission). Write-backs of the mirrors note it in vk_memory.c; the lazily written ones fetch on +// the CPU's first access. +static volatile int32_t ex_results; // VK_RES_* bits +void vk_exec_note_results(int kind) { + __atomic_fetch_or(&ex_results, kind, __ATOMIC_RELAXED); + vk_gpu_passes_result(kind); // NX2NAVI_GPU_PASSES: where in the submission it is done +} +int vk_exec_take_results(void) { return __atomic_exchange_n(&ex_results, 0, __ATOMIC_RELAXED); } + static void guest_written_cb(void *arg) { Range *r = arg; if (r->back) vk_gpu_written_back(r->addr, r->size); @@ -76,6 +87,7 @@ static void note_gpu_write(VkExec *ex, uint64_t addr, uint64_t size) { Range *r = malloc(sizeof *r); if (!r) rt_fatal("vk: out of memory"); *r = (Range){addr, size, vk_mem_mirrored(addr, size)}; + if (!r->back) vk_exec_note_results(VK_RES_WRITE); vk_mem_gpu_pending(addr, size); vk_completion_add(ex->fence, guest_written_cb, r); } @@ -514,6 +526,7 @@ static void oq_segment(VkExec *ex, uint64_t addr, int reset) { if (!s || (oq_nseg && !(s->q = malloc(sizeof *s->q * oq_nseg)))) rt_fatal("vk: out of memory"); s->cap = oq_nseg; s->addr = addr, s->seq = report_seq, s->reset = reset, s->lost = oq_lost, s->n = oq_nseg; + if (addr) vk_exec_note_results(VK_RES_OQ); if (oq_nseg) memcpy(s->q, oq_seg, sizeof *s->q * oq_nseg); if (gs_on >= 0) { // the front pass: the workers add the queries of the group's ops before it __typeof__(GS[0]) *gs = &GS[gs_on]; @@ -728,6 +741,7 @@ static void signal_event(VkExec *ex, const p_signal_event *p) { ev_late++, ev_late_subs += vk_exec_submit_seq - ev_open[i].sub; ev_open[i] = ev_open[--ev_open_n]; } else i++; + vk_exec_note_results(VK_RES_EVENT); EventWrite *e = malloc(sizeof *e); if (!e) rt_fatal("vk: out of memory"); *e = (EventWrite){p->val, p->mode, p->value}; @@ -747,6 +761,7 @@ static void report_counter(VkExec *ex, const p_report_counter *p) { vk_mem_count(m.cls, VK_MUSE_GPU_WRITE, sizeof(NVNcounterData)); if (samples) oq_report(ex, p->addr); else { + vk_exec_note_results(VK_RES_COUNTER); CounterWrite *a = malloc(sizeof *a); if (!a) rt_fatal("vk: out of memory"); *a = (CounterWrite){p->addr, report_seq}; diff --git a/runtime/vk_ffx.cpp b/runtime/vk_ffx.cpp index 7b5eff30..541e0e0a 100644 --- a/runtime/vk_ffx.cpp +++ b/runtime/vk_ffx.cpp @@ -28,6 +28,7 @@ extern "C" { #include "platform/platform.h" #include "vk.h" #include "vk_ffx.h" +#include "vk_fg.h" } namespace { @@ -274,7 +275,8 @@ extern "C" int vk_ffx_available(void) { #endif } -extern "C" int vk_ffx_framegen_available(void) { return vk_ffx_available(); } // the same library, its API calls +// the same library, its API calls; and a queue for the frame generation swapchain's presents (vk_fg_has_queues) +extern "C" int vk_ffx_framegen_available(void) { return vk_ffx_available() && vk_fg_has_queues(); } // NX2NAVI_FSR4 (default 0, live): FSR 4 where it runs (vk_ffx4.cpp) and has not failed in a row, else FSR 3.1. extern "C" int vk_ffx_fsr4(void) { return cfg_bool("NX2NAVI_FSR4") && vk_ffx4_available() && !vk_ffx4_failed(); } diff --git a/runtime/vk_fg.cpp b/runtime/vk_fg.cpp index 25f03cc8..d0a57455 100644 --- a/runtime/vk_fg.cpp +++ b/runtime/vk_fg.cpp @@ -36,6 +36,7 @@ namespace { struct { VkQueue present = VK_NULL_HANDLE, acquire = VK_NULL_HANDLE; + uint32_t pfam = 0, afam = 0; // their queue families ffxContext sc = nullptr; // the swapchain context ffxContext fg = nullptr; // the frame generation context VkExtent2D fg_ext = {0, 0}; @@ -108,6 +109,12 @@ extern "C" int vk_fg_wanted(void) { if (F.sc) return !F.failed && !four; if (four) return 1; if (F.failed) return 0; + if (!F.present) { // no queue to present on besides the game's (vk_device.c says why): the plain swapchain stays + static int told_q; + if (!__atomic_exchange_n(&told_q, 1, __ATOMIC_RELAXED)) + rt_log("fg: AMD FSR 3.1 frame generation is set, but the device got no queue for its presents: off"); + return 0; + } if (vk_ffx_framegen_available()) return 1; static int told; if (!__atomic_exchange_n(&told, 1, __ATOMIC_RELAXED)) @@ -118,11 +125,13 @@ extern "C" int vk_fg_wanted(void) { // Always the three queues: the setting can be switched on during a run, and idle queues cost nothing. extern "C" uint32_t vk_fg_queue_count(void) { return 3; } -extern "C" void vk_fg_queues(VkQueue present, VkQueue acquire) { - F.present = present, F.acquire = acquire; - vk_fg4_queues(present, acquire); +extern "C" void vk_fg_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family) { + F.present = present, F.acquire = acquire, F.pfam = present_family, F.afam = acquire_family; + vk_fg4_queues(present, present_family, acquire, acquire_family); } +extern "C" int vk_fg_has_queues(void) { return F.present != VK_NULL_HANDLE; } + // AMD's swapchain is in place: the window's swapchain handle is AMD's object, and every swapchain call has to go // through AMD's functions. That holds after a failure too, until the present code has replaced the swapchain // (vk_fg_wanted): the plain Vulkan calls must never get that handle. @@ -153,8 +162,8 @@ extern "C" VkResult vk_fg_create_swapchain(const VkSwapchainCreateInfoKHR *ci, V d.createInfo.oldSwapchain = VK_NULL_HANDLE; d.gameQueue = {c->gfx_queue, c->gfx_family, submit_game}; d.asyncComputeQueue = {VK_NULL_HANDLE, c->gfx_family, nullptr}; - d.presentQueue = {F.present, c->gfx_family, nullptr}; - d.imageAcquireQueue = {F.acquire, c->gfx_family, nullptr}; + d.presentQueue = {F.present, F.pfam, nullptr}; + d.imageAcquireQueue = {F.acquire, F.afam, nullptr}; ffxReturnCode_t r = vk_ffx_create(&F.sc, &d.header); if (r != FFX_API_RETURN_OK) { F.sc = nullptr; @@ -278,6 +287,15 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, fgr_note(&F.cuts, F.frame, reset || !F.prepared); if ((reset || !F.prepared) && on) F.cut_frames++; F.prepared = on; + // Every 600 frames, whether this one generates or not: what kept frames from being generated, and how many frames + // AMD's swapchain presented (generated ones included). Logged only from a generating frame, as before, a run whose + // frames never generated (the menu open at the swapchain's second frame, no inputs, no HUD-less picture) said nothing. + if (F.frame == 120 || F.frame % 600 == 0) + rt_log("fg: %llu frames, %llu generated (%llu with the HUD kept out), %llu presents by the swapchain; not generated: " + "%llu menu open, %llu without depth and motion, %llu without the HUD-less picture; %llu after a camera cut", + (unsigned long long)F.frame, (unsigned long long)F.generated, (unsigned long long)F.hudless, + (unsigned long long)vk_fg_presents(sc), (unsigned long long)F.paused, (unsigned long long)F.no_inputs, + (unsigned long long)F.no_hudless, (unsigned long long)F.cut_frames); ffxConfigureDescFrameGeneration cfg = {}; cfg.header.type = FFX_API_CONFIGURE_DESC_TYPE_FRAMEGENERATION; cfg.swapChain = (void *)sc; @@ -321,11 +339,6 @@ extern "C" void vk_fg_frame(VkCommandBuffer cb, VkSwapchainKHR sc, VkFormat fmt, p.motionVectors = image(&mv); r = vk_ffx_dispatch(&F.fg, &p.header); if (r != FFX_API_RETURN_OK) return fail("frame generation preparation", r); - if (F.frame == 2 || F.frame % 3600 == 0) - rt_log("fg: %llu frames presented, %llu generated (%llu with the HUD kept out); paused %llu (menu), %llu without depth " - "and motion, %llu without the HUD-less picture; %llu after a camera cut", (unsigned long long)F.frame, (unsigned long long)F.generated, - (unsigned long long)F.hudless, (unsigned long long)F.paused, (unsigned long long)F.no_inputs, - (unsigned long long)F.no_hudless, (unsigned long long)F.cut_frames); } extern "C" void vk_fg_release(void) { diff --git a/runtime/vk_fg.h b/runtime/vk_fg.h index 31a97cb7..54771529 100644 --- a/runtime/vk_fg.h +++ b/runtime/vk_fg.h @@ -5,7 +5,10 @@ int vk_fg_wanted(void); // NX2NAVI_FRAMEGEN = fsr3 with DLSS or FSR 3.1 upscaling on (live), where FSR 3.1 loads uint32_t vk_fg_queue_count(void); // graphics-family queues the device creates -void vk_fg_queues(VkQueue present, VkQueue acquire); // vk_device.c, after the device exists +// vk_device.c, after the device exists: the queues AMD's swapchain presents and acquires on, and their families (the +// graphics family's own where it has three queues, else queues of another family: AMD's Windows driver gives one) +void vk_fg_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family); +int vk_fg_has_queues(void); // vk_fg_queues got a present queue: frame generation can run here int vk_fg_active(void); // the frame generation swapchain exists int vk_fg_generating(void); // ... and the last frame had generation on (not paused for the menu) // gfx_present.c's swapchain calls while vk_fg_wanted: the first create makes AMD's swapchain context, later ones @@ -34,7 +37,7 @@ int vk_fg4_failed(void); // it failed in this run: FSR 3.1's const char *vk_fg4_name(void); // the frame generation version the loader lists first for the device ("3.1.6"), "" unknown int vk_fg4_active(void); int vk_fg4_generating(void); -void vk_fg4_queues(VkQueue present, VkQueue acquire); +void vk_fg4_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family); VkResult vk_fg4_create_swapchain(const VkSwapchainCreateInfoKHR *ci, VkSwapchainKHR *sc); void vk_fg4_destroy_swapchain(VkSwapchainKHR sc); VkResult vk_fg4_get_images(VkSwapchainKHR sc, uint32_t *n, VkImage *img); diff --git a/runtime/vk_fg4.cpp b/runtime/vk_fg4.cpp index c380e156..6ca06b23 100644 --- a/runtime/vk_fg4.cpp +++ b/runtime/vk_fg4.cpp @@ -28,7 +28,7 @@ extern "C" int vk_fg4_failed(void) { return 0; } extern "C" const char *vk_fg4_name(void) { return ""; } extern "C" int vk_fg4_active(void) { return 0; } extern "C" int vk_fg4_generating(void) { return 0; } -extern "C" void vk_fg4_queues(VkQueue, VkQueue) {} +extern "C" void vk_fg4_queues(VkQueue, uint32_t, VkQueue, uint32_t) {} extern "C" VkResult vk_fg4_create_swapchain(const VkSwapchainCreateInfoKHR *, VkSwapchainKHR *) { return VK_ERROR_FEATURE_NOT_PRESENT; } @@ -68,6 +68,7 @@ namespace { struct { VkQueue present = VK_NULL_HANDLE, acquire = VK_NULL_HANDLE; + uint32_t pfam = 0, afam = 0; // their queue families ffxContext sc = nullptr; // the swapchain context ffxContext fg = nullptr; // the frame generation context VkSwapchainKHR handle = VK_NULL_HANDLE; // the swapchain the context made: remade in place, destroyed after the context @@ -200,7 +201,9 @@ extern "C" int vk_fg4_failed(void) { return F.failed; } extern "C" const char *vk_fg4_name(void) { return g_fg_name; } -extern "C" void vk_fg4_queues(VkQueue present, VkQueue acquire) { F.present = present, F.acquire = acquire; } +extern "C" void vk_fg4_queues(VkQueue present, uint32_t present_family, VkQueue acquire, uint32_t acquire_family) { + F.present = present, F.acquire = acquire, F.pfam = present_family, F.afam = acquire_family; +} // The SDK's swapchain is in place: the window's swapchain handle is its object, and every swapchain call has to go // through its functions, after a failure too, until the present code has taken it down. @@ -230,8 +233,8 @@ extern "C" VkResult vk_fg4_create_swapchain(const VkSwapchainCreateInfoKHR *ci, F.sc_desc.createInfo.oldSwapchain = VK_NULL_HANDLE; F.sc_desc.gameQueue = {c->gfx_queue, c->gfx_family, submit_game}; F.sc_desc.asyncComputeQueue = {VK_NULL_HANDLE, c->gfx_family, nullptr}; - F.sc_desc.presentQueue = {F.present, c->gfx_family, nullptr}; - F.sc_desc.imageAcquireQueue = {F.acquire, c->gfx_family, nullptr}; + F.sc_desc.presentQueue = {F.present, F.pfam, nullptr}; + F.sc_desc.imageAcquireQueue = {F.acquire, F.afam, nullptr}; // the version this code was built against, which the SDK asks for with every context, and a test's version F.sc_ver = {}; F.sc_ver.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_FRAMEGENERATIONSWAPCHAIN_VERSION_VK; diff --git a/runtime/vk_memory.c b/runtime/vk_memory.c index 05b50924..d9d2466c 100644 --- a/runtime/vk_memory.c +++ b/runtime/vk_memory.c @@ -82,6 +82,8 @@ typedef struct PoolRec { FlushRange fl[FLUSH_RANGES + 1]; // flush_lock: nvnMemoryPoolFlushMappedRange pages not copied yet (flush_add) int fl_n; // flush_lock: ranges in fl; read without the lock to skip an empty list uint64_t fl_calls, fl_sum; // flush_lock: the calls and bytes behind fl, for mirror_sync's statistics + uint64_t hot_bytes; // replay: uploads, write-backs and merges since the last hot check (mirror_hot) + uint64_t hot_up; // of them the uploads: pages the CPU wrote (mirror_hot leaves such pools mirrored) // Shadow storage (POOL_IMPORT in shadow mode, "shadow storage" below): buf is a host-visible copy of the guest pages. VmaAllocation salloc; uint8_t *smap; // its mapping; offsets as in buf (from import_lo) @@ -237,8 +239,17 @@ static int alloc_physical(VkCtx *c, PoolRec *r) { VkMemoryRequirements req = {.size = (rb.size + align - 1) & ~(align - 1), .alignment = align, .memoryTypeBits = bits}; VmaAllocationCreateInfo aci = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; - if ((res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL)) != VK_SUCCESS || - (res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) { + res = vmaAllocateMemory(c->vma, &req, &aci, &r->alloc, NULL); + if (res != VK_SUCCESS) { // video memory full (VMA heap limits, vk_device.c): any memory the pool can live in + VmaAllocationCreateInfo any = {.flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, + .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + r->alloc = NULL; + res = vmaAllocateMemory(c->vma, &req, &any, &r->alloc, NULL); + if (res == VK_SUCCESS) + rt_log("vk: physical pool of %#llx bytes: no video memory left, placed in system memory", + (unsigned long long)r->size); + } + if (res != VK_SUCCESS || (res = vmaBindBufferMemory(c->vma, r->alloc, r->buf)) != VK_SUCCESS) { rt_log("vk: physical pool of %#llx bytes: device-local allocation failed (%d)", (unsigned long long)r->size, (int)res); if (r->alloc) vmaFreeMemory(c->vma, r->alloc); vkDestroyBuffer(c->device, r->buf, NULL); @@ -648,6 +659,7 @@ static int dead_pending(void); static void vram_log(uint64_t frame); static void outside_watch(uint64_t frame); +static void up_ts_log(uint64_t frame); static volatile uint64_t mirror_frame; // the frame vk_mem_census last saw, for mirror_sync's log lines @@ -660,7 +672,7 @@ void vk_mem_census(uint64_t frame) { mirror_frame = frame; static uint64_t window_start; if ((rt_trace & RT_TRACE_PERF) && frame % 600 == 0) vram_log(frame), mirror_pool_log(frame); - if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame); + if ((rt_trace & RT_TRACE_PERF) && frame % 60 == 0) outside_watch(frame), up_ts_log(frame), vk_gtl_log(frame, 60); if ((rt_trace & RT_TRACE_PERF) && !vk_mirror_on) dirty_frame(frame); if ((rt_trace & RT_TRACE_PERF) ? frame % 60 == 0 : frame == 600 || frame == 1800) mirror_log(frame), shadow_log(frame); // mirror_log also logs the write-watch line @@ -987,6 +999,43 @@ static uint64_t snap_regions(uint64_t r0, uint64_t r1, uint32_t epoch) { return calls; } +// NX2NAVI_SNAP_WIDEN= (default 8, 0 off): a texture check that has to query widens the query to up to this +// many 2 MiB regions on each side, inside the import pool that holds the texture, so the textures of one pool are +// snapped by one query a submission instead of one each. With freezing off (NX2NAVI_FREEZE_IDLE) an RX 580 in Hyrule +// Field made about 1,130 texture queries a frame, about 6 MiB each on average incl. repeats, 9.4 ms with the pools'. +static int snap_widen_n(void) { + static int n = -1; + if (n < 0) { + const char *e = cfg_get("NX2NAVI_SNAP_WIDEN"); + n = e && *e ? atoi(e) : 8; + if (n < 0) n = 0; + rt_log("mem: a texture check's write-watch query takes up to %d neighbouring 2 MiB regions of its pool on each " + "side (NX2NAVI_SNAP_WIDEN)", n); + } + return n; +} + +// r0..r1 widened inside the import pool that holds [lo, hi); pool_lock taken here (before gen_lock, as everywhere). +static void snap_widen(uint64_t lo, uint64_t hi, uint64_t *r0, uint64_t *r1) { + int w = snap_widen_n(); + if (!w) return; + plat_mutex_lock_shared(&pool_lock); + const pidx_ent *e = pidx_find(&pool_index, (uintptr_t)lo, hi - lo); + const PoolRec *r = e ? e->buf : NULL; + if (r && r->kind == POOL_IMPORT && r->buf && r->import_hi > r->import_lo) { + uint64_t p0 = (r->import_lo + (1ull << SNAP_SHIFT) - 1) >> SNAP_SHIFT; // whole regions of the pool only + uint64_t p1 = r->import_hi >> SNAP_SHIFT; // [p0, p1) + if (p1 > p0) { + uint64_t a = *r0 > (uint64_t)w ? *r0 - (uint64_t)w : 0, b = *r1 + (uint64_t)w; + if (a < p0) a = p0; + if (b > p1 - 1) b = p1 - 1; + if (a < *r0) *r0 = a; + if (b > *r1 && b < SNAP_REGIONS) *r1 = b; + } + } + plat_mutex_unlock_shared(&pool_lock); +} + int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t epoch, uint32_t *now) { if (!size) { *now = since; return 0; } uint64_t lo = addr & ~0xfffull, hi = (addr + size + 0xfff) & ~0xfffull; @@ -998,8 +1047,10 @@ int vk_mem_changed_snap(uint64_t addr, uint64_t size, uint32_t since, uint32_t e if (snap_at[r] != epoch) { excl = 1; break; } if (excl) { plat_mutex_unlock_shared(&gen_lock); + uint64_t w0 = r0, w1 = r1; + snap_widen(lo, hi, &w0, &w1); plat_mutex_lock(&gen_lock); - n_snap_calls += snap_regions(r0, r1, epoch); + n_snap_calls += snap_regions(w0, w1, epoch); } int untracked = 0, changed = 0; for (uint64_t r = r0; r <= r1; r++) untracked |= snap_untracked[r]; @@ -2086,6 +2137,9 @@ static int cregions; static PlatMutex flush_lock = PLAT_MUTEX_INIT; static int flush_cap = FLUSH_RANGES; // ranges a pool keeps; 1 with NX2NAVI_FLUSH_SPAN=1 static int flush_upload; // NX2NAVI_FLUSH_UPLOAD=1: flush ranges are copied where the write watch sees them too +// NX2NAVI_FLUSH_SKIP_CLEAN (flush_clean_bytes in the 0.5.0 and 0.6.0 fixes: flushed ranges upload only the pages the CPU +// wrote) is this default since 0.6.1: a flush range of a write-watched pool is not copied at all; the key stays known so an +// nx2navi.ini that sets it reads without a warning, and does nothing. typedef struct { uint64_t subs, syncs, upload_bytes, upload_copies, first_n, first_bytes, wb_bytes, wb_copies, wb_ranges; @@ -2532,7 +2586,7 @@ static void mirror_sync(PoolRec *r) { for (uint64_t x = a; x < b; x += 1ull << GEN_PAGE_SHIFT) if (ch[(x >> GEN_PAGE_SHIFT) & (GEN_PER_CHUNK - 1)] > since) { range_add(&pend, &npend, &cpend, r, x - lo, 1ull << GEN_PAGE_SHIFT), mrep.page_bytes += 1ull << GEN_PAGE_SHIFT; - r->st_pages += 1ull << GEN_PAGE_SHIFT; + r->st_pages += 1ull << GEN_PAGE_SHIFT, r->hot_bytes += 1ull << GEN_PAGE_SHIFT, r->hot_up += 1ull << GEN_PAGE_SHIFT; } } r->mgen = gen_now; @@ -2547,6 +2601,7 @@ static void mirror_sync(PoolRec *r) { r->fl_n = 0, r->fl_calls = r->fl_sum = 0; plat_mutex_unlock(&flush_lock); mrep.flush_calls += calls, mrep.flush_sum += sum; + r->hot_up += sum; // the CPU wrote them (mirror_hot) for (int i = 0; i < n; i++) { uint64_t a = fl[i].lo > lo ? fl[i].lo : lo, b = fl[i].hi < hi ? fl[i].hi : hi; if (a >= b) continue; @@ -2661,7 +2716,10 @@ void vk_mem_use(VkMemSpan *sp) { return; } // a refused pool not due for another try stays imported: nearly every pool in the field (two have mirrors) - if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) return; + if (r->mstate == MS_REFUSED && !(r->retry_sub && (int32_t)(msub - r->retry_sub) >= 0)) { + vk_mem_note_live(); // the GPU reads this pool's guest memory as it runs (vk_mem_take_live) + return; + } int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); mem_use_locked(sp); @@ -2677,7 +2735,10 @@ static void mem_use_locked(VkMemSpan *sp) { r->mstate = MS_NONE, r->retry_sub = 0; // refuse sets it again } if (r->mstate == MS_NONE) mirror_create(r); - if (r->mstate != MS_ON) return; + if (r->mstate != MS_ON) { // the GPU reads this pool's guest memory as it runs (staged mirror uploads) + vk_mem_note_live(); + return; + } if (r->acc_sub != msub) mirror_sync(r); r->vd_use_sub = msub; sp->buffer = r->mbuf; @@ -2685,7 +2746,11 @@ static void mem_use_locked(VkMemSpan *sp) { void vk_mem_pick(VkMemSpan *sp) { PoolRec *r = sp->pool; - if (!r || r->mstate != MS_ON) return; + if (!r) return; + if (r->mstate != MS_ON) { // guest memory itself, read by the GPU when it runs (vk_mem_take_live) + vk_mem_note_live(); + return; + } if (r->acc_sub != msub) { int lk = vk_exec_shared(); if (lk) plat_mutex_lock(&use_lock); @@ -3298,6 +3363,7 @@ static void gpu_wrote_range(const VkMemSpan *sp, uint64_t size, int shader) { PoolRec *r = sp->pool; if (!r || r->mstate != MS_ON || sp->buffer != r->mbuf || !size) return; r->vd_wrote_sub = msub, r->vd_wrote_n++; + r->hot_bytes += size; // A binding to the end of its pool (no size) can span hundreds of MiB: those stay copies. uint64_t o = 0; if (shader && !(sp->offset & 3) && !(size & 3) && size <= merge_max() && merging()) @@ -3445,7 +3511,88 @@ static void freeze_pass(void) { } } +// NX2NAVI_MIRROR_HOT (percent, 0 off): a mirrored pool whose copies each frame (CPU pages up, GPU writes back) reach this +// share of its size goes back to being read and written by the GPU in system memory. On an RX 580 in Kokiri Forest a +// handful of 6-16 MiB pools the CPU refills every frame and compute shaders write took 66 MiB of uploads and 76 MiB of +// write-backs a frame through the mirrors, the GPU idle 7.8 ms a frame with those copies queued; read in place, only +// the bytes the shaders touch cross the bus. Checked every HOT_FRAMES frames at the start of a submission, when the +// previous one has queued its write-backs and nothing is pending for the next: the pool leaves the mirrored list, the +// device address table points at its imported buffer again, and the mirror is freed once the submissions that may still +// use it are done (vk_defer_buffer, as mirror_evict). Pools under HOT_MIN a frame stay mirrored. +// Only pools the CPU does not write go: a pool read in place gives the GPU the game's bytes as they are when it reads +// them, not as they were at the submit (the mirror's copy). The game refills its per-frame pools for the next frame +// while the replay (asynchronous, up to a frame behind) and the GPU still draw the last one from them: the small grass +// and moss, drawn from instances in such pools, flickered and slid over the ground as the camera turned, and with +// 0.8.0's mesh fallback every mesh flashed (RX 580, Kokiri Forest, 2026-11-11; 18 pools of 2-15 MiB had gone, each +// refilled by the CPU every frame). A pool with more than HOT_UP_MAX of CPU writes a frame stays mirrored. +#define HOT_UP_MAX (64ull << 10) +#define HOT_FRAMES 120 +#define HOT_MIN (1ull << 20) +static int mirror_hot = -1; +static uint64_t hot_frame0, n_demoted; + +static void hot_check(void) { + if (mirror_hot < 0) { + const char *v = cfg_value("NX2NAVI_MIRROR_HOT"); // the settings row's default (25) when unset + int p = v && *v ? atoi(v) : 25; + mirror_hot = p < 0 ? 0 : p > 1000 ? 1000 : p; + if (mirror_hot) rt_log("mem: pools copied %d%% of their size a frame stay in system memory (NX2NAVI_MIRROR_HOT)", mirror_hot); + } + if (!mirror_hot) return; + uint64_t fr = mirror_frame; + if (!hot_frame0 || fr < hot_frame0) { + hot_frame0 = fr ? fr : 1; + return; + } + if (fr - hot_frame0 < HOT_FRAMES || npend || nwbl) return; // nsnl: the previous submission's, recorded already + double frames = (double)(fr - hot_frame0); + hot_frame0 = fr; + PoolRec *out[8]; + VkBuffer dead[8]; + VmaAllocation dead_a[8]; + int nout = 0; + plat_mutex_lock(&pool_lock); + for (int i = 0; i < nm;) { + PoolRec *r = mlist[i]; + uint64_t n = r->import_hi - r->import_lo, per = (uint64_t)(r->hot_bytes / frames), up = (uint64_t)(r->hot_up / frames); + r->hot_bytes = (uint64_t)per; // kept for the log line below + r->hot_up = 0; + if (nout == 8 || per < HOT_MIN || per * 100 < n * (uint64_t)mirror_hot || up > HOT_UP_MAX || !r->mbuf || + (!r->twin && r->gpa_rec)) { + r->hot_bytes = 0, i++; // !twin with a table record: the table's writes are in the mirror alone (mirror_evict) + continue; + } + mem_lazy_release(r->import_lo, r->import_hi - r->import_lo); // what only the mirror holds comes down first + // as mirror_evict, under the exclusive lock (vk_pool_fini takes pools off the list under it), but for good: + // MS_REFUSED with no retry (retry_sub 0), so neither vk_mem_use nor the budget retries mirror it again + mirror_forget(r); // takes r off mlist: i now names the next pool + r->mstate = MS_REFUSED, r->retry_sub = 0; + plat_mutex_lock(&flush_lock); // flushes not copied yet: the pool is read in place from now on + __atomic_store_n(&r->fl_n, 0, __ATOMIC_RELEASE); + r->fl_calls = r->fl_sum = 0; + plat_mutex_unlock(&flush_lock); + // the pages name the imported buffer alone again (the twin was that buffer) + if (r->gpa_rec) r->gpa_rec = vk_gpa_remap_plain(r->import_lo, r->import_hi, r->gpa_rec, r->buf, r->import_lo); + dead[nout] = r->mbuf, dead_a[nout] = r->malloc_; + r->mbuf = VK_NULL_HANDLE, r->malloc_ = NULL; + out[nout++] = r; + } + plat_mutex_unlock(&pool_lock); + for (int k = 0; k < nout; k++) { + PoolRec *r = out[k]; + uint64_t n = r->import_hi - r->import_lo, per = r->hot_bytes; + r->hot_bytes = 0; + vk_defer_buffer(dead[k], dead_a[k]); // after the submission being recorded + char f[160]; + flag_names(r->flags, f, sizeof f); + if (++n_demoted <= 32) + rt_log("mem: pool %#llx (%.1f MiB, %s) copied %.1f MiB a frame through its mirror: system memory from now on", + (unsigned long long)r->lo, n / 1048576.0, f, per / 1048576.0); + } +} + void vk_mem_mirror_begin(void) { + if (vk_mirror_on) hot_check(); if (!vk_mirror_on) { // import mode: the texture checks' snapshots still skip frozen regions if (!freeze_idle) return; plat_mutex_lock(&gen_lock); @@ -3493,28 +3640,51 @@ void vk_mem_mirror_begin(void) { int vk_mem_mirror_pending(void) { return npend || snl_copied < nsnl; } -// NX2NAVI_MIRROR_CAPTURE_KB (default 0, off): the pages the CPU wrote since a VRAM mirror's last sync are copied out of -// guest memory by the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a -// host buffer the submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them -// from there into the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or -// more after the game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, -// values meant for a later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest -// memory is still on the GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over -// the mirror's newer ones), and when its device address table writes reach guest memory as well (a twin: what they -// write is not recorded); so does a range with a page that cannot be read without a fault (mem_copy_readable: a -// lazily armed range). The whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU. +// NX2NAVI_MIRROR_CAPTURE_KB: the pages the CPU wrote since a VRAM mirror's last sync are copied out of guest memory by +// the CPU when the part that needs them is handed to the queue, up to this many KiB a part, into a host buffer the +// submission owns (a chunk of the snapshot pool, given back at its fence); the prologue copies them from there into +// the mirror. Without it the prologue copies them from guest memory when the GPU runs it, a frame or more after the +// game's submit when the GPU is behind: the GPU then reads whatever the game wrote there meanwhile, values meant for a +// later frame. A pool stays with the GPU-time copy while a write-back of its mirror into guest memory is still on the +// GPU (the CPU would read guest bytes that write-back has not reached yet and copy them over the mirror's newer ones), +// and when its device address table writes reach guest memory as well (a twin: what they write is not recorded); so +// does a range with a page that cannot be read without a fault (mem_copy_readable: a lazily armed range). The +// whole-pool copy of a mirror's first use is larger than the limit and stays with the GPU. +// Unset or 0 with NX2NAVI_MIRROR_STAGE=1 (the default here): CAPTURE_STAGE_KB. A submission whose prologue copied +// everything this way reads nothing of guest memory late, so its submit wait ends once the replay has submitted it +// (vk_sync_recorded_wait); one with a range left to the GPU-time copy, or a draw that reads a pool without a mirror, +// is noted (vk_mem_note_live) and its submit wait is the GPU's. The first form of this (0.8.0) copied while the replay +// recorded and left out neither the write-backs in flight nor the twins. +#define CAPTURE_STAGE_KB (64 << 10) +static volatile int32_t mirror_live; // the submission being recorded had the GPU read guest memory + +static int stage_on(void) { + static int on = -1; + if (on < 0) on = rt_knob("MIRROR_STAGE", "1") != 0; + return on; +} + static uint64_t capture_cap(void) { static int64_t cap = -1; if (cap < 0) { int kb = cfg_int("NX2NAVI_MIRROR_CAPTURE_KB"); + if (kb <= 0 && stage_on()) kb = CAPTURE_STAGE_KB; cap = kb > 0 ? (int64_t)kb << 10 : 0; if (cap) rt_log("mem: mirror pages the CPU wrote are copied out of guest memory when their part is handed over, up to " - "%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB)", kb); + "%d KiB a part (NX2NAVI_MIRROR_CAPTURE_KB, NX2NAVI_MIRROR_STAGE)", kb); } return (uint64_t)cap; } +void vk_mem_note_live(void) { __atomic_store_n(&mirror_live, 1, __ATOMIC_RELAXED); } + +int vk_mem_take_live(void) { + int live = __atomic_exchange_n(&mirror_live, 0, __ATOMIC_RELAXED); + if (!vk_mirror_on || shadow_on || !capture_cap()) live = 1; + return live; +} + static MRange *capm; // replay: mirror_capture's merged ranges static uint64_t *caps; // replay: the chunk offset of each, UINT64_MAX for one left to the GPU-time copy static uint8_t **capd, *capok; // replay: mem_copy_readable's destinations and results @@ -3596,6 +3766,7 @@ void vk_mem_mirror_prologue(VkCommandBuffer cb) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT); mirror_capture(cb); + if (npend) vk_mem_note_live(); // left to the GPU-time copy: the GPU reads guest memory late copy_ranges(cb, pend, npend, 1, &mrep.upload_bytes, &mrep.upload_copies); if (snl_copied < nsnl) { // the snapshots read what the uploads above wrote if (npend) mbarrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, @@ -3738,6 +3909,7 @@ int vk_mem_mirror_end(VkCommandBuffer cb, uint64_t fence) { memcpy(f->c, chunks, sizeof(SnapChunk) * (size_t)nchunks); vk_completion_add(fence, snap_free, f); } + if (nsnl || nwbl) vk_exec_note_results(VK_RES_WRITEBACK); // write-backs: guest memory gets the GPU's bytes at the fence if (nwbl) { mbarrier(cb, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_READ_BIT); @@ -4203,6 +4375,9 @@ typedef struct { StgChunk *chunk; // arena chunks this batch's uploads took ranges of (vk_staging_at), the last one open int nchunk, cap_chunk; VkDeviceSize chunk_used; // bytes of the last chunk handed out + uint64_t bytes; // staging bytes of its buffers (upload_cap) + int ts; // its pair of upload timestamps (up_ts_*), -1: not timed + uint64_t sub_ns; // when it was handed to the queue (plat_now_ns), for the GPU timeline (vk_gtl_note) } Batch; enum { DEAD_IMAGE = 1, DEAD_IMAGE_VMA, DEAD_VIEW, DEAD_BUFFER_VMA, DEAD_HOST }; @@ -4216,6 +4391,60 @@ static int open_active; static Batch *inflight; static int ninflight, cap_inflight; static Batch *spare; // completed batches whose command buffer and fence are reused + +// ---- upload GPU time (NX2NAVI_TRACE=perf): a timestamp at the start and the end of each upload batch, read when it +// retires. The replay's GPU time (vk_sync.c) times only its own parts; upload batches go to the queue as submissions of +// their own, with the ASTC decoder's dispatches in them, so their time showed only as GPU idle "with it queued". +// cmd_lock held throughout. +#define UP_TS_PAIRS 256 +static VkQueryPool up_ts_pool; +static int up_ts_state; // 0 not set up, 1 on, -1 off +static uint8_t up_ts_busy[UP_TS_PAIRS]; +static uint32_t up_ts_next; +static uint64_t up_ts_ns, up_ts_n, up_ts_max_ns, up_ts_missed, up_n_batches; // since the last log line + +static int up_ts_open(VkCtx *c, VkCommandBuffer cb) { + if (!(rt_trace & RT_TRACE_PERF) || up_ts_state < 0) return -1; + if (!up_ts_state) { + VkQueryPoolCreateInfo qi = {.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO, .queryType = VK_QUERY_TYPE_TIMESTAMP, + .queryCount = 2 * UP_TS_PAIRS}; + up_ts_state = c->props.limits.timestampComputeAndGraphics && + vkCreateQueryPool(c->device, &qi, NULL, &up_ts_pool) == VK_SUCCESS ? 1 : -1; + if (up_ts_state < 0) return -1; + } + uint32_t p = up_ts_next % UP_TS_PAIRS; + if (up_ts_busy[p]) { up_ts_missed++; return -1; } + up_ts_next++, up_ts_busy[p] = 1; + vkCmdResetQueryPool(cb, up_ts_pool, 2 * p, 2); + vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, up_ts_pool, 2 * p); + return (int)p; +} + +static void up_ts_read(VkCtx *c, Batch *b) { // b retired: its fence signalled + if (b->ts < 0) return; + uint64_t t[2]; + if (vkGetQueryPoolResults(c->device, up_ts_pool, 2 * (uint32_t)b->ts, 2, sizeof t, t, sizeof t[0], + VK_QUERY_RESULT_64_BIT) == VK_SUCCESS && t[1] >= t[0]) { + uint64_t ns = (uint64_t)((double)(t[1] - t[0]) * c->props.limits.timestampPeriod); + up_ts_ns += ns, up_ts_n++; + vk_gtl_note(GTL_UPLOAD, b->sub_ns, t[0], t[1]); + if (ns > up_ts_max_ns) up_ts_max_ns = ns; + } + up_ts_busy[b->ts] = 0; + b->ts = -1; +} + +static void cmd_init(void *p); +static void up_ts_log(uint64_t frame) { // every 60 frames + plat_once(&cmd_once, cmd_init, NULL); + plat_rmutex_lock(&cmd_lock); + rt_log("perf: frame %llu: upload batches %.1f per frame, on the GPU %.2f ms per frame (%llu timed, %.3f ms each, the " + "longest %.2f ms; %llu not timed)", (unsigned long long)frame, up_n_batches / 60.0, up_ts_ns / 60e6, + (unsigned long long)up_ts_n, up_ts_n ? up_ts_ns / 1e6 / (double)up_ts_n : 0.0, up_ts_max_ns / 1e6, + (unsigned long long)up_ts_missed); + up_ts_ns = up_ts_n = up_ts_max_ns = up_ts_missed = up_n_batches = 0; + plat_rmutex_unlock(&cmd_lock); +} static int nspare, cap_spare; static uint64_t next_seq, done_seq; // batches handed out / completed (all batches up to done_seq) static Dead *dead; @@ -4275,6 +4504,7 @@ static void free_staging(VkCtx *c, Batch *b) { else vmaDestroyBuffer(c->vma, b->chunk[i].buf, b->chunk[i].alloc); } b->nchunk = 0, b->chunk_used = 0; + b->bytes = 0; } // The cached staging buffer that fits n with the least waste: at most four times the request, so a small upload does not @@ -4301,6 +4531,7 @@ static void retire(VkCtx *c, int wait_all) { Batch *b = &inflight[k]; if (wait_all) vkWaitForFences(c->device, 1, &b->fence, VK_TRUE, UINT64_MAX); else if (vkGetFenceStatus(c->device, b->fence) != VK_SUCCESS) break; + up_ts_read(c, b); free_staging(c, b); done_seq = b->seq; if (nspare == cap_spare) grow((void **)&spare, &cap_spare, sizeof *spare); @@ -4438,13 +4669,67 @@ VkCommandBuffer vk_upload_begin(void) { VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; vkBeginCommandBuffer(open_b.cb, &bi); + open_b.ts = up_ts_open(c, open_b.cb); open_b.seq = ++next_seq; vk_crumb(open_b.cb, VK_CRUMB_UPLOAD, open_b.seq); open_active = 1; return open_b.cb; } -void vk_upload_end(void) { plat_rmutex_unlock(&cmd_lock); } +// NX2NAVI_UPLOAD_BATCH_MB (default 256; 0 = no limit): once the open batch's staging buffers hold that much, the +// outermost vk_upload_end submits it. A save load made thousands of textures between two replay submissions, and every +// one of them went into one batch: about 1.5 GB of raw ASTC blocks plus the decoder's work in a single submission, +// which AMD's 23.10 driver on an RX 580 refused with VK_ERROR_OUT_OF_DEVICE_MEMORY. Submitted early, a batch still +// precedes the replay submission that uses its textures (the queue keeps order), and its staging is freed as it retires. +static uint64_t upload_cap = UINT64_MAX - 1; // UINT64_MAX - 1: not read yet +static uint64_t n_early; + +void vk_upload_end(void) { + if (upload_cap == UINT64_MAX - 1) { + const char *e = cfg_get("NX2NAVI_UPLOAD_BATCH_MB"); + long mb = e && *e ? atol(e) : 256; + upload_cap = mb > 0 ? (uint64_t)mb << 20 : UINT64_MAX; + if (mb > 0) rt_log("vk: upload batches submitted early above %ld MiB of staging (NX2NAVI_UPLOAD_BATCH_MB)", mb); + else rt_log("vk: upload batches not limited (NX2NAVI_UPLOAD_BATCH_MB=0)"); + } + if (open_active && open_b.bytes >= upload_cap && cmd_lock.depth == 1) { + n_early++; + if (n_early <= 4 || !(n_early & (n_early - 1))) + rt_log("vk: upload batch submitted early with %.0f MiB of staging in %d buffers and %d arena chunks (%llu so far)", + open_b.bytes / 1048576.0, open_b.nstg, open_b.nchunk, (unsigned long long)n_early); + vk_upload_flush(0); + } + plat_rmutex_unlock(&cmd_lock); +} + +// An out-of-memory failure of a submit: every heap's use against its budget (the driver's numbers with +// VK_EXT_memory_budget), VMA's blocks and allocations on it, and the upload batches, so the log names what ran out. +// Takes no lock but the allocator's own: it may run with cmd_lock held. +void vk_mem_oom_report(const char *what, VkResult r) { + if (r != VK_ERROR_OUT_OF_DEVICE_MEMORY && r != VK_ERROR_OUT_OF_HOST_MEMORY) return; + VkCtx *c = vk_ctx(); + if (!c) return; + rt_log("vk: %s failed: out of %s memory (%d)", what, r == VK_ERROR_OUT_OF_DEVICE_MEMORY ? "device" : "host", (int)r); + VkPhysicalDeviceMemoryBudgetPropertiesEXT b = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT}; + VkPhysicalDeviceMemoryProperties2 p2 = {.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, + .pNext = c->has_memory_budget ? &b : NULL}; + vkGetPhysicalDeviceMemoryProperties2(c->phys, &p2); + VmaBudget vb[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(c->vma, vb); + const double M = 1048576.0; + for (uint32_t h = 0; h < c->mem_props.memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + rt_log("vk: heap %u (%s, %.0f MiB): driver use %.0f of budget %.0f MiB%s; vma %u blocks %.0f MiB, %u allocations " + "%.0f MiB", h, c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "device-local" : "host", + c->mem_props.memoryHeaps[h].size / M, c->has_memory_budget ? b.heapUsage[h] / M : vb[h].usage / M, + c->has_memory_budget ? b.heapBudget[h] / M : vb[h].budget / M, + c->has_memory_budget ? "" : " (VMA's estimate)", vb[h].statistics.blockCount, + vb[h].statistics.blockBytes / M, vb[h].statistics.allocationCount, vb[h].statistics.allocationBytes / M); + uint64_t ib = 0; + for (int i = 0; i < ninflight; i++) ib += inflight[i].bytes; + rt_log("vk: upload batch open %d (%d staging buffers, %.0f MiB), %d in flight (%.0f MiB of staging), %llu submitted " + "early; VRAM mirrors %lld (%.0f MiB); objects waiting for release %d", open_active, open_b.nstg, + open_b.bytes / M, ninflight, ib / M, (unsigned long long)n_early, (long long)m_live_n, m_live_bytes / M, ndead); +} // Staging memory comes from a pool of its own that keeps STG_MIN_BLOCKS blocks from its first use (at boot). In the // default pools VMA creates a block under the lock of its memory type (VmaBlockVector::Allocate), so a block made by @@ -4556,6 +4841,7 @@ void *vk_staging(VkDeviceSize size, VkBuffer *buf, VmaAllocation *alloc) { else { if (open_b.nstg == open_b.cap) grow((void **)&open_b.stg, &open_b.cap, sizeof *open_b.stg); open_b.stg[open_b.nstg++] = (Staging){b, a, bi.size}; + open_b.bytes += bi.size; } return info.pMappedData; } @@ -4594,6 +4880,7 @@ void *vk_staging_at(VkDeviceSize size, VkBuffer *buf, VkDeviceSize *off) { } if (open_b.nchunk == open_b.cap_chunk) grow((void **)&open_b.chunk, &open_b.cap_chunk, sizeof *open_b.chunk); open_b.chunk[open_b.nchunk++] = k; + open_b.bytes += ARENA_CHUNK; // the staging the batch holds (upload_cap) at = 0; } StgChunk *k = &open_b.chunk[open_b.nchunk - 1]; @@ -4615,12 +4902,16 @@ uint64_t vk_upload_flush(int wait) { for (int i = 0; i < open_b.nchunk; i++) vmaFlushAllocation(c->vma, open_b.chunk[i].alloc, 0, i + 1 < open_b.nchunk ? VK_WHOLE_SIZE : open_b.chunk_used); vk_crumb(open_b.cb, VK_CRUMB_UPLOAD_END, open_b.seq); + if (open_b.ts >= 0) vkCmdWriteTimestamp(open_b.cb, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, up_ts_pool, 2 * (uint32_t)open_b.ts + 1); + up_n_batches++; vkEndCommandBuffer(open_b.cb); VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &open_b.cb}; vk_queue_lock(); + open_b.sub_ns = plat_now_ns(); VkResult r = vkQueueSubmit(c->gfx_queue, 1, &si, open_b.fence); vk_queue_unlock(); if (r != VK_SUCCESS) { + vk_mem_oom_report("upload submit", r); vk_device_lost_check(r, "the submit of an upload batch"); rt_fatal("vk: upload submit failed (%d)", (int)r); } diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 225bd84f..8e50e882 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -43,6 +43,7 @@ #include #include "cfg.h" +#include "mesh_emu_spirv.h" #include "gfx.h" #include "host.h" #include "nvn_hmap.h" @@ -132,6 +133,8 @@ typedef struct Layout { uint32_t n0; // independent layouts: b[0..n0) are set 0, the rest set 1 VkPipelineLayout half[2]; // GPL libraries: {set 0, null}, {null, set 1} (made on first use) VkDescriptorUpdateTemplate tpl[2]; // per set (make_tpl); VK_NULL_HANDLE: written with write structs + uint8_t emu; // NX2NAVI_MESH_EMULATION: a mesh layout whose task and mesh stages run as compute + VkDescriptorUpdateTemplate tpl_c; // emu: set 0's template for the compute bind point VkDescriptorSetLayoutBinding b[]; } Layout; @@ -2214,6 +2217,14 @@ int vk_pool_marks_enabled(void) { // its depth, sample mask or stencil reference, or already writes memory does not, as early tests would change what it // does. The caller passes early_ok only for a pipeline without alpha to coverage, which also drops samples after the // shader (ensure_init keeps alpha to coverage out of the dynamic state while marks are on, so a pipeline knows). +// A fragment shader's marks are stored where it ends, not where it samples. A store between a sample's coordinates and +// the sample takes the quad's helper lanes out of the code in between on AMD's driver for Polaris: their coordinates +// stayed stale, the derivatives at a triangle's edge were garbage, and edge pixels came out dark (the lines along the +// paths' triangles since 0.7.0). Each access only notes its slot in a Private variable (a register); before each +// return of the entry function, and before a kill, terminate or demote anywhere, every variable is stored to its slot, +// a variable no access set going to the spare slot past the pool (vk_exec_draw.c pool_marks_make). An access in a loop +// or outside the entry function can run more than once with other slots: it notes them in a ring of RING Private +// variables, so a loop's last RING slots are all marked. // *status: 1 marked, 0 no pool texture access to mark, -1 one that could not be marked (*why says what stopped it). // NULL unless marked. static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, uint64_t *out_n, int *status, @@ -2224,7 +2235,10 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui OP_EXECUTION_MODE = 16, OP_EXECUTION_MODE_ID = 331, OP_IMAGE_WRITE = 99, OP_ATOMIC_FIRST = 227, OP_ATOMIC_LAST = 242, OP_KILL = 252, OP_TERMINATE_INVOCATION = 4416, OP_DEMOTE_TO_HELPER = 5380, EM_EARLY_FRAGMENT_TESTS = 9, BI_SAMPLE_MASK = 20, BI_FRAG_DEPTH = 22, BI_FRAG_STENCIL_REF = 5014, - SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35 }; + SC_STORAGE_BUFFER = 12, DEC_BLOCK = 2, DEC_ARRAY_STRIDE = 6, DEC_OFFSET_ = 35, OP_TYPE_BOOL = 20, + OP_LOAD_ = 61, OP_BITCAST = 124, OP_SELECT_ = 169, OP_INOT_EQUAL = 171, OP_LOOP_MERGE = 246, OP_LABEL_ = 248, + OP_RETURN = 253, OP_RETURN_VALUE = 254, SC_PRIVATE = 6, LOOP_DEPTH = 64, OP_TYPE_ARRAY_ = 28, OP_IADD = 128, + OP_BITWISE_AND = 199, OP_ULESS_THAN = 176, OP_BRANCH = 249, OP_BRANCH_CONDITIONAL = 250, OP_PHI = 245 }; *status = 0, *why = NULL; if (n < 5) return NULL; uint32_t bound = w[3]; @@ -2233,11 +2247,19 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui uint8_t *set2 = calloc(bound, 1); if (!bind || !set2) return free(bind), free(set2), *status = -1, *why = "no memory", NULL; uint64_t ann_end = 0, fn_start = 0, em_end = 0; // em_end: past the entry points and execution modes - uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0; + uint32_t uint_t = 0, c0 = 0, c1 = 0, ptr_u = 0, fs_entry = 0, int_t = 0, bool_t = 0; int vertex_stage = 0, fragment_stage = 0, entries = 0, early = early_ok; + uint32_t *rtype = calloc(bound, sizeof *rtype); // per id: its result type, where the defer path needs it + uint8_t *is_type = calloc(bound, 1); + if (!rtype || !is_type) + return free(bind), free(set2), free(rtype), free(is_type), *status = -1, *why = "no memory", NULL; +#define POOL_MARK_FREE() (free(bind), free(set2), free(rtype), free(is_type), free(site_def)) + // per site: 0 stored in place; else deferred, bit 1 the index is signed (a bitcast), bit 2 a ring (in a loop or + // outside the entry function) + uint8_t *site_def = NULL; for (uint64_t i = 5; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; - if (!l || i + l > n) return free(bind), free(set2), *status = -1, *why = "an instruction past the module's end", NULL; + if (!l || i + l > n) return POOL_MARK_FREE(), *status = -1, *why = "an instruction past the module's end", NULL; if (op == SPV_OP_FUNCTION) { fn_start = i; break; } if (op == SPV_OP_ENTRY_POINT || op == OP_EXECUTION_MODE || op == OP_EXECUTION_MODE_ID) em_end = i + l; if (op == SPV_OP_ENTRY_POINT && l >= 3) { @@ -2256,6 +2278,10 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui early = 0; // the guest's own storage buffers: its stores would change under early tests } else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 0) { uint_t = w[i + 1]; + } else if (op == SPV_OP_TYPE_INT && l == 4 && w[i + 2] == 32 && w[i + 3] == 1) { + int_t = w[i + 1]; + } else if (op == OP_TYPE_BOOL && l == 2) { + bool_t = w[i + 1]; } else if (op == SPV_OP_CONSTANT && l == 4 && uint_t && w[i + 1] == uint_t) { if (w[i + 3] == 0 && !c0) c0 = w[i + 2]; if (w[i + 3] == 1 && !c1) c1 = w[i + 2]; @@ -2265,50 +2291,95 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui if (op == SPV_OP_DECORATE || op == OP_MEMBER_DECORATE_ || op == OP_DECORATION_GROUP || op == OP_GROUP_DECORATE || op == OP_GROUP_MEMBER_DECORATE || op == OP_DECORATE_ID || op == OP_DECORATE_STRING || op == OP_MEMBER_DECORATE_STRING) ann_end = i + l; + if (op >= 19 && op <= 39 && l >= 2 && w[i + 1] < bound) is_type[w[i + 1]] = 1; // OpTypeVoid .. OpTypeForwardPointer + else if (op == SPV_OP_CONSTANT && l >= 4 && w[i + 2] < bound) rtype[w[i + 2]] = w[i + 1]; i += l; } // pool texture variables: set 2, a binding below VK_POOL_KINDS (spirv_pool_rebind); one already at the marks // binding means the module was marked before - uint64_t sites = 0; + enum { RING = 8 }; + uint64_t sites = 0, deferred = 0, rings = 0, exits = 0; int marked = 0; for (uint32_t id = 0; id < bound; id++) if (set2[id] && bind[id] == VK_POOL_MARK_BINDING + 1) marked = 1; + site_def = fn_start && !marked ? calloc((n - fn_start) / 5 + 1, 1) : NULL; // an access chain is 5 words or more + if (fn_start && !marked && !site_def) return POOL_MARK_FREE(), *status = -1, *why = "no memory", NULL; + uint32_t loops[LOOP_DEPTH], fn = 0; + int depth = 0; for (uint64_t i = fn_start; fn_start && !marked && i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (!l || i + l > n) break; + if (l >= 3 && w[i + 1] < bound && is_type[w[i + 1]] && w[i + 2] < bound) rtype[w[i + 2]] = w[i + 1]; + if (op == SPV_OP_FUNCTION && l >= 3) fn = w[i + 2], depth = 0; + else if (op == OP_LOOP_MERGE && l >= 3) depth < LOOP_DEPTH ? (void)(loops[depth++] = w[i + 1]) : (void)depth++; + else if (op == OP_LABEL_ && l >= 2 && depth > 0 && depth <= LOOP_DEPTH && loops[depth - 1] == w[i + 1]) depth--; if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && - bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) + bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) { + uint32_t ix = w[i + 4], t = ix < bound ? rtype[ix] : 0; + if (fragment_stage && entries == 1 && fs_entry && t && (t == uint_t || t == int_t)) { + int ring = fn != fs_entry || depth; + site_def[sites] = (uint8_t)(1 | (t == int_t ? 2 : 0) | (ring ? 4 : 0)); + ring ? rings++ : deferred++; + } sites++; - else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE || - (op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST)) + } else if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || op == OP_IMAGE_WRITE || + (op >= OP_ATOMIC_FIRST && op <= OP_ATOMIC_LAST)) early = 0; + if (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || + (fn == fs_entry && (op == OP_RETURN || op == OP_RETURN_VALUE))) + exits++; i += l; } - if (marked || !sites) return free(bind), free(set2), NULL; + if (marked || !sites) return POOL_MARK_FREE(), NULL; + uint64_t stores = deferred + rings; // what each way out stores: a variable, or a ring's entries in a loop + if (stores && (!exits || exits * (deferred + 3 * rings) > 8192)) // no way out to store at, or code that would grow + memset(site_def, 0, sites), deferred = rings = stores = 0; // too much *why = w[1] < 0x00010300 || w[1] > 0x00010600 ? "a SPIR-V version before 1.3 or after 1.6" : !ann_end || !fn_start ? "no decorations or no function" : vertex_stage && !marks_vs_ok ? "the GPU allows no stores in vertex-pipeline shaders" : fragment_stage && !marks_fs_ok ? "the GPU allows no stores in fragment shaders" : NULL; - if (*why) return free(bind), free(set2), *status = -1, NULL; + if (*why) return POOL_MARK_FREE(), *status = -1, NULL; early = early && fragment_stage && entries == 1 && em_end && em_end < ann_end; // From SPIR-V 1.4 on an entry point lists every global variable its functions use, not only the inputs and outputs // (the translator's mesh shaders are 1.4): each entry point gets the marks variable, a word more. int iface = w[1] >= 0x00010400; - // the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site - uint64_t on = n + 3 + (uint64_t)entries + 20 + 30 + 9 * sites; + // the mode, a word per entry point, decorations, at most 30 words of new types and the variable, 9 a site; deferred: + // a word per entry point and variable, 80 words of types and constants, then per site its variables and the store of + // its slot (ring: 10 + 29), and per way out 29 words a variable and 8 * 29 a ring (its loop is 83) + uint64_t on = n + 3 + (uint64_t)entries * (1 + deferred + 2 * rings) + 20 + 30 + 9 * sites + 80 + deferred * 12 + + rings * 40 + (deferred * 29 + rings * RING * 29) * exits; uint32_t *o = malloc(on * 4); - if (!o) return free(bind), free(set2), *status = -1, *why = "no memory", NULL; + if (!o) return POOL_MARK_FREE(), *status = -1, *why = "no memory", NULL; uint32_t next = bound; uint32_t new_uint = uint_t ? 0 : (uint_t = next++), new_c0 = c0 ? 0 : (c0 = next++), new_c1 = c1 ? 0 : (c1 = next++); uint32_t rta = next++, st = next++, ptr_st = next++, var = next++, new_ptr_u = ptr_u ? 0 : (ptr_u = next++); + // deferred: the types, constants and variables (pv0: one per site; ra0, rc0: a ring and its count per ring site) + uint32_t new_bool = 0, ptr_pv = 0, c_none = 0, c_spare = 0, pv0 = 0, ring_t = 0, ptr_ring = 0, ring_none = 0, + cr[RING] = {0}, c_ring = 0, ra0 = 0, rc0 = 0; + if (stores) { + new_bool = bool_t ? 0 : (bool_t = next++); + ptr_pv = next++, c_none = next++, c_spare = next++, pv0 = next, next += (uint32_t)deferred; + if (rings) { + ring_t = next++, ptr_ring = next++, ring_none = next++, c_ring = next++; + for (int j = 0; j < RING; j++) cr[j] = next++; + ra0 = next, next += (uint32_t)rings, rc0 = next, next += (uint32_t)rings; + } + } uint64_t p = 5; memcpy(o, w, 5 * 4); for (uint64_t i = 5; i < ann_end;) { // the header up to the annotations' end: entry points widened, the mode added uint32_t op = w[i] & 0xffff, l = w[i] >> 16; if (early && i == em_end) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = fs_entry, o[p++] = EM_EARLY_FRAGMENT_TESTS; memcpy(o + p, w + i, l * 4); - if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) o[p] = (l + 1) << 16 | op, o[p + l] = var, p++; + if (iface && op == SPV_OP_ENTRY_POINT && l >= 3) { + uint32_t nv = 0, *at = o + p + l + 1; // the marks buffer, then pv0.., ra0.. and rc0.. + o[p] = (uint32_t)(l + 1 + deferred + 2 * rings) << 16 | op, o[p + l] = var, p++; + for (uint32_t k = 0; k < deferred; k++) at[nv++] = pv0 + k; + for (uint32_t k = 0; k < rings; k++) at[nv++] = ra0 + k; + for (uint32_t k = 0; k < rings; k++) at[nv++] = rc0 + k; + p += nv; + } p += l; i += l; } @@ -2326,19 +2397,109 @@ static uint32_t *spirv_pool_mark(const uint32_t *w, uint64_t n, int early_ok, ui o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_st, o[p++] = SC_STORAGE_BUFFER, o[p++] = st; if (new_ptr_u) o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_u, o[p++] = SC_STORAGE_BUFFER, o[p++] = uint_t; o[p++] = 4u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_st, o[p++] = var, o[p++] = SC_STORAGE_BUFFER; + if (stores) { + if (new_bool) o[p++] = 2u << 16 | OP_TYPE_BOOL, o[p++] = bool_t; + o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_pv, o[p++] = SC_PRIVATE, o[p++] = uint_t; + o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_none, o[p++] = 0xffffffffu; + o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_spare, o[p++] = VK_POOL_TEX; + if (rings) { + for (int j = 0; j < RING; j++) o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = cr[j], o[p++] = (uint32_t)j; + o[p++] = 4u << 16 | SPV_OP_CONSTANT, o[p++] = uint_t, o[p++] = c_ring, o[p++] = RING; + o[p++] = 4u << 16 | OP_TYPE_ARRAY_, o[p++] = ring_t, o[p++] = uint_t, o[p++] = c_ring; + o[p++] = 4u << 16 | SPV_OP_TYPE_POINTER, o[p++] = ptr_ring, o[p++] = SC_PRIVATE, o[p++] = ring_t; + o[p++] = (3u + RING) << 16 | SPV_OP_CONSTANT_COMPOSITE, o[p++] = ring_t, o[p++] = ring_none; + for (int j = 0; j < RING; j++) o[p++] = c_none; + } + for (uint32_t k = 0; k < deferred; k++) + o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_pv, o[p++] = pv0 + k, o[p++] = SC_PRIVATE, o[p++] = c_none; + for (uint32_t k = 0; k < rings; k++) + o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_ring, o[p++] = ra0 + k, o[p++] = SC_PRIVATE, o[p++] = ring_none; + for (uint32_t k = 0; k < rings; k++) + o[p++] = 5u << 16 | SPV_OP_VARIABLE, o[p++] = ptr_pv, o[p++] = rc0 + k, o[p++] = SC_PRIVATE, o[p++] = c0; + } + uint64_t site = 0; + uint32_t k_def = 0, k_ring = 0, cur = 0; // cur: the label of the block being copied + fn = 0; for (uint64_t i = fn_start; i < n;) { uint32_t op = w[i] & 0xffff, l = w[i] >> 16; + if (op == SPV_OP_FUNCTION && l >= 3) fn = w[i + 2]; + if (op == OP_LABEL_ && l >= 2) cur = w[i + 1]; + if (stores && (op == OP_KILL || op == OP_TERMINATE_INVOCATION || op == OP_DEMOTE_TO_HELPER || + (fn == fs_entry && (op == OP_RETURN || op == OP_RETURN_VALUE)))) { + // marks[the slot, or the spare when none was noted] = 1 +#define POOL_MARK_STORE(from) \ + do { \ + uint32_t v_ = next++, ok_ = next++, ix_ = next++, at_ = next++; \ + o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = v_, o[p++] = (from); \ + o[p++] = 5u << 16 | OP_INOT_EQUAL, o[p++] = bool_t, o[p++] = ok_, o[p++] = v_, o[p++] = c_none; \ + o[p++] = 6u << 16 | OP_SELECT_, o[p++] = uint_t, o[p++] = ix_, o[p++] = ok_, o[p++] = v_, o[p++] = c_spare; \ + o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at_, o[p++] = var, o[p++] = c0, o[p++] = ix_; \ + o[p++] = 3u << 16 | OP_STORE_, o[p++] = at_, o[p++] = c1; \ + } while (0) + for (uint32_t k = 0; k < deferred; k++) POOL_MARK_STORE(pv0 + k); + for (uint32_t r = 0; r < rings; r++) { + if (op == OP_DEMOTE_TO_HELPER) { // not the block's end: every entry, without a loop + for (int j = 0; j < RING; j++) { + uint32_t from = next++; + o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = from, o[p++] = ra0 + r, o[p++] = cr[j]; + POOL_MARK_STORE(from); + } + continue; + } + // the block ends here (a return, kill or terminate), so it can end in a loop over the entries noted: + // for (j = 0; j < min(count, RING); j++) mark ring[j] + uint32_t cnt = next++, lt = next++, lim = next++, hd = next++, j = next++, mg = next++, ct = next++, + cond = next++, body = next++, from = next++, jn = next++; + o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = cnt, o[p++] = rc0 + r; + o[p++] = 5u << 16 | OP_ULESS_THAN, o[p++] = bool_t, o[p++] = lt, o[p++] = cnt, o[p++] = c_ring; + o[p++] = 6u << 16 | OP_SELECT_, o[p++] = uint_t, o[p++] = lim, o[p++] = lt, o[p++] = cnt, o[p++] = c_ring; + o[p++] = 2u << 16 | OP_BRANCH, o[p++] = hd; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = hd; + o[p++] = 7u << 16 | OP_PHI, o[p++] = uint_t, o[p++] = j, o[p++] = c0, o[p++] = cur, o[p++] = jn, o[p++] = ct; + o[p++] = 5u << 16 | OP_ULESS_THAN, o[p++] = bool_t, o[p++] = cond, o[p++] = j, o[p++] = lim; + o[p++] = 4u << 16 | OP_LOOP_MERGE, o[p++] = mg, o[p++] = ct, o[p++] = 0; + o[p++] = 4u << 16 | OP_BRANCH_CONDITIONAL, o[p++] = cond, o[p++] = body, o[p++] = mg; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = body; + o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = from, o[p++] = ra0 + r, o[p++] = j; + POOL_MARK_STORE(from); + o[p++] = 2u << 16 | OP_BRANCH, o[p++] = ct; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = ct; + o[p++] = 5u << 16 | OP_IADD, o[p++] = uint_t, o[p++] = jn, o[p++] = j, o[p++] = c1; + o[p++] = 2u << 16 | OP_BRANCH, o[p++] = hd; + o[p++] = 2u << 16 | OP_LABEL_, o[p++] = mg; + cur = mg; + } +#undef POOL_MARK_STORE + } memcpy(o + p, w + i, l * 4), p += l; if ((op == OP_ACCESS_CHAIN_ || op == OP_IN_BOUNDS_ACCESS_CHAIN) && l >= 5 && w[i + 3] < bound && set2[w[i + 3]] && bind[w[i + 3]] && bind[w[i + 3]] - 1 < VK_POOL_KINDS) { - uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access - o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4]; - o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1; + uint8_t d = site_def[site++]; + if (d) { // the slot noted for the stores at the end + uint32_t ix = w[i + 4]; + if (d & 2) o[p++] = 4u << 16 | OP_BITCAST, o[p++] = uint_t, o[p++] = next, o[p++] = ix, ix = next++; + if (d & 4) { // ring[count % RING] = slot, count + 1 + uint32_t ra = ra0 + k_ring, rc = rc0 + k_ring++, cnt = next++, j = next++, at = next++, inc = next++; + o[p++] = 4u << 16 | OP_LOAD_, o[p++] = uint_t, o[p++] = cnt, o[p++] = rc; + o[p++] = 5u << 16 | OP_BITWISE_AND, o[p++] = uint_t, o[p++] = j, o[p++] = cnt, o[p++] = cr[RING - 1]; + o[p++] = 5u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_pv, o[p++] = at, o[p++] = ra, o[p++] = j; + o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = ix; + o[p++] = 5u << 16 | OP_IADD, o[p++] = uint_t, o[p++] = inc, o[p++] = cnt, o[p++] = c1; + o[p++] = 3u << 16 | OP_STORE_, o[p++] = rc, o[p++] = inc; + } else { + o[p++] = 3u << 16 | OP_STORE_, o[p++] = pv0 + k_def++, o[p++] = ix; + } + } else { + uint32_t at = next++; // marks[slot] = 1, slot being the pool index of this access + o[p++] = 6u << 16 | OP_ACCESS_CHAIN_, o[p++] = ptr_u, o[p++] = at, o[p++] = var, o[p++] = c0, o[p++] = w[i + 4]; + o[p++] = 3u << 16 | OP_STORE_, o[p++] = at, o[p++] = c1; + } } i += l; } o[3] = next; - free(bind), free(set2); + POOL_MARK_FREE(); +#undef POOL_MARK_FREE *out_n = p; *status = 1; return o; @@ -2500,9 +2661,9 @@ static const char *origin_name(const GfxShader *s) { // The shaders of the pipeline this thread builds, for the crash notes (platform.h plat_crash_context): the first 8 // bytes of each cache key, as gfx_shader.c names its cache files. static void crash_note(const char *what, const GfxShader *a, const GfxShader *b, const GfxShader *c) { - static __thread char buf[160]; + static __thread char buf[320]; char *o = buf; - o += snprintf(o, 40, "building %s", what); + o += snprintf(o, 160, "building %s", what); const GfxShader *sh[3] = {a, b, c}; for (int i = 0; i < 3; i++) if (sh[i]) { @@ -2596,8 +2757,8 @@ static int layout_covers(const VkPipeState *s, const Layout *l, uint32_t fs_off, // layouts hold the compute stage only). VK_NULL_HANDLE when templates are off, for an empty set, for a push set // without vkCmdPushDescriptorSetWithTemplateKHR or when creation fails: the replay then writes the set with write // structs. -static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, - int push, VkPipelineLayout pl, uint32_t set) { +static VkDescriptorUpdateTemplate make_tpl_bp(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, + int push, VkPipelineLayout pl, uint32_t set, int bp) { if (!P.tpl || !n || (push && !P.tpl_push)) return VK_NULL_HANDLE; VkDescriptorUpdateTemplateEntry e[MAX_BINDINGS]; size_t off = 0; @@ -2612,7 +2773,8 @@ static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b .pDescriptorUpdateEntries = e, .templateType = push ? VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR : VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET, .descriptorSetLayout = sl, - .pipelineBindPoint = stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, + .pipelineBindPoint = bp >= 0 ? (VkPipelineBindPoint)bp + : stages & VK_SHADER_STAGE_COMPUTE_BIT ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, .pipelineLayout = pl, .set = set}; VkDescriptorUpdateTemplate t; if (P.tpl_create(P.device, &ci, NULL, &t) != VK_SUCCESS) { @@ -2621,6 +2783,10 @@ static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b } return t; } +static VkDescriptorUpdateTemplate make_tpl(const VkDescriptorSetLayoutBinding *b, uint32_t n, VkDescriptorSetLayout sl, + int push, VkPipelineLayout pl, uint32_t set) { + return make_tpl_bp(b, n, sl, push, pl, set, -1); +} // Finds or creates the layout of a binding list (sorted here); NULL on failure. Caller holds P.lock. // Sorts a binding list by binding and hashes it into *h; the descriptor total, or -1 when a binding repeats. @@ -2675,6 +2841,158 @@ static Layout *layout_get(Bindings *B) { return l; } +// NX2NAVI_MESH_EMULATION (GPUs without VK_EXT_mesh_shader): the layout of an emulated mesh pipeline. The task and +// mesh stages' bindings are visible to the compute stage (they run as compute pipelines of this layout), the +// fragment stage's stay; push constants (MESH_EMU_PC_BYTES) for the compute stages and the generated vertex shader; +// set 0 gets a template per bind point. Caller holds P.lock. +static int mesh_emu_on(void) { + static int on = -1; + if (on < 0) on = rt_knob("MESH_EMULATION", "1"); + return on; +} + +static Layout *layout_get_emu(Bindings *B) { + for (uint32_t i = 0; i < B->n; i++) { + VkShaderStageFlags f = B->b[i].stageFlags; + if (f & (VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT)) + f = (f & ~(VK_SHADER_STAGE_MESH_BIT_EXT | VK_SHADER_STAGE_TASK_BIT_EXT)) | VK_SHADER_STAGE_COMPUTE_BIT; + B->b[i].stageFlags = f; + B->pb[i].stages = f; + } + uint64_t h = fnv(FNV0, "mesh-emu", 8); + int64_t total = bindings_sort(B, &h); + if (total < 0) return NULL; + Layout **head = &layouts[h % LAYOUT_BUCKETS]; + for (Layout *l = *head; l; l = l->next) + if (l->hash == h && l->emu && l->n == B->n && !memcmp(l->b, B->b, B->n * sizeof *B->b)) return l; + Layout *l = calloc(1, sizeof *l + B->n * sizeof *B->b); + if (!l) return NULL; + l->hash = h, l->n = B->n, l->emu = 1; + memcpy(l->b, B->b, B->n * sizeof *B->b); + l->push = total <= (int64_t)P.max_push; + VkDescriptorSetLayoutCreateInfo si = {.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .flags = l->push ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0, + .bindingCount = l->n, .pBindings = l->b}; + if (vkCreateDescriptorSetLayout(P.device, &si, NULL, &l->set) != VK_SUCCESS) { + free(l); + return NULL; + } + VkDescriptorSetLayout sets3[3] = {l->set, vk_empty_set_layout(), vk_pool_set_layout()}; + VkPushConstantRange pcr = {VK_SHADER_STAGE_COMPUTE_BIT | VK_SHADER_STAGE_VERTEX_BIT, 0, MESH_EMU_PC_BYTES}; + VkPipelineLayoutCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, .setLayoutCount = 3, + .pSetLayouts = sets3, .pushConstantRangeCount = 1, .pPushConstantRanges = &pcr}; + if (vkCreatePipelineLayout(P.device, &pi, NULL, &l->layout) != VK_SUCCESS) { + vkDestroyDescriptorSetLayout(P.device, l->set, NULL); + free(l); + return NULL; + } + l->tpl[0] = make_tpl_bp(l->b, l->n, l->set, l->push, l->layout, 0, VK_PIPELINE_BIND_POINT_GRAPHICS); + l->tpl_c = make_tpl_bp(l->b, l->n, l->set, l->push, l->layout, 0, VK_PIPELINE_BIND_POINT_COMPUTE); + l->next = *head; + *head = l; + P.stats.layouts++; + return l; +} + +// The compute pipelines of an emulated mesh pipeline, per (mesh, task, layout): made by the pipeline's build +// (create_graphics, any thread) and found by its draws. ok 0: the stages could not be converted or built. +typedef struct MeshEmuEnt { + struct MeshEmuEnt *next; + const GfxShader *ms, *ts; + VkPipelineLayout layout; + int ok; + VkPipeline mesh, task; + MeshEmuInfo mi, ti; +} MeshEmuEnt; +// Each draw of an emulated mesh pipeline looks its entry up (vk_pipeline_mesh_emu), so the lookup takes no lock: an +// entry is complete before it is published at its bucket's head (release) and is never removed. emu_lock only orders +// the publishing. The build (two compute pipelines, up to seconds each in the driver on a first run) is done outside +// it: held across the build, as before, it stopped every draw of the replay that looked up any entry meanwhile, 2.5 s +// of the replay thread's samples in a first 0.8.0 run in Kokiri Forest on an RX 580 (2026-11-11). +#define EMU_BUCKETS 256 +static MeshEmuEnt *emu_ents[EMU_BUCKETS]; +static PlatMutex emu_lock = PLAT_MUTEX_INIT; + +static uint32_t emu_bucket(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout) { + uint64_t h = (uint64_t)(uintptr_t)ms * 0x9e3779b97f4a7c15ull ^ (uint64_t)(uintptr_t)ts * 0xc2b2ae3d27d4eb4full ^ + (uint64_t)(uintptr_t)layout * 0x165667b19e3779f9ull; + return (uint32_t)(h >> 40) % EMU_BUCKETS; +} + +static MeshEmuEnt *emu_find(MeshEmuEnt *e, const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout) { + while (e && !(e->ms == ms && e->ts == ts && e->layout == layout)) e = __atomic_load_n(&e->next, __ATOMIC_ACQUIRE); + return e; +} + +static VkPipeline emu_compute(const uint32_t *w, size_t n, VkPipelineLayout layout) { + VkShaderModule m = make_module(w, n, 0); + if (!m) return VK_NULL_HANDLE; + VkComputePipelineCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, + .stage = {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .stage = VK_SHADER_STAGE_COMPUTE_BIT, .module = m, .pName = "main"}, + .layout = layout}; + VkPipeline p = VK_NULL_HANDLE; + if (vkCreateComputePipelines(P.device, P.cache, 1, &ci, NULL, &p) != VK_SUCCESS) p = VK_NULL_HANDLE; + vkDestroyShaderModule(P.device, m, NULL); + return p; +} + +static const MeshEmuEnt *emu_get(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, uint32_t toff, int create) { + MeshEmuEnt **head = &emu_ents[emu_bucket(ms, ts, layout)]; + MeshEmuEnt *e = emu_find(__atomic_load_n(head, __ATOMIC_ACQUIRE), ms, ts, layout); + if (e || !create) return e; + e = calloc(1, sizeof *e); + if (!e) return NULL; + e->ms = ms, e->ts = ts, e->layout = layout; + const char *why = NULL; + uint64_t n = 0; + size_t cn = 0; + const uint32_t *w = gfx_shader_spirv(ms, &n); + uint32_t *c = w ? mesh_emu_compute(w, (size_t)n, &e->mi, &cn, &why) : NULL; + if (c && (e->mesh = emu_compute(c, cn, layout))) e->ok = 1; + else if (!why) why = "the mesh stage's compute pipeline was not created"; + free(c); + if (e->ok && e->mi.payload_words && !ts) e->ok = 0, why = "a mesh stage that reads a task payload without a task stage"; + if (e->ok && ts) { + w = gfx_shader_spirv(ts, &n); + c = w ? mesh_emu_compute(w, (size_t)n, &e->ti, &cn, &why) : NULL; + uint32_t *r = c && toff ? spirv_rebind(c, cn, toff) : NULL; // the task stage's bindings follow the fragment stage's + if (c && (e->task = emu_compute(r ? r : c, cn, layout))) { + if (e->ti.payload_words != e->mi.payload_words) e->ok = 0, why = "task and mesh payload sizes differ"; + } else { + e->ok = 0; + if (!why) why = "the task stage's compute pipeline was not created"; + } + free(r), free(c); + } + plat_mutex_lock(&emu_lock); + MeshEmuEnt *had = emu_find(*head, ms, ts, layout); // another build of the same stages published first: keep that + if (!had) { + static int logged; + if (!e->ok && logged++ < 16) rt_log("vk_pipeline: mesh emulation: a mesh program is not emulated: %s", why ? why : "?"); + e->next = *head; + __atomic_store_n(head, e, __ATOMIC_RELEASE); + } + plat_mutex_unlock(&emu_lock); + if (had) { + if (e->mesh) vkDestroyPipeline(P.device, e->mesh, NULL); + if (e->task) vkDestroyPipeline(P.device, e->task, NULL); + free(e); + return had; + } + return e; +} + +int vk_pipeline_mesh_emu(const GfxShader *ms, const GfxShader *ts, VkPipelineLayout layout, VkPipeline *mesh, + VkPipeline *task, MeshEmuInfo *mi) { + const MeshEmuEnt *e = emu_get(ms, ts, layout, 0, 0); + if (!e || !e->ok) return 0; + *mesh = e->mesh, *task = e->task; + *mi = e->mi; + if (ts) mi->max_mesh = e->ti.max_mesh; + return 1; +} + // Independent layouts (P.indep): the vertex stage's bindings in set 0, the fragment stage's in set 1 (empty without a // fragment shader), each with its own support buffer at binding 0, the pipeline layout created with // VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT so stage libraries can use it. Only the P.push_stage set is a @@ -2801,6 +3119,7 @@ static Pipe *pipe_add(const void *key, size_t size, uint64_t h, VkPipeline pipel p->info.n_sets = l && l->set1 ? 2 : 1; p->info.pool = B ? B->pool : 0; if (l) p->info.tpl[0] = l->tpl[0], p->info.tpl[1] = l->tpl[1]; + if (l && l->emu) p->info.emu = 1, p->info.emu_tpl_c = l->tpl_c; p->lay = l; Pipe **head = &pipes[h % PIPE_BUCKETS]; p->next = *head; @@ -2972,7 +3291,8 @@ static void gkey_fill(const VkPipeState *s, GKey *k) { o[3] = (uint8_t)blend_factor(b->src_a), o[4] = (uint8_t)blend_factor(b->dst_a), o[5] = (uint8_t)blend_op(b->eq_a); } } - if (is_mesh(s)) k->topo_class = 2; // mesh draws: no vertex input, the mesh stage declares its primitives + // mesh draws: no vertex input, the mesh stage declares its primitives (emulated point meshes draw point lists) + if (is_mesh(s)) k->topo_class = s->prim == NVN_DRAW_PRIMITIVE_POINTS ? 0 : 2; if (!P.dyn_vi && !is_mesh(s)) { VtxInput v; vertex_input(s, &v); @@ -3142,11 +3462,31 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay uint64_t vw, fw, vm_hash = 0, fm_hash = 0; // the hashes: of the final modules, for the PIPE_FEEDBACK 2 line const uint32_t *vcode = gfx_shader_spirv(s->vs, &vw); uint32_t *vpatched = NULL, *fpatched = NULL; - int mesh = is_mesh(s); + int mesh = is_mesh(s), emu = mesh && l->emu; + // the water pair's vertex edit is for a translated vertex stage: an emulated mesh draw's generated one has none int wmode = mesh || !l->set1 ? 0 : water_mode(s, k); // the edited fragment modules are in their set 1 form uint64_t wn = 0; const uint32_t *vwater = water_pair_vertex(vcode, vw, wmode, &wn); if (vwater) vcode = vwater, vw = wn; + if (emu) { // NX2NAVI_MESH_EMULATION: the task and mesh stages as compute pipelines, a generated vertex shader here + uint32_t toff = fs_off + (s->fs ? binding_span(s->fs) - 1 : 0); + const MeshEmuEnt *me = emu_get(s->vs, s->ts, l->layout, toff, 1); + if (!me || !me->ok) return VK_NULL_HANDLE; + GfxShaderIo fin[32]; + uint8_t ty[32], cp[32]; + uint32_t need = s->fs ? gfx_shader_inputs(s->fs, fin) : 0; + for (int i = 0; i < 32; i++) { + ty[i] = need >> i & 1 && !fin[i].packed && fin[i].type <= GFX_TYPE_UINT ? (uint8_t)fin[i].type : 0; + cp[i] = need >> i & 1 && !fin[i].packed && fin[i].comps ? (uint8_t)fin[i].comps : 4; + } + size_t vn = 0; + uint64_t fsw = 0; + const uint32_t *fcode0 = s->fs ? gfx_shader_spirv(s->fs, &fsw) : NULL; + vpatched = mesh_emu_vs(&me->mi, need, ty, cp, fcode0, (size_t)fsw, &vn); + if (!vpatched) return VK_NULL_HANDLE; + vcode = vpatched, vw = vn; + mesh = 0; // a vertex pipeline from here on + } VkShaderModule tm = VK_NULL_HANDLE; if (mesh && s->ts) { // the task stage's bindings follow the fragment stage's (pipe_miss) uint64_t tw; @@ -3168,14 +3508,15 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay } if (s->fs) { GfxShaderIo fin[32], vout[32]; - // a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none + // a mesh stage's outputs from its module (spirv_mesh_outputs): the translation reflects none; an emulated one + // (mesh_emu) writes what the fragment stage reads int points = 0; uint32_t need = gfx_shader_inputs(s->fs, fin), missing = 0, - have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : gfx_shader_outputs(s->vs, vout); + have = mesh ? (vcode ? spirv_mesh_outputs(vcode, vw, &points) : ~0u) : emu ? ~0u : gfx_shader_outputs(s->vs, vout); for (int i = 0; i < 32; i++) { if (!(need >> i & 1) || fin[i].packed) continue; if (!(have >> i & 1)) missing |= 1u << i; - else if (!mesh && vout[i].type != fin[i].type) + else if (!mesh && !emu && vout[i].type != fin[i].type) LOG_ONCE("vk_pipeline: varying %d is type %d in the vertex stage, %d in the fragment stage", i, vout[i].type, fin[i].type); } if (missing && !l->set1 && !mesh) { @@ -3224,7 +3565,7 @@ static VkPipeline create_graphics(const VkPipeState *s, const GKey *k, const Lay VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rqv, rqf; uint64_t sw; const uint32_t *scode = gfx_shader_spirv(s->vs, &sw); // vcode may be a patched copy that is freed by now - pin_subgroup32(&stages[0], &rqv, scode, sw); + if (!emu) pin_subgroup32(&stages[0], &rqv, scode, sw); if (s->fs) pin_subgroup32(&stages[1], &rqf, gfx_shader_spirv(s->fs, &sw), sw); VkVertexInputAttributeDescription va[32]; VkVertexInputBindingDescription vb[17]; @@ -3502,7 +3843,9 @@ static VkPipeline lib_create(int kind, const VkPipeState *s, const GKey *k, VkPi VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; pi.pNext = fb_chain(&fci, &fb, pi.pNext); - crash_note("a pipeline library", s->vs, s->fs, NULL); + static const char *const kind_name[4] = {"a vertex input library", "a pre-rasterization library", + "a fragment shader library", "a fragment output library"}; + crash_note(kind_name[kind], s->vs, s->fs, NULL); if (pipe_create(1, &pi, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); if (pipe && (kind == LIB_PR || kind == LIB_FS)) fb_note(FB_LIBRARY, &fb); // the shader libraries: the others compile nothing @@ -3734,7 +4077,12 @@ static VkPipeline gpl_link(const VkPipeState *s, const GKey *k, const Layout *lc .layout = l->layout}; if (build) plat_mutex_unlock(&P.lock); uint64_t t1 = qpc(); - crash_note("a library link", s->vs, s->fs, NULL); + char what[160]; // the parts that differ between links: a link that crashes the driver names its case + snprintf(what, sizeof what, + "a library link (fragment shader %s, %u colour targets, depth format %u, stencil format %u, %u samples, " + "vertex outputs added %#x)", s->fs ? "yes" : "none", k->n_color, k->depth, k->stencil, k->samples, + pk.missing); + crash_note(what, s->vs, s->fs, NULL); if (vkCreateGraphicsPipelines(P.device, VK_NULL_HANDLE, 1, &pi, NULL, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; plat_crash_context(NULL); double ms = ms_since(t1); // the link alone: library builds are counted apart @@ -4554,11 +4902,12 @@ static Pipe *pipe_miss(const VkPipeState *s, const GKey *k, uint64_t h, int mode // vertex pipelines). They go to the workers like the others: built here, each new one stopped the frame for its // whole compile (0.26-0.51 s in Kokiri Forest), and they are recorded for the boot prewarm (keys_note). uint32_t fspan = s->fs ? binding_span(s->fs) : 1; - if (!vk_ctx()->has_mesh_shader) why = "the GPU has no mesh shaders"; + int emu = !vk_ctx()->has_mesh_shader && mesh_emu_on(); + if (!vk_ctx()->has_mesh_shader && !emu) why = "the GPU has no mesh shaders (mesh_emulation = 1 emulates them)"; else if ((s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS) || (s->ts && gfx_shader_stage(s->ts) != GFX_STAGE_TS)) why = "stage mismatch"; else if (!add_stage(&B, s->vs, 0) || (s->fs && !add_stage(&B, s->fs, span - 1)) || (s->ts && !add_stage(&B, s->ts, span - 1 + fspan - 1))) why = "too many bindings"; - else if (!(l = layout_get(&B))) why = "descriptor layout creation failed"; + else if (!(l = emu ? layout_get_emu(&B) : layout_get(&B))) why = "descriptor layout creation failed"; } else if (gfx_shader_stage(s->vs) != GFX_STAGE_VS || (s->fs && gfx_shader_stage(s->fs) != GFX_STAGE_FS)) why = "stage mismatch"; else if (k->topo_class == 3) why = "patches need tessellation stages, which are not translated"; else if (P.indep) { @@ -5021,8 +5370,12 @@ static void prewarm_run(void) { VkPipeState s = r[i].s; s.vs = vs, s.fs = fs, s.ts = ts; // a mesh pipeline of the shared list (keys_open_seed) on a GPU without mesh shaders, which draws them through - // the mesh fallback: none to build, as the preparation skips it (seed_pipelines) - if (is_mesh(&s) && !vk_ctx()->has_mesh_shader) { + // the mesh fallback: none to build, as the preparation skips it (seed_pipelines). With the mesh emulation + // (NX2NAVI_MESH_EMULATION) it is a vertex pipeline and its compute stages (pipe_miss: layout_get_emu), built + // here like the others: left to the first draw, each took 113-830 ms on an RX 580 and the draws waiting for it + // were skipped (22 of the 48 pipelines missed entering Kokiri Forest, 0.9.1), frames held back or shown without + // them. + if (is_mesh(&s) && !vk_ctx()->has_mesh_shader && !mesh_emu_on()) { dropped++; continue; } diff --git a/runtime/vk_rec.c b/runtime/vk_rec.c index 97098330..e9fea21e 100644 --- a/runtime/vk_rec.c +++ b/runtime/vk_rec.c @@ -166,10 +166,15 @@ void rec_begin(VkCommandBuffer cb, uint64_t fence) { // past it). Keeping objects until their submission completed and ending a part from the recorder // (rec_end_cb) fixed the white scene on an RX 6700 XT, but not this: 0.2.8 turned the recorder on for every // GPU and the AMD reports came back the same day. NX2NAVI_VK_RECORDER=1 turns it on for an A/B run. + // On AMD GPUs without mesh shaders (GCN: RX 400/500, Vega; RDNA 1) it is on: every report above is from an + // RDNA 2-4 card, and those all have mesh shaders. An RX 580 ran with it and NX2NAVI_REC_CHECK + // (no hazards), the replay's part of the game's wait after a frame's first submission 6.9 ms instead of 12. const char *set = cfg_get("NX2NAVI_VK_RECORDER"); - mode = rt_knob("VK_RECORDER", vk_caps.amd ? "0" : "1"); - if (!mode && vk_caps.amd && !(set && *set)) - rt_log("vk_rec: no recorder thread on AMD GPUs (NX2NAVI_VK_RECORDER=1 turns it on)"); + int amd_off = vk_caps.amd && vk_ctx()->has_mesh_shader; + mode = rt_knob("VK_RECORDER", amd_off ? "0" : "1"); + if (!mode && amd_off && !(set && *set)) + rt_log("vk_rec: no recorder thread on AMD GPUs with mesh shaders (RDNA 2 and newer; NX2NAVI_VK_RECORDER=1 " + "turns it on)"); if (mode) { plat_event_init(&wake_ev, 0, 0); rec_prio = plat_thread_priority_get(); diff --git a/runtime/vk_sync.c b/runtime/vk_sync.c index 11b3db00..ae107252 100644 --- a/runtime/vk_sync.c +++ b/runtime/vk_sync.c @@ -16,6 +16,7 @@ #include "gfx.h" #include "host.h" #include "nvn_api.h" +#include "prof.h" #include "vk.h" #include "platform/platform.h" @@ -47,6 +48,21 @@ static PlatCond cv_work = PLAT_COND_INIT; // a submission, a callback or stop static PlatCond cv_done = PLAT_COND_INIT; // done advanced static uint64_t issued, done; // newest submitted value; newest value whose side effects ran static uint64_t reserved; // asynchronous replay: newest value handed out before its submission (>= issued) +// Staged mirror uploads (vk_memory.c "staged mirror uploads"): per submission value, whether its GPU still reads guest +// memory itself (value << 1 | live), under st_lock. A value not found counts as live. +#define LIVE_RING 4096 +static uint64_t live_of[LIVE_RING]; +// Per submission value: the newest submission up to it whose GPU reads guest memory itself (live_of), 0 none. The +// submit wait is for the frame's first submission, but the game's writes after it race with every submission before it +// that still reads guest memory: those of the frame before (the instances of its grass and moss, read through the +// device address table) were often still on the GPU when the wait ended at the first one's hand-over, and the game +// wrote the next frame's instances under them (flicker and sliding foliage, RX 580, 2026-11-11). The wait goes on until +// that one is done. +static uint64_t live_last_of[LIVE_RING], live_last; +// per value (index & LIVE_RING - 1): the newest value up to it whose completion hands results to the CPU (vk_exec_take_results) +static uint64_t res_of[LIVE_RING], res_last; +static uint8_t res_kinds[LIVE_RING]; // per value: its VK_RES_* kinds +static int res_mask = 0x1f; // NX2NAVI_SUBMIT_FINISH_KINDS: the kinds the wait counts static int stop; static Pending *queue; // side effects in the order they were added static int qhead, qn, qcap; @@ -71,15 +87,26 @@ static void results_run(uint64_t upto); // H is guarded by the queue lock. NX2NAVI_SUBMIT_BATCH=0 (or 1) hands each to the queue at once; =N holds up to N. #define HOLD_CAP 16 #define HOLD_NS 500000ull +// Command buffers of one submission: the mirror prologue, the parts cut before the part with their pre-pass command +// buffers (CHAIN_MAX), the part's pre-pass command buffer, the part (vk_sync_pre, vk_sync_cut, chain_infos). +#define CHAIN_MAX 64 +#define SUB_CBS (CHAIN_MAX + PART_CBS + 3) static struct { int n; VkSubmitInfo2 si[HOLD_CAP]; - VkCommandBufferSubmitInfo cbi[HOLD_CAP][PART_CBS + 2]; // the prologue and the part's command buffers + VkCommandBufferSubmitInfo cbi[HOLD_CAP][SUB_CBS]; VkSemaphoreSubmitInfo sig[HOLD_CAP][2]; uint64_t last; // the newest timeline value a held submission signals, 0 none uint64_t batches, subs; // vkQueueSubmit2 calls and the submissions they carried } H; static volatile int32_t held_n; // H.n, for readers without the lock +// The GPU timeline (vk_gtl_note): the timestamp pairs of the held submissions, stamped with the time they really go to +// the queue (ts_queue_ns), as a held part waits here and not in the queue. hold_pend: closed by ts_submitted for the +// submission queue_submit takes next (sub_lock); hold_pairs: those of the held ones (queue lock). +#define HOLD_PAIRS 512 +static uint32_t hold_pend[HOLD_PAIRS], hold_pairs[HOLD_PAIRS]; +static int nhold_pend, nhold_pairs; +static uint64_t ts_queue_ns[1024]; // per pair (TS_PAIRS): when its submission went to the queue static volatile uint64_t held_t0; // when the oldest held submission was held (kern_now_ns) static volatile uint64_t queued_v; // the newest timeline value handed to the queue static int hold_max = -1; @@ -238,11 +265,15 @@ static void hold_flush_locked(VkCtx *c) { } VkResult r = lost_at_due() ? VK_ERROR_DEVICE_LOST : vkQueueSubmit2(c->gfx_queue, (uint32_t)H.n, H.si, VK_NULL_HANDLE); if (r != VK_SUCCESS) { + vk_mem_oom_report("replay submit", r); vk_device_lost_check(r, "the submit of the replay's command buffers"); rt_fatal("vk: vkQueueSubmit2 of %d held submissions up to value %llu failed (%d)", H.n, (unsigned long long)H.last, (int)r); } if (H.last) __atomic_store_n(&queued_v, H.last, __ATOMIC_RELEASE); + uint64_t now = plat_now_ns(); + for (int k = 0; k < nhold_pairs; k++) ts_queue_ns[hold_pairs[k]] = now; + nhold_pairs = 0; H.batches++, H.subs += (uint64_t)H.n; H.n = 0, H.last = 0; __atomic_store_n(&held_n, 0, __ATOMIC_RELEASE); @@ -269,8 +300,11 @@ static void queue_submit(VkCtx *c, const VkSubmitInfo2 *si, uint64_t value) { vk_queue_lock_raw(); perf_end(PERF_FENCE, p0); if (H.n == HOLD_CAP) hold_flush_locked(c); + for (int k = 0; k < nhold_pend && nhold_pairs < HOLD_PAIRS; k++) hold_pairs[nhold_pairs++] = hold_pend[k]; + nhold_pend = 0; int i = H.n++; H.si[i] = *si; + if (si->commandBufferInfoCount > SUB_CBS) rt_fatal("vk: a submission of %u command buffers", si->commandBufferInfoCount); if (si->commandBufferInfoCount) memcpy(H.cbi[i], si->pCommandBufferInfos, sizeof H.cbi[i][0] * si->commandBufferInfoCount); H.si[i].pCommandBufferInfos = H.cbi[i]; @@ -309,6 +343,7 @@ static void hold_release(uint64_t value, int aged, int try_only) { // part's interval to the busy time, counting overlap with the previous part once. Off (one log line) when the queue // family has no timestamp bits or the period is 0. Upload batches (vk_memory.c) are not timed. #define TS_PAIRS 1024 +_Static_assert(TS_PAIRS == sizeof ts_queue_ns / sizeof ts_queue_ns[0], "ts_queue_ns has a slot per pair"); static VkQueryPool ts_pool; static int ts_state; // 0 not set up, 1 on, -1 off static double ts_period; // ns per tick @@ -325,13 +360,19 @@ static volatile int64_t ts_busy_all; // the previous one had ended (the GPU waited for the CPU: starved) or before (it waited for untimed work or a wait). static uint64_t ts_sub_ns[TS_PAIRS]; // CPU time the part was handed to vkQueueSubmit2 // A pair to read at its value. mirror: the copies of a mirror prologue (vk_sync.c prologue), counted apart as well. -typedef struct { uint32_t pair; int mirror; uint64_t fence; } TsRead; -static TsRead *ts_unsent[4]; // closed and not yet submitted (sub_lock) +// mirror: GTL_PART, GTL_MIRROR or GTL_PRE (the GPU timeline's kinds, below); cb: the command buffer it times. +typedef struct { uint32_t pair; int mirror; uint64_t fence; VkCommandBuffer cb; } TsRead; +// closed and not yet submitted (sub_lock): with cuts a submission carries up to SUB_CBS command buffers, each timed; +// with room for 4 the rest went to the completion queue at once, without a submit time and out of the GPU's order, and +// their gaps counted as idle "with it queued" +#define TS_UNSENT (2 * SUB_CBS) +static TsRead *ts_unsent[TS_UNSENT]; static int ts_n_unsent; static volatile int64_t ts_mirror_ns; // of ts_busy_ns, the mirror prologues' copies static uint64_t ts_last_end_ns; // completion thread: ts_last_end on the CPU clock static volatile int64_t ts_idle_ns, ts_starved_ns, ts_gaps, ts_max_gap_ns, ts_queued_ns; -static int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when it was taken +static volatile int64_t cal_tick, cal_ns, cal_at; // completion thread: a calibrated (GPU tick, CPU ns) pair and when + // it was taken; the GPU timeline's notes read them from other threads static PFN_vkGetCalibratedTimestampsKHR cal_fn; // On the first submission that wants timing (perf_gpu_timing: NX2NAVI_TRACE=perf or the full performance overlay). @@ -399,19 +440,200 @@ static void max64(volatile int64_t *m, int64_t v) { static void ts_read_cb(void *arg); +// ---- the GPU timeline (NX2NAVI_TRACE=perf, vk.h vk_gtl_note): every timed piece of GPU work, with the CPU time it was +// handed to the queue and its GPU start and end on the CPU clock (calibrated timestamps). The replay's parts, their +// mirror prologues and pre-pass command buffers come from here, the upload batches from vk_memory.c, the presents from +// gfx_present.c. Every 60 frames (vk_gtl_log) the pieces are put in GPU order and each gap between them is split: +// starved, nothing handed to the queue yet, or queued, the next piece handed over and not started; the queued part is +// summed by the kind of the piece that started after it. A gap with work queued is a wait inside the queue (a +// semaphore, the present's acquire) or the driver's and the system's delay to start work on an idle GPU. +#define GTL_N 16384 +typedef struct { int kind; int64_t call, sub, start, end; } GtlEv; // call: handed to vk_sync's submit; sub: to the queue +static GtlEv gtl[GTL_N]; +static int gtl_n; +static uint64_t gtl_dropped; +static PlatMutex gtl_lock = PLAT_MUTEX_INIT; +static const char *const gtl_names[GTL_KINDS] = {"replay parts", "mirror prologues", "pre-pass buffers", "upload batches", + "presents"}; + +static void vk_gtl_note_ns(int kind, int64_t call, int64_t sub, int64_t start, int64_t end) { + if (!(rt_trace & RT_TRACE_PERF) || kind < 0 || kind >= GTL_KINDS || !sub || end < start) return; + plat_mutex_lock(>l_lock); + if (gtl_n < GTL_N) gtl[gtl_n++] = (GtlEv){kind, call ? call : sub, sub, start, end}; + else gtl_dropped++; + plat_mutex_unlock(>l_lock); +} + +void vk_gtl_note(int kind, uint64_t sub_ns, uint64_t start_tick, uint64_t end_tick) { + if (!(rt_trace & RT_TRACE_PERF) || ts_state != 1 || !__atomic_load_n(&cal_at, __ATOMIC_ACQUIRE)) return; + uint64_t s = start_tick & ts_mask, e = end_tick & ts_mask; + if (e < s) return; + vk_gtl_note_ns(kind, (int64_t)sub_ns, (int64_t)sub_ns, tick_ns(s), tick_ns(e)); +} + +// ---- the game's waits for GPU work (NX2NAVI_TRACE=perf): for each submission value, when the game handed it to the +// replay (vk_sync_reserve; the submit itself without asynchronous replay), when the replay had recorded and submitted it +// (issued) and when it was done (GPU and completion). A fence wait of any thread but the replay's is split into its +// time until the waited submission was submitted (the replay still recording it or those before it) and its time after +// (the GPU, the submit batching's hold, the completion thread). Every 60 frames with the GPU timeline (vk_gtl_log). +#define GW_RING 4096 +typedef struct { uint64_t v, res, iss, done; } GwSub; +static GwSub gw[GW_RING]; +static PlatMutex gw_lock = PLAT_MUTEX_INIT; +static uint64_t gw_n, gw_ns, gw_rep_ns, gw_gpu_ns, gw_lead_ns, gw_age, gw_before, gw_lat_rep, gw_lat_gpu, gw_lat_n; +static uint64_t gw_max_ns, gw_unknown, gw_rec_n, gw_rec_done, gw_rec_self, gw_need_back, gw_need_n; +static uint64_t gw_need_kind[5], gw_skip_kind[5]; // VK_RES_KINDS // gw_rec: vk_sync_recorded_wait, those it ended + +static GwSub *gw_at(uint64_t v) { // gw_lock held + GwSub *e = &gw[v & (GW_RING - 1)]; + if (e->v != v) *e = (GwSub){v, 0, 0, 0}; + return e; +} + +static void gw_mark(uint64_t v, int what) { + if (!(rt_trace & RT_TRACE_PERF) || !v) return; + uint64_t now = plat_now_ns(); + plat_mutex_lock(&gw_lock); + GwSub *e = gw_at(v); + if (what == 0) e->res = now; + else if (what == 1) e->iss = now; + else if (!e->done) e->done = now; + plat_mutex_unlock(&gw_lock); +} + +// the waits: t0 to t1 for value; newest: the newest value handed out at t0, sent: the newest submitted at t0 +static void gw_wait(uint64_t value, uint64_t t0, uint64_t t1, uint64_t newest, uint64_t sent) { + if (!(rt_trace & RT_TRACE_PERF) || t1 <= t0 || plat_thread_id() == prof_replay_tid) return; + uint64_t d = t1 - t0; + plat_mutex_lock(&gw_lock); + GwSub e = gw[value & (GW_RING - 1)]; + gw_n++, gw_ns += d; + if (d > gw_max_ns) gw_max_ns = d; + if (e.v != value || !e.iss) { + gw_unknown++, gw_gpu_ns += d; + } else { + uint64_t rep = e.iss > t0 ? (e.iss < t1 ? e.iss : t1) - t0 : 0; + gw_rep_ns += rep, gw_gpu_ns += d - rep; + if (e.res && e.res <= t0) gw_lead_ns += t0 - e.res; + if (e.res && e.done && e.iss >= e.res && e.done >= e.iss) + gw_lat_rep += e.iss - e.res, gw_lat_gpu += e.done - e.iss, gw_lat_n++; + } + if (newest > value) gw_age += newest - value; + if (value > sent + 1) gw_before += value - sent - 1; + plat_mutex_unlock(&gw_lock); +} + +static void gw_log(uint64_t frame, int frames) { + plat_mutex_lock(&gw_lock); + uint64_t n = gw_n, ns = gw_ns, rep = gw_rep_ns, gpu = gw_gpu_ns, lead = gw_lead_ns, age = gw_age, before = gw_before, + lr = gw_lat_rep, lg = gw_lat_gpu, ln = gw_lat_n, mx = gw_max_ns, unk = gw_unknown, rn = gw_rec_n, + rd = gw_rec_done, rs = gw_rec_self; + gw_n = gw_ns = gw_rep_ns = gw_gpu_ns = gw_lead_ns = gw_age = gw_before = gw_lat_rep = gw_lat_gpu = gw_lat_n = 0; + uint64_t nb = gw_need_back, nn = gw_need_n, nk[5], sk[5]; + memcpy(nk, gw_need_kind, sizeof nk), memcpy(sk, gw_skip_kind, sizeof sk); + memset(gw_need_kind, 0, sizeof gw_need_kind), memset(gw_skip_kind, 0, sizeof gw_skip_kind); + gw_max_ns = gw_unknown = gw_rec_n = gw_rec_done = gw_rec_self = gw_need_back = gw_need_n = 0; + plat_mutex_unlock(&gw_lock); + if (!n) { + rt_log("perf: frame %llu: waits for GPU work outside the replay: none", (unsigned long long)frame); + return; + } + double f = frames; + rt_log("perf: frame %llu: waits for GPU work outside the replay: %.1f per frame, %.2f ms per frame (longest %.2f ms): " + "%.2f ms until the replay had submitted the waited submission (%.2f submissions still before it in the replay " + "on average), %.2f ms after (GPU, submit batching hold, completion; %llu without a record); the waited one was " + "%.2f submissions behind the newest and handed to the replay %.2f ms before the wait; per waited submission " + "%.2f ms from hand-over to submit and %.2f ms from submit to done", + (unsigned long long)frame, n / f, ns / 1e6 / f, mx / 1e6, rep / 1e6 / f, (double)before / (double)n, + gpu / 1e6 / f, (unsigned long long)unk, (double)age / (double)n, lead / 1e6 / (double)n, + ln ? lr / 1e6 / (double)ln : 0.0, ln ? lg / 1e6 / (double)ln : 0.0); + if (rn) rt_log("perf: frame %llu: submit waits: %llu, %llu of them without the GPU's run of the submission itself (%llu of " + "those waited for it anyway: it hands the CPU results), the rest went on to the GPU (the submission's GPU " + "read guest memory itself)", (unsigned long long)frame, (unsigned long long)rn, (unsigned long long)rd, + (unsigned long long)rs); + if (nn) rt_log("perf: frame %llu: the submission the submit wait waited for was %.2f submissions before the waited one; " + "its kinds: events %llu, counter reports %llu, occlusion reports %llu, unmirrored writes %llu, write-backs " + "%llu; the submissions after it (left out by the mask): %llu, %llu, %llu, %llu, %llu", + (unsigned long long)frame, (double)nb / (double)nn, (unsigned long long)nk[0], (unsigned long long)nk[1], + (unsigned long long)nk[2], (unsigned long long)nk[3], (unsigned long long)nk[4], (unsigned long long)sk[0], + (unsigned long long)sk[1], (unsigned long long)sk[2], (unsigned long long)sk[3], (unsigned long long)sk[4]); +} + +static int gtl_cmp(const void *a, const void *b) { + int64_t x = ((const GtlEv *)a)->start, y = ((const GtlEv *)b)->start; + return x < y ? -1 : x > y; +} + +void vk_gtl_log(uint64_t frame, int frames) { + static GtlEv ev[GTL_N]; + gw_log(frame, frames); + plat_mutex_lock(>l_lock); + int n = gtl_n; + memcpy(ev, gtl, sizeof ev[0] * (size_t)n); + gtl_n = 0; + uint64_t dropped = gtl_dropped; + gtl_dropped = 0; + plat_mutex_unlock(>l_lock); + if (n < 2 || frames < 1) return; + qsort(ev, (size_t)n, sizeof ev[0], gtl_cmp); + double busy = 0, starved = 0, queued = 0, q_by[GTL_KINDS] = {0}, lat[GTL_KINDS] = {0}, gpu[GTL_KINDS] = {0}; + double held = 0, held_idle = 0; // replay pieces: held in the submit batching; of it, while the GPU was idle + int big[GTL_KINDS] = {0}, nlat[GTL_KINDS] = {0}, cnt[GTL_KINDS] = {0}; + int64_t end = ev[0].start; + for (int i = 0; i < n; i++) { + const GtlEv *e = &ev[i]; + cnt[e->kind]++, gpu[e->kind] += (double)(e->end - e->start); + held += (double)(e->sub - e->call); + if (e->start > end) { // a gap before e + int64_t gap = e->start - end, s = e->sub > end ? e->sub - end : 0; + int64_t h0 = e->call > end ? e->call : end, h1 = e->sub < e->start ? e->sub : e->start; + if (h1 > h0) held_idle += (double)(h1 - h0); + if (s > gap) s = gap; + starved += (double)s, queued += (double)(gap - s), q_by[e->kind] += (double)(gap - s); + if (gap - s > 500000) big[e->kind]++; + if (e->sub >= end) lat[e->kind] += (double)(e->start - e->sub), nlat[e->kind]++; // handed to an idle GPU + } + int64_t from = e->start > end ? e->start : end; + if (e->end > from) busy += (double)(e->end - from), end = e->end; + } + double f = 1e6 * frames; + char by[512], la[512], gp[512]; + int a = 0, b = 0, g = 0; + for (int k = 0; k < GTL_KINDS; k++) { + if (!cnt[k]) continue; + a += snprintf(by + a, sizeof by - (size_t)a, "%s%s %.2f ms (%.1f gaps over 0.5 ms)", a ? ", " : "", gtl_names[k], + q_by[k] / f, (double)big[k] / frames); + b += snprintf(la + b, sizeof la - (size_t)b, "%s%s %.2f ms (%d)", b ? ", " : "", gtl_names[k], + nlat[k] ? lat[k] / nlat[k] / 1e6 : 0.0, nlat[k]); + g += snprintf(gp + g, sizeof gp - (size_t)g, "%s%s %.1f x %.2f ms", g ? ", " : "", gtl_names[k], + (double)cnt[k] / frames, gpu[k] / f); + } + rt_log("perf: frame %llu: GPU timeline per frame: busy %.2f ms; idle %.2f ms starved (nothing in the queue; %.2f ms of " + "it while the next piece was held in the submit batching) and %.2f ms with work in the queue, before %s; held " + "in the submit batching %.2f ms in all", (unsigned long long)frame, busy / f, starved / f, held_idle / f, + queued / f, by, held / f); + rt_log("perf: frame %llu: GPU timeline: pieces per frame and their GPU time: %s; from hand-over to start on an idle " + "GPU, on average: %s; %d pieces, %llu dropped", (unsigned long long)frame, gp, la, n, (unsigned long long)dropped); +} + // Just before vkQueueSubmit2 (sub_lock held): the parts closed since the last submit leave the CPU now. Their reads // are queued in the order the GPU runs them: a mirror prologue is closed after the part it was made for but runs // first. Read after its part, its copies counted neither as busy time nor as a gap of their own: they showed up as // idle time with work queued, 7-8 ms a frame over PCIe 3.0 in Kokiri Forest, and the frame as CPU-bound. -static void ts_submitted(void) { +// cbi: the submission's command buffers in queue order; their pairs are read in that order (a part's pre-pass command +// buffer and a cut part's are closed in another order than they run). +static void ts_submitted(const VkCommandBufferSubmitInfo *cbi, uint32_t ncb) { uint64_t now = plat_now_ns(); gfx_tl_note('S', now, cur_fence, 0); - for (int pass = 0; pass < 2; pass++) + for (uint32_t k = 0; k <= ncb; k++) // k == ncb: whatever no command buffer named, last for (int i = 0; i < ts_n_unsent; i++) { TsRead *t = ts_unsent[i]; - if (!t->mirror != pass) continue; // the prologues first + if (!t || (k < ncb ? t->cb != cbi[k].commandBuffer : 0)) continue; ts_sub_ns[t->pair] = now; + ts_queue_ns[t->pair] = 0; + if (nhold_pend < HOLD_PAIRS) hold_pend[nhold_pend++] = t->pair; vk_completion_add(t->fence, ts_read_cb, t); + ts_unsent[i] = NULL; } ts_n_unsent = 0; } @@ -435,12 +657,14 @@ static void ts_read_cb(void *arg) { // completion thread, in submission order if (sub && sn > sub) plat_add64(&ts_queued_ns, sn - sub); gfx_tl_note('G', (uint64_t)sn, (uint64_t)en, (uint64_t)sub); if (en > ts_last_end_ns) ts_last_end_ns = en; + int64_t q = (int64_t)ts_queue_ns[t->pair]; + vk_gtl_note_ns(t->mirror, sub, q ? q : sub, sn, en); } uint64_t from = s > ts_last_end ? s : ts_last_end; if (e > from) { int64_t ns = (int64_t)((double)(e - from) * ts_period); plat_add64(&ts_busy_ns, ns); - if (t->mirror) plat_add64(&ts_mirror_ns, ns); + if (t->mirror == GTL_MIRROR) plat_add64(&ts_mirror_ns, ns); plat_add64(&ts_busy_all, ns); perf_gpu_busy((uint64_t)ns); } @@ -558,8 +782,8 @@ static void ts_close(VkCommandBuffer cb, int pair, uint64_t fence, int mirror) { ts_sub_ns[pair] = 0; TsRead *t = malloc(sizeof *t); if (!t) rt_fatal("vk: out of memory"); - *t = (TsRead){(uint32_t)pair, mirror, fence}; - if (ts_n_unsent < 4) ts_unsent[ts_n_unsent++] = t; + *t = (TsRead){(uint32_t)pair, mirror, fence, cb}; + if (ts_n_unsent < TS_UNSENT) ts_unsent[ts_n_unsent++] = t; else vk_completion_add(fence, ts_read_cb, t); // more than a part and its prologue between two submits } @@ -595,6 +819,8 @@ static void gpu_stats(PerfGpu *g) { g->mirror_ns = (uint64_t)plat_xchg64(&ts_mirror_ns, 0); } +WEAK void vk_exec_mesh_flush(VkCommandBuffer cb) { (void)cb; } // without vk_exec_draw.c: nothing is queued + // Weak stand-in for vk_exec.c's replay: only empty recordings ({OP_BEGIN, OP_END}) can be submitted. WEAK int vk_exec_record(VkCommandBuffer cb, const uint64_t *handles, int n, uint64_t fence) { (void)cb, (void)fence; @@ -686,8 +912,10 @@ static void completion_main(void *arg) { // finished submissions held (vk_mem_collect: upload batches, destroyed images, views and buffers, each // checked against the timeline itself) is nothing they read, and it takes cmd_lock, which the upload // path holds, so done waited behind uploads. + uint64_t was = done; __atomic_store_n(&done, reached, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_done); + for (uint64_t v = reached - was > GW_RING ? reached - GW_RING + 1 : was + 1; v <= reached; v++) gw_mark(v, 2); plat_mutex_unlock(&st_lock); vk_mem_collect(); plat_mutex_lock(&st_lock); @@ -763,6 +991,69 @@ static CmdSlot *take_slot(VkCtx *c, uint64_t fence) { return &slots[cur_slot]; } +// ---- the part's pre-pass command buffer (vk.h vk_sync_pre; vk_exec_draw.c mesh emulation, NX2NAVI_MESH_EMU_BATCH) +// Begun at the first call of a part, submitted right before the part (after the mirror prologue) and ended there. Its +// own pool: with the recorder on, the recorder thread records the part (cmd_pool) while the replay records this. +static VkCommandPool pre_pool; +static CmdSlot *pre_slots; +static int npre, pre_pair = -1; +static VkCommandBuffer pre_cb; + +VkCommandBuffer vk_sync_pre(void) { + if (pre_cb) return pre_cb; + VkCtx *c = vk_ctx(); + if (!pre_pool) { + VkCommandPoolCreateInfo pi = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, + .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .queueFamilyIndex = c->gfx_family}; + if (vkCreateCommandPool(c->device, &pi, NULL, &pre_pool) != VK_SUCCESS) return VK_NULL_HANDLE; + } + uint64_t gpu = 0; + vkGetSemaphoreCounterValue(c->device, c->timeline, &gpu); + int i = 0; + while (i < npre && pre_slots[i].fence > gpu) i++; + if (i == npre) { + CmdSlot *n = realloc(pre_slots, sizeof *n * (size_t)(npre + 1)); + if (!n) return VK_NULL_HANDLE; + pre_slots = n; + VkCommandBufferAllocateInfo ai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pre_pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; + if (vkAllocateCommandBuffers(c->device, &ai, &pre_slots[npre].cb) != VK_SUCCESS) return VK_NULL_HANDLE; + npre++; + } + pre_slots[i].fence = cur_fence; // the submission's value: its parts are done by then + VkCommandBuffer cb = pre_slots[i].cb; + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(cb, &bi); + pre_pair = ts_open(cb); + // everything earlier in queue order (the parts before, the mirror prologue) before what it records + VkMemoryBarrier2 mb = {.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; + VkDependencyInfo di = {.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .memoryBarrierCount = 1, .pMemoryBarriers = &mb}; + vkCmdPipelineBarrier2(cb, &di); + pre_cb = cb; + return cb; +} + +// Command buffers of the submission that a cut (vk_sync_cut) ended and that go to the queue with the next part, in +// order before it: a cut part's pre-pass command buffer, then the part. +static VkCommandBuffer chain[CHAIN_MAX]; +static int nchain; + +// The pre-pass command buffer of the part that goes now, ended; VK_NULL_HANDLE without one. +static VkCommandBuffer pre_end(uint64_t fence) { + VkCommandBuffer cb = pre_cb; + if (!cb) return VK_NULL_HANDLE; + vk_exec_mesh_flush(cb); // the stages of the pass's emulated draws, by stage + ts_close(cb, pre_pair, fence, GTL_PRE); + pre_pair = -1; + vkEndCommandBuffer(cb); + pre_cb = VK_NULL_HANDLE; + return cb; +} + // The copies the replay's first accesses to VRAM mirrors queued (vk_memory.c), in a command buffer of their own that is // submitted right before the part that needs them and timed like a part; VK_NULL_HANDLE when there are none. After the // part was recorded, sub_lock held; may move slots[]. @@ -783,26 +1074,59 @@ static VkCommandBuffer prologue(VkCtx *c, uint64_t fence) { return cb; } -// The command buffers of the part that ends now, in queue order, into cbi: the prologue (when there is one), the -// part's earlier pieces (vk_sync_append) and, with `with_last`, its last command buffer. Empties the list. -static uint32_t part_list(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer last, int with_last) { +// What goes to the queue for a part: the mirror prologue, the parts cut before it with their pre-pass command buffers +// and pieces (the chain, emptied), the part's own pre-pass command buffer, the part's earlier pieces (vk_sync_append: +// the replay's and its workers', emptied), the part's last command buffer. Any of them may be missing. +static uint32_t chain_infos(VkCommandBufferSubmitInfo *cbi, VkCommandBuffer pro, VkCommandBuffer pre, VkCommandBuffer cb) { uint32_t n = 0; - VkCommandBufferSubmitInfo one = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}; - if (pro) one.commandBuffer = pro, cbi[n++] = one; +#define CHAIN_ADD(x) \ + if (x) cbi[n++] = (VkCommandBufferSubmitInfo){.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .commandBuffer = (x)} + CHAIN_ADD(pro); + for (int k = 0; k < nchain; k++) CHAIN_ADD(chain[k]); + CHAIN_ADD(pre); for (int i = 0; i < npart_cbs; i++) { if (!part_cbs[i]) rt_fatal("vk: a replay worker's command buffer is missing from a part"); - one.commandBuffer = part_cbs[i], cbi[n++] = one; + CHAIN_ADD(part_cbs[i]); } - if (with_last) one.commandBuffer = last, cbi[n++] = one; - npart_cbs = 0; + CHAIN_ADD(cb); +#undef CHAIN_ADD + nchain = 0, npart_cbs = 0; return n; } +// A cut: the part ends and the replay goes on in a new command buffer, as at a split, but nothing goes to the queue: +// the part (its pre-pass command buffer, its pieces and itself) waits in the chain for the next part that does. The +// texture pool set stays as it is, as nothing of the submission has been handed to the GPU. VK_NULL_HANDLE when the +// chain is full. +VkCommandBuffer vk_sync_cut(void) { + if (nchain + npart_cbs + 2 > CHAIN_MAX) return VK_NULL_HANDLE; + for (int i = 0; i < npart_cbs; i++) + if (!part_cbs[i]) return VK_NULL_HANDLE; // a replay worker's piece not filled yet: the part goes on + VkCtx *c = vk_ctx(); + VkCommandBuffer cb = slots[cur_slot].cb; + vk_gpu_passes_part(cb, 1); + ts_end(cb, cur_fence); + if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); + VkCommandBuffer pre = pre_end(cur_fence); + if (pre) chain[nchain++] = pre; + for (int i = 0; i < npart_cbs; i++) chain[nchain++] = part_cbs[i]; + npart_cbs = 0; + chain[nchain++] = cb; + CmdSlot *s = take_slot(c, cur_fence); // the cut part keeps its slot until the value is reached + VkCommandBufferBeginInfo bi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT}; + vkBeginCommandBuffer(s->cb, &bi); + ts_begin(s->cb); + rec_begin(s->cb, cur_fence); + vk_gpu_passes_part(s->cb, 0); + return s->cb; +} + // The replay workers (vk_exec.c "replay workers"): the replay's command buffer being recorded ends and joins the // part's list, then n places for the command buffers the workers record, in that order, and a new command buffer of // the same part is begun and returned: the same timeline value, timed and submitted with the rest, so the workers' // commands run between the replay's commands before and after. The places are filled (vk_sync_append_fill) before the -// part is submitted, which part_list checks. vk_sync_append_room: whether the list has room for n more. +// part is submitted, which chain_infos checks. vk_sync_append_room: whether the list has room for n more. int vk_sync_append_room(int n) { return npart_cbs + n + 1 <= PART_CBS; } VkCommandBuffer vk_sync_append_later(int n, int *at) { @@ -864,7 +1188,6 @@ VkCommandBuffer vk_sync_worker_begin(int w) { } void vk_sync_worker_end(VkCommandBuffer cb) { vkEndCommandBuffer(cb); } - static VkCommandBuffer split_part(int exec); VkCommandBuffer vk_sync_split(void) { return split_part(0); } // The part that ends here is the last with the submission's own commands: it signals the executed timeline. @@ -877,15 +1200,15 @@ static VkCommandBuffer split_part(int exec) { ts_end(cb, cur_fence); if (!rec_end_cb(cb, 0, 0)) vkEndCommandBuffer(cb); // every command of the part is in cb, ended by its recorder vk_upload_flush(0); // the uploads this part asked for precede it in queue order - VkCommandBuffer pro = prologue(c, cur_fence); - VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = part_list(cbi, pro, cb, 1), - .pCommandBufferInfos = cbi}; + VkCommandBuffer pro = prologue(c, cur_fence), pre = pre_end(cur_fence); + VkCommandBufferSubmitInfo cbi[SUB_CBS]; + uint32_t ncb = chain_infos(cbi, pro, pre, cb); // the mirror copies, the cut parts, the pre-pass work, the part + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi}; VkSemaphoreSubmitInfo esig = {.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = exec_tl, .value = cur_fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}; if (exec && exec_tl && exec_sig < cur_fence) si.signalSemaphoreInfoCount = 1, si.pSignalSemaphoreInfos = &esig, exec_sig = cur_fence; - ts_submitted(); + ts_submitted(cbi, ncb); queue_submit(c, &si, 0); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); vk_mem_note_submit(cur_fence); // objects destroyed from here on may be in use by the part just submitted @@ -908,6 +1231,7 @@ uint64_t vk_sync_reserve(void) { if (reserved < issued) reserved = issued; uint64_t v = ++reserved; plat_mutex_unlock(&st_lock); + gw_mark(v, 0); return v; } @@ -939,6 +1263,7 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { vk_crumb(s->cb, VK_CRUMB_SUBMIT, fence); ts_begin(s->cb); rec_begin(s->cb, fence); + vk_gpu_passes_fence(fence); // the submission the parts and results that follow belong to vk_gpu_passes_part(s->cb, 0); int recorded = 1; if (own) own->record(s->cb, own->arg); @@ -953,10 +1278,11 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { if (cpu0) perf_replay_cpu(plat_thread_cpu_ns(me) - cpu0); // After the recording: the uploads it asked for (and every earlier one) precede it in queue order. vk_upload_flush(0); - VkCommandBuffer pro = prologue(c, fence); + VkCommandBuffer pro = prologue(c, fence), pre = pre_end(fence); s = &slots[cur_slot]; - VkCommandBufferSubmitInfo cbi[PART_CBS + 2]; - uint32_t ncbi = part_list(cbi, pro, s->cb, recorded); + VkCommandBufferSubmitInfo cbi[SUB_CBS]; + int sent = recorded || nchain || npart_cbs; // a cut part or a piece goes with the part after it, which goes too + uint32_t ncb = chain_infos(cbi, pro, pre, sent ? s->cb : VK_NULL_HANDLE); // the timeline, and the executed timeline unless a part of this submission signalled it already (split_part) VkSemaphoreSubmitInfo sig[2] = {{.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = c->timeline, .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}, @@ -964,17 +1290,28 @@ static uint64_t submit(const uint64_t *handles, int n, const OwnCommands *own) { .value = fence, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT}}; int both = exec_tl && exec_sig < fence; if (both) exec_sig = fence; - VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncbi, - .pCommandBufferInfos = cbi, .signalSemaphoreInfoCount = both ? 2u : 1u, - .pSignalSemaphoreInfos = sig}; - ts_submitted(); + VkSubmitInfo2 si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .commandBufferInfoCount = ncb, .pCommandBufferInfos = cbi, + .signalSemaphoreInfoCount = both ? 2u : 1u, .pSignalSemaphoreInfos = sig}; + ts_submitted(cbi, ncb); queue_submit(c, &si, fence); // held or handed to the queue (submit batching) vk_mem_lazy_handed(); - s->fence = recorded ? fence : 0; + s->fence = sent ? fence : 0; vk_mem_note_submit(fence); + gw_mark(fence, 1); + // after the last part's prologue: every staged copy of the submission is taken; a probe's submit leaves the note to + // the replay's submission it may have come in the middle of + int live = own ? 1 : vk_mem_take_live(); + int res = own ? 0x1f : vk_exec_take_results(); plat_mutex_lock(&st_lock); + live_of[fence & (LIVE_RING - 1)] = fence << 1 | (uint64_t)(live != 0); + if (live) live_last = fence; + live_last_of[fence & (LIVE_RING - 1)] = live_last; + res_kinds[fence & (LIVE_RING - 1)] = (uint8_t)res; + if (res & res_mask) res_last = fence; + res_of[fence & (LIVE_RING - 1)] = res_last; __atomic_store_n(&issued, fence, __ATOMIC_RELEASE); plat_cond_broadcast(&cv_work); + plat_cond_broadcast(&cv_done); // vk_sync_recorded_wait plat_mutex_unlock(&st_lock); static unsigned subs; if (++subs % 4096 == 0) { @@ -1049,9 +1386,94 @@ static void results_run(uint64_t upto) { static int fence_wait_impl(uint64_t value, uint64_t timeout_ns); + +// The game's wait after a frame's first submission (nvn_api_core.c NX2NAVI_SUBMIT_FINISH): until the replay has +// submitted value, which took every copy of guest memory the submission reads; then also until the GPU has run it when +// its GPU reads guest memory itself (vk_mem_take_live) or NX2NAVI_SUBMIT_FINISH_GPU=1. 1 when reached. +int vk_sync_result_mask(void) { return res_mask; } + +int vk_sync_recorded_wait(uint64_t value, uint64_t timeout_ns, int (*gpu_wait)(uint64_t, uint64_t)) { + vk_sync_start(); + static int gpu = -1, res_mode = 1; + if (gpu < 0) { + res_mode = rt_knob("SUBMIT_FINISH_RESULTS", "1"); + const char *km = cfg_get("NX2NAVI_SUBMIT_FINISH_KINDS"); // a number (rt_knob reads only 0, 1 and ab) + res_mask = (km && *km ? (int)strtol(km, NULL, 0) : 31) & 0x1f; + rt_log("vk: result kinds the submit wait counts: %#x (NX2NAVI_SUBMIT_FINISH_KINDS: 1 events, 2 counter reports, " + "4 occlusion reports, 8 GPU writes of unmirrored memory, 16 mirror write-backs)", res_mask); + rt_log("vk: the wait after a frame's first submission also waits for %s (NX2NAVI_SUBMIT_FINISH_RESULTS=%d)", + res_mode == 2 ? "every submission before it" : res_mode ? "the newest submission up to it that hands the " + "CPU results at its completion (events, counter reports, write-backs)" : "nothing more", res_mode); + gpu = rt_knob("SUBMIT_FINISH_GPU", "0") != 0; + rt_log("vk: the wait after a frame's first submission ends %s (NX2NAVI_SUBMIT_FINISH_GPU=%d)", + gpu ? "when the GPU has run it" : "when the replay has submitted it, or the GPU has run it where it reads guest " + "memory itself", gpu); + } + if (__atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value) return 1; + if (gpu) return gpu_wait(value, timeout_ns); + uint64_t w0 = plat_now_ns(), newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE); + uint64_t sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE); + int live = 1, ok = 1; + plat_mutex_lock(&st_lock); + if (value > issued && value > reserved) ok = 0; // never handed over + while (ok && issued < value && done < value) { + if (timeout_ns < (1ull << 62)) { + uint64_t el = plat_now_ns() - w0; + if (el >= timeout_ns) { ok = 0; break; } + plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); + } else { + plat_cond_wait(&cv_done, &st_lock); + } + } + uint64_t need = 0; // the newest submission up to value whose completion hands the CPU results + if (ok) { + live = done < value && live_of[value & (LIVE_RING - 1)] != (value << 1); + int known = live_of[value & (LIVE_RING - 1)] >> 1 == value; + need = known ? res_of[value & (LIVE_RING - 1)] : value; + if (res_mode == 2) need = value - 1; + if (!res_mode) need = 0; + uint64_t lneed = known ? live_last_of[value & (LIVE_RING - 1)] : value; // the newest reading guest memory + if (lneed > need) need = lneed; + while (!live && need && done < need) { + if (timeout_ns < (1ull << 62)) { + uint64_t el = plat_now_ns() - w0; + if (el >= timeout_ns) { ok = 0; break; } + plat_cond_wait_ns(&cv_done, &st_lock, timeout_ns - el); + } else { + plat_cond_wait(&cv_done, &st_lock); + } + } + } + plat_mutex_unlock(&st_lock); + uint64_t w1 = plat_now_ns(); + if (w1 > w0) { + if (!live) gw_wait(value, w0, w1, newest > sent ? newest : sent, sent); // a GPU wait counts there + plat_mutex_lock(&gw_lock); + gw_rec_n++; + if (!live) gw_rec_done++; + if (!live && need == value) gw_rec_self++; + if (!live && need) vk_gpu_passes_waited(need); + if (!live && need && need <= value) { + gw_need_back += value - need, gw_need_n++; + int k = res_kinds[need & (LIVE_RING - 1)]; + for (int b = 0; b < VK_RES_KINDS; b++) if (k >> b & 1) gw_need_kind[b]++; + if (need < value) { // what the submissions between need and value had that the mask leaves out + for (uint64_t v = need + 1; v <= value && v - need < 64; v++) + for (int b = 0; b < VK_RES_KINDS; b++) if (res_kinds[v & (LIVE_RING - 1)] >> b & 1) gw_skip_kind[b]++; + } + } + plat_mutex_unlock(&gw_lock); + } + if (!ok) return 0; + return live ? gpu_wait(value, timeout_ns) : 1; +} + int vk_sync_fence_wait(uint64_t value, uint64_t timeout_ns) { uint64_t p0 = perf_begin(); + uint64_t newest = __atomic_load_n(&reserved, __ATOMIC_ACQUIRE), sent = __atomic_load_n(&issued, __ATOMIC_ACQUIRE); + uint64_t w0 = __atomic_load_n(&done, __ATOMIC_ACQUIRE) >= value ? 0 : plat_now_ns(); int ok = fence_wait_impl(value, timeout_ns); + if (w0 && ok) gw_wait(value, w0, plat_now_ns(), newest > sent ? newest : sent, sent); perf_end(PERF_FENCE, p0); perf_gpu_wait(p0); return ok; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 8d18c0bd..c2972f8a 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -114,6 +114,9 @@ typedef struct Tex { VmaAllocation flat3d_alloc; uint32_t flat3d_up, flat3d_layer; // n_up and the layer of the copy's last refresh uint64_t flat3d_wr; // wr_fence then + uint64_t used_frame; // presented frame (gfx_counters.frames) when a draw binding or lookup last named it + uint8_t in_rt; // in rts[] (bound into pool memory, the GPU wrote it) + uint8_t clobbered; // another image of another layout wrote over its pool memory since its own last write } Tex; // A slot keeps what vk_texture_by_id resolved for it, so a draw binding skips the key lookup and the address chain: the @@ -1097,11 +1100,56 @@ static uint32_t detail_skip(const tex_t *t, const TexKey *k, const TexVkFormat * return most >= 2 && levels >= 4 && w >= DETAIL_LOW_MIN && h >= DETAIL_LOW_MIN ? 2 : 1; } +// Where texture images go when video memory runs short. An RX 580 (8 GiB, AMD 23.10) shows its memory as a 6 GiB +// device-local heap and a 2 GiB device-local, host-visible one. Asked for device-local memory without a budget, VMA +// took the first heap every time: a save load put 5.6 GiB into it against a budget of 5.1 GiB while the other heap +// held 69 MiB, and the next submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY. Each step below allocates within the +// heaps' budgets (VK_EXT_memory_budget): device-local memory of any heap, then any memory the image allows (system +// memory, slower to sample); only the last step allocates past the budget, as before. +static uint64_t placed[VK_MAX_MEMORY_HEAPS], n_spilled; + +static void note_placement(VkCtx *c, VmaAllocation a, int step) { // tex_lock held + VmaAllocationInfo info; + vmaGetAllocationInfo(c->vma, a, &info); + uint32_t heap = c->mem_props.memoryTypes[info.memoryType].heapIndex; + if (heap >= VK_MAX_MEMORY_HEAPS) return; + uint64_t n = ++placed[heap]; + int first_heap = 1; + for (uint32_t h = 0; h < heap; h++) + if (c->mem_props.memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) first_heap = 0; + if (step == 0 && first_heap && (c->mem_props.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT)) return; + if (step) n_spilled++; + if (n <= 4 || !(n & (n - 1))) + rt_log("vk: texture memory: %llu images in heap %u (%s)%s", (unsigned long long)n, heap, + c->mem_props.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ? "video memory" : "system memory", + step == 2 ? ", past the budget" : ""); +} + +static VkResult image_alloc(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocationCreateFlags own, VkImage *img, + VmaAllocation *a) { + static const struct { VkMemoryPropertyFlags req; VmaAllocationCreateFlags flags; } step[3] = { + {VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT}, + {0, VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT}, + {0, 0}}; + VkResult r = VK_ERROR_OUT_OF_DEVICE_MEMORY; + for (int i = 0; i < 3; i++) { + VmaAllocationCreateInfo ai = {.flags = step[i].flags | own, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, + .requiredFlags = step[i].req}; + r = vmaCreateImage(c->vma, ci, &ai, img, a, NULL); + if (r == VK_SUCCESS) { + note_placement(c, *a, i); + return r; + } + } + return r; +} + // A texture whose content the CPU provides (tex_mirrored), up to SUBALLOC_MAX, takes memory from VMA's device blocks: // the driver prefers a dedicated allocation for most images, and vmaCreateImage follows that, so a scene load // (about 2700 new textures in one frame entering Kokiri Forest) made as many vkAllocateMemory calls, 43% of that // 680 ms frame. vmaAllocateMemory with the plain requirements sub-allocates instead, from the blocks the headroom -// thread keeps ahead (vk_headroom.c). Render targets keep vmaCreateImage and the driver's preference (vma_image). +// thread keeps ahead (vk_headroom.c), within the budget: past it, the next device-local memory type that has room +// (image_alloc in vma_image). Render targets keep vmaCreateImage and the driver's preference (vma_image). // VK_NULL_HANDLE when the image does not take that memory. enum { SUBALLOC_MAX = 32 << 20 }; static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocation *alloc) { @@ -1113,9 +1161,11 @@ static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocati VkImageMemoryRequirementsInfo2 ri = {.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, .image = img}; vkGetImageMemoryRequirements2(c->device, &ri, &mr2); VkMemoryRequirements mr = mr2.memoryRequirements; - VmaAllocationCreateInfo ai = {.requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; + VmaAllocationCreateInfo ai = {.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT}; int ok = !dr.requiresDedicatedAllocation && mr.size <= SUBALLOC_MAX && vmaAllocateMemory(c->vma, &mr, &ai, alloc, NULL) == VK_SUCCESS; + if (ok) note_placement(c, *alloc, 0); if (ok && vmaBindImageMemory(c->vma, *alloc, img) != VK_SUCCESS) { vmaFreeMemory(c->vma, *alloc); ok = 0; @@ -1127,18 +1177,92 @@ static VkImage suballoc_image(VkCtx *c, const VkImageCreateInfo *ci, VmaAllocati return img; } -// An image made with vmaCreateImage. One of DEDICATED_MIN or more gets memory of its own: freed inside one of VMA's -// 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, against the budget and the -// system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the block emptied; its own -// memory goes back when it does. +// An image made with vmaCreateImage, by image_alloc's steps. One of DEDICATED_MIN or more gets memory of its own: +// freed inside one of VMA's 256 MiB blocks it left a hole the driver went on counting, with the rest of the block, +// against the budget and the system's commit (0.5 to 1.8 GiB of such free space in a session at 5120x2160), until the +// block emptied; its own memory goes back when it does. static VkResult vma_image(VkCtx *c, const VkImageCreateInfo *ci, const ImgKey *ik, VkImage *img, VmaAllocation *alloc) { - VmaAllocationCreateInfo ai = {.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}; const VkMemoryRequirements *known = nvn_hm_get(&img_reqs, ik); - if ((known ? known->size : image_size(ci)) >= DEDICATED_MIN) ai.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; - return vmaCreateImage(c->vma, ci, &ai, img, alloc, NULL); + VmaAllocationCreateFlags own = (known ? known->size : image_size(ci)) >= DEDICATED_MIN + ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0; + return image_alloc(c, ci, own, img, alloc); } // defer_init: the caller records the image's first transition (init_image or alias_copy). +// ---------------------------------------------------------------- overwritten pool memory (NX2NAVI_TEX_CLOBBER) +// The game places transient render targets over each other in its memory pools, and the replay binds their images +// into the same memory (bind_pool_memory). An image of another format or size that renders over an image's memory +// leaves it holding the other image's texels in the GPU's own tiled and compressed layout, which reads back as +// unrelated numbers. Ocarina's volumetric fog reads its 360x202 RGBA32F fog-volume texture at the start of a frame, +// before it draws it again; at the end of the previous frame the 1440x810 R11G11B10F bloom target had been drawn over +// that memory, and the fog lighting took those numbers as volume distances and types: teal splotches in the fog of +// Kokiri Forest, on AMD and NVIDIA. An image read while in that state gets memory of its own from then on +// (private_keys), so it keeps what it last drew; the read that found it takes an empty image once. +// NX2NAVI_TEX_CLOBBER=0 keeps such images in pool memory. +static Tex **rts; // pool-bound images the GPU wrote, sorted by address +static int nrts, crts; +static uint64_t max_rt; // largest storage size in rts[] +static nvn_hmap private_keys = {.klen = sizeof(TexKey)}; +static int clobber_fix = -1; +static volatile int32_t n_unaliased; +static uint64_t n_unaliased_period, n_unalias_kept; // since the last perf line: given own memory, left in pool memory + +static int rts_lower(uint64_t a) { + int lo = 0, hi = nrts; + while (lo < hi) { + int m = (lo + hi) / 2; + if (rts[m]->pub.addr < a) lo = m + 1; + else hi = m; + } + return lo; +} + +static void rt_add(Tex *e) { + if (e->in_rt) return; + if (nrts == crts) { + crts = crts ? 2 * crts : 256; + if (!(rts = realloc(rts, sizeof *rts * (size_t)crts))) rt_fatal("vk: out of memory for the render target list"); + } + int i = rts_lower(e->pub.addr); + memmove(rts + i + 1, rts + i, sizeof *rts * (size_t)(nrts - i)); + rts[i] = e; + nrts++; + if (e->t.L.size > max_rt) max_rt = e->t.L.size; + e->in_rt = 1; +} + +static void rt_remove(Tex *e) { + if (!e->in_rt) return; + for (int i = rts_lower(e->pub.addr); i < nrts && rts[i]->pub.addr == e->pub.addr; i++) + if (rts[i] == e) { + memmove(rts + i, rts + i + 1, sizeof *rts * (size_t)(nrts - i - 1)); + nrts--; + break; + } + e->in_rt = 0; +} + +static int alias_compatible(const Tex *e, const Tex *src, const char **why); + +// e, bound into pool memory, was just written by the GPU: its own content is current, and the pool-bound images of +// another layout whose storage overlaps it no longer hold theirs. +static void mark_clobbered(Tex *e) { + if (clobber_fix < 0) clobber_fix = cfg_bool("NX2NAVI_TEX_CLOBBER"); + e->clobbered = 0; + if (!clobber_fix || !e->bound_to) return; + rt_add(e); + uint64_t lo = e->pub.addr, hi = lo + e->t.L.size; + for (int i = rts_lower(lo > max_rt ? lo - max_rt : 0); i < nrts && rts[i]->pub.addr < hi; i++) { + Tex *o = rts[i]; + if (o == e || o->clobbered || o->bound_to != e->bound_to || o->pub.addr + o->t.L.size <= lo) continue; + const char *why; // the same texels through both: a compatible image of the same shape (another size tiles apart) + if (o->pub.image == e->pub.image || (alias_compatible(o, e, &why) && o->pub.width == e->pub.width && + o->pub.height == e->pub.height && o->pub.depth == e->pub.depth)) + continue; + o->clobbered = 1; + } +} + static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { static volatile int32_t n_unmapped, n_unsupported, n_failed, n_unbound, n_thick; VkCtx *c = vk_ctx(); @@ -1250,7 +1374,9 @@ static Tex *create_tex(const tex_t *t, const TexKey *k, int defer_init) { e->pub.addr = k->addr; img_key_of(&ci, &e->ik); VkImage img = VK_NULL_HANDLE; - if (scale == 1 && nvn_gpu_only(k->addr)) { // storage in a physical or virtual pool: alias the pool memory (section 3.2) + // storage in a physical or virtual pool: alias the pool memory (section 3.2), unless an image of another layout + // overwrote it there before it was read (private_keys) + if (scale == 1 && nvn_gpu_only(k->addr) && !nvn_hm_get(&private_keys, k)) { const VkMemoryRequirements *q = nvn_hm_get(&img_reqs, &e->ik); VkMemSpan sp; if (q && !pool_fits(c, e, q, L.size, &sp)) { // it would not bind there (img_reqs) @@ -1657,41 +1783,6 @@ int vk_texture_slot_retry_locked(int id) { static void retire_remove(Tex *e); -static void destroy_tex(Tex *e) { - if (e->retiring) retire_remove(e); - pool_unlist(e); - pool_forget(e); - own_remove(e); - gwin_remove(e); - n_site[0]++; - tex_gen_bump(e->key.addr); - nvn_hm_del(&by_key, &e->key); - Tex *head = nvn_hm_get(&by_addr, &e->key.addr); - if (head == e) { - if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at); - else nvn_hm_del(&by_addr, &e->key.addr); - } else { - for (Tex *p = head; p; p = p->next_at) - if (p->next_at == e) { p->next_at = e->next_at; break; } - } - if (e->released) nreleased--, rel_unlink(e); - if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) - vk_defer_image(e->pub.image, e->alloc); - if (e->flat3d) { // its 3D copy goes the same way - vk_exec_image_gone(e->flat3d->image); - vk_defer_image(e->flat3d->image, e->flat3d_alloc); - e->flat3d->gone = 1; - vk_defer_free(e->flat3d); - } - free(e->snap); - // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a - // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on - // whichever thread released the texture. Freed here, a pass that began again read the freed record, cached a view of - // the dead image (vk_exec_target_view), and a later image given the same handle drew through it into freed memory. - e->pub.gone = 1; - vk_defer_free(e); -} - // ---- a flat texture sampled as a 3D one // Maxwell samples a texture by its own descriptor, whatever the shader's instruction says: a 2D texture under a 3D // sample gives its texel at (u, v), and the third coordinate does nothing. Vulkan has no 3D view of a 2D image, so @@ -1765,7 +1856,6 @@ const VkTexImage *vk_texture_flat3d_locked(const VkTexImage *img, uint32_t layer if (cb) vk_upload_end(); return e->flat3d; } - // ---- origins (NX2NAVI_TRACE=perf; a run without it tests the flag and counts the drops, nothing more) // A texture drawn without its content leaves nothing in the log: no call fails and the image exists. What can still be // told afterwards is where the entry came from, so each entry keeps how it was made and how its content started @@ -2005,6 +2095,85 @@ static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { return k < (int)n ? k : (int)n - 1; } +// NX2NAVI_LOG_TEX_NEW= (diagnostics): from that presented frame on, the first LOG_NEW_MAX textures made are +// logged with what was at their address and, when the same key was destroyed before, how many frames ago and from +// where (destroy_tex's caller, an address the map file names: "destroy_tex at" gives the reference). For the steady +// creations a still scene shows (about seven a frame in Kokiri Forest), each with its image, memory and upload. +#define LOG_NEW_MAX 400 +#define GONE_N 4096 +static struct { uint64_t h, frame; void *ra; } gone[GONE_N]; +static uint32_t gone_n; +static long long log_new_at = -2; +static int log_new_n; + +static void log_new(const Tex *e, const TexKey *k) { // tex_lock held + if (log_new_at == -2) { + const char *v = cfg_get("NX2NAVI_LOG_TEX_NEW"); + log_new_at = v && *v ? atoll(v) : -1; + } + uint64_t now = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + if (log_new_at < 0 || now < (uint64_t)log_new_at || log_new_n >= LOG_NEW_MAX) return; + log_new_n++; + int live = 0, rel = 0; + for (const Tex *p = nvn_hm_get(&by_addr, &k->addr); p; p = p->next_at) + if (p != e) p->released ? rel++ : live++; + uint64_t h = key_hash(k); + char was[160] = "never destroyed before"; + for (uint32_t i = 0; i < GONE_N && i < gone_n; i++) { + uint32_t j = (gone_n - 1 - i) & (GONE_N - 1); + if (gone[j].h == h) { + snprintf(was, sizeof was, "the same key destroyed %llu frames ago from %p (destroy_tex at %p)", + (unsigned long long)(now - gone[j].frame), gone[j].ra, (void *)destroy_tex); + break; + } + } + rt_log("vk: new texture frame %llu: %#llx %ux%ux%u nvn fmt %#x target %d levels %u%s%s; at its address %d live, %d " + "released; %s", (unsigned long long)now, (unsigned long long)k->addr, e->pub.width, e->pub.height, + e->pub.depth, k->d.format, k->d.target, e->pub.levels, e->pub.mirrored ? ", mirrored" : "", + e->alloc ? ", own memory" : "", live, rel, was); +} + +__attribute__((noinline)) static void destroy_tex(Tex *e) { + if (log_new_at >= 0) { // NX2NAVI_LOG_TEX_NEW: who destroyed which key, for the log of new textures + uint32_t j = gone_n++ & (GONE_N - 1); + gone[j].h = key_hash(&e->key), gone[j].frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); + gone[j].ra = __builtin_return_address(0); + } + if (e->retiring) retire_remove(e); + pool_unlist(e); + pool_forget(e); + own_remove(e); + gwin_remove(e); + rt_remove(e); + n_site[0]++; + tex_gen_bump(e->key.addr); + nvn_hm_del(&by_key, &e->key); + Tex *head = nvn_hm_get(&by_addr, &e->key.addr); + if (head == e) { + if (e->next_at) nvn_hm_put(&by_addr, &e->key.addr, e->next_at); + else nvn_hm_del(&by_addr, &e->key.addr); + } else { + for (Tex *p = head; p; p = p->next_at) + if (p->next_at == e) { p->next_at = e->next_at; break; } + } + if (e->released) nreleased--, rel_unlink(e); + if (e->pub.image) vk_exec_image_gone(e->pub.image); // none when another texture took it (create_tex, reuse) + vk_defer_image(e->pub.image, e->alloc); + if (e->flat3d) { // its 3D copy goes the same way + vk_exec_image_gone(e->flat3d->image); + vk_defer_image(e->flat3d->image, e->flat3d_alloc); + e->flat3d->gone = 1; + vk_defer_free(e->flat3d); + } + free(e->snap); + // The record goes once the GPU is past the submission being recorded, as its image: the replay keeps &e->pub as a + // render target for the rest of a command buffer (vk_exec.c resolve_target) and DLSS for the frame, and this runs on + // whichever thread released the texture. Freed here, a pass that began again read the freed record, cached a view of + // the dead image (vk_exec_target_view), and a later image given the same handle drew through it into freed memory. + e->pub.gone = 1; + vk_defer_free(e); +} + // Released entries with another descriptor at the same address are replaced; past RELEASED_MAX released entries one // goes at each release (it comes back on demand, a mirrored one re-uploaded from its storage). NX2navi // finalizes and initializes transient textures every frame, so a released entry is mostly taken back within frames. @@ -2069,7 +2238,6 @@ static int others_at(char *out, size_t n, const Tex *e, const Tex *skip) { #define RELEASED_HARD 16384 #define SWEEP_LOOK 64 #define SWEEP_UNSAMPLED PRESENTS(120) - static int comes_back_intact(const Tex *e); static uint64_t stale_after(const Tex *e); @@ -3998,6 +4166,12 @@ static Tex *entry_of(const tex_t *t) { if (pool_ctx) return NULL; retire_cancel(e); } + if (!e && private_keys.n) // another swizzle of an image that has memory of its own (unalias): the same image + for (Tex *p = nvn_hm_get(&by_addr, &k.addr); p; p = p->next_at) + if (!p->released && !p->bound_to && nvn_hm_get(&private_keys, &p->key) && same_but_swizzle(&p->key, &k)) { + touch(p); + return p; + } if (!e) { if (held_out(&k) || create_held(&k)) return NULL; uint64_t c0 = kern_now_ns(); // the entry's creation (VK_TS_CREATE): image, memory, first transition @@ -4015,6 +4189,7 @@ static Tex *entry_of(const tex_t *t) { nvn_hm_put(&by_key, &k, e); n_site[1]++; tex_gen_bump(k.addr); + log_new(e, &k); vk_texstat_time(VK_TS_CREATE, c0); e->samp_frame = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED); // new: not disused before it is drawn } @@ -4128,6 +4303,7 @@ void vk_texture_gpu_written(const tex_t *t) { } pool_gpu_owned(e); e->pub.gpu_owned = 1; + mark_clobbered(e); e->chash_ok = 0; // the image now holds the GPU's content: matching storage bytes no longer mean "unchanged" if (e->pub.skip) { // texture detail leaves out what the GPU can write (detail_skip); said if it ever does static volatile int32_t n_skip_written; @@ -4524,6 +4700,33 @@ uint32_t vk_sampler_gen(void) { return __atomic_load_n(&samp_gen, __ATOMIC_RELAX int vk_texture_slot_count_locked(void) { return ntslots; } int vk_sampler_slot_count_locked(void) { return nsslots; } +// e was read while marked clobbered (mark_clobbered). It and the descriptors that differ from it only in swizzle (the +// same texels: the game renders through one and samples through another) are dropped, and come back as one image in +// memory of its own (create_tex, entry_of). A live descriptor of another format or geometry over its storage reads +// those bytes through the pool memory, which an image of its own would no longer update: e then stays in pool memory +// (the clobbered read is what it was). 1 when e was dropped. +static int unalias(Tex *e) { + e->clobbered = 0; + Tex *sib[16]; + int n = 0; + for (Tex *p = nvn_hm_get(&by_addr, &e->key.addr); p; p = p->next_at) { + if (p == e || p->released) continue; + if (!same_but_swizzle(&p->key, &e->key) || n == 16) { + n_unalias_kept++; + return 0; + } + sib[n++] = p; + } + if (n_unaliased++ < 32) + rt_log("vk: texture %#llx (%ux%u nvn fmt %#x) was read after another image drew over its memory: it gets memory " + "of its own", (unsigned long long)e->pub.addr, e->pub.width, e->pub.height, e->key.d.format); + n_unaliased_period++; + nvn_hm_put(&private_keys, &e->key, (void *)1); + for (int i = 0; i < n; i++) destroy_tex(sib[i]); + destroy_tex(e); // bumps the address generation: every slot naming it or a sibling resolves again + return 1; +} + const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { lock(); const VkTexImage *r = vk_texture_by_id_locked(id, view, has_view); @@ -4534,6 +4737,7 @@ const VkTexImage *vk_texture_by_id(int id, view_t *view, int *has_view) { const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { if (id < 0 || id >= ntslots || !tslots[id].used) return NULL; TexSlot *s = &tslots[id]; +again:; uint32_t g = agen[sinfo[id] & 0xffff]; // resolved again: an entry at its address was created, dropped or released, or the slot registered (s->ce // cleared); an empty result is not kept, as image creation may succeed later, except a held one for the pool @@ -4554,6 +4758,10 @@ const VkTexImage *vk_texture_by_id_locked(int id, view_t *view, int *has_view) { // a draw binding: the texture is in use, whether or not the pool's refresh checked it first in this submission // (by list that check is no lookup, vk_texture_pool_fresh_locked, so the stamp of the use is left here too) if (e && !pool_ctx) touch(e); + if (e && e->clobbered && unalias(e)) { // read after another image overwrote its pool memory: own memory from now on + s->ce = NULL; + goto again; + } // Bindings of one submit share one check, answered from the submit's write-watch snapshot (guest writes during a // replay race the GPU read on hardware too). After one check an entry is either uploaded or never uploads (not // mirrored, GPU-owned or no CPU layout), so a slot that checked its entry in this submit has nothing left to do. A @@ -4624,9 +4832,11 @@ void vk_texture_new_submit(void) { if ((rt_trace & RT_TRACE_PERF) && f != log_at && tex_alias > 0) { log_at = f; rt_log("perf: frame %llu: texture aliases: content copied %llu (%llu from a released texture), filled from storage " - "%llu, left uninitialised %llu; taken back with their own content %llu", + "%llu, left uninitialised %llu; taken back with their own content %llu; read after an overwrite: given own " + "memory %llu, kept in pool memory %llu", (unsigned long long)f * 60, (unsigned long long)n_alias_copied, (unsigned long long)n_alias_released, - (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept); + (unsigned long long)n_alias_upload, (unsigned long long)n_alias_uninit, (unsigned long long)n_alias_kept, + (unsigned long long)n_unaliased_period, (unsigned long long)n_unalias_kept); n_alias_copied = n_alias_upload = n_alias_uninit = n_alias_kept = n_alias_released = 0; if (n_dinit_inline || n_dinit_upload) rt_log("perf: frame %llu: new depth images started at the replay's position %llu (%llu ended an open pass), " @@ -4638,6 +4848,7 @@ void vk_texture_new_submit(void) { "holds dropped by a mark that another entry at the address had taken %llu", (unsigned long long)f * 60, (unsigned long long)n_disuse_shared, (unsigned long long)n_disuse_diverted); n_disuse_shared = n_disuse_diverted = 0; + n_unaliased_period = n_unalias_kept = 0; rt_log("perf: frame %llu: texture re-uploads %llu (%.1f MiB, %.2f ms), after WriteTexels %llu (%.1f MiB, %.2f ms); " "skipped with unchanged bytes %llu (%.1f MiB)", (unsigned long long)f * 60, (unsigned long long)ru_n, ru_bytes / 1048576.0, ru_ns / 1e6, diff --git a/runtime/vk_vma_budget.c b/runtime/vk_vma_budget.c new file mode 100644 index 00000000..a0b7f971 --- /dev/null +++ b/runtime/vk_vma_budget.c @@ -0,0 +1,124 @@ +// Video memory by the heaps' live budgets, for every allocation through VMA (vk.h maps vmaCreateImage, vmaCreateBuffer +// and vmaAllocateMemory here). +// +// An RX 580 (8 GiB, AMD 23.10) has a 6 GiB device-local heap and a 2 GiB device-local, host-visible one. The heap limits +// of vk_device.c (vram_heap_limits) are fixed when the device is made: each heap's budget then, less a tenth for the +// memory VMA does not see. That memory grows later: AMD's FidelityFX (FSR 3.1 upscaling contexts, up to six, and frame +// generation with its swapchain and context) allocates its own, outside VMA. With FSR 3.1 and frame generation on at +// 1080p it reached 882 MiB, and heap 0 held 4644 MiB of VMA's beside it: 5526 of a 5526 MiB budget, while heap 2 had +// 1757 MiB free. The next upload submit failed with VK_ERROR_OUT_OF_DEVICE_MEMORY and the game ended (2026-11-11). +// Only texture images looked at the budget (vk_texture.c image_alloc); render targets, pool memory, mirrors and the +// rest were placed in heap 0 up to VMA's fixed limit. +// +// Here each allocation leaves out the memory types of a device-local heap whose use, as the driver reports it +// (VK_EXT_memory_budget: VMA's budget, which counts the memory VMA does not see), is within the reserve of its budget, +// and VMA places it in the other device-local heap, or system memory, as it does past the fixed limits. The reserve +// (NX2NAVI_VRAM_RESERVE_MB, default 512, 0 off) is room left for allocations VMA does not make, which can come at +// once: AMD's contexts are made when the upscaler or the frame generation setting changes. An allocation that finds +// no room that way is made as asked, as before. Allocations from a VMA pool and those for one memory type alone (the +// headroom's anchors, vk_headroom.c) are made as asked. +#define VK_VMA_BUDGET_IMPL +#include +#include "vk.h" +#include "cfg.h" +#include "host.h" +#include "platform/platform.h" + +static VkDeviceSize reserve_bytes(void) { + static int64_t r = -1; + if (r < 0) { + const char *e = cfg_get("NX2NAVI_VRAM_RESERVE_MB"); + long mb = e && *e ? atol(e) : 512; + r = (int64_t)(mb < 0 ? 0 : mb) << 20; + } + return (VkDeviceSize)r; +} + +static volatile int64_t n_moved; // allocations placed outside a full heap +static volatile uint32_t full_seen; // heaps already logged as full (bit per heap) + +// The memory types of bits that may take a new allocation: those of device-local heaps within reserve of their budget +// are left out. Returns bits itself when nothing is left out or nothing would be left. +static uint32_t budget_bits(VmaAllocator vma, uint32_t bits) { + VkDeviceSize reserve = reserve_bytes(); + if (!reserve) return bits; + const VkPhysicalDeviceMemoryProperties *mp; + vmaGetMemoryProperties(vma, &mp); + VmaBudget b[VK_MAX_MEMORY_HEAPS]; + vmaGetHeapBudgets(vma, b); + uint32_t full = 0; + for (uint32_t h = 0; h < mp->memoryHeapCount && h < VK_MAX_MEMORY_HEAPS; h++) + if ((mp->memoryHeaps[h].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) && b[h].budget && + b[h].usage + reserve > b[h].budget) + full |= 1u << h; + if (!full) return bits; + uint32_t out = 0; + for (uint32_t t = 0; t < mp->memoryTypeCount; t++) + if ((bits >> t & 1) && !(full >> mp->memoryTypes[t].heapIndex & 1)) out |= 1u << t; + if (!out || out == bits) return bits; + uint32_t told = __atomic_fetch_or(&full_seen, full, __ATOMIC_RELAXED); + for (uint32_t h = 0; h < VK_MAX_MEMORY_HEAPS; h++) + if ((full >> h & 1) && !(told >> h & 1)) + rt_log("vk: video memory heap %u full: %llu of its %llu MiB budget in use (%llu MiB of it VMA's), less %llu MiB " + "kept free (NX2NAVI_VRAM_RESERVE_MB): new allocations go to the other heaps while it is", h, + (unsigned long long)(b[h].usage >> 20), (unsigned long long)(b[h].budget >> 20), + (unsigned long long)(b[h].statistics.blockBytes >> 20), (unsigned long long)(reserve >> 20)); + return out; +} + +static void note_moved(void) { + int64_t n = __atomic_add_fetch(&n_moved, 1, __ATOMIC_RELAXED); + if (!(n & (n - 1)) && n >= 16) + rt_log("vk: video memory: %lld allocations placed outside a full heap so far", (long long)n); +} + +uint64_t vk_vma_moved(void) { return (uint64_t)__atomic_load_n(&n_moved, __ATOMIC_RELAXED); } + +// Device-local types of bits (0: all), from the allocator's properties: whether the request could land in a heap +// budget_bits leaves out +static uint32_t all_bits(VmaAllocator vma, uint32_t bits) { + if (bits) return bits; + const VkPhysicalDeviceMemoryProperties *mp; + vmaGetMemoryProperties(vma, &mp); + return mp->memoryTypeCount >= 32 ? ~0u : (1u << mp->memoryTypeCount) - 1; +} + +VkResult vk_vma_create_image(VmaAllocator vma, const VkImageCreateInfo *ici, const VmaAllocationCreateInfo *ai, VkImage *img, + VmaAllocation *a, VmaAllocationInfo *info) { + if (!ai->pool) { + uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); + if (nb != bits) { + VmaAllocationCreateInfo b = *ai; + b.memoryTypeBits = nb; + if (vmaCreateImage(vma, ici, &b, img, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + } + } + return vmaCreateImage(vma, ici, ai, img, a, info); +} + +VkResult vk_vma_create_buffer(VmaAllocator vma, const VkBufferCreateInfo *bci, const VmaAllocationCreateInfo *ai, + VkBuffer *buf, VmaAllocation *a, VmaAllocationInfo *info) { + if (!ai->pool) { + uint32_t bits = all_bits(vma, ai->memoryTypeBits), nb = budget_bits(vma, bits); + if (nb != bits) { + VmaAllocationCreateInfo b = *ai; + b.memoryTypeBits = nb; + if (vmaCreateBuffer(vma, bci, &b, buf, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + } + } + return vmaCreateBuffer(vma, bci, ai, buf, a, info); +} + +VkResult vk_vma_allocate_memory(VmaAllocator vma, const VkMemoryRequirements *req, const VmaAllocationCreateInfo *ai, + VmaAllocation *a, VmaAllocationInfo *info) { + uint32_t bits = req->memoryTypeBits & all_bits(vma, ai->memoryTypeBits); + if (!ai->pool && (bits & (bits - 1))) { // more than one type: one alone is asked for on purpose + uint32_t nb = budget_bits(vma, bits); + if (nb != bits) { + VkMemoryRequirements r = *req; + r.memoryTypeBits = nb; + if (vmaAllocateMemory(vma, &r, ai, a, info) == VK_SUCCESS) return note_moved(), VK_SUCCESS; + } + } + return vmaAllocateMemory(vma, req, ai, a, info); +} diff --git a/tools/make_game.py b/tools/make_game.py index 91dc0c17..36fa9556 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -1425,10 +1425,30 @@ def fast_clang(msys, clang, env): return fast if mine and clang_ident(fast, env) == mine else None +def guest_opt_ini(out): + """NX2NAVI_GUEST_OPT from nx2navi.ini next to the exe (the menu's Lama patches tab writes it there) when the + environment does not set it: the next build of the game folder out takes it.""" + if os.environ.get('NX2NAVI_GUEST_OPT', '').strip(): + return + try: + for line in (Path(out) / 'nx2navi.ini').read_text(encoding='utf-8', errors='replace').splitlines(): + key, eq, value = line.partition('=') + key = key.strip().lower() + if eq and key in ('guest_opt', 'nx2navi_guest_opt'): + value = value.split('#')[0].split(';')[0].strip().strip('"') + if value: + os.environ['NX2NAVI_GUEST_OPT'] = value + except OSError: + pass + + def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): """fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers, - and its command, which is part of its objects' keys, stays as it was.""" + and its command, which is part of its objects' keys, stays as it was. NX2NAVI_GUEST_OPT (an environment variable, + or guest_opt in nx2navi.ini) 1 builds the lifted code at -O1; by default -O2: the game's own threads run it and + bound the frame on a slower CPU; the build takes several times longer.""" + guest_opt = '-O1' if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '1' else '-O2' # -O2 unless guest_opt = 1 def cmd_for(src, dst): cpp = src.suffix in ('.cpp', '.cxx') # the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict @@ -1440,7 +1460,7 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): cc = [str(fast), f'--sysroot={msys}'] if guest and fast else [str(clangxx if cpp else clang)] # -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally declspec = ['-fdeclspec'] if cpp and not WIN else [] - return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', '-O1' if guest else '-O2', + return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', guest_opt if guest else '-O2', *march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)] return cmd_for @@ -1681,6 +1701,20 @@ def retune_due(obj, msys): return True +def write_runtime_map(full, dst): + """The linker's map (the build cache's, which tools/prof_report.py reads) without the lines of the lifted game + functions (f_): the runtime's and the libraries' symbols, a few MiB, next to the exe to send with a crash log + or a NX2NAVI_PROFILE profile of the render thread.""" + lifted = re.compile(r'\bf_[0-9a-f]{4,}\b') # any line naming a lifted function: its symbol or its section + try: + with open(full, encoding='utf-8', errors='replace') as i, open(dst, 'w', encoding='utf-8') as o: + for line in i: + if not lifted.search(line): + o.write(line) + except OSError: + pass + + def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): """romfs_job: the RomFS step still running beside this one (a first build). Everything that does not include romfs_config.h compiles meanwhile; the RomFS index and the few files that include it follow once it is done. @@ -1707,6 +1741,9 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): cid, cid_old = compiler_ids(clang, env) # fast_clang's compiler writes the same objects: not part of the key now = cpu_level(clang, env) level = lifted_level(obj, now, update) + guest_opt_ini(out) + if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2': # runtime_cmd's guest_opt + say(' lifted code at -O2 (NX2NAVI_GUEST_OPT): a long build') if level == now: say(f' the lifted code is built for x86-64-v{level}, this processor\'s level' if level else ' the lifted code is built for generic x86-64: the processor\'s level could not be read') @@ -1794,7 +1831,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): stamp = [[o.name, o.stat().st_size, o.stat().st_mtime_ns] for o in objs] if prebuilt: stamp.append([prebuilt[0].name, prebuilt[0].stat().st_size, sha256_file(prebuilt[0])]) - link_key = sha256_bytes(json.dumps([LINK_LIBS, ld, cid] + stamp).encode()) + link_key = sha256_bytes(json.dumps([LINK_LIBS, ld, cid, 'map'] + stamp).encode()) state_link = cache_dir / 'link.key' if exe.is_file() and state_link.is_file() and state_link.read_text(encoding='utf-8') == link_key: say(f' {EXE_NAME} is current, not linked again') @@ -1814,6 +1851,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): except PermissionError: tmp.unlink(missing_ok=True) fail(f'{exe} is in use: close the game and try again') + write_runtime_map(cache_dir / 'ocarina.map', out / 'nx2navi.map') # next to the exe, to send with a crash log state_link.write_text(link_key, encoding='utf-8') copy_if_changed(gen / 'romfs-index.bin', out / 'romfs-index.bin') xlate = xlate_source()