diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 300a38bc..1a7c7803 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -123,6 +123,9 @@ # vk_dyn_a2c = 1 # Pin the shader subgroup size to 32 when the GPU allows it. # vk_subgroup_pin = 1 +# Keep a shader's threads together after divergent code where the game's shaders expect it (maximal reconvergence, +# when the driver supports it). 0 turns it off; leaves then go missing from the foliage's colour pass. +# vk_reconverge = 1 [display] # Device profile: auto (the Steam Deck when this is one), desktop or deck. On a Steam Deck the keys this file does not diff --git a/runtime/cfg.c b/runtime/cfg.c index 3dde3222..85175bf2 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -59,7 +59,7 @@ static const char *const known[] = { "SHOTS_FULL", "SPEED_PROBE", "SPIN", "SPIN_HANDOFF_US", "SPLIT_DRAWS", "SPLIT_US", "SPV_OVERRIDE", "STICK_DEADZONE", "STICK_SENS", "SWKBD_ANSWER", "SET_AT", "SWKBD_TEST", "TAA", "TEST_PAD", "TEST_PAD_AT", "TEST_PREPARE_SHADERS", "TEX_ALIAS", "TEX_MEMO", "TEX_VRAM_PRESSURE", "TEX_RELEASED_MB", "TEX_COMMIT_FREE_MB", "TIMER_1MS", "TRACE", "TXD_TRANSPOSE", "UBO_SNAPSHOT", "UPSCALER", "VK_DYN_A2C", "VK_DYN_VERTEX", "VK_NOOPT", "VK_NO_FLOAT64", - "VK_PRESENT", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", + "VK_PRESENT", "VK_RECONVERGE", "VK_RECORDER", "VK_SSBO_ALIGN", "VK_SUBGROUP", "VK_SUBGROUP_PIN", "VK_VALIDATION", "VOLUME", "VRAM_HEADROOM", "VS_OUTS", "WALK_KEY", "WALK_SCALE", "WB_EXACT", "WB_FENCE_LIMIT", "WB_MERGE", "WINDOW_SIZE", "XLATE", "XLATE_ID", "XLATE_KEEP", "XLATE_PROCS", "ZSTD_CHECK", "ZSTD_NATIVE", }; @@ -100,7 +100,7 @@ static const struct { const char *key, *section; } sections[] = { {"TEX_ALIAS", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, {"BIND_SKIP", "performance"}, {"SHADER_MEMO", "performance"}, {"PIPE_MEMO", "performance"}, {"TEX_MEMO", "performance"}, {"DESC_TEMPLATE", "performance"}, {"PUSH_SKIP", "performance"}, {"SPLIT_DRAWS", "performance"}, {"SPLIT_US", "performance"}, {"VK_VALIDATION", "performance"}, - {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, + {"VK_DYN_VERTEX", "performance"}, {"VK_DYN_A2C", "performance"}, {"VK_SUBGROUP_PIN", "performance"}, {"VK_RECONVERGE", "performance"}, {"TRACE", "diagnostics"}, {"FLIGHT", "diagnostics"}, {"PASS_CRUMBS", "diagnostics"}, {"FLIGHT_SPIKE_MS", "diagnostics"}, {"WB_FENCE_LIMIT", "diagnostics"}, {"WB_MERGE", "diagnostics"}, {"WB_EXACT", "diagnostics"}, {"MV_SURVEY", "diagnostics"}, @@ -183,6 +183,7 @@ static const CfgMeta metas[] = { // Vulkan {"VK_VALIDATION", "0", "0|1", RESTART}, {"VK_DYN_VERTEX", "", "0|1", RESTART}, {"VK_DYN_A2C", "", "0|1", RESTART}, {"VK_SUBGROUP_PIN", "1", "0|1", RESTART}, + {"VK_RECONVERGE", "1", "0|1", RESTART}, // vk_pipeline.c spirv_reconverge {"VK_NOOPT", "0", "0|1", RESTART}, // vk_pipeline.c noopt_flag // diagnostics {"TRACE", "", NULL, RESTART}, {"FLIGHT", "0", "0|1", RESTART}, {"FLIGHT_SPIKE_MS", "100", "0..60000", RESTART}, diff --git a/runtime/vk.h b/runtime/vk.h index 1ca4f7ac..11f05eee 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -37,7 +37,8 @@ typedef struct VkCtx { has_astc_ldr, // textureCompressionASTC_LDR (enabled when reported) has_calibrated, // VK_KHR/EXT_calibrated_timestamps (GPU idle split in the perf line) has_hdr_metadata, // VK_EXT_hdr_metadata (vk_hdr.c vk_hdr_metadata) - has_mesh_shader, has_mesh_queries, has_vrs; // VK_EXT_mesh_shader with taskShader and meshShader (NVN2 task/mesh stages) + has_mesh_shader, has_mesh_queries, has_vrs, // VK_EXT_mesh_shader with taskShader and meshShader (NVN2 task/mesh stages) + has_maximal_reconvergence; // VK_KHR_shader_maximal_reconvergence (vk_pipeline.c spirv_reconverge) // Feedback loops (maxwell.md 1.9, vk_exec_draw.c feedback_prepare): a draw samples a target of its open pass that // an earlier draw of the pass wrote. NX2NAVI_FEEDBACK=0: VK_FEEDBACK_OFF (sampled in GENERAL without a barrier, the // old behaviour); VK_FEEDBACK_LAYOUT with VK_EXT_attachment_feedback_loop_layout and dynamicRenderingLocalRead diff --git a/runtime/vk_device.c b/runtime/vk_device.c index fdaf30bf..a1cb26a9 100644 --- a/runtime/vk_device.c +++ b/runtime/vk_device.c @@ -234,6 +234,7 @@ typedef struct { VkPhysicalDeviceDynamicRenderingLocalReadFeaturesKHR lr; VkPhysicalDeviceMeshShaderFeaturesEXT mesh; VkPhysicalDeviceFragmentShadingRateFeaturesKHR vrs; + VkPhysicalDeviceShaderMaximalReconvergenceFeaturesKHR mrc; } FeatureChain; static int create_device(void) { @@ -272,6 +273,10 @@ static int create_device(void) { OPTIONAL_EXT(has_mesh_shader, VK_EXT_MESH_SHADER_EXTENSION_NAME, mesh, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MESH_SHADER_FEATURES_EXT); // The foliage's far shadow cascades shade one fragment per 2x2 pixels (vk_exec_draw.c mesh_shading_rate) OPTIONAL_EXT(has_vrs, VK_KHR_FRAGMENT_SHADING_RATE_EXTENSION_NAME, vrs, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADING_RATE_FEATURES_KHR); + // Guest shaders that use warp-wide operations after divergent code run with their threads reconverged + // (vk_pipeline.c spirv_reconverge) + OPTIONAL_EXT(has_maximal_reconvergence, VK_KHR_SHADER_MAXIMAL_RECONVERGENCE_EXTENSION_NAME, mrc, + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MAXIMAL_RECONVERGENCE_FEATURES_KHR); // Feedback loops (vk.h VkCtx.feedback): the layout, and barriers inside dynamic rendering int has_fbl = 0, has_lr = 0; if (has_ext(ep, ne, VK_EXT_ATTACHMENT_FEEDBACK_LOOP_LAYOUT_EXTENSION_NAME) && @@ -365,6 +370,7 @@ static int create_device(void) { if (!c->mesh.meshShader || !c->mesh.taskShader) g_vk.has_mesh_shader = 0; rt_log("vk: mesh shaders: %s", g_vk.has_mesh_shader ? "VK_EXT_mesh_shader" : "not supported (task/mesh draws are skipped)"); } // vkGetDeviceFaultInfoEXT needs the feature, not just the extension + if (g_vk.has_maximal_reconvergence && !c->mrc.shaderMaximalReconvergence) g_vk.has_maximal_reconvergence = 0; g_vk.has_buffer_device_address = c->f12.bufferDeviceAddress; g_vk.has_astc_ldr = h->textureCompressionASTC_LDR; { diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 19eb10ec..7e5aec67 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -79,6 +79,7 @@ static struct { int cache_control; // pipelineCreationCacheControl: a create can ask for the pipeline caches only int pv_last; // provokingVertexLast enabled: NVN's default is the last vertex VkShaderStageFlags sg32_stages; // stages that accept requiredSubgroupSize = 32 (0: not available) + int reconverge; // modules with subgroup ops get MaximallyReconvergesKHR (spirv_reconverge) VkDynamicState dyn[32]; uint32_t n_dyn; PFN_vkCmdPushDescriptorSetKHR push; @@ -237,6 +238,11 @@ static int ensure_init(void) { rt_log("vk_pipeline: subgroup size %u-%u, requiredSubgroupSizeStages %#x, pinning %u on %#x", p13.minSubgroupSize, p13.maxSubgroupSize, (unsigned)p13.requiredSubgroupSizeStages, vk_caps.sg_size, (unsigned)P.sg32_stages); } + // A warp reconverges where the guest compiler put it back together; Vulkan guarantees that only with maximal + // reconvergence (spirv_reconverge). NX2NAVI_VK_RECONVERGE=0 leaves the modules as translated. + P.reconverge = c->has_maximal_reconvergence && rt_knob("VK_RECONVERGE", "1") == 1; + rt_log("vk_pipeline: maximal reconvergence for shaders with subgroup ops: %s", + P.reconverge ? "on" : c->has_maximal_reconvergence ? "off (NX2NAVI_VK_RECONVERGE=0)" : "not supported by the driver"); P.f_full_subgroups = v13.computeFullSubgroups; P.cache_control = v13.pipelineCreationCacheControl; P.f_logic_op = f2.features.logicOp; @@ -867,6 +873,46 @@ static int uses_subgroup_ops(const uint32_t *w, uint64_t words) { return 0; } +// Guest code runs warp-wide operations (ballot, shuffle, vote) where the guest compiler had the warp reconverged, for +// example right after a loop that each lane leaves at a different iteration. Vulkan only promises the same lanes at the +// same point with the MaximallyReconvergesKHR execution mode. Without it a driver may run the lanes that left the loop +// early through the operations on their own, with a ballot or scan that sees only some of the lanes: a mesh program +// that hands out output slots from one could give two primitives the same slot. This adds the extension and the mode +// for each entry point of a module that declares subgroup capabilities. NULL: nothing to change (or no memory). +static uint32_t *spirv_reconverge(const uint32_t *w, uint64_t n, uint64_t *out_n) { + enum { OP_EXTENSION = 10, OP_EXECUTION_MODE = 16, OP_CAPABILITY = 17, MODE_MAXIMALLY_RECONVERGES = 6023 }; + static const char ext[] = "SPV_KHR_maximal_reconvergence"; + const uint32_t ext_words = (uint32_t)(sizeof ext + 3) / 4; // the string with its terminating zero, padded + if (n < 5 || !uses_subgroup_ops(w, n)) return NULL; + uint64_t caps_end = 0, eps_end = 0; + uint32_t eps[16], neps = 0; + for (uint64_t i = 5; i < n;) { + uint32_t op = w[i] & 0xffff, l = w[i] >> 16; + if (!l || i + l > n) return NULL; + if (op == OP_CAPABILITY) caps_end = i + l; + else if (op == SPV_OP_ENTRY_POINT && l >= 3) { + if (neps == 16) return NULL; + eps[neps++] = w[i + 2], eps_end = i + l; + } else if (op == OP_EXECUTION_MODE && l >= 3 && w[i + 2] == MODE_MAXIMALLY_RECONVERGES) return NULL; + else if (op == SPV_OP_FUNCTION) break; + i += l; + } + if (!caps_end || !neps || eps_end < caps_end) return NULL; + uint64_t on = n + 1 + ext_words + 3ull * neps; + uint32_t *o = malloc(on * 4); + if (!o) return NULL; + uint64_t p = 0; + memcpy(o, w, caps_end * 4), p = caps_end; + o[p++] = (1 + ext_words) << 16 | OP_EXTENSION; + memset(o + p, 0, ext_words * 4); + memcpy(o + p, ext, sizeof ext), p += ext_words; + memcpy(o + p, w + caps_end, (eps_end - caps_end) * 4), p += eps_end - caps_end; + for (uint32_t k = 0; k < neps; k++) o[p++] = 3u << 16 | OP_EXECUTION_MODE, o[p++] = eps[k], o[p++] = MODE_MAXIMALLY_RECONVERGES; + memcpy(o + p, w + eps_end, (n - eps_end) * 4), p += n - eps_end; + *out_n = p; + return o; +} + // Chain {size 32} into a stage create info when the stage uses subgroup ops and the driver allows the pin. static void pin_subgroup32(VkPipelineShaderStageCreateInfo *st, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *rq, const uint32_t *w, uint64_t words) { @@ -952,6 +998,12 @@ static VkShaderModule make_module(const uint32_t *w, uint64_t words) { free(fixed); w = fixed = pooled; } + uint64_t rc_n = 0; + uint32_t *rc = P.reconverge ? spirv_reconverge(w, words, &rc_n) : NULL; + if (rc) { + free(fixed); + w = fixed = rc, words = rc_n; + } VkShaderModuleCreateInfo ci = {.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO, .codeSize = (size_t)words * 4, .pCode = w}; VkShaderModule m = VK_NULL_HANDLE; VkResult r = vkCreateShaderModule(P.device, &ci, NULL, &m);