Make the ASTC scratch trim only try the upload lock, so it can no longer deadlock with an upload.

vk_astc_scratch_trim runs at every present and held the decoder's lock while vk_defer_buffer waited for cmd_lock.
The upload path takes the two locks the other way round: vk_upload_begin takes cmd_lock, then vk_astc_record and
scratch_get take the decoder's lock. A present that trimmed while another thread was between vk_upload_begin and
scratch_get therefore hung both threads. Without the hang, the present still waited for as long as an upload flush
held cmd_lock through its fence waits. vk_memory.c gets vk_defer_buffer_try, which defers only when it gets cmd_lock
at once. When the lock is busy the trim leaves the scratch as it is, and the next present tries again.

tests/astc_trim_check builds the real vk_astc.c with platform/common.c's recursive lock. It checks that the trim gives
way while another thread holds cmd_lock, that it goes through once the lock is free, and that the forced interleaving
ends. Before the change the check reported the deadlock (exit 2). Checks that passed: astc_trim_check, pass_cc_check
on every runtime file (vk.h changed), no new -Wall -Wextra warnings in vk_astc.c and vk_memory.c, and the pass's
test matrix (35 ok; the 6 failures are the baseline's).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-13 17:05:51 -05:00
parent a473e763b2
commit b755cdc536
5 changed files with 269 additions and 3 deletions

View File

@@ -567,6 +567,8 @@ void vk_defer_view(VkImageView view);
void vk_defer_free(void *p);
int vk_exec_image_gone(VkImage image); // texture lock held: the replay's views of image go with it; returns how many
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
// As vk_defer_buffer without waiting: 0 when another thread holds the upload lock, and nothing is deferred.
int vk_defer_buffer_try(VkBuffer buf, VmaAllocation alloc);
// ---- ASTC (vk_astc.c). NX2NAVI_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init

View File

@@ -564,8 +564,11 @@ static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int
// SCRATCH_SHRINK_NS, without waiting for the next ASTC upload, which is when scratch_get makes it small again: in a
// session of 2026-11-13 the decoder held 224 MiB for the 70 s after an area load and from a later load to the session's
// end. Nothing is made here: the next request makes the usual scratch, as after scratch_get's own shrink. The buffer
// is freed once the upload batches recorded so far completed (vk_defer_buffer), as there. Only tried: scratch_get holds
// g_lock while VMA makes a buffer, up to 19 ms for a new block, and the present does not wait for that.
// is freed once the upload batches recorded so far completed (vk_defer_buffer_try), as there. Both locks are only
// tried. scratch_get holds g_lock while VMA makes a buffer, up to 19 ms for a new block. The upload path takes
// cmd_lock first (vk_memory.c vk_upload_begin) and g_lock inside it (vk_astc_record, scratch_get), so waiting for
// cmd_lock while holding g_lock would deadlock with an upload, and waiting at all would hold the present for as long
// as an upload flush waits for its fences. When either lock is busy the trim gives way; the next present tries again.
// NX2NAVI_ASTC_SCRATCH_TRIM=0: the grown scratch stays until the next upload, as in 0.10.1.
void vk_astc_scratch_trim(void) {
static int on = -1;
@@ -574,7 +577,10 @@ void vk_astc_scratch_trim(void) {
if (__atomic_load_n(&S.size, __ATOMIC_RELAXED) <= SCRATCH_MIN || !plat_mutex_trylock(&g_lock)) return;
VkCtx *c = vk_ctx();
if (S.buf && c && S.dev == c->device && S.size > SCRATCH_MIN && kern_now_ns() - S.big_ns > SCRATCH_SHRINK_NS) {
vk_defer_buffer(S.buf, S.alloc);
if (!vk_defer_buffer_try(S.buf, S.alloc)) {
plat_mutex_unlock(&g_lock);
return;
}
S.buf = VK_NULL_HANDLE, S.alloc = NULL, S.size = 0, S.cb = VK_NULL_HANDLE;
n_scratch_trim++;
}

View File

@@ -6016,6 +6016,16 @@ void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NU
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
}
// For a caller that holds a lock the upload path takes inside cmd_lock: vk_astc.c vk_astc_scratch_trim holds the
// decoder's lock, which vk_astc.c scratch_get takes under vk_upload_begin's cmd_lock, so waiting for cmd_lock there
// would reverse that order. defer takes cmd_lock again, which the recursive lock allows.
int vk_defer_buffer_try(VkBuffer buf, VmaAllocation alloc) {
plat_once(&cmd_once, cmd_init, NULL);
if (!plat_rmutex_trylock(&cmd_lock)) return 0;
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
plat_rmutex_unlock(&cmd_lock);
return 1;
}
void vk_mem_note_submit(uint64_t timeline_value) {
plat_once(&cmd_once, cmd_init, NULL);