Make the ASTC scratch trim only try the upload lock, so it can no longer deadlock with an upload.

vk_astc_scratch_trim runs at every present and held the decoder's lock while vk_defer_buffer waited for cmd_lock.
The upload path takes the two locks the other way round: vk_upload_begin takes cmd_lock, then vk_astc_record and
scratch_get take the decoder's lock. A present that trimmed while another thread was between vk_upload_begin and
scratch_get therefore hung both threads. Without the hang, the present still waited for as long as an upload flush
held cmd_lock through its fence waits. vk_memory.c gets vk_defer_buffer_try, which defers only when it gets cmd_lock
at once. When the lock is busy the trim leaves the scratch as it is, and the next present tries again.

tests/astc_trim_check builds the real vk_astc.c with platform/common.c's recursive lock. It checks that the trim gives
way while another thread holds cmd_lock, that it goes through once the lock is free, and that the forced interleaving
ends. Before the change the check reported the deadlock (exit 2). Checks that passed: astc_trim_check, pass_cc_check
on every runtime file (vk.h changed), no new -Wall -Wextra warnings in vk_astc.c and vk_memory.c, and the pass's
test matrix (35 ok; the 6 failures are the baseline's).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-13 17:05:51 -05:00
parent a473e763b2
commit b755cdc536
5 changed files with 269 additions and 3 deletions

View File

@@ -567,6 +567,8 @@ void vk_defer_view(VkImageView view);
void vk_defer_free(void *p);
int vk_exec_image_gone(VkImage image); // texture lock held: the replay's views of image go with it; returns how many
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc);
// As vk_defer_buffer without waiting: 0 when another thread holds the upload lock, and nothing is deferred.
int vk_defer_buffer_try(VkBuffer buf, VmaAllocation alloc);
// ---- ASTC (vk_astc.c). NX2NAVI_ASTC = auto | gpu | native | cpu | check; see vk_astc.c.
enum { VK_ASTC_CPU, VK_ASTC_GPU, VK_ASTC_NATIVE, VK_ASTC_CHECK };
int vk_astc_mode(void); // decided once per device (logged); VK_ASTC_CPU before vk_init

View File

@@ -564,8 +564,11 @@ static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int
// SCRATCH_SHRINK_NS, without waiting for the next ASTC upload, which is when scratch_get makes it small again: in a
// session of 2026-11-13 the decoder held 224 MiB for the 70 s after an area load and from a later load to the session's
// end. Nothing is made here: the next request makes the usual scratch, as after scratch_get's own shrink. The buffer
// is freed once the upload batches recorded so far completed (vk_defer_buffer), as there. Only tried: scratch_get holds
// g_lock while VMA makes a buffer, up to 19 ms for a new block, and the present does not wait for that.
// is freed once the upload batches recorded so far completed (vk_defer_buffer_try), as there. Both locks are only
// tried. scratch_get holds g_lock while VMA makes a buffer, up to 19 ms for a new block. The upload path takes
// cmd_lock first (vk_memory.c vk_upload_begin) and g_lock inside it (vk_astc_record, scratch_get), so waiting for
// cmd_lock while holding g_lock would deadlock with an upload, and waiting at all would hold the present for as long
// as an upload flush waits for its fences. When either lock is busy the trim gives way; the next present tries again.
// NX2NAVI_ASTC_SCRATCH_TRIM=0: the grown scratch stays until the next upload, as in 0.10.1.
void vk_astc_scratch_trim(void) {
static int on = -1;
@@ -574,7 +577,10 @@ void vk_astc_scratch_trim(void) {
if (__atomic_load_n(&S.size, __ATOMIC_RELAXED) <= SCRATCH_MIN || !plat_mutex_trylock(&g_lock)) return;
VkCtx *c = vk_ctx();
if (S.buf && c && S.dev == c->device && S.size > SCRATCH_MIN && kern_now_ns() - S.big_ns > SCRATCH_SHRINK_NS) {
vk_defer_buffer(S.buf, S.alloc);
if (!vk_defer_buffer_try(S.buf, S.alloc)) {
plat_mutex_unlock(&g_lock);
return;
}
S.buf = VK_NULL_HANDLE, S.alloc = NULL, S.size = 0, S.cb = VK_NULL_HANDLE;
n_scratch_trim++;
}

View File

@@ -6016,6 +6016,16 @@ void vk_defer_free(void *p) { if (p) defer(DEAD_HOST, (uint64_t)(uintptr_t)p, NU
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
}
// For a caller that holds a lock the upload path takes inside cmd_lock: vk_astc.c vk_astc_scratch_trim holds the
// decoder's lock, which vk_astc.c scratch_get takes under vk_upload_begin's cmd_lock, so waiting for cmd_lock there
// would reverse that order. defer takes cmd_lock again, which the recursive lock allows.
int vk_defer_buffer_try(VkBuffer buf, VmaAllocation alloc) {
plat_once(&cmd_once, cmd_init, NULL);
if (!plat_rmutex_trylock(&cmd_lock)) return 0;
if (buf) defer(DEAD_BUFFER_VMA, (uint64_t)(uintptr_t)buf, alloc);
plat_rmutex_unlock(&cmd_lock);
return 1;
}
void vk_mem_note_submit(uint64_t timeline_value) {
plat_once(&cmd_once, cmd_init, NULL);

171
tests/astc_trim_check.c Normal file
View File

@@ -0,0 +1,171 @@
// runtime/vk_astc.c vk_astc_scratch_trim against the upload path's lock order. Built and run by
// tests/astc_trim_check.py; prints each failed check and exits 1 on any, 2 when two threads stay stuck.
// The upload path takes the upload lock first (vk_memory.c vk_upload_begin: cmd_lock) and the decoder's lock inside it
// (vk_astc_record: g_lock shared for the pipelines, then scratch_get: g_lock). The trim runs at every present and holds
// g_lock, so it may only try cmd_lock (vk_defer_buffer_try): waiting for it there deadlocks with an upload.
// The platform's mutexes are SRW locks as in runtime/platform/win32.c; the recursive cmd_lock is
// runtime/platform/common.c's, linked in. vk_defer_buffer and vk_defer_buffer_try stand in for vk_memory.c's, which
// take cmd_lock as its defer does.
#include "vk_astc.c"
#include <windows.h>
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
// ---- the platform: SRW locks and condition variables (runtime/platform/win32.c), thread ids for common.c
void plat_mutex_init(PlatMutex *m) { InitializeSRWLock((PSRWLOCK)m); }
void plat_mutex_lock(PlatMutex *m) { AcquireSRWLockExclusive((PSRWLOCK)m); }
void plat_mutex_lock_at(PlatMutex *m, void *site) { (void)site, AcquireSRWLockExclusive((PSRWLOCK)m); }
int plat_mutex_trylock(PlatMutex *m) { return TryAcquireSRWLockExclusive((PSRWLOCK)m) != 0; }
void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive((PSRWLOCK)m); }
void plat_mutex_lock_shared(PlatMutex *m) { AcquireSRWLockShared((PSRWLOCK)m); }
void plat_mutex_unlock_shared(PlatMutex *m) { ReleaseSRWLockShared((PSRWLOCK)m); }
void plat_cond_init(PlatCond *c) { InitializeConditionVariable((PCONDITION_VARIABLE)c); }
void plat_cond_wait(PlatCond *c, PlatMutex *m) {
SleepConditionVariableSRW((PCONDITION_VARIABLE)c, (PSRWLOCK)m, INFINITE, 0);
}
int plat_cond_wait_ns(PlatCond *c, PlatMutex *m, uint64_t ns) {
return SleepConditionVariableSRW((PCONDITION_VARIABLE)c, (PSRWLOCK)m, (DWORD)(ns / 1000000), 0) != 0;
}
void plat_cond_broadcast(PlatCond *c) { WakeAllConditionVariable((PCONDITION_VARIABLE)c); }
void plat_cond_signal(PlatCond *c) { WakeConditionVariable((PCONDITION_VARIABLE)c); }
uint32_t plat_thread_id(void) { return (uint32_t)GetCurrentThreadId(); }
// ---- the rest of the runtime vk_astc.c names on this path
static uint64_t fake_now;
uint64_t kern_now_ns(void) { return __atomic_load_n(&fake_now, __ATOMIC_ACQUIRE); }
int cfg_bool(const char *k) { (void)k; return 1; } // NX2NAVI_ASTC_SCRATCH_KEEP=1, the default
int rt_knob(const char *k, const char *def) { (void)k; return atoi(def); }
void rt_log(const char *fmt, ...) { (void)fmt; }
void vk_texstat_time(int kind, uint64_t t0) { (void)kind, (void)t0; }
static VkCtx ctx;
VkCtx *vk_ctx(void) { return &ctx; }
static uintptr_t next_handle = 0x1000;
VkResult vk_vma_create_buffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai,
VkBuffer *b, VmaAllocation *al, VmaAllocationInfo *info, const char *file) {
(void)a, (void)bi, (void)ai, (void)info, (void)file;
*b = (VkBuffer)(next_handle += 0x10), *al = (VmaAllocation)(next_handle += 0x10);
return VK_SUCCESS;
}
// ---- vk_memory.c's deferral: defer appends under cmd_lock (recursive)
static PlatRMutex cmd_lock = PLAT_RMUTEX_INIT;
static volatile LONG deferred;
static void defer_locked(void) {
plat_rmutex_lock(&cmd_lock);
InterlockedIncrement(&deferred);
plat_rmutex_unlock(&cmd_lock);
}
void vk_defer_buffer(VkBuffer buf, VmaAllocation alloc) {
(void)alloc;
if (buf) defer_locked();
}
int vk_defer_buffer_try(VkBuffer buf, VmaAllocation alloc) {
(void)alloc;
if (!plat_rmutex_trylock(&cmd_lock)) return 0;
if (buf) defer_locked();
plat_rmutex_unlock(&cmd_lock);
return 1;
}
// A large texture grows the scratch (under cmd_lock, as an upload asks for it), then 6 s pass without one.
static void grow(void) {
int reused;
plat_rmutex_lock(&cmd_lock);
scratch_get(&ctx, (VkCommandBuffer)0x2222, 80ull << 20, &reused);
plat_rmutex_unlock(&cmd_lock);
__atomic_add_fetch(&fake_now, 6000000000ull, __ATOMIC_ACQ_REL);
}
// ---- (a) another thread holds cmd_lock: the trim returns at once and leaves the scratch
static volatile LONG holding, release_hold, trim_done;
static DWORD WINAPI holder_thread(void *arg) {
(void)arg;
plat_rmutex_lock(&cmd_lock);
InterlockedExchange(&holding, 1);
while (!release_hold) Sleep(1);
plat_rmutex_unlock(&cmd_lock);
return 0;
}
static DWORD WINAPI trim_thread(void *arg) {
(void)arg;
vk_astc_scratch_trim();
InterlockedExchange(&trim_done, 1);
return 0;
}
// ---- (c) the upload holds cmd_lock, the present's trim comes, then the upload asks for the scratch
static volatile LONG upload_holds, trim_returned, upload_done;
static DWORD WINAPI upload_thread(void *arg) {
(void)arg;
plat_rmutex_lock(&cmd_lock); // vk_upload_begin
InterlockedExchange(&upload_holds, 1);
for (int i = 0; i < 200 && !trim_returned; i++) Sleep(1); // the staging copy: the present's trim comes meanwhile
Sleep(20); // a blocking trim is now waiting for cmd_lock
plat_mutex_lock_shared(&g_lock); // vk_astc_record: the pipelines
plat_mutex_unlock_shared(&g_lock);
int reused;
scratch_get(&ctx, (VkCommandBuffer)0x2222, 4u << 20, &reused);
plat_rmutex_unlock(&cmd_lock); // vk_upload_end
InterlockedExchange(&upload_done, 1);
return 0;
}
static DWORD WINAPI present_thread(void *arg) {
(void)arg;
while (!upload_holds) Sleep(1);
vk_astc_scratch_trim();
InterlockedExchange(&trim_returned, 1);
return 0;
}
int main(void) {
ctx.device = (VkDevice)0x1111;
__atomic_store_n(&fake_now, 100000000000ull, __ATOMIC_RELEASE);
// (a)
grow();
CHECK(S.size == 96ull << 20, "the large texture left a scratch of %llu MiB, not 96",
(unsigned long long)(S.size >> 20));
LONG d0 = deferred;
HANDLE h = CreateThread(NULL, 0, holder_thread, NULL, 0, NULL);
while (!holding) Sleep(1);
HANDLE t = CreateThread(NULL, 0, trim_thread, NULL, 0, NULL);
DWORD w = WaitForSingleObject(t, 100);
CHECK(w == WAIT_OBJECT_0, "the trim waited for the upload lock another thread holds");
InterlockedExchange(&release_hold, 1);
WaitForSingleObject(t, 3000);
WaitForSingleObject(h, 3000);
if (w == WAIT_OBJECT_0) {
CHECK(S.size == 96ull << 20 && S.buf, "the trim that gave way still dropped the scratch (%llu MiB)",
(unsigned long long)(S.size >> 20));
CHECK(n_scratch_trim == 0 && deferred == d0, "the trim that gave way counted %llu trims, %ld deferrals",
(unsigned long long)n_scratch_trim, deferred - d0);
}
// (b) cmd_lock free: the next present's trim goes through
uint64_t trims = n_scratch_trim;
d0 = deferred;
vk_astc_scratch_trim();
CHECK(S.size == 0 && !S.buf, "with the upload lock free the scratch stayed (%llu MiB)",
(unsigned long long)(S.size >> 20));
CHECK(n_scratch_trim == trims + 1 && deferred == d0 + 1, "with the upload lock free: %llu trims, %ld deferrals",
(unsigned long long)(n_scratch_trim - trims), deferred - d0);
// (c) the interleaving that deadlocked: both threads end
grow();
HANDLE th[2] = {CreateThread(NULL, 0, upload_thread, NULL, 0, NULL),
CreateThread(NULL, 0, present_thread, NULL, 0, NULL)};
if (WaitForMultipleObjects(2, th, TRUE, 3000) == WAIT_TIMEOUT) {
printf("FAIL: deadlock: after 3 s the upload has not got its scratch (done %ld) and the trim has not ended "
"(returned %ld)\n", upload_done, trim_returned);
fflush(stdout);
ExitProcess(2);
}
CHECK(S.buf && S.size == 32ull << 20, "after the upload the scratch is %llu MiB, not the usual 32",
(unsigned long long)(S.size >> 20));
if (fails) return printf("astc_trim_check: %d checks failed\n", fails), 1;
printf("astc_trim_check: all checks passed\n");
return 0;
}

77
tests/astc_trim_check.py Normal file
View File

@@ -0,0 +1,77 @@
"""Build and run tests/astc_trim_check.c: runtime/vk_astc.c vk_astc_scratch_trim, which every present runs, against the
upload path's lock order (cmd_lock, then the decoder's lock). The trim gives way while another thread holds the upload
lock, goes through once it is free, and never deadlocks with an upload that asks for the scratch meanwhile.
No GPU, no game files, any Python. The recursive lock is runtime/platform/common.c's; the rest of the runtime that
vk_astc.c names is stubbed (each stub returns 0), and the compute shaders' SPIR-V headers, when tools/build_shaders.py
has not made them, are stand-ins written to the test's build folder.
Usage: astc_trim_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
# The symbols vk_astc.c names that neither the check nor platform/common.c defines. None runs on the trim's path.
STUBS = ['cfg_get', 'cfg_value', 'plat_mem_info', 'plat_now_ns', 'rc_vkCmdBindPipeline', 'rc_vkCmdCopyBuffer',
'rc_vkCmdCopyBufferToImage', 'rc_vkCmdDispatch', 'rc_vkCmdPipelineBarrier2', 'rc_vkCmdPushConstants',
'rc_vkCmdWriteTimestamp2', 'rec_push', 'rt_fatal', 'rt_nvn_api_version', 'vkCreateComputePipelines',
'vkCreateDescriptorSetLayout', 'vkCreatePipelineLayout', 'vkCreateQueryPool', 'vkCreateShaderModule',
'vkDestroyDescriptorSetLayout', 'vkDestroyPipeline', 'vkDestroyPipelineLayout', 'vkDestroyQueryPool',
'vkDestroyShaderModule', 'vkGetPhysicalDeviceFeatures2', 'vkGetPhysicalDeviceFormatProperties',
'vkGetPhysicalDeviceQueueFamilyProperties', 'vkGetQueryPoolResults', 'vkResetQueryPool', 'vk_crumb', 'vk_dpa',
'vk_staging', 'vk_vma_destroy_buffer']
SHADERS = ['astc_decode', 'bc3_encode']
def compiler():
"""$CC, else MSYS2 UCRT64's clang, else clang on PATH (as tests/diff/common.py picks it, without its Unicorn)."""
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def vma_include():
"""The folder holding vk_mem_alloc.h: $VMA_INCLUDE, else MSYS2 UCRT64's."""
if os.environ.get('VMA_INCLUDE'):
return os.environ['VMA_INCLUDE']
return str(MSYS_BIN.parent / 'include' / 'vma')
def main():
if os.name != 'nt':
print('astc_trim_check: Windows only (the check uses SRW locks and Windows threads); skipped')
return 0
work = ROOT / 'build' / 'test' / 'astc_trim'
work.mkdir(parents=True, exist_ok=True)
stubs = work / 'stubs.c'
stubs.write_text('// Stand-ins written by tests/astc_trim_check.py: each returns 0.\n' +
''.join(f'long long {s}() {{ return 0; }}\n' for s in STUBS))
for name in SHADERS: # only used when runtime/<name>_spv.h is missing: the runtime folder is searched first
(work / f'{name}_spv.h').write_text('// A stand-in written by tests/astc_trim_check.py.\n#pragma once\n'
f'#include <stdint.h>\nstatic const uint32_t {name}_spv[] = {{0}};\n')
exe = work / 'astc_trim_check.exe'
env = dict(os.environ)
if MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
rt = ROOT / 'runtime'
subprocess.check_call([compiler(), '-std=gnu11', '-O1', '-g', '-w', '-fno-strict-aliasing', '-march=x86-64-v3',
'-I', str(rt), '-I', str(work), '-I', vma_include(),
str(ROOT / 'tests' / 'astc_trim_check.c'), str(rt / 'platform' / 'common.c'), str(stubs),
'-o', str(exe), '-lws2_32'], env=env)
try:
return subprocess.run([str(exe)], env=env, timeout=60).returncode
except subprocess.TimeoutExpired:
print('astc_trim_check: FAIL: the check did not end within 60 s')
return 1
if __name__ == '__main__':
sys.exit(main())