From bc28416fc098af1dd22ba140a09915d086e77c97 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Fri, 13 Nov 2026 00:39:28 -0500 Subject: [PATCH] Destroy a fast-linked pipeline once its optimised build has replaced it and the GPU is done with it. With the graphics pipeline library a draw's first pipeline is linked from stage libraries, and the optimised build replaces it when ready. The fast link was kept until the process ended, so each pipeline a run drew held the driver memory of two: one play session replaced 907 fast links while the video memory outside the allocator, where the driver keeps compiled pipelines, grew by 491 MiB, and a 68,400-frame session grew it from 1,672 to 2,485 MiB. The replaced fast link is now destroyed on the completion thread once the GPU is past the submission being recorded and one after it. The draws read the pipeline from the cache entry each time they record and keep no handle past their submission, so none can name the old one later. The gpl log line counts the fast links given back. Co-Authored-By: Claude Opus 5.5 --- runtime/vk_pipeline.c | 48 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index a789656d..302f989b 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -55,6 +55,7 @@ #include "tex.h" #include "vk_build_gate.h" #include "vk_cache_file.h" +#include "vk_exec.h" #include "vk_save_retry.h" #include "vk_warm_hint.h" @@ -153,7 +154,7 @@ typedef struct Pipe { int building; // a background job owns info.pipeline (under P.lock) struct Job *job; // that job while building const struct Layout *lay; // graphics: the layout it was entered with - VkPipeline fast; // GPL: the fast-linked pipeline (kept until shutdown: recorded draws use it) + VkPipeline fast; // GPL: the fast-linked pipeline, until the optimised one replaces it (fast_retire) uint8_t fast_tried; uint8_t link_queued; // a waiting draw queued its fast link to the workers (LibJob p), not yet run uint8_t used; // compute: a dispatch of this run asked for it (vk_pipeline_compute) @@ -4955,6 +4956,37 @@ int vk_pipeline_probe_buildgate(void) { return fails ? 1 : 0; } +// ---- fast-linked pipelines the optimised build replaces +// With the graphics pipeline library a draw first gets a pipeline linked from stage libraries (p->fast), and the +// optimised build of the same state replaces it in p->info.pipeline when it is done. The fast link was kept until the +// process ended, so every pipeline a run drew held its fast link's driver memory as well: one play session of 13,800 +// frames made 1,108 fast links and replaced 907 of them, while the video memory outside the allocator, where the +// driver keeps compiled pipelines, grew by 491 MiB (RTX 5090), and a 68,400-frame session grew it from 1,672 to +// 2,485 MiB. A replaced fast link is destroyed now, once the GPU is past every submission that may have recorded it. +// The draws read p->info.pipeline when they record (vk_exec_draw.c, the pipeline memo reads it again each time) and +// keep no handle past the submission they record, so after the store below a draw can only have the old handle in +// the submission being recorded, or in an earlier one: vk_sync_issued() + 1 at most. The destroy waits for one more +// submission than that, for a recording thread that runs a submission behind (vk_rec.c). It runs on the completion +// thread (vk_completion_add), which holds no lock of ours while it calls it; a lost device or the end of the process +// leaves it to the driver, as everything else. +static uint64_t fast_retired; // P.lock: fast links destroyed so far + +static void fast_destroy_cb(void *arg) { + vkDestroyPipeline(P.device, (VkPipeline)(uintptr_t)arg, NULL); +} + +// P.lock held, p->info.pipeline just replaced by the optimised pipeline: the fast link goes once the GPU is done with it. +static void fast_retire(Pipe *p) { + VkPipeline old = p->fast; + if (!old || old == p->info.pipeline) return; + p->fast = VK_NULL_HANDLE; + __atomic_thread_fence(__ATOMIC_SEQ_CST); // the new handle is visible before the submission count is read + uint64_t rf = vk_exec_recording_fence(), at = vk_sync_issued() + 2; + if (rf + 1 > at) at = rf + 1; + vk_completion_add(at, fast_destroy_cb, (void *)(uintptr_t)old); + fast_retired++; +} + // The whole build of job j, which the caller took off its queue, and the job's end: its pipeline is entered, the // threads waiting for it are woken and j is freed. lowrun: 1 on a pool worker that took j from the low queue. Caller // holds P.lock, which is released for the build. @@ -4962,9 +4994,9 @@ static void job_build(Job *j, int lowrun) { plat_mutex_unlock(&P.lock); VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off); plat_mutex_lock(&P.lock); - if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one + if (pipe || !j->p->fast) __atomic_store_n(&j->p->info.pipeline, pipe, __ATOMIC_SEQ_CST); // a failed optimised build keeps the fast-linked one ready_note(j->p, 0); - if (pipe && j->p->fast) P.g.swaps++; + if (pipe && j->p->fast) P.g.swaps++, fast_retire(j->p); j->p->building = 0; j->p->job = NULL; if (pipe) { @@ -5557,9 +5589,9 @@ static void warm_builder(void *arg) { plat_mutex_unlock(&P.lock); VkPipeline pipe = create_graphics(&j->s, &j->k, j->l, j->fs_off); plat_mutex_lock(&P.lock); - if (pipe || !j->p->fast) j->p->info.pipeline = pipe; // a failed optimised build keeps the fast-linked one + if (pipe || !j->p->fast) __atomic_store_n(&j->p->info.pipeline, pipe, __ATOMIC_SEQ_CST); // a failed optimised build keeps the fast-linked one ready_note(j->p, 0); - if (pipe && j->p->fast) P.g.swaps++; + if (pipe && j->p->fast) P.g.swaps++, fast_retire(j->p); j->p->building = 0; j->p->job = NULL; if (pipe) { @@ -8625,13 +8657,15 @@ static int save_with(uint32_t wait_ms, int exiting) { "max %.2f ms, %llu failed), %llu lookups by a draw found a stage library not made in this process yet and " "left it to a worker (whether the driver's cache has it shows in the library times below), %llu optimised " "pipelines swapped in (%llu queued " - "low after their fast link), %llu drawing-thread links over 5 ms; %llu waiting draws built their libraries " + "low after their fast link; %llu replaced fast links given back), %llu drawing-thread links over 5 ms; %llu " + "waiting draws built their libraries " "(avg %.2f ms, max %.2f ms)", (unsigned long long)P.g.links, (unsigned long long)P.g.wlinks, (unsigned long long)P.g.flinks, (unsigned long long)P.g.fb_links, P.g.links ? P.g.link_ms / P.g.links : 0.0, P.g.link_max, (unsigned long long)P.g.link_fail, (unsigned long long)P.g.defer, - (unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)P.g.slow, + (unsigned long long)P.g.swaps, (unsigned long long)P.g.requeued, (unsigned long long)fast_retired, + (unsigned long long)P.g.slow, (unsigned long long)P.g.dlinks, P.g.dlinks ? P.g.dlink_ms / P.g.dlinks : 0.0, P.g.dlink_max); uint64_t rn = P.g.ready_n[0] + P.g.ready_n[1] + P.g.ready_n[2] + P.g.ready_n[3] + P.g.ready_n[4] + P.g.ready_n[5]; rt_log("vk_pipeline: gpl: pipelines a draw missed were usable after under 4 ms %llu, 4-8 ms %llu, 8-17 ms %llu, "