diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index e1cbaf26..084c7025 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -1799,13 +1799,15 @@ static void build_key(const uint8_t code_hash[32], const GfxShaderAnswers *a, co } static size_t build_request(char *o0, const char *hex, const char *stage, const char *code_path, const uint8_t *c1, - uint32_t c1_len, const GfxShaderAnswers *a, const uint8_t *control) { + uint32_t c1_len, const GfxShaderAnswers *a, const uint8_t *control, uint8_t *options_hash) { char *o = o0; o += sprintf(o, "{\"job\":{\"key\":\""); json_esc(&o, hex); - o += sprintf(o, "\",\"stage\":\"%s\",\"debug\":false,\"code\":\"", stage); + o += sprintf(o, "\",\"code\":\""); json_esc(&o, code_path); - o += sprintf(o, "\",\"c1\":\""); + o += sprintf(o, "\","); + const char *options = o; + o += sprintf(o, "\"stage\":\"%s\",\"debug\":false,\"c1\":\"", stage); for (uint32_t i = 0; i < c1_len; i++) o += sprintf(o, "%02x", c1[i]); o += sprintf(o, "\""); if (a) { @@ -1884,6 +1886,7 @@ static size_t build_request(char *o0, const char *hex, const char *stage, const if (end >= 8 * (ntex + nimg)) o += sprintf(o, ",\"nvn2_table\":[%d,%d,%d]", end - 8 * (ntex + nimg), ntex, nimg); } } + if (options_hash) rt_sha256(options, (size_t)(o - options), options_hash); o += sprintf(o, "},\"out\":\""); json_esc(&o, cache_dir); o += sprintf(o, "\"}"); @@ -2216,8 +2219,90 @@ static int in_load(const uint8_t iid[32], Inputs *in) { // ---- translation +typedef struct PreparedJobs PreparedJobs; +static _Thread_local PreparedJobs *preparing; + +// Only preparation workers share translator results. Logical shader and input IDs continue to include the full +// guest program. The child sees the bytes of the code file and the serialized options in build_request. +typedef struct PreparedJob { + struct PreparedJob *next; + uint8_t digest[32]; + PlatCond done; + int busy, ready; + char cache_key[65], reflection_key[65]; +} PreparedJob; + +struct PreparedJobs { + PlatMutex lock; + PreparedJob *buckets[BUCKETS]; + volatile int64_t reused; +}; + +// One worker owns a cold request. A failed owner leaves it available for the next worker to retry. +static PreparedJob *prepared_enter(const uint8_t options_hash[32], const uint8_t *bytes, size_t n, PreparedJob **owner) { + uint8_t parts[3][32], digest[32]; + memcpy(parts[0], xlate_id, 32); + rt_sha256(bytes, n, parts[1]); + memcpy(parts[2], options_hash, 32); + rt_sha256(parts, sizeof parts, digest); + uint32_t slot; + memcpy(&slot, digest, sizeof slot); + PreparedJob **bucket = &preparing->buckets[slot % BUCKETS]; + plat_mutex_lock(&preparing->lock); + PreparedJob *j = *bucket; + while (j && memcmp(j->digest, digest, 32)) j = j->next; + if (!j && (j = calloc(1, sizeof *j))) { + memcpy(j->digest, digest, 32); + plat_cond_init(&j->done); + j->next = *bucket; + *bucket = j; + } + if (j) { + while (j->busy) plat_cond_wait(&j->done, &preparing->lock); + if (!j->ready) { + j->busy = 1; + *owner = j; + } + } + plat_mutex_unlock(&preparing->lock); + return *owner ? NULL : j; +} + +static void prepared_finish(PreparedJob *owner, const GfxShader *s, const uint8_t request_key[32]) { + plat_mutex_lock(&preparing->lock); + if (s) { + key_hex(s->key, owner->cache_key); + key_hex(request_key, owner->reflection_key); + owner->ready = 1; + } + owner->busy = 0; + plat_cond_broadcast(&owner->done); + plat_mutex_unlock(&preparing->lock); +} + +// Copy the unmodified disk SPIR-V. disk_load_from applies the per-install HDR substitution in memory. +// The translator embeds its job key in reflection, so each logical job gets the key it would have emitted. +static GfxShader *prepared_copy(const PreparedJob *j, const Req *r, const char *hex, const char *spv, const char *res) { + char path[PATH_CAP + 80]; + uint64_t ns = 0, nr = 0; + snprintf(path, sizeof path, "%s/%s.res.json", cache_dir, j->cache_key); + uint8_t *reflection = read_file(path, &nr); + if (!reflection) return NULL; + const char *end = (char *)reflection + nr; + const char *id = json_key((char *)reflection, end, "key"); + if (!id || end - id < 66 || *id != '"' || id[65] != '"' || memcmp(id + 1, j->reflection_key, 64)) + return free(reflection), NULL; + memcpy((char *)id + 1, hex, 64); + snprintf(path, sizeof path, "%s/%s.spv", cache_dir, j->cache_key); + uint8_t *raw = read_file(path, &ns); + int ok = raw && plat_file_write_cached(spv, raw, (size_t)ns) && plat_file_write_cached(res, reflection, (size_t)nr); + free(raw); + free(reflection); + return ok ? disk_load_from(r->key, r->iid, 0, 1) : NULL; +} + static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *control, const uint8_t *code, - const GfxShaderAnswers *a) { + const GfxShaderAnswers *a, PreparedJob **owner, int *published) { const uint8_t *key = r->key; GfxShader *s; UseMask m; @@ -2235,39 +2320,47 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co snprintf(code_path, PATH_CAP + 80, "%s/%s.code", cache_dir, hex); snprintf(spv_path, PATH_CAP + 80, "%s/%s.spv", cache_dir, hex); snprintf(res_path, PATH_CAP + 80, "%s/%s.res.json", cache_dir, hex); - if (!write_file(code_path, code + skip, (size_t)(r->length - skip))) return fail("cannot write the code file", hex); - size_t cap = 2 * PATH_CAP * 6 + 2 * r->c1_len + 8192; + size_t cap = 2 * PATH_CAP * 6 + 2 * (size_t)r->c1_len + 8192; char *req = malloc(cap), reply[LINE_MAX_LEN]; - if (!req) return plat_file_delete(code_path), fail("out of memory", hex); - build_request(req, hex, r->stage_name, code_path, r->c1, r->c1_len, a, control); - int ok = svc_request(req, reply, sizeof reply); + if (!req) return fail("out of memory", hex); // NX2NAVI_XLATE_KEEP=1: a last-frame variant's job stays as .job (one request line) with its code, to run the // translator on by hand (nx2navi_xlate --serve < .job); 2 keeps every job translated in this run (delete a // key's .res.json and .spv from the cache to have it translated, and kept, again) - static int keep = -1; - if (keep < 0) { - const char *k = cfg_get("NX2NAVI_XLATE_KEEP"); - keep = k && *k ? atoi(k) : 0; - } - // A failed translation keeps its job and code too (rare; what a translator fix is tested on). - int failed = ok && strncmp(reply, "ok ", 3); - int kept = keep >= 2 || (keep && a && a->mvprev) || failed; - if (kept) { - char job_path[PATH_CAP + 80]; - snprintf(job_path, sizeof job_path, "%s/%s.job", cache_dir, hex); - write_file(job_path, req, strlen(req)); - } - free(req); - if (!kept) plat_file_delete(code_path); - if (!ok) return fail("translator unavailable", hex); - if (strncmp(reply, "ok ", 3)) { - plat_file_delete(spv_path), plat_file_delete(res_path); - if (r->sm86) plat_inc64(&n_sm86); - return fail(reply, hex); + const char *k = cfg_get("NX2NAVI_XLATE_KEEP"); + int keep = k && *k ? atoi(k) : 0; + uint8_t options[32]; + int share = preparing && !keep; + build_request(req, hex, r->stage_name, code_path, r->c1, r->c1_len, a, control, share ? options : NULL); + size_t code_len = (size_t)(r->length - skip); + PreparedJob *prior = share ? prepared_enter(options, code + skip, code_len, owner) : NULL; + s = prior ? prepared_copy(prior, r, hex, spv_path, res_path) : NULL; + int reused = s != NULL; + if (s) { + free(req); + } else { + if (!write_file(code_path, code + skip, code_len)) return free(req), fail("cannot write the code file", hex); + int ok = svc_request(req, reply, sizeof reply); + // Keep a failed translation's job and code for diagnosis. + int failed = ok && strncmp(reply, "ok ", 3); + int kept = keep >= 2 || (keep && a && a->mvprev) || failed; + if (kept) { + char job_path[PATH_CAP + 80]; + snprintf(job_path, sizeof job_path, "%s/%s.job", cache_dir, hex); + write_file(job_path, req, strlen(req)); + } + free(req); + if (!kept) plat_file_delete(code_path); + if (!ok) return fail("translator unavailable", hex); + if (failed) { + plat_file_delete(spv_path), plat_file_delete(res_path); + if (r->sm86) plat_inc64(&n_sm86); + return fail(reply, hex); + } + if (!(s = disk_load_from(key, r->iid, 0, 1))) return fail("translator output missing or malformed", hex); } // Not flushed (see "Durability" at the top): the translator wrote them whole, and disk_load checks both. if (cache_flush()) plat_file_flush(spv_path), plat_file_flush(res_path); - if (!(s = disk_load(key, r->iid))) return fail("translator output missing or malformed", hex); + *published = 1; if (!r->final) { // name the files after the cache key, remember the mask m = use_of(s); build_key(r->code_hash, a, &m, mk, miid); @@ -2280,6 +2373,7 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co if (moved) { use_store(r->code_hash, &m); } else { // keep the mask for this run only + *published = 0; plat_file_delete(spv_path), plat_file_delete(res_path), rt_log("gfx_shader: cannot rename %s", hex); plat_mutex_lock(&cache_lock); use_put(r->code_hash, 1, &m); @@ -2288,7 +2382,8 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co memcpy(s->key, mk, 32), memcpy(s->iid, miid, 32); } in_store(s->iid, stage, control, code, r, a); - plat_inc64(&n_xlated); + if (reused) plat_inc64(&preparing->reused); + else plat_inc64(&n_xlated); s->fresh = 1; return mem_insert(s); } @@ -2300,7 +2395,10 @@ static const GfxShader *translate(const Req *r, int stage, const uint8_t *contro int away = kern_core_away(); const uint8_t *id = r->final ? r->key : r->code_hash; flight_enter(id); - const GfxShader *s = translate_one(r, stage, control, code, a); + PreparedJob *owner = NULL; + int published = 0; + const GfxShader *s = translate_one(r, stage, control, code, a, &owner, &published); + if (owner) prepared_finish(owner, published ? s : NULL, r->key); flight_leave(id); kern_core_back(&away); return s; @@ -2755,7 +2853,7 @@ int gfx_shader_write_code(const GfxShader *s, const char *path) { char *req = st ? malloc(cap) : NULL, job[PATH_CAP + 80], hex[65]; if (ok && req) { key_hex(s->key, hex); - build_request(req, hex, st, path, in.c1, in.c1_len, &in.a, in.control); + build_request(req, hex, st, path, in.c1, in.c1_len, &in.a, in.control, NULL); snprintf(job, sizeof job, "%s.job", path); write_file(job, req, strlen(req)); } @@ -3041,6 +3139,7 @@ typedef struct { PrgList progs; const uint8_t *ans; size_t n_ans; + PreparedJobs jobs; uint8_t (*iid)[32]; // per answers record: the resolved input id, zero on failure volatile int64_t next, done, missing, failed, refined; } Prep; @@ -3054,9 +3153,10 @@ static void prg_collect(const uint8_t *rec, void *ctx) { static void prep_worker(void *arg) { Prep *p = arg; + preparing = &p->jobs; for (;;) { size_t i = (size_t)plat_inc64(&p->next) - 1; - if (i >= p->n_ans) return; + if (i >= p->n_ans) break; const uint8_t *r = p->ans + i * ANS_REC; uint8_t key_rec[12 + 32]; memcpy(key_rec + 12, r + 4, 32); @@ -3078,6 +3178,7 @@ static void prep_worker(void *arg) { else if (p->iid) memcpy(p->iid[i], s->iid, 32); plat_inc64(&p->done); } + preparing = NULL; } // Translates every record of ans_path whose program is in prg_path on `threads` threads with up to `procs` translators. @@ -3101,6 +3202,7 @@ static long long prepare_files(const char *ans_path, const char *prg_path, int t prg_walk(pb, pn, prg_collect, &p.progs); qsort(p.progs.idx, p.progs.n, sizeof *p.progs.idx, prg_cmp); plat_inc32(&C.suspended); // a preparation translates everything under the current translator, carrying nothing + plat_mutex_init(&p.jobs.lock); uint64_t xl0 = (uint64_t)n_xlated; threads = threads < 1 ? 1 : threads > 64 ? 64 : threads; svc_set_limit(procs); @@ -3127,6 +3229,15 @@ static long long prepare_files(const char *ans_path, const char *prg_path, int t "cached; %lld default variants with refined answers; %.0f ms", p.n_ans, what, p.progs.n, (unsigned long long)((uint64_t)n_xlated - xl0), (long long)p.failed, (long long)p.missing, (long long)p.refined, (double)plat_now_ms() - t0); + rt_log("gfx_shader: prepare: %lld equivalent translator requests reused", (long long)p.jobs.reused); + for (int b = 0; b < BUCKETS; b++) { + PreparedJob *j = p.jobs.buckets[b]; + while (j) { + PreparedJob *next = j->next; + free(j); + j = next; + } + } free(p.progs.idx), free(pb); if (ans_out) *ans_out = ab; else free(ab); @@ -3394,7 +3505,7 @@ static int carry_translate(const Req *r, const Inputs *in, char *why, size_t nwh size_t cap = 2 * PATH_CAP * 6 + 2 * (size_t)r->c1_len + 8192; char *req = malloc(cap), reply[LINE_MAX_LEN]; if (!req) return plat_file_delete(code_path), snprintf(why, nwhy, "out of memory"), -1; - build_request(req, hex, r->stage_name, code_path, r->c1, r->c1_len, &in->a, in->control); + build_request(req, hex, r->stage_name, code_path, r->c1, r->c1_len, &in->a, in->control, NULL); int ok = carry_exchange(req, reply, sizeof reply), refused = ok && strncmp(reply, "ok ", 3); if (refused) { // as translate_one: the job and its code stay, for a translator fix to be tested on char job_path[PATH_CAP + 80]; diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index b5bceef9..433369f4 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -6277,10 +6277,53 @@ static uint32_t *spirv_fog_keep_froxels(const uint32_t *w, uint64_t n) { return o; } +// Successful cache-only warm requests. Input ids can differ while their final SPIR-V and pipeline layouts match. +// Only preparation that destroys its pipeline may reuse these entries; a dispatch still needs its own live handle. +// The code is hashed after overrides and fog changes. Subgroup requirements follow that code and the fixed device +// capabilities. Layout handles and creation flags distinguish otherwise identical shader modules. +typedef struct { + uint8_t code[32]; + VkPipelineLayout layout; + uint64_t words; + VkPipelineCreateFlags flags; + VkPipelineShaderStageCreateFlags stage_flags; +} ComputeWarmKey; +typedef struct ComputeWarmEntry { + ComputeWarmKey key; + struct ComputeWarmEntry *next; +} ComputeWarmEntry; +#define COMPUTE_WARM_BUCKETS 1024 +static ComputeWarmEntry *compute_warm[COMPUTE_WARM_BUCKETS]; +static PlatMutex compute_warm_lock = PLAT_MUTEX_INIT; +static uint64_t compute_warm_reused; + +// No in-flight entries: concurrent identical requests may both build, but a failed request never suppresses a retry. +static int compute_warm_remember(const ComputeWarmKey *key, int completed) { + uint64_t h = fnv(FNV0, key, sizeof *key); + plat_mutex_lock(&compute_warm_lock); + ComputeWarmEntry **head = &compute_warm[h % COMPUTE_WARM_BUCKETS], *e = *head; + while (e && memcmp(&e->key, key, sizeof *key)) e = e->next; + int have = e != NULL; + if (!have && completed && (e = malloc(sizeof *e))) e->key = *key, e->next = *head, *head = e; + plat_mutex_unlock(&compute_warm_lock); + return have; +} + +static void compute_warm_clear(void) { + for (size_t i = 0; i < COMPUTE_WARM_BUCKETS; i++) { + for (ComputeWarmEntry *e = compute_warm[i], *n; e; e = n) n = e->next, free(e); + compute_warm[i] = NULL; + } +} + // A compute pipeline for cs in layout l; VK_NULL_HANDLE on failure. No lock needed (P.cache is driver-synchronised). // cached_only (needs P.cache_control): from P.cache or the driver's own disk cache, never compiled; VK_NULL_HANDLE when // neither holds it (the driver returns VK_PIPELINE_COMPILE_REQUIRED, a success code, with no pipeline). -static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only) { +// already_warm is only supplied by callers that need cache coverage, not a live pipeline: 1 and VK_NULL_HANDLE mean +// an identical request already succeeded in this process. NULL preserves normal creation and has no hashing cost. +// NX2NAVI_COMPUTE_WARM_REUSE=0 disables this reuse for comparisons without changing the prepared inventory. +static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cached_only, int *already_warm) { + if (already_warm) *already_warm = 0; uint64_t words; const uint32_t *code = gfx_shader_spirv(cs, &words); uint64_t ow = 0; @@ -6303,12 +6346,28 @@ static VkPipeline compute_create(const GfxShader *cs, const Layout *l, int cache uint32_t lx = local_size_x(code, words); if (P.f_full_subgroups && vk_caps.sg_size && lx && lx % vk_caps.sg_size == 0 && uses_subgroup_ops(code, words)) ci.stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT; + ComputeWarmKey warm_key = {0}; + const char *reuse = already_warm ? cfg_get("NX2NAVI_COMPUTE_WARM_REUSE") : NULL; + int remember = already_warm && (!reuse || strcmp(reuse, "0")); + if (remember) { + rt_sha256(code, (size_t)words * 4, warm_key.code); + warm_key.layout = l->layout, warm_key.words = words; + warm_key.flags = ci.flags, warm_key.stage_flags = ci.stage.flags; + if (compute_warm_remember(&warm_key, 0)) { + *already_warm = 1; + __atomic_fetch_add(&compute_warm_reused, 1, __ATOMIC_RELAXED); + vkDestroyShaderModule(P.device, m, NULL); + free(over); + return VK_NULL_HANDLE; + } + } VkPipeline pipe = VK_NULL_HANDLE; VkPipelineCreationFeedbackCreateInfo fci; VkPipelineCreationFeedback fb; ci.pNext = fb_chain(&fci, &fb, ci.pNext); crash_note("a compute pipeline", cs, NULL, NULL); if (pipe_create(2, &ci, &pipe) != VK_SUCCESS) pipe = VK_NULL_HANDLE; + if (pipe && remember) compute_warm_remember(&warm_key, 1); plat_crash_context(NULL); if (pipe) fb_note(FB_COMPUTE, &fb); cc_hit = !pipe ? -1 : pb_last ? 1 @@ -6936,7 +6995,7 @@ VkPipeline vk_pipeline_compute(const GfxShader *cs, const VkPipeInfo **info) { plat_mutex_unlock(&P.lock); // the compile runs without the lock uint64_t p0 = perf_begin(); double c0 = now_ms(); - pipe = compute_create(cs, l, 0); + pipe = compute_create(cs, l, 0, NULL); double cms = now_ms() - c0; int hit = cc_hit; perf_end(PERF_PIPE, p0); @@ -7186,7 +7245,7 @@ static void bg_main(void *arg) { if (P.cache && vkCreatePipelineCache(P.device, &oci, NULL, &own) != VK_SUCCESS) own = VK_NULL_HANDLE; double t0 = now_ms(); own_cache = own; // without one the build goes into P.cache as before - VkPipeline pipe = compute_create(cs, l, 0); + VkPipeline pipe = compute_create(cs, l, 0, NULL); own_cache = VK_NULL_HANDLE; double ms = now_ms() - t0; if (held >= 0) gate_leave(held); @@ -7230,7 +7289,7 @@ static void bg_start(void) { // keep 0: the pipeline is built for the pipeline cache only and destroyed at once, unless a dispatch came to wait for // it meanwhile; a shader over LB.big_words is then not built at all (comment at compute-used.bin). cs may be a copy // the caller releases afterwards when keep is 0: nothing here holds on to it then. -static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) { +static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep, int private_shader) { struct { const GfxShader *cs; uint64_t compute; } k = {cs, 1}; // no padding: the key is hashed and compared as bytes uint64_t h = key_hash(&k, sizeof k); Bindings B = {0}; @@ -7273,7 +7332,10 @@ static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) { } else if (lb_idle()) { plat_thread_priority(PLAT_PRIO_NORMAL); } - VkPipeline pipe = compute_create(cs, l, big); + int reused = 0; + // A private shader cannot have a dispatch waiting on its cs_flight slot. Shared/kept shaders always create their + // handle so a dispatch that arrives while the build is in flight still receives the pipeline it is waiting for. + VkPipeline pipe = compute_create(cs, l, big, !keep && private_shader ? &reused : NULL); double ms = now_ms() - t0; int compiled = pipe && cc_hit != 1, kept = 0; lb_lock(); @@ -7293,7 +7355,7 @@ static void lb_compute(const GfxShader *cs, const uint8_t iid[32], int keep) { lb_unlock(); if (!pipe && big && bg_on()) bg_queue(cs, iid, words); else if (!pipe && big) lb_defer(iid, words); - else lb_count(2, !pipe ? 2 : have ? 1 : kept ? 0 : 3, ms); + else lb_count(2, reused ? 1 : !pipe ? 2 : have ? 1 : kept ? 0 : 3, ms); } // The pre-rasterization (vertex) or fragment-shader library of one shader with the LB state. @@ -7428,7 +7490,7 @@ static void lb_worker(void *arg) { LB.noshader++; plat_mutex_unlock(&LB.stat); } else if (gfx_shader_stage(sh) == GFX_STAGE_CS) { - lb_compute(sh, LB.e[i].iid, keep); + lb_compute(sh, LB.e[i].iid, keep, owned); } else if (!owned && (gfx_shader_stage(sh) == GFX_STAGE_VS || gfx_shader_stage(sh) == GFX_STAGE_FS)) { if (P.gpl) lb_library(sh, gfx_shader_stage(sh) == GFX_STAGE_VS, 1); // a kept library points at sh: never a copy else lb_skip_lib(); @@ -7911,9 +7973,10 @@ static void lb_warm_worker(void *arg) { Layout *l = compute_layout(sh, &B, &why); lb_unlock(); double t0 = now_ms(); - VkPipeline pipe = l ? compute_create(sh, l, 0) : VK_NULL_HANDLE; + int reused = 0; + VkPipeline pipe = l ? compute_create(sh, l, 0, &reused) : VK_NULL_HANDLE; if (pipe) vkDestroyPipeline(P.device, pipe, NULL); - lb_count(2, pipe ? 3 : 2, now_ms() - t0); + lb_count(2, reused ? 1 : pipe ? 3 : 2, now_ms() - t0); } else if (st == GFX_STAGE_VS || st == GFX_STAGE_FS) { if (P.gpl) lb_library(sh, st == GFX_STAGE_VS, 0); else lb_skip_lib(); @@ -8552,6 +8615,9 @@ static int cache_save(uint32_t wait_ms, int exiting) { // Returns cache_save's answer. static int save_with(uint32_t wait_ms, int exiting) { if (P.ready) keys_flush(), pb_flush(), cu_flush(), cf_log(); + uint64_t reused = __atomic_load_n(&compute_warm_reused, __ATOMIC_RELAXED); + if (reused) rt_log("vk_pipeline: %llu duplicate compute warm requests reused a successful identical request", + (unsigned long long)reused); if (P.ready && P.gpl) { rt_log("vk_pipeline: gpl: %llu fast links (%llu by workers, %llu of those by the library workers for skipped draws; " "%llu of pipelines with a feedback loop; avg %.2f ms, " @@ -8691,6 +8757,7 @@ void vk_pipeline_autosave(void) { void vk_pipeline_shutdown(void) { if (!P.ready) return; lb_stop(); + compute_warm_clear(); workers_drain(); // jobs point at layouts and Pipe entries freed below vk_pipeline_save(); plat_mutex_lock(&P.lock); diff --git a/tests/compute_warm_reuse_check.py b/tests/compute_warm_reuse_check.py new file mode 100644 index 00000000..1ed2986c --- /dev/null +++ b/tests/compute_warm_reuse_check.py @@ -0,0 +1,113 @@ +"""Exercise production compute creation with a fake driver: reuse must never suppress a required live pipeline.""" +from pathlib import Path +import subprocess +import sys + +root = Path(__file__).resolve().parents[1] +source = (root / 'runtime/vk_pipeline.c').read_text() +start = source.index('typedef struct {\n uint8_t code[32];\n VkPipelineLayout layout;') +end = source.index('// The layout of a compute shader in *B;', start) +production = source[start:end] +harness = r''' +#include +#include +#include +#include +#include +#include +#include +#include "sha256.h" +typedef SRWLOCK PlatMutex; +#define PLAT_MUTEX_INIT SRWLOCK_INIT +#define FNV0 0xcbf29ce484222325ull +#define FB_COMPUTE 0 +typedef struct { uint32_t code[8]; } GfxShader; +typedef struct { VkPipelineLayout layout; } Layout; +static struct { int f_full_subgroups, feedback; VkDevice device; VkPipelineCache cache; } P; +static struct { int sg_size; } vk_caps; +static int cc_hit, pb_last, calls, modules, fail_next, disabled, override_word; +static uint64_t mm_hash; +static VkPipelineCreateFlags test_flags; +static uint64_t fnv(uint64_t h, const void *p, size_t n) { + for (size_t i=0;icode; } +static uint32_t *spv_override(const uint32_t *code, uint64_t n, uint64_t *out) { + if (!override_word) return NULL; + uint32_t *p=malloc(n*4);memcpy(p,code,n*4);p[7]=override_word;*out=n;return p; +} +static uint32_t *spirv_fog_keep_froxels(const uint32_t *code, uint64_t n) { return NULL; } +static VkShaderModule make_module(const uint32_t *code, uint64_t n, int stage) { + modules++;return (VkShaderModule)(uintptr_t)modules; +} +VKAPI_ATTR void VKAPI_CALL vkDestroyShaderModule(VkDevice d,VkShaderModule m,const VkAllocationCallbacks *a) { modules--; } +static VkPipelineCreateFlags noopt_flag(void) { return test_flags; } +static void pin_subgroup32(VkPipelineShaderStageCreateInfo *ci, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *r, + const uint32_t *w,uint64_t n) {} +static uint32_t local_size_x(const uint32_t *w,uint64_t n) { return 32; } +static int uses_subgroup_ops(const uint32_t *w,uint64_t n) { return 1; } +static const char *cfg_get(const char *k) { return disabled ? "0" : NULL; } +static void *fb_chain(VkPipelineCreationFeedbackCreateInfo *c,VkPipelineCreationFeedback *f,const void *next) { + memset(f,0,sizeof *f);return NULL; +} +static void fb_note(int kind, const VkPipelineCreationFeedback *f) {} +static void crash_note(const char *w,const GfxShader *s,const GfxShader *b,const GfxShader *c) {} +static void plat_crash_context(const char *c) {} +static void rt_log(const char *fmt,...) {} +static VkResult pipe_create(int kind, void *info, VkPipeline *out) { + calls++; + if (fail_next) { fail_next=0;*out=VK_NULL_HANDLE;return VK_ERROR_UNKNOWN; } + *out=(VkPipeline)(uintptr_t)calls;return VK_SUCCESS; +} +''' +harness += production +harness += r''' +int main(void) { + GfxShader a={{0x07230203,1,2,3,4,5,6,7}}, b=a; + Layout l={(VkPipelineLayout)(uintptr_t)1}, other={(VkPipelineLayout)(uintptr_t)2}; + int reused=-1; + assert(compute_create(&a,&l,0,&reused) && !reused && calls==1); + assert(!compute_create(&b,&l,0,&reused) && reused && calls==1); // distinct shader objects, same request + assert(compute_create(&b,&l,0,NULL) && calls==2); // real dispatch still obtains a handle + assert(compute_create(&b,&other,0,&reused) && !reused && calls==3); + test_flags=VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT; + assert(compute_create(&b,&l,0,&reused) && !reused && calls==4); + test_flags=0;b.code[7]++; + fail_next=1; + assert(!compute_create(&b,&l,0,&reused) && !reused && calls==5); + assert(compute_create(&b,&l,0,&reused) && !reused && calls==6); // failure is not cached + assert(!compute_create(&b,&l,0,&reused) && reused && calls==6); + assert(compute_create(&b,&l,1,&reused) && !reused && calls==7); // cache-only creation flags differ + override_word=19; + assert(compute_create(&a,&l,0,&reused) && !reused && calls==8); // final code, not original code + override_word=20; + assert(compute_create(&a,&l,0,&reused) && !reused && calls==9); + override_word=19; + assert(!compute_create(&a,&l,0,&reused) && reused && calls==9); + override_word=0;P.f_full_subgroups=1;vk_caps.sg_size=32; + assert(compute_create(&a,&l,0,&reused) && !reused && calls==10); + disabled=1; + assert(compute_create(&a,&l,0,&reused) && !reused && calls==11); + disabled=0; + assert(!compute_create(&a,&l,0,&reused) && reused && calls==11); + compute_warm_clear(); + assert(compute_create(&a,&l,0,&reused) && !reused && calls==12); + compute_warm_clear(); + assert(modules==0); + puts("compute warm reuse: equivalent requests, live handles, layout/flags/code separation, failures and cleanup passed"); + return 0; +} +''' +out = root / 'build/test/compute_warm_reuse' +out.mkdir(parents=True, exist_ok=True) +c = out / 'check.c' +c.write_text(harness) +clang = Path(sys.argv[1]) if len(sys.argv) > 1 else Path('C:/msys64/ucrt64/bin/clang.exe') +exe = out / 'check.exe' +subprocess.run([str(clang), '-std=gnu11', '-O1', '-I', str(root / 'runtime'), str(c), + str(root / 'runtime/sha256.c'), '-o', str(exe)], check=True) +subprocess.run([str(exe)], check=True) diff --git a/tests/prepared_request_reuse_check.py b/tests/prepared_request_reuse_check.py new file mode 100644 index 00000000..fc6d9201 --- /dev/null +++ b/tests/prepared_request_reuse_check.py @@ -0,0 +1,91 @@ +"""Compare preparation request sharing with independent jobs using six synthetic shaders. + +Pass the candidate executable, the translator, and --reference-exe for a paired build. +No game data or installed cache is used. All output stays in build/. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import struct +import subprocess +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[1] + +def seed(directory): + programs = bytearray(struct.pack('<8sII', b'NX2NSPRG', 1, 0)) + answers = bytearray(struct.pack('<8sII', b'NX2NSANS', 1, 216)) + exit_op = struct.pack('<4I', 0x794d, 0, 0x03800000, 0x000fea00) + for tag, group, code, c1 in [(0, 1, exit_op, b''), (1, 1, exit_op, b''), (2, 1, exit_op, b''), + (0, 2, exit_op, b''), (0, 1, 2 * exit_op, b''), (0, 1, exit_op, bytes(16))]: + control = bytearray(0x880) + struct.pack_into('