Reuse equivalent shader translation and compute preparation work

This commit is contained in:
automata.
2026-11-11 17:55:06 -05:00
committed by chrg6
parent 6590f68f6e
commit 9e23aec717
4 changed files with 426 additions and 44 deletions

View File

@@ -0,0 +1,113 @@
"""Exercise production compute creation with a fake driver: reuse must never suppress a required live pipeline."""
from pathlib import Path
import subprocess
import sys
root = Path(__file__).resolve().parents[1]
source = (root / 'runtime/vk_pipeline.c').read_text()
start = source.index('typedef struct {\n uint8_t code[32];\n VkPipelineLayout layout;')
end = source.index('// The layout of a compute shader in *B;', start)
production = source[start:end]
harness = r'''
#include <windows.h>
#include <vulkan/vulkan.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "sha256.h"
typedef SRWLOCK PlatMutex;
#define PLAT_MUTEX_INIT SRWLOCK_INIT
#define FNV0 0xcbf29ce484222325ull
#define FB_COMPUTE 0
typedef struct { uint32_t code[8]; } GfxShader;
typedef struct { VkPipelineLayout layout; } Layout;
static struct { int f_full_subgroups, feedback; VkDevice device; VkPipelineCache cache; } P;
static struct { int sg_size; } vk_caps;
static int cc_hit, pb_last, calls, modules, fail_next, disabled, override_word;
static uint64_t mm_hash;
static VkPipelineCreateFlags test_flags;
static uint64_t fnv(uint64_t h, const void *p, size_t n) {
for (size_t i=0;i<n;i++) h=(h^((const unsigned char *)p)[i])*0x100000001b3ull;
return h;
}
static void plat_mutex_lock(PlatMutex *m) { AcquireSRWLockExclusive(m); }
static void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive(m); }
static void rt_sha256(const void *p, size_t n, uint8_t out[32]) { sha256(p,n,out); }
static const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *n) { *n=8;return s->code; }
static uint32_t *spv_override(const uint32_t *code, uint64_t n, uint64_t *out) {
if (!override_word) return NULL;
uint32_t *p=malloc(n*4);memcpy(p,code,n*4);p[7]=override_word;*out=n;return p;
}
static uint32_t *spirv_fog_keep_froxels(const uint32_t *code, uint64_t n) { return NULL; }
static VkShaderModule make_module(const uint32_t *code, uint64_t n, int stage) {
modules++;return (VkShaderModule)(uintptr_t)modules;
}
VKAPI_ATTR void VKAPI_CALL vkDestroyShaderModule(VkDevice d,VkShaderModule m,const VkAllocationCallbacks *a) { modules--; }
static VkPipelineCreateFlags noopt_flag(void) { return test_flags; }
static void pin_subgroup32(VkPipelineShaderStageCreateInfo *ci, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *r,
const uint32_t *w,uint64_t n) {}
static uint32_t local_size_x(const uint32_t *w,uint64_t n) { return 32; }
static int uses_subgroup_ops(const uint32_t *w,uint64_t n) { return 1; }
static const char *cfg_get(const char *k) { return disabled ? "0" : NULL; }
static void *fb_chain(VkPipelineCreationFeedbackCreateInfo *c,VkPipelineCreationFeedback *f,const void *next) {
memset(f,0,sizeof *f);return NULL;
}
static void fb_note(int kind, const VkPipelineCreationFeedback *f) {}
static void crash_note(const char *w,const GfxShader *s,const GfxShader *b,const GfxShader *c) {}
static void plat_crash_context(const char *c) {}
static void rt_log(const char *fmt,...) {}
static VkResult pipe_create(int kind, void *info, VkPipeline *out) {
calls++;
if (fail_next) { fail_next=0;*out=VK_NULL_HANDLE;return VK_ERROR_UNKNOWN; }
*out=(VkPipeline)(uintptr_t)calls;return VK_SUCCESS;
}
'''
harness += production
harness += r'''
int main(void) {
GfxShader a={{0x07230203,1,2,3,4,5,6,7}}, b=a;
Layout l={(VkPipelineLayout)(uintptr_t)1}, other={(VkPipelineLayout)(uintptr_t)2};
int reused=-1;
assert(compute_create(&a,&l,0,&reused) && !reused && calls==1);
assert(!compute_create(&b,&l,0,&reused) && reused && calls==1); // distinct shader objects, same request
assert(compute_create(&b,&l,0,NULL) && calls==2); // real dispatch still obtains a handle
assert(compute_create(&b,&other,0,&reused) && !reused && calls==3);
test_flags=VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;
assert(compute_create(&b,&l,0,&reused) && !reused && calls==4);
test_flags=0;b.code[7]++;
fail_next=1;
assert(!compute_create(&b,&l,0,&reused) && !reused && calls==5);
assert(compute_create(&b,&l,0,&reused) && !reused && calls==6); // failure is not cached
assert(!compute_create(&b,&l,0,&reused) && reused && calls==6);
assert(compute_create(&b,&l,1,&reused) && !reused && calls==7); // cache-only creation flags differ
override_word=19;
assert(compute_create(&a,&l,0,&reused) && !reused && calls==8); // final code, not original code
override_word=20;
assert(compute_create(&a,&l,0,&reused) && !reused && calls==9);
override_word=19;
assert(!compute_create(&a,&l,0,&reused) && reused && calls==9);
override_word=0;P.f_full_subgroups=1;vk_caps.sg_size=32;
assert(compute_create(&a,&l,0,&reused) && !reused && calls==10);
disabled=1;
assert(compute_create(&a,&l,0,&reused) && !reused && calls==11);
disabled=0;
assert(!compute_create(&a,&l,0,&reused) && reused && calls==11);
compute_warm_clear();
assert(compute_create(&a,&l,0,&reused) && !reused && calls==12);
compute_warm_clear();
assert(modules==0);
puts("compute warm reuse: equivalent requests, live handles, layout/flags/code separation, failures and cleanup passed");
return 0;
}
'''
out = root / 'build/test/compute_warm_reuse'
out.mkdir(parents=True, exist_ok=True)
c = out / 'check.c'
c.write_text(harness)
clang = Path(sys.argv[1]) if len(sys.argv) > 1 else Path('C:/msys64/ucrt64/bin/clang.exe')
exe = out / 'check.exe'
subprocess.run([str(clang), '-std=gnu11', '-O1', '-I', str(root / 'runtime'), str(c),
str(root / 'runtime/sha256.c'), '-o', str(exe)], check=True)
subprocess.run([str(exe)], check=True)

View File

@@ -0,0 +1,91 @@
"""Compare preparation request sharing with independent jobs using six synthetic shaders.
Pass the candidate executable, the translator, and --reference-exe for a paired build.
No game data or installed cache is used. All output stays in build/.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import struct
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[1]
def seed(directory):
programs = bytearray(struct.pack('<8sII', b'NX2NSPRG', 1, 0))
answers = bytearray(struct.pack('<8sII', b'NX2NSANS', 1, 216))
exit_op = struct.pack('<4I', 0x794d, 0, 0x03800000, 0x000fea00)
for tag, group, code, c1 in [(0, 1, exit_op, b''), (1, 1, exit_op, b''), (2, 1, exit_op, b''),
(0, 2, exit_op, b''), (0, 1, 2 * exit_op, b''), (0, 1, exit_op, bytes(16))]:
control = bytearray(0x880)
struct.pack_into('<I', control, 0, 0x98761234)
control[0x50] = tag
struct.pack_into('<I', control, 0x38, len(code))
struct.pack_into('<II', control, 0x40, len(code), len(code) + len(c1))
struct.pack_into('<3I', control, 0x58, group, 1, 1)
full = code + c1
key = hashlib.sha256(struct.pack('<I', 5) + control + struct.pack('<Q', len(full)) + full
+ struct.pack('<I', len(c1)) + c1).digest()
programs += struct.pack('<III', 5, len(full), len(c1)) + key + control + full + c1
answers += struct.pack('<I', 5) + key + bytes(32 + 147 + 1)
(directory / 'seed-programs.bin').write_bytes(programs)
(directory / 'seed-answers.bin').write_bytes(answers)
def snapshot(cache):
return {str(p.relative_to(cache)): p.read_bytes() for p in cache.rglob('*')
if p.is_file() and (p.suffix in ('.spv', '.use', '.in') or p.name.endswith('.res.json'))}
def run(exe, translator, folder, keep=False):
env = dict(os.environ, NX2NAVI_XLATE=str(translator), NX2NAVI_XLATE_ID='request-reuse-test',
NX2NAVI_XLATE_KEEP='2' if keep else '0', NX2NAVI_SHADER_CARRY='0',
NX2NAVI_PREWARM_XLATE='4', NX2NAVI_PIPELINE_CACHE=str(folder/'pipeline-cache.bin'),
NX2NAVI_SHADER_CACHE=str(folder/'shader-cache'))
t=time.perf_counter()
result=subprocess.run([str(exe),'--prepare-shaders'],cwd=folder,env=env,capture_output=True,timeout=90)
elapsed=time.perf_counter()-t
log=(result.stdout+result.stderr).decode(errors='replace')
(folder/('warm.log' if (folder/'run.log').exists() else 'run.log')).write_text(log)
assert result.returncode == 0, log
translated=re.search(r'6 seed variants of 6 programs: (\d+) translated, 0 failed',log)
assert translated,log
return int(translated[1]),elapsed,log
def main():
ap=argparse.ArgumentParser(description=__doc__)
ap.add_argument('game',type=Path)
ap.add_argument('translator',type=Path)
ap.add_argument('--reference-exe',type=Path)
ap.add_argument('--out',type=Path,default=ROOT/'build/request-reuse-check')
args=ap.parse_args()
args.out.mkdir(parents=True,exist_ok=True)
exe=args.game.resolve(); ref=(args.reference_exe or args.game).resolve(); translator=args.translator.resolve()
results=[]
for iteration in range(3):
folders=[args.out.resolve()/f'{iteration}-{n}' for n in ('independent','shared')]
for folder in folders:
folder.mkdir(exist_ok=False)
seed(folder)
count0,time0,_=run(ref,translator,folders[0],keep=True)
count1,time1,log1=run(exe,translator,folders[1])
assert count0==6 and count1==4 and '2 equivalent translator requests reused' in log1,log1
assert snapshot(folders[0]/'shader-cache')==snapshot(folders[1]/'shader-cache')
assert (folders[0]/'shader-prepared.bin').read_bytes()==(folders[1]/'shader-prepared.bin').read_bytes()
before=snapshot(folders[1]/'shader-cache')
count2,_,_=run(exe,translator,folders[1])
assert count2==0 and before==snapshot(folders[1]/'shader-cache')
results.append(dict(independent_jobs=count0,shared_jobs=count1,independent_seconds=time0,shared_seconds=time1,
identical_outputs=True,warm_jobs=count2))
(args.out/'results.json').write_text(json.dumps(results,indent=2))
print(json.dumps(results,indent=2))
if __name__=='__main__':
main()