mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 15:49:57 -04:00
Reuse equivalent shader translation and compute preparation work
This commit is contained in:
113
tests/compute_warm_reuse_check.py
Normal file
113
tests/compute_warm_reuse_check.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""Exercise production compute creation with a fake driver: reuse must never suppress a required live pipeline."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
source = (root / 'runtime/vk_pipeline.c').read_text()
|
||||
start = source.index('typedef struct {\n uint8_t code[32];\n VkPipelineLayout layout;')
|
||||
end = source.index('// The layout of a compute shader in *B;', start)
|
||||
production = source[start:end]
|
||||
harness = r'''
|
||||
#include <windows.h>
|
||||
#include <vulkan/vulkan.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include "sha256.h"
|
||||
typedef SRWLOCK PlatMutex;
|
||||
#define PLAT_MUTEX_INIT SRWLOCK_INIT
|
||||
#define FNV0 0xcbf29ce484222325ull
|
||||
#define FB_COMPUTE 0
|
||||
typedef struct { uint32_t code[8]; } GfxShader;
|
||||
typedef struct { VkPipelineLayout layout; } Layout;
|
||||
static struct { int f_full_subgroups, feedback; VkDevice device; VkPipelineCache cache; } P;
|
||||
static struct { int sg_size; } vk_caps;
|
||||
static int cc_hit, pb_last, calls, modules, fail_next, disabled, override_word;
|
||||
static uint64_t mm_hash;
|
||||
static VkPipelineCreateFlags test_flags;
|
||||
static uint64_t fnv(uint64_t h, const void *p, size_t n) {
|
||||
for (size_t i=0;i<n;i++) h=(h^((const unsigned char *)p)[i])*0x100000001b3ull;
|
||||
return h;
|
||||
}
|
||||
static void plat_mutex_lock(PlatMutex *m) { AcquireSRWLockExclusive(m); }
|
||||
static void plat_mutex_unlock(PlatMutex *m) { ReleaseSRWLockExclusive(m); }
|
||||
static void rt_sha256(const void *p, size_t n, uint8_t out[32]) { sha256(p,n,out); }
|
||||
static const uint32_t *gfx_shader_spirv(const GfxShader *s, uint64_t *n) { *n=8;return s->code; }
|
||||
static uint32_t *spv_override(const uint32_t *code, uint64_t n, uint64_t *out) {
|
||||
if (!override_word) return NULL;
|
||||
uint32_t *p=malloc(n*4);memcpy(p,code,n*4);p[7]=override_word;*out=n;return p;
|
||||
}
|
||||
static uint32_t *spirv_fog_keep_froxels(const uint32_t *code, uint64_t n) { return NULL; }
|
||||
static VkShaderModule make_module(const uint32_t *code, uint64_t n, int stage) {
|
||||
modules++;return (VkShaderModule)(uintptr_t)modules;
|
||||
}
|
||||
VKAPI_ATTR void VKAPI_CALL vkDestroyShaderModule(VkDevice d,VkShaderModule m,const VkAllocationCallbacks *a) { modules--; }
|
||||
static VkPipelineCreateFlags noopt_flag(void) { return test_flags; }
|
||||
static void pin_subgroup32(VkPipelineShaderStageCreateInfo *ci, VkPipelineShaderStageRequiredSubgroupSizeCreateInfo *r,
|
||||
const uint32_t *w,uint64_t n) {}
|
||||
static uint32_t local_size_x(const uint32_t *w,uint64_t n) { return 32; }
|
||||
static int uses_subgroup_ops(const uint32_t *w,uint64_t n) { return 1; }
|
||||
static const char *cfg_get(const char *k) { return disabled ? "0" : NULL; }
|
||||
static void *fb_chain(VkPipelineCreationFeedbackCreateInfo *c,VkPipelineCreationFeedback *f,const void *next) {
|
||||
memset(f,0,sizeof *f);return NULL;
|
||||
}
|
||||
static void fb_note(int kind, const VkPipelineCreationFeedback *f) {}
|
||||
static void crash_note(const char *w,const GfxShader *s,const GfxShader *b,const GfxShader *c) {}
|
||||
static void plat_crash_context(const char *c) {}
|
||||
static void rt_log(const char *fmt,...) {}
|
||||
static VkResult pipe_create(int kind, void *info, VkPipeline *out) {
|
||||
calls++;
|
||||
if (fail_next) { fail_next=0;*out=VK_NULL_HANDLE;return VK_ERROR_UNKNOWN; }
|
||||
*out=(VkPipeline)(uintptr_t)calls;return VK_SUCCESS;
|
||||
}
|
||||
'''
|
||||
harness += production
|
||||
harness += r'''
|
||||
int main(void) {
|
||||
GfxShader a={{0x07230203,1,2,3,4,5,6,7}}, b=a;
|
||||
Layout l={(VkPipelineLayout)(uintptr_t)1}, other={(VkPipelineLayout)(uintptr_t)2};
|
||||
int reused=-1;
|
||||
assert(compute_create(&a,&l,0,&reused) && !reused && calls==1);
|
||||
assert(!compute_create(&b,&l,0,&reused) && reused && calls==1); // distinct shader objects, same request
|
||||
assert(compute_create(&b,&l,0,NULL) && calls==2); // real dispatch still obtains a handle
|
||||
assert(compute_create(&b,&other,0,&reused) && !reused && calls==3);
|
||||
test_flags=VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;
|
||||
assert(compute_create(&b,&l,0,&reused) && !reused && calls==4);
|
||||
test_flags=0;b.code[7]++;
|
||||
fail_next=1;
|
||||
assert(!compute_create(&b,&l,0,&reused) && !reused && calls==5);
|
||||
assert(compute_create(&b,&l,0,&reused) && !reused && calls==6); // failure is not cached
|
||||
assert(!compute_create(&b,&l,0,&reused) && reused && calls==6);
|
||||
assert(compute_create(&b,&l,1,&reused) && !reused && calls==7); // cache-only creation flags differ
|
||||
override_word=19;
|
||||
assert(compute_create(&a,&l,0,&reused) && !reused && calls==8); // final code, not original code
|
||||
override_word=20;
|
||||
assert(compute_create(&a,&l,0,&reused) && !reused && calls==9);
|
||||
override_word=19;
|
||||
assert(!compute_create(&a,&l,0,&reused) && reused && calls==9);
|
||||
override_word=0;P.f_full_subgroups=1;vk_caps.sg_size=32;
|
||||
assert(compute_create(&a,&l,0,&reused) && !reused && calls==10);
|
||||
disabled=1;
|
||||
assert(compute_create(&a,&l,0,&reused) && !reused && calls==11);
|
||||
disabled=0;
|
||||
assert(!compute_create(&a,&l,0,&reused) && reused && calls==11);
|
||||
compute_warm_clear();
|
||||
assert(compute_create(&a,&l,0,&reused) && !reused && calls==12);
|
||||
compute_warm_clear();
|
||||
assert(modules==0);
|
||||
puts("compute warm reuse: equivalent requests, live handles, layout/flags/code separation, failures and cleanup passed");
|
||||
return 0;
|
||||
}
|
||||
'''
|
||||
out = root / 'build/test/compute_warm_reuse'
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
c = out / 'check.c'
|
||||
c.write_text(harness)
|
||||
clang = Path(sys.argv[1]) if len(sys.argv) > 1 else Path('C:/msys64/ucrt64/bin/clang.exe')
|
||||
exe = out / 'check.exe'
|
||||
subprocess.run([str(clang), '-std=gnu11', '-O1', '-I', str(root / 'runtime'), str(c),
|
||||
str(root / 'runtime/sha256.c'), '-o', str(exe)], check=True)
|
||||
subprocess.run([str(exe)], check=True)
|
||||
91
tests/prepared_request_reuse_check.py
Normal file
91
tests/prepared_request_reuse_check.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""Compare preparation request sharing with independent jobs using six synthetic shaders.
|
||||
|
||||
Pass the candidate executable, the translator, and --reference-exe for a paired build.
|
||||
No game data or installed cache is used. All output stays in build/.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def seed(directory):
|
||||
programs = bytearray(struct.pack('<8sII', b'NX2NSPRG', 1, 0))
|
||||
answers = bytearray(struct.pack('<8sII', b'NX2NSANS', 1, 216))
|
||||
exit_op = struct.pack('<4I', 0x794d, 0, 0x03800000, 0x000fea00)
|
||||
for tag, group, code, c1 in [(0, 1, exit_op, b''), (1, 1, exit_op, b''), (2, 1, exit_op, b''),
|
||||
(0, 2, exit_op, b''), (0, 1, 2 * exit_op, b''), (0, 1, exit_op, bytes(16))]:
|
||||
control = bytearray(0x880)
|
||||
struct.pack_into('<I', control, 0, 0x98761234)
|
||||
control[0x50] = tag
|
||||
struct.pack_into('<I', control, 0x38, len(code))
|
||||
struct.pack_into('<II', control, 0x40, len(code), len(code) + len(c1))
|
||||
struct.pack_into('<3I', control, 0x58, group, 1, 1)
|
||||
full = code + c1
|
||||
key = hashlib.sha256(struct.pack('<I', 5) + control + struct.pack('<Q', len(full)) + full
|
||||
+ struct.pack('<I', len(c1)) + c1).digest()
|
||||
programs += struct.pack('<III', 5, len(full), len(c1)) + key + control + full + c1
|
||||
answers += struct.pack('<I', 5) + key + bytes(32 + 147 + 1)
|
||||
(directory / 'seed-programs.bin').write_bytes(programs)
|
||||
(directory / 'seed-answers.bin').write_bytes(answers)
|
||||
|
||||
|
||||
def snapshot(cache):
|
||||
return {str(p.relative_to(cache)): p.read_bytes() for p in cache.rglob('*')
|
||||
if p.is_file() and (p.suffix in ('.spv', '.use', '.in') or p.name.endswith('.res.json'))}
|
||||
|
||||
|
||||
def run(exe, translator, folder, keep=False):
|
||||
env = dict(os.environ, NX2NAVI_XLATE=str(translator), NX2NAVI_XLATE_ID='request-reuse-test',
|
||||
NX2NAVI_XLATE_KEEP='2' if keep else '0', NX2NAVI_SHADER_CARRY='0',
|
||||
NX2NAVI_PREWARM_XLATE='4', NX2NAVI_PIPELINE_CACHE=str(folder/'pipeline-cache.bin'),
|
||||
NX2NAVI_SHADER_CACHE=str(folder/'shader-cache'))
|
||||
t=time.perf_counter()
|
||||
result=subprocess.run([str(exe),'--prepare-shaders'],cwd=folder,env=env,capture_output=True,timeout=90)
|
||||
elapsed=time.perf_counter()-t
|
||||
log=(result.stdout+result.stderr).decode(errors='replace')
|
||||
(folder/('warm.log' if (folder/'run.log').exists() else 'run.log')).write_text(log)
|
||||
assert result.returncode == 0, log
|
||||
translated=re.search(r'6 seed variants of 6 programs: (\d+) translated, 0 failed',log)
|
||||
assert translated,log
|
||||
return int(translated[1]),elapsed,log
|
||||
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument('game',type=Path)
|
||||
ap.add_argument('translator',type=Path)
|
||||
ap.add_argument('--reference-exe',type=Path)
|
||||
ap.add_argument('--out',type=Path,default=ROOT/'build/request-reuse-check')
|
||||
args=ap.parse_args()
|
||||
args.out.mkdir(parents=True,exist_ok=True)
|
||||
exe=args.game.resolve(); ref=(args.reference_exe or args.game).resolve(); translator=args.translator.resolve()
|
||||
results=[]
|
||||
for iteration in range(3):
|
||||
folders=[args.out.resolve()/f'{iteration}-{n}' for n in ('independent','shared')]
|
||||
for folder in folders:
|
||||
folder.mkdir(exist_ok=False)
|
||||
seed(folder)
|
||||
count0,time0,_=run(ref,translator,folders[0],keep=True)
|
||||
count1,time1,log1=run(exe,translator,folders[1])
|
||||
assert count0==6 and count1==4 and '2 equivalent translator requests reused' in log1,log1
|
||||
assert snapshot(folders[0]/'shader-cache')==snapshot(folders[1]/'shader-cache')
|
||||
assert (folders[0]/'shader-prepared.bin').read_bytes()==(folders[1]/'shader-prepared.bin').read_bytes()
|
||||
before=snapshot(folders[1]/'shader-cache')
|
||||
count2,_,_=run(exe,translator,folders[1])
|
||||
assert count2==0 and before==snapshot(folders[1]/'shader-cache')
|
||||
results.append(dict(independent_jobs=count0,shared_jobs=count1,independent_seconds=time0,shared_seconds=time1,
|
||||
identical_outputs=True,warm_jobs=count2))
|
||||
(args.out/'results.json').write_text(json.dumps(results,indent=2))
|
||||
print(json.dumps(results,indent=2))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user