mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 13:59:56 -04:00
Stop flushing the shader cache files every read checks, so shader preparation translates twice as fast.
Each translation flushed its .spv and .res.json to the disk, wrote its .use mask with a flushed, written-through rename and renamed its outputs written through: about six waits for the drive per shader. An NVMe drive does about 1,100 flushed files a second, a SATA SSD or a hard drive far fewer, so the translators sat mostly idle. Every read already checks these files (spirv_complete, reflection_valid, the .use magic and size) and a torn one is translated again, so they are now written whole but not flushed (plat_file_write_cached, plat_file_replace without durable). The .in files stay flushed: they hold the guest's code and only their header and size are checked. 4,683 shaders of the install's preparation on a Ryzen 7 7700X and a Samsung 990 PRO: translation 36.4 and 37.2 s before, 18.4 and 18.9 s after; every .spv, .res.json, .use and .in file and shader-prepared.bin are the same. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,15 @@
|
||||
// input id, can be translated again under a new translator without guest memory (gfx_shader_load_input,
|
||||
// gfx_shader_prewarm).
|
||||
//
|
||||
// Durability. Every file appears whole: it is written under another name and renamed. Only the .in files are also
|
||||
// flushed to the disk before their rename (plat_file_write_atomic): they hold the guest's code, which nothing else can
|
||||
// give back once the game is not running, and in_load checks only their header and size, so a power cut that tore one
|
||||
// would have a wrong shader translated from it. The .spv, .res.json and .use files are not flushed
|
||||
// (plat_file_write_cached, plat_file_replace without durable): every read checks them (spirv_complete,
|
||||
// reflection_valid, the .use magic and size) and a torn one is translated again. A flush waits for the drive, 1-5 ms
|
||||
// on an NVMe drive and far longer on a SATA SSD or a hard drive, and the preparation of an install translates about
|
||||
// 36,000 shaders: flushing these files too held its translation at half speed on an NVMe drive.
|
||||
//
|
||||
// Reflection: every shader that enters the cache has its .res.json parsed into GfxShaderRes arrays and its SPIR-V
|
||||
// scanned for interface Locations and the workgroup size (gfx.h accessors at the end of the file).
|
||||
//
|
||||
@@ -1058,7 +1067,7 @@ static void use_store(const uint8_t h[32], const UseMask *m) {
|
||||
char p[PATH_CAP + 80];
|
||||
use_path(h, p);
|
||||
uint32_t f[5] = {USE_MAGIC, USE_VERSION, m->slots, m->attrs, m->fouts};
|
||||
plat_file_write_atomic(p, f, sizeof f);
|
||||
plat_file_write_cached(p, f, sizeof f); // not flushed: a torn mask fails its magic and size check (see the top)
|
||||
plat_mutex_lock(&cache_lock);
|
||||
use_put(h, 1, m);
|
||||
plat_mutex_unlock(&cache_lock);
|
||||
@@ -1423,7 +1432,8 @@ static void in_path(const uint8_t iid[32], char *p) {
|
||||
snprintf(p, PATH_CAP + 80, "%s/in/%s.in", cache_dir, hex);
|
||||
}
|
||||
|
||||
// Stores the inputs of a translation unless they are stored already; the file appears whole (written, then renamed).
|
||||
// Stores the inputs of a translation unless they are stored already; the file appears whole (written, then renamed)
|
||||
// and is on the disk before its rename, the one cache file that is (see "Durability" at the top).
|
||||
static void in_store(const uint8_t iid[32], int stage, const uint8_t *control, const uint8_t *code, const Req *r,
|
||||
const GfxShaderAnswers *a) {
|
||||
char p[PATH_CAP + 80], tmp[PATH_CAP + 120];
|
||||
@@ -1518,7 +1528,7 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co
|
||||
if (r->sm86) plat_inc64(&n_sm86);
|
||||
return fail(reply, hex);
|
||||
}
|
||||
plat_file_flush(spv_path), plat_file_flush(res_path); // the translator wrote them; on disk before use or rename
|
||||
// Not flushed (see "Durability" at the top): the translator wrote them whole, and disk_load checks both.
|
||||
if (!(s = disk_load(key, r->iid))) return fail("translator output missing or malformed", hex);
|
||||
if (!r->final) { // name the files after the cache key, remember the mask
|
||||
m = use_of(s);
|
||||
@@ -1526,9 +1536,9 @@ static const GfxShader *translate_one(const Req *r, int stage, const uint8_t *co
|
||||
char mhex[65], to[PATH_CAP + 80];
|
||||
key_hex(mk, mhex);
|
||||
snprintf(to, PATH_CAP + 80, "%s/%s.spv", cache_dir, mhex);
|
||||
int moved = plat_file_replace(spv_path, to);
|
||||
int moved = plat_file_replace(spv_path, to, 0);
|
||||
snprintf(to, PATH_CAP + 80, "%s/%s.res.json", cache_dir, mhex);
|
||||
moved = moved && plat_file_replace(res_path, to);
|
||||
moved = moved && plat_file_replace(res_path, to, 0);
|
||||
if (moved) {
|
||||
use_store(r->code_hash, &m);
|
||||
} else { // keep the mask for this run only
|
||||
|
||||
@@ -123,7 +123,7 @@ static void log_open(int game) {
|
||||
if (i > 1) snprintf(from, sizeof from, "%s/nx2navi.%d.log", dir, i - 1);
|
||||
else snprintf(from, sizeof from, "%s/nx2navi.log", dir);
|
||||
snprintf(to, sizeof to, "%s/nx2navi.%d.log", dir, i);
|
||||
if (plat_file_exists(from)) plat_file_replace(from, to);
|
||||
if (plat_file_exists(from)) plat_file_replace(from, to, 1);
|
||||
}
|
||||
snprintf(g_log_path, sizeof g_log_path, "%s/nx2navi.log", dir);
|
||||
g_log_file = plat_append_open(g_log_path);
|
||||
|
||||
@@ -755,9 +755,9 @@ static void sync_dir_of(const char *path) {
|
||||
}
|
||||
}
|
||||
|
||||
int plat_file_replace(const char *from, const char *to) {
|
||||
int plat_file_replace(const char *from, const char *to, int durable) {
|
||||
if (rename(from, to)) return 0;
|
||||
sync_dir_of(to);
|
||||
if (durable) sync_dir_of(to);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -804,7 +804,8 @@ void plat_unlock_file(const char *path) {
|
||||
plat_mutex_unlock(&lock_lock);
|
||||
}
|
||||
|
||||
int plat_file_write_atomic(const char *path, const void *data, size_t n) {
|
||||
// path.<pid>.<tid>.tmp renamed over path; durable: the file and then its folder are synced to the disk.
|
||||
static int write_swap(const char *path, const void *data, size_t n, int durable) {
|
||||
// A rename replaces a read-only file here (only the folder's permission counts) but not on Windows: a file the
|
||||
// player made read-only stays as it is on both.
|
||||
if (access(path, W_OK) && errno != ENOENT) return 0;
|
||||
@@ -813,16 +814,20 @@ int plat_file_write_atomic(const char *path, const void *data, size_t n) {
|
||||
if (len < 0 || len >= (int)sizeof tmp) return 0;
|
||||
int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
|
||||
if (fd < 0) return 0;
|
||||
int ok = write_all(fd, data, n) && fsync(fd) == 0;
|
||||
int ok = write_all(fd, data, n) && (!durable || fsync(fd) == 0);
|
||||
ok = close(fd) == 0 && ok;
|
||||
if (ok && !rename(tmp, path)) {
|
||||
sync_dir_of(path);
|
||||
if (durable) sync_dir_of(path);
|
||||
return 1;
|
||||
}
|
||||
unlink(tmp);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int plat_file_write_atomic(const char *path, const void *data, size_t n) { return write_swap(path, data, n, 1); }
|
||||
|
||||
int plat_file_write_cached(const char *path, const void *data, size_t n) { return write_swap(path, data, n, 0); }
|
||||
|
||||
int plat_file_flush(const char *path) {
|
||||
int fd = open(path, O_WRONLY | O_CLOEXEC);
|
||||
if (fd < 0) return 0;
|
||||
|
||||
@@ -306,10 +306,15 @@ void plat_file_mkdir_p(const char *path); // creates every missi
|
||||
// Whole file into a malloc'd buffer with a NUL after the last byte; NULL when missing, empty or over 1 GiB.
|
||||
uint8_t *plat_file_read(const char *path, uint64_t *size);
|
||||
int plat_file_write(const char *path, const void *data, size_t n); // create or truncate; 1 when all n bytes landed
|
||||
int plat_file_replace(const char *from, const char *to); // rename, replacing to (write-through); 1 on success
|
||||
// Renames from over to, replacing it; durable: the rename is on the disk at return (written through). 1 on success.
|
||||
int plat_file_replace(const char *from, const char *to, int durable);
|
||||
// Writes path.<pid>.<tid>.tmp, flushes it to the disk and renames it over path: a crash leaves the old file or the new
|
||||
// one, never a torn or zero-filled one. 1 on success.
|
||||
int plat_file_write_atomic(const char *path, const void *data, size_t n);
|
||||
// The same without waiting for the disk: a crash of the process still leaves the old file or the new one, but a power
|
||||
// cut can leave a torn or zero-filled one. For files whose readers check them and that can be made again (the shader
|
||||
// cache). Each flush costs 1-5 ms on an NVMe drive and much more on a SATA SSD or a hard drive. 1 on success.
|
||||
int plat_file_write_cached(const char *path, const void *data, size_t n);
|
||||
// Takes the file at path (made when missing) as a lock no other process can take, held until this process ends:
|
||||
// 1 held (again, when this process holds it already), 0 another process holds it, -1 the file cannot be made.
|
||||
int plat_lock_file(const char *path);
|
||||
|
||||
@@ -605,10 +605,10 @@ int plat_append_write(PlatAppend *f, const void *data, size_t n) {
|
||||
return WriteFile((HANDLE)f, data, (DWORD)n, &put, &o) && put == n;
|
||||
}
|
||||
|
||||
int plat_file_replace(const char *from, const char *to) {
|
||||
int plat_file_replace(const char *from, const char *to, int durable) {
|
||||
wchar_t a[WPATH], b[WPATH];
|
||||
if (!widen(from, a, WPATH) || !widen(to, b, WPATH)) return 0;
|
||||
return MoveFileExW(a, b, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) != 0;
|
||||
return MoveFileExW(a, b, MOVEFILE_REPLACE_EXISTING | (durable ? MOVEFILE_WRITE_THROUGH : 0)) != 0;
|
||||
}
|
||||
|
||||
static char lock_held[4][1100];
|
||||
@@ -654,7 +654,8 @@ void plat_unlock_file(const char *path) {
|
||||
plat_mutex_unlock(&lock_lock);
|
||||
}
|
||||
|
||||
int plat_file_write_atomic(const char *path, const void *data, size_t n) {
|
||||
// path.<pid>.<tid>.tmp renamed over path; durable: flushed to the disk before the rename, which is written through.
|
||||
static int write_swap(const char *path, const void *data, size_t n, int durable) {
|
||||
char tmp[WPATH + 48];
|
||||
wchar_t a[WPATH], b[WPATH];
|
||||
int len = snprintf(tmp, sizeof tmp, "%s.%u.%u.tmp", path, (unsigned)GetCurrentProcessId(), (unsigned)GetCurrentThreadId());
|
||||
@@ -662,13 +663,17 @@ int plat_file_write_atomic(const char *path, const void *data, size_t n) {
|
||||
HANDLE h = CreateFileW(a, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
|
||||
if (h == INVALID_HANDLE_VALUE) return 0;
|
||||
DWORD put = 0;
|
||||
int ok = WriteFile(h, data, (DWORD)n, &put, NULL) && put == n && FlushFileBuffers(h);
|
||||
int ok = WriteFile(h, data, (DWORD)n, &put, NULL) && put == n && (!durable || FlushFileBuffers(h));
|
||||
CloseHandle(h);
|
||||
if (ok && MoveFileExW(a, b, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) return 1;
|
||||
if (ok && MoveFileExW(a, b, MOVEFILE_REPLACE_EXISTING | (durable ? MOVEFILE_WRITE_THROUGH : 0))) return 1;
|
||||
DeleteFileW(a);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int plat_file_write_atomic(const char *path, const void *data, size_t n) { return write_swap(path, data, n, 1); }
|
||||
|
||||
int plat_file_write_cached(const char *path, const void *data, size_t n) { return write_swap(path, data, n, 0); }
|
||||
|
||||
int plat_file_flush(const char *path) {
|
||||
wchar_t w[WPATH];
|
||||
if (!widen(path, w, WPATH)) return 0;
|
||||
|
||||
Reference in New Issue
Block a user