Read a pipeline cache larger than 2 GB on Windows, and log every refused file with its size.

cache_load measured pipeline-cache.bin with ftell into a long, which is 32 bits on Windows: a file past 2 GB measured as -1, so the run started with an empty cache and the log said nothing, not even that the file was over pipeline_cache_max_mb. The read now lives in runtime/vk_cache_file.h: the length comes from the 64-bit tell, the read runs in 64 MB pieces, and a refusal returns its reason and the file's size, which cache_load logs. The Windows atomic write no longer cuts a block of 4 GB or more to a 32-bit count.

tests/pipeline_cache_file_check.py reads a sparse file just over 2 GB through the header. With --probe-prewarm and such a file carrying this GPU's header, the log reads "holds 2048 MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB 512" under the default limit and "2048.0 MB read" with the limit off; the exe before this change logged nothing for the same file.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 04:29:31 -05:00
parent 84b0672ec2
commit 871b06e2e7
6 changed files with 247 additions and 17 deletions

View File

@@ -389,6 +389,11 @@ A change to what the library warm logs when the driver's cache of stage librarie
`runtime/vk_pipeline.c` `lw_main`) passes `python tests\warm_hint_check.py`, which builds the header alone and checks `runtime/vk_pipeline.c` `lw_main`) passes `python tests\warm_hint_check.py`, which builds the header alone and checks
the added text by GPU vendor and by why the cache was cold. No game files. the added text by GPU vendor and by why the cache was cold. No game files.
A change to how `pipeline-cache.bin` is read (`runtime/vk_cache_file.h`, `runtime/vk_pipeline.c` `cache_load`) passes
`python tests\pipeline_cache_file_check.py`, which builds the header alone and reads a sparse file just over 2 GB
through it: measured and read whole without a size limit, refused by its real size under one. No game files, no GPU;
about 2 GB of memory for a few seconds.
A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c` A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c`
`vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and
its reason by platform, by GPU driver and by what the player set. No game files, no GPU. its reason by platform, by GPU driver and by what the player set. No game files, no GPU.

View File

@@ -702,8 +702,15 @@ static int write_swap(const char *path, const void *data, size_t n, int durable)
if (len < 0 || len >= (int)sizeof tmp || !widen(tmp, a, WPATH) || !widen(path, b, WPATH)) return 0; if (len < 0 || len >= (int)sizeof tmp || !widen(tmp, a, WPATH) || !widen(path, b, WPATH)) return 0;
HANDLE h = CreateFileW(a, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); HANDLE h = CreateFileW(a, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (h == INVALID_HANDLE_VALUE) return 0; if (h == INVALID_HANDLE_VALUE) return 0;
DWORD put = 0; // In pieces: WriteFile takes a 32-bit count, so a block of 4 GB or more (a pipeline cache can grow that far)
int ok = WriteFile(h, data, (DWORD)n, &put, NULL) && put == n && (!durable || FlushFileBuffers(h)); // would otherwise be cut to its low 32 bits and the write refused.
int ok = 1;
for (size_t at = 0; ok && at < n;) {
DWORD piece = n - at > (1u << 30) ? (1u << 30) : (DWORD)(n - at), put = 0;
ok = WriteFile(h, (const char *)data + at, piece, &put, NULL) && put == piece;
at += piece;
}
ok = ok && (!durable || FlushFileBuffers(h));
CloseHandle(h); CloseHandle(h);
if (ok && MoveFileExW(a, b, MOVEFILE_REPLACE_EXISTING | (durable ? MOVEFILE_WRITE_THROUGH : 0))) return 1; if (ok && MoveFileExW(a, b, MOVEFILE_REPLACE_EXISTING | (durable ? MOVEFILE_WRITE_THROUGH : 0))) return 1;
DeleteFileW(a); DeleteFileW(a);

66
runtime/vk_cache_file.h Normal file
View File

@@ -0,0 +1,66 @@
// Reads pipeline-cache.bin into memory for cache_load (runtime/vk_pipeline.c), with 64-bit sizes on every platform.
// ftell returns a long, which is 32 bits on Windows: a file past 2 GB measured as -1 there, and the run started with an
// empty cache without a word in the log. The length comes from the 64-bit tell, the read runs in pieces no C library
// call has to take whole, and every refusal names its reason and the file's size, so cache_load can log it.
// tests/pipeline_cache_file_check.py reads a sparse file just over 2 GB through it.
#pragma once
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
enum {
CACHE_FILE_OK,
CACHE_FILE_NONE, // no such file: a first run
CACHE_FILE_UNSIZED, // the file's length could not be measured
CACHE_FILE_SHORT, // not longer than the header every cache starts with
CACHE_FILE_OVER_CAP, // longer than the limit; not read
CACHE_FILE_NO_MEMORY, // no block of the file's size
CACHE_FILE_READ_FAIL, // the read ended early
};
#define CACHE_FILE_PIECE ((size_t)64 << 20)
static inline const char *cache_file_why(int why) {
switch (why) {
case CACHE_FILE_UNSIZED: return "its size cannot be measured";
case CACHE_FILE_SHORT: return "it is shorter than a pipeline cache header";
case CACHE_FILE_OVER_CAP: return "it is over the size limit";
case CACHE_FILE_NO_MEMORY: return "there is no memory for a block of its size";
case CACHE_FILE_READ_FAIL: return "reading it ended early";
default: return "";
}
}
// The whole file in a malloc block, or NULL with *why set. *len is the file's length in bytes whenever it could be
// measured, read or not. min: a file of at most this many bytes is refused; cap: so is one longer than cap (0: no limit),
// before any of it is read.
static inline void *cache_file_read(const char *path, uint64_t min, uint64_t cap, uint64_t *len, int *why) {
*len = 0;
FILE *f = fopen(path, "rb");
if (!f) return *why = CACHE_FILE_NONE, NULL;
#ifdef _WIN32
int64_t n = _fseeki64(f, 0, SEEK_END) ? -1 : _ftelli64(f);
if (n >= 0 && _fseeki64(f, 0, SEEK_SET)) n = -1;
#else
int64_t n = fseeko(f, 0, SEEK_END) ? -1 : (int64_t)ftello(f);
if (n >= 0 && fseeko(f, 0, SEEK_SET)) n = -1;
#endif
void *data = NULL;
if (n < 0) *why = CACHE_FILE_UNSIZED;
else if (*len = (uint64_t)n, (uint64_t)n <= min) *why = CACHE_FILE_SHORT;
else if (cap && (uint64_t)n > cap) *why = CACHE_FILE_OVER_CAP;
else if ((uint64_t)n > SIZE_MAX || !(data = malloc((size_t)n))) *why = CACHE_FILE_NO_MEMORY;
else {
size_t got = 0, want = (size_t)n;
while (got < want) {
size_t piece = want - got < CACHE_FILE_PIECE ? want - got : CACHE_FILE_PIECE;
size_t r = fread((char *)data + got, 1, piece, f);
got += r;
if (r != piece) break;
}
if (got == want) *why = CACHE_FILE_OK;
else free(data), data = NULL, *why = CACHE_FILE_READ_FAIL;
}
fclose(f);
return data;
}

View File

@@ -48,6 +48,7 @@
#include "nvn_hmap.h" #include "nvn_hmap.h"
#include "platform/platform.h" #include "platform/platform.h"
#include "tex.h" #include "tex.h"
#include "vk_cache_file.h"
#include "vk_gpa.h" #include "vk_gpa.h"
#include "vk_warm_hint.h" #include "vk_warm_hint.h"
@@ -601,33 +602,32 @@ static void cache_load(VkCtx *c) {
} }
void *data = NULL; void *data = NULL;
size_t size = 0; size_t size = 0;
FILE *f = P.cache_path[0] ? fopen(P.cache_path, "rb") : NULL; if (P.cache_path[0]) {
if (f) {
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
// NX2NAVI_PIPELINE_CACHE_MAX_MB (default 512; 0 no limit): a larger file starts the run empty, and the run's // NX2NAVI_PIPELINE_CACHE_MAX_MB (default 512; 0 no limit): a larger file starts the run empty, and the run's
// saves replace it. The file only grows: entries of earlier translators and builds stay, and on NVIDIA // saves replace it. The file only grows: entries of earlier translators and builds stay, and on NVIDIA
// (621.01) an entry another executable wrote is neither used nor written again, so a renamed or updated exe // (621.01) an entry another executable wrote is neither used nor written again, so a renamed or updated exe
// compiles while the file keeps its size (2026-11-09: 650 MB, 15-18 % of the pipelines read from it). The // compiles while the file keeps its size (2026-11-09: 650 MB, 15-18 % of the pipelines read from it). The
// whole file sits in process memory twice at the boot (this read and the driver's copy) and is copied out // whole file sits in process memory twice at the boot (this read and the driver's copy) and is copied out
// at every save, which a 16 GB machine near its commit limit cannot spare. The size is checked before the // at every save, which a 16 GB machine near its commit limit cannot spare. The size is checked before the
// read, so a file that is dropped never takes its size in memory. // read, so a file that is dropped never takes its size in memory. Sizes are 64-bit all the way
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_MAX_MB"), NULL, 10) << 20; // (vk_cache_file.h cache_file_read): a file past 2 GB is read with the limit raised or off, and refused by
if (len > 0 && cap && (uint64_t)len > cap) { // its real size otherwise.
uint64_t cap = (uint64_t)strtoull(cfg_value("NX2NAVI_PIPELINE_CACHE_MAX_MB"), NULL, 10) << 20, len = 0;
int why = CACHE_FILE_NONE;
data = cache_file_read(P.cache_path, sizeof(VkPipelineCacheHeaderVersionOne), cap, &len, &why);
if (data) size = (size_t)len;
else if (why == CACHE_FILE_OVER_CAP)
rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; this run " rt_log("vk_pipeline: %s holds %.0f MB, over NX2NAVI_PIPELINE_CACHE_MAX_MB %llu: starting empty; this run "
"rewrites it", P.cache_path, len / 1048576.0, (unsigned long long)(cap >> 20)); "rewrites it", P.cache_path, len / 1048576.0, (unsigned long long)(cap >> 20));
len = 0; else if (why != CACHE_FILE_NONE)
} rt_log("vk_pipeline: %s (%llu bytes, %.1f MB) is not read: %s; starting empty", P.cache_path,
if (len > (long)sizeof(VkPipelineCacheHeaderVersionOne) && (data = malloc((size_t)len)) && (unsigned long long)len, len / 1048576.0, cache_file_why(why));
fread(data, 1, (size_t)len, f) == (size_t)len)
size = (size_t)len;
fclose(f);
VkPipelineCacheHeaderVersionOne h; VkPipelineCacheHeaderVersionOne h;
if (size) memcpy(&h, data, sizeof h); if (size) memcpy(&h, data, sizeof h);
if (size && (h.headerVersion != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || h.vendorID != c->props.vendorID || if (size && (h.headerVersion != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || h.vendorID != c->props.vendorID ||
h.deviceID != c->props.deviceID || memcmp(h.pipelineCacheUUID, c->props.pipelineCacheUUID, VK_UUID_SIZE))) { h.deviceID != c->props.deviceID || memcmp(h.pipelineCacheUUID, c->props.pipelineCacheUUID, VK_UUID_SIZE))) {
rt_log("vk_pipeline: %s was written by another device or driver; starting empty", P.cache_path); rt_log("vk_pipeline: %s (%.1f MB) was written by another device or driver; starting empty", P.cache_path,
size / 1048576.0);
size = 0; size = 0;
} }
} }

View File

@@ -0,0 +1,114 @@
// runtime/vk_cache_file.h: the read of pipeline-cache.bin (runtime/vk_pipeline.c cache_load) with 64-bit sizes. Makes a
// sparse file just over 2 GB in the folder given as argv[1], with a marker at its start and at its end, and checks that
// it is measured and read whole, that a size limit refuses it by its real size without reading it, and the small cases.
// Built and run by tests/pipeline_cache_file_check.py; prints each failed check and exits 1 on any.
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "vk_cache_file.h"
#ifdef _WIN32
#include <windows.h>
#include <winioctl.h>
#else
#include <fcntl.h>
#include <unistd.h>
#endif
static int fails;
#define CHECK(c, ...) do { if (!(c)) { fails++; printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)
#define HEADER 32 // the size of the header every pipeline cache starts with (VkPipelineCacheHeaderVersionOne)
// A file of len bytes: head at its start, tail as its last bytes, a hole between them that takes no disk space.
static int make_sparse(const char *path, uint64_t len, const char *head, const char *tail) {
size_t hn = strlen(head), tn = strlen(tail);
#ifdef _WIN32
HANDLE h = CreateFileA(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (h == INVALID_HANDLE_VALUE) return 0;
DWORD got = 0;
LARGE_INTEGER at = {.QuadPart = (LONGLONG)(len - tn)};
int ok = DeviceIoControl(h, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &got, NULL) &&
WriteFile(h, head, (DWORD)hn, &got, NULL) && got == hn && SetFilePointerEx(h, at, NULL, FILE_BEGIN) &&
WriteFile(h, tail, (DWORD)tn, &got, NULL) && got == tn;
CloseHandle(h);
return ok;
#else
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
if (fd < 0) return 0;
int ok = write(fd, head, hn) == (ssize_t)hn && pwrite(fd, tail, tn, (off_t)(len - tn)) == (ssize_t)tn;
close(fd);
return ok;
#endif
}
int main(int argc, char **argv) {
if (argc < 2) return printf("usage: pipeline_cache_file_check <folder>\n"), 2;
char path[1024];
uint64_t len = 0;
int why = -1;
// no file: a first run, which is not a refusal
snprintf(path, sizeof path, "%s/none.bin", argv[1]);
remove(path);
CHECK(!cache_file_read(path, HEADER, 0, &len, &why) && why == CACHE_FILE_NONE && len == 0, "no file: why %d", why);
// a file of a header alone holds no pipeline: refused with its size
snprintf(path, sizeof path, "%s/short.bin", argv[1]);
FILE *f = fopen(path, "wb");
for (int i = 0; f && i < HEADER; i++) fputc(i, f);
if (f) fclose(f);
CHECK(!cache_file_read(path, HEADER, 0, &len, &why) && why == CACHE_FILE_SHORT && len == HEADER,
"header only: why %d, len %llu", why, (unsigned long long)len);
remove(path);
// a small file: read whole
f = fopen(path, "wb");
for (int i = 0; f && i < 4096; i++) fputc(i & 0xff, f);
if (f) fclose(f);
uint8_t *d = cache_file_read(path, HEADER, 1 << 20, &len, &why);
CHECK(d && why == CACHE_FILE_OK && len == 4096 && d[4095] == 0xff && d[257] == 1, "small file: why %d, len %llu", why,
(unsigned long long)len);
free(d);
remove(path);
// just over 2 GB: a 32-bit tell measures it as negative
const uint64_t big = (2ull << 30) + 4096;
snprintf(path, sizeof path, "%s/big.bin", argv[1]);
if (!make_sparse(path, big, "HEAD-OF-THE-CACHE", "TAIL-OF-THE-CACHE")) {
printf("FAIL: cannot make the sparse file %s\n", path);
remove(path);
return 1;
}
f = fopen(path, "rb");
fseek(f, 0, SEEK_END);
long l32 = ftell(f);
fclose(f);
printf("pipeline cache file: %llu bytes; ftell into a long (%d bits here) gives %ld\n", (unsigned long long)big,
(int)sizeof(long) * 8, l32);
// under a limit: refused before any read, and its real size comes back for the log line
d = cache_file_read(path, HEADER, 512ull << 20, &len, &why);
CHECK(!d && why == CACHE_FILE_OVER_CAP && len == big, "over the limit: why %d, len %llu", why, (unsigned long long)len);
CHECK(strlen(cache_file_why(why)) > 0, "a refusal has words for the log");
free(d);
// with no limit, and with a limit above its size: read whole
for (int pass = 0; pass < 2; pass++) {
d = cache_file_read(path, HEADER, pass ? 4096ull << 20 : 0, &len, &why);
CHECK(d && why == CACHE_FILE_OK && len == big, "pass %d: over 2 GB: why %d (%s), len %llu", pass, why,
cache_file_why(why), (unsigned long long)len);
if (d) {
CHECK(!memcmp(d, "HEAD-OF-THE-CACHE", 17), "pass %d: the start of the file is the start of the block", pass);
CHECK(!memcmp(d + big - 17, "TAIL-OF-THE-CACHE", 17), "pass %d: the last bytes are past 2 GB in the block", pass);
CHECK(d[1ull << 31] == 0 && d[(1ull << 30) + 5] == 0, "pass %d: the hole reads as zeros", pass);
}
free(d);
}
remove(path);
if (!fails) printf("pipeline cache file: all checks passed\n");
return fails != 0;
}

View File

@@ -0,0 +1,38 @@
"""Build and run tests/pipeline_cache_file_check.c: runtime/vk_cache_file.h, the read of pipeline-cache.bin with 64-bit
sizes. The test makes a sparse file just over 2 GB in build/test/pipeline_cache_file (it takes no disk space, and about
2 GB of memory for the seconds the read lasts) and removes it.
Usage: pipeline_cache_file_check.py
"""
import os
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
def compiler():
if os.environ.get('CC'):
return os.environ['CC']
if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
return str(MSYS_BIN / 'clang.exe')
return shutil.which('clang') or 'clang'
def main():
work = ROOT / 'build' / 'test' / 'pipeline_cache_file'
work.mkdir(parents=True, exist_ok=True)
exe = work / ('pipeline_cache_file_check.exe' if os.name == 'nt' else 'pipeline_cache_file_check')
env = dict(os.environ)
if os.name == 'nt' and MSYS_BIN.exists():
env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
subprocess.check_call([compiler(), '-O1', '-Wall', '-Werror', '-I', str(ROOT / 'runtime'),
str(ROOT / 'tests' / 'pipeline_cache_file_check.c'), '-o', str(exe)], env=env)
return subprocess.run([str(exe), str(work)], env=env, timeout=600).returncode
if __name__ == '__main__':
sys.exit(main())