Stop serving carried translations whose texture pool array has the earlier translator's length.

Installs updated from 0.10.0 or 0.10.1 hold translations of the 0.9.2 translator, which declares the texture pool with
32768 entries and clamps every handle to slot 32767. The carry lookups (gfx_shader.c carry_load, carry_find_iid) served
such a translation for the whole session, so every texture the game numbers from 32768 up (Ocarina's pool has 33068)
was drawn with whatever slot 32767 held: the see-through ground of issues #569 and #824 stayed on those installs after
the update. carry_usable now refuses a previous translator's translation that has a texture pool array (reflection
pool 1) of 32768 entries; the lookup goes on to the next previous identity and then to a translation under the current
translator, so the boot prewarm translates the shaders it names on the boot screen and the rest are translated at first
use. The first refusal is logged, and the saves and the exit log how many there were. get_one now checks its failed
keys before the previous translators' disk entries, so a shader whose translation failed does not read a refused entry
from the disk at every call.

Not reproduced before the change. tests/carry_pool_check.py (new) cuts the carry lookups out of gfx_shader.c and checks
that an earlier-length translation is freed and passed over, also in the owned branch, while others are served;
gfx_shader.c compiles with no new warnings.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/building.md

@@ -648,6 +648,11 @@ A change to the layout of `shader-prepared.bin` (`runtime/prp_file.h`, `runtime/
 the header alone, checks the parse of both versions and has `make_game.py` read the list the C side wrote. No game
 files, no GPU.
 
+A change to which translations of a previous translator are served (`runtime/gfx_shader.c` `carry_usable`,
+`carry_load`, `carry_find_iid`) passes `python tests\carry_pool_check.py`, which cuts the carry lookups out of
+`gfx_shader.c` and runs them against stand-in caches: a translation whose texture pool array has the earlier length
+(32768) is freed and the lookup goes on, one with the current length or no pool is served. No game files, no GPU.
+
 A change to where the lazy write-back is on by default (`runtime/wb_lazy_choice.h`, `runtime/vk_memory.c`
 `vk_mem_lazy_decide`) passes `python tests\wb_lazy_check.py`, which builds the header alone and checks the choice and
 its reason by platform, by GPU driver and by what the player set. No game files, no GPU.

runtime/gfx_shader.c

@@ -29,7 +29,8 @@
 // current key on disk, the previous keys on disk, then a translation (carry_find). A shader served from a previous
 // translator thus keeps that translation for the rest of the session, deliberately: no pipeline is built a second
 // time for it, and the pass below needs no way to swap shaders under the renderer. A shader first met after the pass
-// translated it gets the new translation.
+// translated it gets the new translation. A translation that indexes the texture pool with the earlier translator's
+// length is never served (carry_usable): its clamp would draw the textures numbered past that length with another.
 // The background pass (gfx_shader_carry_start, after the boot prewarm) translates every stored input that has a
 // previous translator's entry again under the current identity, on a translator child of its own that runs at idle
 // priority, outside the pool, and waits while on-demand translations are queued or running. It writes cache files only;
@@ -201,6 +202,8 @@ static struct {
     int n_dropped;
     volatile int64_t served;                               // shaders served from a previous translator this session
     int64_t served_logged;
+    volatile int64_t refused;                              // previous translations not used (carry_usable)
+    int64_t refused_logged;
 } C;
 static PlatMutex ids_lock = PLAT_MUTEX_INIT;               // identities.txt rewrites, C.dropped
 
@@ -1962,10 +1965,38 @@ static int use_lookup(const uint8_t id[32], const uint8_t h[32], UseMask *m) {
     return known;
 }
 
+// The texture pool length of the earlier translator (vk.h VK_POOL_TEX_CARRIED). Its pool arrays clamp every handle to
+// slot 32767, so a texture the game numbers from 32768 up (Ocarina's pool has 33068 descriptors) is drawn with whatever
+// slot 32767 holds: ground seen through after an area load. The current translator declares VK_POOL_TEX (40960).
+#define POOL_TEX_SHORT 32768
+
+// Whether s declares a texture pool array (reflection "pool" 1) of the earlier length.
+static int pool_short(const GfxShader *s) {
+    for (int k = 0; k < s->nres[GFX_RES_TEX]; k++)
+        if (s->res[GFX_RES_TEX][k].pool == 1 && s->res[GFX_RES_TEX][k].count == POOL_TEX_SHORT) return 1;
+    return 0;
+}
+
+// Whether previous identity j's translation s may serve. One that indexes the texture pool with the earlier length is
+// not used: the lookup goes on to the next previous identity and then to a translation under the current one (the boot
+// prewarm translates the shaders it names on the boot screen, others are translated at first use). Served, it would
+// keep the clamp for the whole session (see "Carry" at the top). Counted for gfx_shader_record_flush; the first logged.
+static int carry_usable(const GfxShader *s, int j) {
+    if (!pool_short(s)) return 1;
+    if (plat_inc64(&C.refused) == 1) {
+        const uint8_t *id = C.prev[j];
+        rt_log("gfx_shader: carry: translations of a previous translator (%02x%02x%02x%02x) that index the texture pool "
+               "with %d entries are not used (textures numbered from %d up would read slot %d); those shaders are "
+               "translated again", id[0], id[1], id[2], id[3], POOL_TEX_SHORT, POOL_TEX_SHORT, POOL_TEX_SHORT - 1);
+    }
+    return 0;
+}
+
 // Previous identity j's entry `key` from the disk cache into the memory cache; NULL when the cache has no valid one.
-// stage (GFX_STAGE_*, -1 any): an entry of another stage than the request's is not used. Every entry passes
-// disk_load's checks (whole SPIR-V, a .res.json naming its stage, both from the pack or both loose), which matters most
-// for compute shaders: the replay thread dispatches one the moment it has it (vk_exec_draw.c shader_for).
+// stage (GFX_STAGE_*, -1 any): an entry of another stage than the request's is not used, nor one carry_usable refuses.
+// Every entry passes disk_load's checks (whole SPIR-V, a .res.json naming its stage, both from the pack or both loose),
+// which matters most for compute shaders: the replay thread dispatches one the moment it has it (vk_exec_draw.c
+// shader_for).
 static GfxShader *carry_load(const uint8_t key[32], const uint8_t iid[32], int stage, int j) {
     GfxShader *s = disk_load(key, iid);
     if (!s) return NULL;
@@ -1976,6 +2007,10 @@ static GfxShader *carry_load(const uint8_t key[32], const uint8_t iid[32], int s
         shader_free(s);
         return NULL;
     }
+    if (!carry_usable(s, j)) {
+        shader_free(s);
+        return NULL;
+    }
     s->carried = (uint8_t)(j + 1);
     return mem_insert(s);
 }
@@ -2009,7 +2044,10 @@ static GfxShader *carry_find_iid(const uint8_t iid[32], int disk, int *owned) {
             continue;
         }
         if (!owned) s = carry_load(key, iid, -1, j);
-        else if ((s = disk_load(key, iid))) s->carried = (uint8_t)(j + 1), *owned = 1;
+        else if ((s = disk_load(key, iid))) {
+            if (carry_usable(s, j)) s->carried = (uint8_t)(j + 1), *owned = 1;
+            else shader_free(s), s = NULL;
+        }
         if (s) return s;
     }
     return NULL;
@@ -2308,7 +2346,9 @@ static int failed_slot(const uint8_t key[32], int insert) {
 }
 
 // The lookup order with carry (see "Carry" at the top): the current key in memory, a previous translator's entry in
-// memory, the current key on disk, a previous translator's entry on disk.
+// memory, the current key on disk, a previous translator's entry on disk. A key whose translation failed skips the
+// last: that lookup found nothing usable before the translation, and a refused entry (carry_usable) would otherwise be
+// read from the disk again at every call (a compute shader's at every dispatch, vk_exec_draw.c shader_for).
 static const GfxShader *get_one(const Req *r, int stage, const uint8_t *control, const uint8_t *code,
                                 const GfxShaderAnswers *a) {
     GfxShader *s = mem_find(r->key);
@@ -2316,8 +2356,8 @@ static const GfxShader *get_one(const Req *r, int stage, const uint8_t *control,
     paths();
     if (!cache_dir[0]) return fail("no cache directory", NULL);
     if (r->final && (s = disk_load(r->key, r->iid))) return plat_inc64(&n_hits), mem_insert(s);
-    if ((s = carry_find(r->code_hash, a, stage, 1))) return plat_inc64(&n_hits), s;
     if (failed_slot(r->key, 0)) return NULL;
+    if ((s = carry_find(r->code_hash, a, stage, 1))) return plat_inc64(&n_hits), s;
     const GfxShader *t = translate(r, stage, control, code, a);
     if (!t) failed_slot(r->key, 1);
     return t;
@@ -3314,9 +3354,12 @@ static PlatMutex rec_io = PLAT_MUTEX_INIT;   // the record files: appends in ord
 void gfx_shader_record_flush(int force) {
     static volatile int64_t last;
     int64_t now = (int64_t)plat_now_ms();
-    int64_t served = __atomic_load_n(&C.served, __ATOMIC_RELAXED);
+    int64_t served = __atomic_load_n(&C.served, __ATOMIC_RELAXED), refused = __atomic_load_n(&C.refused, __ATOMIC_RELAXED);
     if (force && served != __atomic_exchange_n(&C.served_logged, served, __ATOMIC_RELAXED))   // the saves and the exit
         rt_log("gfx_shader: carry: %lld shaders served from a previous translator this session", (long long)served);
+    if (force && refused != __atomic_exchange_n(&C.refused_logged, refused, __ATOMIC_RELAXED))
+        rt_log("gfx_shader: carry: a previous translator's translation was not used %lld times this session (texture "
+               "pool of %d entries)", (long long)refused, POOL_TEX_SHORT);
     if (!RC.on || (!force && now - last < 2000)) return;
     last = now;
     // The files are written outside RC.lock, which the render thread takes to record a new answer.

tests/carry_pool_check.py

@@ -0,0 +1,187 @@
+"""Check runtime/gfx_shader.c's carry refusal: a previous translator's translation whose texture pool array has the
+earlier length (32768, vk.h VK_POOL_TEX_CARRIED) is never served, because its clamp draws every texture numbered from
+32768 up with slot 32767. The check cuts pool_short, carry_usable, carry_load, carry_find and carry_find_iid out of
+gfx_shader.c, compiles them against stand-ins for the disk and memory caches, and walks the lookups: a refused entry
+is freed and the lookup goes on to the next previous identity, the owned branch (gfx_shader_load_private) frees it as
+well, and entries with the current length, the sampler pool or no pool are served as before.
+
+No GPU, no game files, any Python.
+
+Usage: carry_pool_check.py
+"""
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
+
+
+def compiler():
+    """$CC, else MSYS2 UCRT64's clang, else clang on PATH."""
+    if os.environ.get('CC'):
+        return os.environ['CC']
+    if os.name == 'nt' and (MSYS_BIN / 'clang.exe').exists():
+        return str(MSYS_BIN / 'clang.exe')
+    return shutil.which('clang') or 'clang'
+
+
+HARNESS = r'''
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include "gfx.h"
+
+typedef struct GfxShader {
+    GfxShaderRes *res[GFX_RES_KINDS];
+    int nres[GFX_RES_KINDS];
+    int stage;
+    uint8_t carried;
+    int pool_len;                 /* the stand-in disk entry's texture pool length, -1 none */
+    int keyj;                     /* the previous identity whose key it was read under */
+} GfxShader;
+typedef struct { uint32_t slots, attrs, fouts; } UseMask;
+
+
+static struct {
+    int n_prev;
+    uint8_t prev[8][32], prev_eff[8][32];
+    volatile int64_t refused;
+} C;
+static int carry_on(void) { return C.n_prev > 0; }
+static int64_t plat_inc64(volatile int64_t *p) { return ++*p; }
+static int32_t plat_inc32(volatile int32_t *p) { return ++*p; }
+static int logs;
+static char last_log[512];
+void rt_log(const char *fmt, ...) {
+    logs++;
+    snprintf(last_log, sizeof last_log, "%s", fmt);
+}
+
+/* The disk cache: entry j of the previous identities (key byte 0 = j) holds a shader whose texture pool array has
+   disk_len[j] entries (0: no entry), of stage disk_stage[j]. */
+static int disk_len[8], disk_stage[8], loads, frees, live;
+static GfxShader *mem;            /* the memory cache: at most one shader here */
+static void key_of_input_id(const uint8_t id[32], const uint8_t iid[32], uint8_t key[32]) {
+    memset(key, 0, 32);
+    key[0] = id[0];
+}
+static void iid_of(const uint8_t h[32], const GfxShaderAnswers *a, const UseMask *m, uint8_t iid[32]) { memset(iid, 0, 32); }
+static int use_lookup(const uint8_t id[32], const uint8_t h[32], UseMask *m) { return 1; }
+static GfxShader *mem_find(const uint8_t key[32]) { return mem && mem->keyj == key[0] ? mem : NULL; }
+static GfxShader *mem_insert(GfxShader *s) { mem = s; return s; }
+static void shader_free(GfxShader *s) { frees++, live--; free(s->res[GFX_RES_TEX]); free(s); }
+static GfxShader *disk_load(const uint8_t key[32], const uint8_t *iid) {
+    int j = key[0];
+    if (!disk_len[j]) return NULL;
+    loads++, live++;
+    GfxShader *s = calloc(1, sizeof *s);
+    s->stage = disk_stage[j];
+    s->keyj = j;
+    s->pool_len = disk_len[j];
+    s->res[GFX_RES_TEX] = calloc(3, sizeof(GfxShaderRes));
+    s->nres[GFX_RES_TEX] = 3;
+    s->res[GFX_RES_TEX][0] = (GfxShaderRes){.slot = 2, .count = 1};                  /* a bound texture */
+    s->res[GFX_RES_TEX][1] = (GfxShaderRes){.slot = -1, .count = 4096, .pool = 2};   /* the sampler pool */
+    if (disk_len[j] > 0) s->res[GFX_RES_TEX][2] = (GfxShaderRes){.slot = -1, .count = (uint16_t)disk_len[j], .pool = 1};
+    else s->nres[GFX_RES_TEX] = 2;
+    return s;
+}
+'''
+
+MAIN = r'''
+static void reset(void) {
+    if (mem) shader_free(mem), mem = NULL;
+    for (int j = 0; j < 8; j++) disk_len[j] = 0, disk_stage[j] = 1;
+    C.refused = 0, logs = 0, loads = 0, frees = 0, live = 0;
+}
+
+int main(void) {
+    uint8_t h[32] = {0}, iid[32] = {0};
+    GfxShaderAnswers a = {0};
+    C.n_prev = 2;
+    C.prev[0][0] = 0x4d, C.prev_eff[0][0] = 0;   /* key byte 0 = identity index */
+    C.prev[1][0] = 0x9c, C.prev_eff[1][0] = 1;
+
+    /* The earlier length: refused, freed, nothing served; counted once, logged once. */
+    reset();
+    disk_len[0] = 32768;
+    assert(!carry_find(h, &a, 1, 1));
+    assert(loads == 1 && frees == 1 && live == 0 && !mem);
+    assert(C.refused == 1 && logs == 1 && strstr(last_log, "texture pool"));
+    assert(!carry_find_iid(iid, 1, NULL));
+    assert(C.refused == 2 && logs == 1 && live == 0);
+
+    /* The lookup goes on to the next previous identity, whose translation has the current length. */
+    reset();
+    disk_len[0] = 32768, disk_len[1] = 40960;
+    GfxShader *s = carry_find(h, &a, 1, 1);
+    assert(s && s->pool_len == 40960 && s->carried == 2 && mem == s);
+    assert(C.refused == 1 && frees == 1 && live == 1);
+
+    /* The current length, a shader with only the sampler pool and bound textures: served as before. */
+    reset();
+    disk_len[0] = 40960;
+    s = carry_find(h, &a, 1, 1);
+    assert(s && s->carried == 1 && C.refused == 0 && logs == 0);
+    reset();
+    disk_len[0] = -1;   /* no texture pool array */
+    s = carry_find_iid(iid, 1, NULL);
+    assert(s && s->carried == 1 && C.refused == 0);
+
+    /* A stage mismatch is still refused for its own reason, before the pool is looked at. */
+    reset();
+    disk_len[0] = 32768, disk_stage[0] = 2;
+    assert(!carry_find(h, &a, 1, 1) && C.refused == 0 && frees == 1);
+
+    /* The owned branch (gfx_shader_load_private): refused and freed, or handed over with *owned 1. */
+    reset();
+    disk_len[0] = 32768;
+    int owned = 0;
+    assert(!carry_find_iid(iid, 1, &owned) && !owned && C.refused == 1 && frees == 1 && live == 0);
+    reset();
+    disk_len[0] = 32768, disk_len[1] = 40960;
+    owned = 0;
+    s = carry_find_iid(iid, 1, &owned);
+    assert(s && owned && s->carried == 2 && !mem && C.refused == 1 && live == 1);
+    shader_free(s);
+
+    /* Memory only (disk 0) never reads the disk. */
+    reset();
+    disk_len[0] = 32768;
+    assert(!carry_find(h, &a, 1, 0) && !carry_find_iid(iid, 0, NULL) && loads == 0);
+
+    reset();
+    printf("carry pool check: earlier-length pool translations refused and freed, the lookup goes on, others served\n");
+    return 0;
+}
+'''
+
+
+def cut(source, start, end):
+    i = source.index(start)
+    return source[i:source.index(end, i)]
+
+
+def main():
+    src = (ROOT / 'runtime' / 'gfx_shader.c').read_text(encoding='utf-8')
+    production = cut(src, '#define POOL_TEX_SHORT', '// Whether the cache holds a previous translator\'s entry of input iid.')
+    work = ROOT / 'build' / 'test' / 'carry_pool'
+    work.mkdir(parents=True, exist_ok=True)
+    c = work / 'check.c'
+    c.write_text(HARNESS + production + MAIN, encoding='utf-8')
+    exe = work / ('check.exe' if os.name == 'nt' else 'check')
+    env = dict(os.environ)
+    if os.name == 'nt' and MSYS_BIN.exists():
+        env['PATH'] = str(MSYS_BIN) + os.pathsep + env.get('PATH', '')
+    subprocess.check_call([compiler(), '-O1', '-g', '-Wall', '-Wno-unused-function', '-I', str(ROOT / 'runtime'),
+                           str(c), '-o', str(exe)], env=env)
+    return subprocess.run([str(exe)], env=env, timeout=60).returncode
+
+
+if __name__ == '__main__':
+    sys.exit(main())