Show Prepare the Shaders moving in the launcher: the game prints its counts and the step sends them on.

The step sat at 0 % until it was done, an hour on some PCs, because nothing it ran reported progress to the launcher. nx2navi.exe --prepare-shaders now prints how many shaders it has translated and how many it has built into the driver's cache every 2 s. make_game.py reads them as they come and fills the bar, the translation the first half and the build the second, with the counts in the step's text. An event every 10 s keeps the text current while nothing is printed, and after 3 minutes without a count moving it says so, and that the step is still running.

The selftest draws the step as a tenth screen. Its made-up update was version 0.3.11, so from 0.4.0 on the update-ready screen showed Up to Date and was no longer drawn; it is now a minor version past the launcher running it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BSHQpLnScF9sNv4LNAaaS3
This commit is contained in:
Claude
2026-11-09 08:32:13 +00:00
parent 754ded99e0
commit b1421faaf9
7 changed files with 166 additions and 14 deletions

View File

@@ -89,7 +89,10 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too
- The lifted code is compiled for the processor's own level, x86-64-v3 (AVX2, FMA) or v2 (`cpu_level`): the game
runs on the PC that builds it, and a generic build would do every fused multiply-add of the game in software.
3. **Prepare the Shaders**: make_game.py's last step translates every shader of the game into the game's shader
cache and the driver's own cache (`tools/prepare_shaders.py`), so play does not stop to compile them.
cache and the driver's own cache (`tools/prepare_shaders.py`), so play does not stop to compile them. Its bar
follows the game's own counts, printed every 2 s: the translation fills the first half, the build into the
driver's cache the second. An update every 10 s keeps the step's text current while nothing is printed, and after
3 minutes without a count moving it says so, and that the step is still running.
4. The game's Cetra Display is decoded for the window.
## Updating

View File

@@ -221,6 +221,8 @@ class Job:
self.skip(sid, 'Kept' if sid in ('exefs', 'romfs') else 'Unchanged')
if link_step and sid in ('compile', 'compile_changed'):
self.skip(link_step, 'Unchanged')
elif kind == 'progress' and sid and 'fraction' in ev: # a step that words its own progress (shaders)
self.progress(sid, float(ev.get('fraction') or 0), ev.get('detail', ''))
elif kind == 'progress' and sid:
done, total, unit = ev.get('done', 0), max(ev.get('total', 1), 1), ev.get('unit', '')
if unit == 'bytes':

View File

@@ -6,7 +6,7 @@ import sys
import time
from pathlib import Path
from .. import feed, jobs
from .. import feed, jobs, versions
def grab_gdi(x, y, w, h, path):
@@ -69,6 +69,16 @@ def grab(app, path):
return False
def ahead_of(version):
"""A version a minor step past version: the made-up release must be newer than the launcher drawing it, or the
update screens show "Up to Date" instead (a fixed 0.3.11 did from 0.4.0 on)."""
try:
nums, _ = versions.parse(version)
except ValueError:
return '99.0.0'
return f'{nums[0]}.{nums[1] + 1}.0'
def fake_release(version='0.3.11'):
m = {'format': 1, 'version': version, 'app': {'name': f'NX2navi-app-{version}.zip', 'size': 1, 'sha256': '0'}}
notes = ('- Each program module comes from the .nup package that holds it whole.\n'
@@ -105,11 +115,11 @@ def states(app):
app.show('home', 0)
def update_ready():
app.release = fake_release()
app.release = fake_release(ahead_of(ctx.version))
app.show('home', 1)
def updating():
j = jobs.UpdateDownloadJob(ctx, fake_release())
j = jobs.UpdateDownloadJob(ctx, fake_release(ahead_of(ctx.version)))
for sid, val in (('download', 'Done · 0:06'), ('verify', 'Checksum OK')):
j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0
j.skip('tools_update', 'Unchanged')
@@ -139,6 +149,19 @@ def states(app):
def uninstall():
app.show('uninstall', 0)
def shaders(): # the counts and the wording tools/make_game.py shader_progress sends
j = jobs.InstallJob(ctx, s['dump'], s['game_dir'])
for sid, val in (('tools', 'Done · 0:48'), ('check', 'Version 1.1.1'), ('exefs', 'Done · 0:04'),
('romfs', 'Done · 0:31'), ('lift', 'Done · 0:48'), ('compile', 'Done · 4:12')):
j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0
j.begin('shaders')
j.progress('shaders', 0.5 + 0.5 * 6214 / 20480, 'Built 6,214 of 20,480 shaders for your graphics card')
j.started = time.time() - 611
for ln in (' shaders: translated 20480 of 20480, 141 s', ' shaders: built 6214 of 20480, 402 s'):
j.lines.append(ln)
app.job, app.job_return, app.job_back = j, 'home', 'first_run'
app.show('job')
def import_saves():
app.release, app.message = None, ''
app.save_message = ('Save imported: 3 files from 0100000000010000. Your old save is in '
@@ -147,7 +170,8 @@ def states(app):
return [('1-first-run', first_run), ('2-installing', installing), ('3-home', home),
('4-update-ready', update_ready), ('5-updating', updating), ('6-settings', settings),
('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves)]
('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves),
('10-shaders', shaders)]
def run(app, shots):

View File

@@ -2221,10 +2221,18 @@ static long long prepare_files(const char *ans_path, const char *prg_path, int t
for (int i = 0; i < threads; i++)
if ((t[k] = plat_thread_new(prep_worker, &p, "shader prepare"))) k++;
if (!k) prep_worker(&p);
for (int m = 0; m < k; m++)
while (!plat_thread_join_ns(t[m], 5000000000u))
rt_log("gfx_shader: prepare: %lld of %zu pairs, %llu translated, %.0f ms", (long long)p.done, p.n_ans,
(unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0);
// every 2 s a count on stdout for tools/make_game.py (step_shaders reads "prepare-shaders: progress" lines and shows
// them in the launcher, which otherwise sat at 0 % for the whole step), and every 10 s one in the log
for (int m = 0, polls = 0; m < k; m++)
while (!plat_thread_join_ns(t[m], 2000000000u)) {
printf("prepare-shaders: progress translate %lld %zu\n", (long long)p.done, p.n_ans);
fflush(stdout);
if (++polls % 5 == 0)
rt_log("gfx_shader: prepare: %lld of %zu pairs, %llu translated, %.0f ms", (long long)p.done, p.n_ans,
(unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0);
}
printf("prepare-shaders: progress translate %zu %zu\n", p.n_ans, p.n_ans);
fflush(stdout);
svc_set_limit(svc_runtime_limit());
rt_log("gfx_shader: prepare: %zu %s of %zu programs: %llu translated, %lld failed, %lld without a program, the rest "
"cached; %lld default variants with refined answers; %.0f ms", p.n_ans, what, p.progs.n,

View File

@@ -3570,7 +3570,18 @@ static void lb_warm_prepared(const char *dir) {
for (int i = 0; i < cpus && i < JR_SLOTS; i++)
if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++;
if (!k) lb_warm_worker((void *)(intptr_t)0);
for (int i = 0; i < k; i++) plat_thread_join(t[i]);
// every 2 s a count on stdout for tools/make_game.py (step_shaders), as gfx_shader.c prepare_files prints for the
// translation: a driver building for a long time still shows the step moving
for (int i = 0; i < k; i++)
while (!plat_thread_join_ns(t[i], 2000000000u)) {
plat_mutex_lock(&LB.stat);
uint64_t d = LB.done;
plat_mutex_unlock(&LB.stat);
printf("prepare-shaders: progress build %llu %zu\n", (unsigned long long)d, n);
fflush(stdout);
}
printf("prepare-shaders: progress build %zu %zu\n", n, n);
fflush(stdout);
if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned
lb_report("preparation done");
// with gpl off only compute variants are built: say so, rather than count the skipped ones as built

View File

@@ -907,6 +907,43 @@ class MakeGameTest(Tmp):
steps = json.loads((self.out / 'build-cache' / 'state.json').read_text())['steps']
self.assertEqual(set(steps), {'exefs', 'romfs', 'lift', 'compile'})
def test_shader_progress_reaches_the_launcher(self):
"""The shaders step turns nx2navi.exe's counts into progress (translation the first half of the bar, the
driver build the second), sends one every few seconds while nothing is printed, says when the counts stop
moving, and passes its other lines on to the log."""
kit = self.tmp / 'kit'
(kit / 'tools').mkdir(parents=True)
(kit / 'tools' / 'prepare_shaders.py').write_text(
'import time\n'
'print("reading the inventory", flush=True)\n'
'time.sleep(0.6)\n'
'for d in (0, 50, 100):\n'
' print(f"prepare-shaders: progress translate {d} 100", flush=True)\n'
' time.sleep(0.05)\n'
'print("prepare-shaders: progress build 10 100", flush=True)\n'
'time.sleep(0.6)\n'
'print("prepare-shaders: progress build 100 100", flush=True)\n')
for name, value in (('KIT', kit), ('EVENTS', True), ('SHADER_BEAT_S', 0.1), ('SHADER_STALL_S', 0.3)):
self.addCleanup(setattr, self.mg, name, getattr(self.mg, name))
setattr(self.mg, name, value)
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
self.mg.step_shaders(self.out)
lines = buf.getvalue().splitlines()
events = [json.loads(ln[2:]) for ln in lines if ln.startswith('@@')]
self.assertIn('reading the inventory', lines)
self.assertFalse([ln for ln in lines if ln.startswith('prepare-shaders: progress')])
self.assertTrue(all(e['ev'] == 'progress' and e['step'] == 'shaders' for e in events))
fractions = [e['fraction'] for e in events]
self.assertEqual(fractions, sorted(fractions))
self.assertEqual(fractions[-1], 1.0)
details = [e['detail'] for e in events]
self.assertTrue(any(d.startswith("Reading the game's shaders") for d in details))
self.assertIn('Translated 50 of 100 shaders', details)
self.assertGreaterEqual(sum(1 for e in events if e['fraction'] == 0.55), 2) # heartbeats while the build waits
self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details))
self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25)
def test_romfs_failure_stops_the_build(self):
ran, lifting, failed = [], threading.Event(), threading.Event()
@@ -1310,7 +1347,7 @@ class WindowTest(Tmp):
r = subprocess.run([sys.executable, '-m', 'launcher', '--root', str(self.tmp / 'w'), '--selftest'],
cwd=ROOT, capture_output=True, text=True, timeout=180)
self.assertEqual(r.returncode, 0, r.stderr)
self.assertIn('9 screens drawn', r.stdout)
self.assertIn('10 screens drawn', r.stdout)
if __name__ == '__main__':

View File

@@ -983,11 +983,78 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
# ---------------------------------------------------------------- main
SHADER_PROGRESS = re.compile(r'^prepare-shaders: progress (translate|build) (\d+) (\d+)\s*$')
SHADER_STALL_S = 180 # no count moved for this long: the launcher says so (the step still runs)
SHADER_BEAT_S = 10 # a progress event at least this often while the step runs
def shader_progress(phase, done, total, quiet_s, elapsed_s):
"""The shaders step's progress event fields: the translation fills the first half of the bar, the build into the
driver's cache the second (nx2navi.exe prints both counts, runtime/gfx_shader.c prepare_files and
runtime/vk_pipeline.c lb_warm_prepared); before either count, reading the game's shaders. quiet_s: seconds since a
count last moved, said once it passes SHADER_STALL_S, so a slow step never looks frozen without a word. The
detail ends without a full stop: the launcher adds one (launcher/ui/app.py)."""
clock = f'{int(elapsed_s) // 60}:{int(elapsed_s) % 60:02d}'
if phase is None:
fraction, detail = 0.0, f'Reading the game\'s shaders ({clock} so far)'
elif phase == 'translate':
fraction, detail = 0.5 * done / max(total, 1), f'Translated {done:,} of {total:,} shaders'
else:
fraction, detail = 0.5 + 0.5 * done / max(total, 1), f'Built {done:,} of {total:,} shaders for your graphics card'
if quiet_s >= SHADER_STALL_S:
detail += (f'. No change for {int(quiet_s) // 60} min, but it is still running: one large shader can take '
'a while')
return {'step': 'shaders', 'fraction': round(fraction, 4), 'detail': detail}
def step_shaders(out):
"""Every shader of the game files into the game's shader cache and the driver's own cache, before the first start."""
cmd = [sys.executable, str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
"""Every shader of the game files into the game's shader cache and the driver's own cache, before the first start.
Its output is read as it comes: the counts nx2navi.exe prints become progress events, the other lines go on to
the log, and while nothing moves an event every few seconds keeps saying the step is running and for how long."""
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
'--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe']
if subprocess.run(cmd).returncode:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8',
errors='replace', bufsize=1)
t0 = time.monotonic()
state = {'phase': None, 'done': 0, 'total': 0, 'moved': t0}
lock = threading.Lock()
stop = threading.Event()
def send():
now = time.monotonic()
emit('progress', **shader_progress(state['phase'], state['done'], state['total'], now - state['moved'],
now - t0))
def heartbeat():
while not stop.wait(SHADER_BEAT_S):
with lock:
send()
beat = threading.Thread(target=heartbeat, daemon=True)
beat.start()
last_log = 0.0
try:
for line in proc.stdout:
m = SHADER_PROGRESS.match(line)
if not m:
say(line.rstrip('\n'))
continue
phase, done, total = m.group(1), int(m.group(2)), int(m.group(3))
with lock:
if (phase, done) != (state['phase'], state['done']):
state['moved'] = time.monotonic()
state.update(phase=phase, done=done, total=total)
send()
if time.monotonic() - last_log >= 30 or done == total: # the log gets a count now and then, not each one
last_log = time.monotonic()
say(f' shaders: {"translated" if phase == "translate" else "built"} {done} of {total}, '
f'{time.monotonic() - t0:.0f} s')
finally:
rc = proc.wait()
proc.stdout.close()
stop.set()
beat.join()
if rc:
fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it '
'meets them. Run the build again to retry this step')
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them