From b1421faaf9f95a77675c0a213936e1e590c74d1d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 08:32:13 +0000 Subject: [PATCH] Show Prepare the Shaders moving in the launcher: the game prints its counts and the step sends them on. The step sat at 0 % until it was done, an hour on some PCs, because nothing it ran reported progress to the launcher. nx2navi.exe --prepare-shaders now prints how many shaders it has translated and how many it has built into the driver's cache every 2 s. make_game.py reads them as they come and fills the bar, the translation the first half and the build the second, with the counts in the step's text. An event every 10 s keeps the text current while nothing is printed, and after 3 minutes without a count moving it says so, and that the step is still running. The selftest draws the step as a tenth screen. Its made-up update was version 0.3.11, so from 0.4.0 on the update-ready screen showed Up to Date and was no longer drawn; it is now a minor version past the launcher running it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BSHQpLnScF9sNv4LNAaaS3 --- docs/launcher.md | 5 ++- launcher/jobs.py | 2 ++ launcher/ui/selftest.py | 32 +++++++++++++++--- runtime/gfx_shader.c | 16 ++++++--- runtime/vk_pipeline.c | 13 +++++++- tests/launcher_test.py | 39 +++++++++++++++++++++- tools/make_game.py | 73 +++++++++++++++++++++++++++++++++++++++-- 7 files changed, 166 insertions(+), 14 deletions(-) diff --git a/docs/launcher.md b/docs/launcher.md index bdeaaf59..4673bfd6 100644 --- a/docs/launcher.md +++ b/docs/launcher.md @@ -89,7 +89,10 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too - The lifted code is compiled for the processor's own level, x86-64-v3 (AVX2, FMA) or v2 (`cpu_level`): the game runs on the PC that builds it, and a generic build would do every fused multiply-add of the game in software. 3. **Prepare the Shaders**: make_game.py's last step translates every shader of the game into the game's shader - cache and the driver's own cache (`tools/prepare_shaders.py`), so play does not stop to compile them. + cache and the driver's own cache (`tools/prepare_shaders.py`), so play does not stop to compile them. Its bar + follows the game's own counts, printed every 2 s: the translation fills the first half, the build into the + driver's cache the second. An update every 10 s keeps the step's text current while nothing is printed, and after + 3 minutes without a count moving it says so, and that the step is still running. 4. The game's Cetra Display is decoded for the window. ## Updating diff --git a/launcher/jobs.py b/launcher/jobs.py index 2a7dd00f..aca09917 100644 --- a/launcher/jobs.py +++ b/launcher/jobs.py @@ -221,6 +221,8 @@ class Job: self.skip(sid, 'Kept' if sid in ('exefs', 'romfs') else 'Unchanged') if link_step and sid in ('compile', 'compile_changed'): self.skip(link_step, 'Unchanged') + elif kind == 'progress' and sid and 'fraction' in ev: # a step that words its own progress (shaders) + self.progress(sid, float(ev.get('fraction') or 0), ev.get('detail', '')) elif kind == 'progress' and sid: done, total, unit = ev.get('done', 0), max(ev.get('total', 1), 1), ev.get('unit', '') if unit == 'bytes': diff --git a/launcher/ui/selftest.py b/launcher/ui/selftest.py index 3e51b2c1..09b3cbf5 100644 --- a/launcher/ui/selftest.py +++ b/launcher/ui/selftest.py @@ -6,7 +6,7 @@ import sys import time from pathlib import Path -from .. import feed, jobs +from .. import feed, jobs, versions def grab_gdi(x, y, w, h, path): @@ -69,6 +69,16 @@ def grab(app, path): return False +def ahead_of(version): + """A version a minor step past version: the made-up release must be newer than the launcher drawing it, or the + update screens show "Up to Date" instead (a fixed 0.3.11 did from 0.4.0 on).""" + try: + nums, _ = versions.parse(version) + except ValueError: + return '99.0.0' + return f'{nums[0]}.{nums[1] + 1}.0' + + def fake_release(version='0.3.11'): m = {'format': 1, 'version': version, 'app': {'name': f'NX2navi-app-{version}.zip', 'size': 1, 'sha256': '0'}} notes = ('- Each program module comes from the .nup package that holds it whole.\n' @@ -105,11 +115,11 @@ def states(app): app.show('home', 0) def update_ready(): - app.release = fake_release() + app.release = fake_release(ahead_of(ctx.version)) app.show('home', 1) def updating(): - j = jobs.UpdateDownloadJob(ctx, fake_release()) + j = jobs.UpdateDownloadJob(ctx, fake_release(ahead_of(ctx.version))) for sid, val in (('download', 'Done · 0:06'), ('verify', 'Checksum OK')): j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0 j.skip('tools_update', 'Unchanged') @@ -139,6 +149,19 @@ def states(app): def uninstall(): app.show('uninstall', 0) + def shaders(): # the counts and the wording tools/make_game.py shader_progress sends + j = jobs.InstallJob(ctx, s['dump'], s['game_dir']) + for sid, val in (('tools', 'Done · 0:48'), ('check', 'Version 1.1.1'), ('exefs', 'Done · 0:04'), + ('romfs', 'Done · 0:31'), ('lift', 'Done · 0:48'), ('compile', 'Done · 4:12')): + j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0 + j.begin('shaders') + j.progress('shaders', 0.5 + 0.5 * 6214 / 20480, 'Built 6,214 of 20,480 shaders for your graphics card') + j.started = time.time() - 611 + for ln in (' shaders: translated 20480 of 20480, 141 s', ' shaders: built 6214 of 20480, 402 s'): + j.lines.append(ln) + app.job, app.job_return, app.job_back = j, 'home', 'first_run' + app.show('job') + def import_saves(): app.release, app.message = None, '' app.save_message = ('Save imported: 3 files from 0100000000010000. Your old save is in ' @@ -147,7 +170,8 @@ def states(app): return [('1-first-run', first_run), ('2-installing', installing), ('3-home', home), ('4-update-ready', update_ready), ('5-updating', updating), ('6-settings', settings), - ('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves)] + ('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves), + ('10-shaders', shaders)] def run(app, shots): diff --git a/runtime/gfx_shader.c b/runtime/gfx_shader.c index 4a517b0e..a4e6c2b3 100644 --- a/runtime/gfx_shader.c +++ b/runtime/gfx_shader.c @@ -2221,10 +2221,18 @@ static long long prepare_files(const char *ans_path, const char *prg_path, int t for (int i = 0; i < threads; i++) if ((t[k] = plat_thread_new(prep_worker, &p, "shader prepare"))) k++; if (!k) prep_worker(&p); - for (int m = 0; m < k; m++) - while (!plat_thread_join_ns(t[m], 5000000000u)) - rt_log("gfx_shader: prepare: %lld of %zu pairs, %llu translated, %.0f ms", (long long)p.done, p.n_ans, - (unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0); + // every 2 s a count on stdout for tools/make_game.py (step_shaders reads "prepare-shaders: progress" lines and shows + // them in the launcher, which otherwise sat at 0 % for the whole step), and every 10 s one in the log + for (int m = 0, polls = 0; m < k; m++) + while (!plat_thread_join_ns(t[m], 2000000000u)) { + printf("prepare-shaders: progress translate %lld %zu\n", (long long)p.done, p.n_ans); + fflush(stdout); + if (++polls % 5 == 0) + rt_log("gfx_shader: prepare: %lld of %zu pairs, %llu translated, %.0f ms", (long long)p.done, p.n_ans, + (unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0); + } + printf("prepare-shaders: progress translate %zu %zu\n", p.n_ans, p.n_ans); + fflush(stdout); svc_set_limit(svc_runtime_limit()); rt_log("gfx_shader: prepare: %zu %s of %zu programs: %llu translated, %lld failed, %lld without a program, the rest " "cached; %lld default variants with refined answers; %.0f ms", p.n_ans, what, p.progs.n, diff --git a/runtime/vk_pipeline.c b/runtime/vk_pipeline.c index 4bdf3662..19eb10ec 100644 --- a/runtime/vk_pipeline.c +++ b/runtime/vk_pipeline.c @@ -3570,7 +3570,18 @@ static void lb_warm_prepared(const char *dir) { for (int i = 0; i < cpus && i < JR_SLOTS; i++) if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++; if (!k) lb_warm_worker((void *)(intptr_t)0); - for (int i = 0; i < k; i++) plat_thread_join(t[i]); + // every 2 s a count on stdout for tools/make_game.py (step_shaders), as gfx_shader.c prepare_files prints for the + // translation: a driver building for a long time still shows the step moving + for (int i = 0; i < k; i++) + while (!plat_thread_join_ns(t[i], 2000000000u)) { + plat_mutex_lock(&LB.stat); + uint64_t d = LB.done; + plat_mutex_unlock(&LB.stat); + printf("prepare-shaders: progress build %llu %zu\n", (unsigned long long)d, n); + fflush(stdout); + } + printf("prepare-shaders: progress build %zu %zu\n", n, n); + fflush(stdout); if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned lb_report("preparation done"); // with gpl off only compute variants are built: say so, rather than count the skipped ones as built diff --git a/tests/launcher_test.py b/tests/launcher_test.py index af32e20a..225bc894 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -907,6 +907,43 @@ class MakeGameTest(Tmp): steps = json.loads((self.out / 'build-cache' / 'state.json').read_text())['steps'] self.assertEqual(set(steps), {'exefs', 'romfs', 'lift', 'compile'}) + def test_shader_progress_reaches_the_launcher(self): + """The shaders step turns nx2navi.exe's counts into progress (translation the first half of the bar, the + driver build the second), sends one every few seconds while nothing is printed, says when the counts stop + moving, and passes its other lines on to the log.""" + kit = self.tmp / 'kit' + (kit / 'tools').mkdir(parents=True) + (kit / 'tools' / 'prepare_shaders.py').write_text( + 'import time\n' + 'print("reading the inventory", flush=True)\n' + 'time.sleep(0.6)\n' + 'for d in (0, 50, 100):\n' + ' print(f"prepare-shaders: progress translate {d} 100", flush=True)\n' + ' time.sleep(0.05)\n' + 'print("prepare-shaders: progress build 10 100", flush=True)\n' + 'time.sleep(0.6)\n' + 'print("prepare-shaders: progress build 100 100", flush=True)\n') + for name, value in (('KIT', kit), ('EVENTS', True), ('SHADER_BEAT_S', 0.1), ('SHADER_STALL_S', 0.3)): + self.addCleanup(setattr, self.mg, name, getattr(self.mg, name)) + setattr(self.mg, name, value) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + self.mg.step_shaders(self.out) + lines = buf.getvalue().splitlines() + events = [json.loads(ln[2:]) for ln in lines if ln.startswith('@@')] + self.assertIn('reading the inventory', lines) + self.assertFalse([ln for ln in lines if ln.startswith('prepare-shaders: progress')]) + self.assertTrue(all(e['ev'] == 'progress' and e['step'] == 'shaders' for e in events)) + fractions = [e['fraction'] for e in events] + self.assertEqual(fractions, sorted(fractions)) + self.assertEqual(fractions[-1], 1.0) + details = [e['detail'] for e in events] + self.assertTrue(any(d.startswith("Reading the game's shaders") for d in details)) + self.assertIn('Translated 50 of 100 shaders', details) + self.assertGreaterEqual(sum(1 for e in events if e['fraction'] == 0.55), 2) # heartbeats while the build waits + self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details)) + self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25) + def test_romfs_failure_stops_the_build(self): ran, lifting, failed = [], threading.Event(), threading.Event() @@ -1310,7 +1347,7 @@ class WindowTest(Tmp): r = subprocess.run([sys.executable, '-m', 'launcher', '--root', str(self.tmp / 'w'), '--selftest'], cwd=ROOT, capture_output=True, text=True, timeout=180) self.assertEqual(r.returncode, 0, r.stderr) - self.assertIn('9 screens drawn', r.stdout) + self.assertIn('10 screens drawn', r.stdout) if __name__ == '__main__': diff --git a/tools/make_game.py b/tools/make_game.py index 277c369a..d11568d0 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -983,11 +983,78 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None): # ---------------------------------------------------------------- main +SHADER_PROGRESS = re.compile(r'^prepare-shaders: progress (translate|build) (\d+) (\d+)\s*$') +SHADER_STALL_S = 180 # no count moved for this long: the launcher says so (the step still runs) +SHADER_BEAT_S = 10 # a progress event at least this often while the step runs + + +def shader_progress(phase, done, total, quiet_s, elapsed_s): + """The shaders step's progress event fields: the translation fills the first half of the bar, the build into the + driver's cache the second (nx2navi.exe prints both counts, runtime/gfx_shader.c prepare_files and + runtime/vk_pipeline.c lb_warm_prepared); before either count, reading the game's shaders. quiet_s: seconds since a + count last moved, said once it passes SHADER_STALL_S, so a slow step never looks frozen without a word. The + detail ends without a full stop: the launcher adds one (launcher/ui/app.py).""" + clock = f'{int(elapsed_s) // 60}:{int(elapsed_s) % 60:02d}' + if phase is None: + fraction, detail = 0.0, f'Reading the game\'s shaders ({clock} so far)' + elif phase == 'translate': + fraction, detail = 0.5 * done / max(total, 1), f'Translated {done:,} of {total:,} shaders' + else: + fraction, detail = 0.5 + 0.5 * done / max(total, 1), f'Built {done:,} of {total:,} shaders for your graphics card' + if quiet_s >= SHADER_STALL_S: + detail += (f'. No change for {int(quiet_s) // 60} min, but it is still running: one large shader can take ' + 'a while') + return {'step': 'shaders', 'fraction': round(fraction, 4), 'detail': detail} + + def step_shaders(out): - """Every shader of the game files into the game's shader cache and the driver's own cache, before the first start.""" - cmd = [sys.executable, str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), + """Every shader of the game files into the game's shader cache and the driver's own cache, before the first start. + Its output is read as it comes: the counts nx2navi.exe prints become progress events, the other lines go on to + the log, and while nothing moves an event every few seconds keeps saying the step is running and for how long.""" + cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'), '--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe'] - if subprocess.run(cmd).returncode: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8', + errors='replace', bufsize=1) + t0 = time.monotonic() + state = {'phase': None, 'done': 0, 'total': 0, 'moved': t0} + lock = threading.Lock() + stop = threading.Event() + + def send(): + now = time.monotonic() + emit('progress', **shader_progress(state['phase'], state['done'], state['total'], now - state['moved'], + now - t0)) + + def heartbeat(): + while not stop.wait(SHADER_BEAT_S): + with lock: + send() + + beat = threading.Thread(target=heartbeat, daemon=True) + beat.start() + last_log = 0.0 + try: + for line in proc.stdout: + m = SHADER_PROGRESS.match(line) + if not m: + say(line.rstrip('\n')) + continue + phase, done, total = m.group(1), int(m.group(2)), int(m.group(3)) + with lock: + if (phase, done) != (state['phase'], state['done']): + state['moved'] = time.monotonic() + state.update(phase=phase, done=done, total=total) + send() + if time.monotonic() - last_log >= 30 or done == total: # the log gets a count now and then, not each one + last_log = time.monotonic() + say(f' shaders: {"translated" if phase == "translate" else "built"} {done} of {total}, ' + f'{time.monotonic() - t0:.0f} s') + finally: + rc = proc.wait() + proc.stdout.close() + stop.set() + beat.join() + if rc: fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it ' 'meets them. Run the build again to retry this step') for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them