mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-17 04:59:57 -04:00
Show Prepare the Shaders moving in the launcher: the game prints its counts and the step sends them on.
The step sat at 0 % until it was done, an hour on some PCs, because nothing it ran reported progress to the launcher. nx2navi.exe --prepare-shaders now prints how many shaders it has translated and how many it has built into the driver's cache every 2 s. make_game.py reads them as they come and fills the bar, the translation the first half and the build the second, with the counts in the step's text. An event every 10 s keeps the text current while nothing is printed, and after 3 minutes without a count moving it says so, and that the step is still running. The selftest draws the step as a tenth screen. Its made-up update was version 0.3.11, so from 0.4.0 on the update-ready screen showed Up to Date and was no longer drawn; it is now a minor version past the launcher running it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BSHQpLnScF9sNv4LNAaaS3
This commit is contained in:
@@ -89,7 +89,10 @@ A path the Windows ANSI code page cannot hold breaks the MSYS2 tools, so the too
|
||||
- The lifted code is compiled for the processor's own level, x86-64-v3 (AVX2, FMA) or v2 (`cpu_level`): the game
|
||||
runs on the PC that builds it, and a generic build would do every fused multiply-add of the game in software.
|
||||
3. **Prepare the Shaders**: make_game.py's last step translates every shader of the game into the game's shader
|
||||
cache and the driver's own cache (`tools/prepare_shaders.py`), so play does not stop to compile them.
|
||||
cache and the driver's own cache (`tools/prepare_shaders.py`), so play does not stop to compile them. Its bar
|
||||
follows the game's own counts, printed every 2 s: the translation fills the first half, the build into the
|
||||
driver's cache the second. An update every 10 s keeps the step's text current while nothing is printed, and after
|
||||
3 minutes without a count moving it says so, and that the step is still running.
|
||||
4. The game's Cetra Display is decoded for the window.
|
||||
|
||||
## Updating
|
||||
|
||||
@@ -221,6 +221,8 @@ class Job:
|
||||
self.skip(sid, 'Kept' if sid in ('exefs', 'romfs') else 'Unchanged')
|
||||
if link_step and sid in ('compile', 'compile_changed'):
|
||||
self.skip(link_step, 'Unchanged')
|
||||
elif kind == 'progress' and sid and 'fraction' in ev: # a step that words its own progress (shaders)
|
||||
self.progress(sid, float(ev.get('fraction') or 0), ev.get('detail', ''))
|
||||
elif kind == 'progress' and sid:
|
||||
done, total, unit = ev.get('done', 0), max(ev.get('total', 1), 1), ev.get('unit', '')
|
||||
if unit == 'bytes':
|
||||
|
||||
@@ -6,7 +6,7 @@ import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .. import feed, jobs
|
||||
from .. import feed, jobs, versions
|
||||
|
||||
|
||||
def grab_gdi(x, y, w, h, path):
|
||||
@@ -69,6 +69,16 @@ def grab(app, path):
|
||||
return False
|
||||
|
||||
|
||||
def ahead_of(version):
|
||||
"""A version a minor step past version: the made-up release must be newer than the launcher drawing it, or the
|
||||
update screens show "Up to Date" instead (a fixed 0.3.11 did from 0.4.0 on)."""
|
||||
try:
|
||||
nums, _ = versions.parse(version)
|
||||
except ValueError:
|
||||
return '99.0.0'
|
||||
return f'{nums[0]}.{nums[1] + 1}.0'
|
||||
|
||||
|
||||
def fake_release(version='0.3.11'):
|
||||
m = {'format': 1, 'version': version, 'app': {'name': f'NX2navi-app-{version}.zip', 'size': 1, 'sha256': '0'}}
|
||||
notes = ('- Each program module comes from the .nup package that holds it whole.\n'
|
||||
@@ -105,11 +115,11 @@ def states(app):
|
||||
app.show('home', 0)
|
||||
|
||||
def update_ready():
|
||||
app.release = fake_release()
|
||||
app.release = fake_release(ahead_of(ctx.version))
|
||||
app.show('home', 1)
|
||||
|
||||
def updating():
|
||||
j = jobs.UpdateDownloadJob(ctx, fake_release())
|
||||
j = jobs.UpdateDownloadJob(ctx, fake_release(ahead_of(ctx.version)))
|
||||
for sid, val in (('download', 'Done · 0:06'), ('verify', 'Checksum OK')):
|
||||
j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0
|
||||
j.skip('tools_update', 'Unchanged')
|
||||
@@ -139,6 +149,19 @@ def states(app):
|
||||
def uninstall():
|
||||
app.show('uninstall', 0)
|
||||
|
||||
def shaders(): # the counts and the wording tools/make_game.py shader_progress sends
|
||||
j = jobs.InstallJob(ctx, s['dump'], s['game_dir'])
|
||||
for sid, val in (('tools', 'Done · 0:48'), ('check', 'Version 1.1.1'), ('exefs', 'Done · 0:04'),
|
||||
('romfs', 'Done · 0:31'), ('lift', 'Done · 0:48'), ('compile', 'Done · 4:12')):
|
||||
j.step(sid).state, j.step(sid).value, j.step(sid).fraction = jobs.DONE, val, 1.0
|
||||
j.begin('shaders')
|
||||
j.progress('shaders', 0.5 + 0.5 * 6214 / 20480, 'Built 6,214 of 20,480 shaders for your graphics card')
|
||||
j.started = time.time() - 611
|
||||
for ln in (' shaders: translated 20480 of 20480, 141 s', ' shaders: built 6214 of 20480, 402 s'):
|
||||
j.lines.append(ln)
|
||||
app.job, app.job_return, app.job_back = j, 'home', 'first_run'
|
||||
app.show('job')
|
||||
|
||||
def import_saves():
|
||||
app.release, app.message = None, ''
|
||||
app.save_message = ('Save imported: 3 files from 0100000000010000. Your old save is in '
|
||||
@@ -147,7 +170,8 @@ def states(app):
|
||||
|
||||
return [('1-first-run', first_run), ('2-installing', installing), ('3-home', home),
|
||||
('4-update-ready', update_ready), ('5-updating', updating), ('6-settings', settings),
|
||||
('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves)]
|
||||
('7-build-failed', failed), ('8-uninstall', uninstall), ('9-import-saves', import_saves),
|
||||
('10-shaders', shaders)]
|
||||
|
||||
|
||||
def run(app, shots):
|
||||
|
||||
@@ -2221,10 +2221,18 @@ static long long prepare_files(const char *ans_path, const char *prg_path, int t
|
||||
for (int i = 0; i < threads; i++)
|
||||
if ((t[k] = plat_thread_new(prep_worker, &p, "shader prepare"))) k++;
|
||||
if (!k) prep_worker(&p);
|
||||
for (int m = 0; m < k; m++)
|
||||
while (!plat_thread_join_ns(t[m], 5000000000u))
|
||||
rt_log("gfx_shader: prepare: %lld of %zu pairs, %llu translated, %.0f ms", (long long)p.done, p.n_ans,
|
||||
(unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0);
|
||||
// every 2 s a count on stdout for tools/make_game.py (step_shaders reads "prepare-shaders: progress" lines and shows
|
||||
// them in the launcher, which otherwise sat at 0 % for the whole step), and every 10 s one in the log
|
||||
for (int m = 0, polls = 0; m < k; m++)
|
||||
while (!plat_thread_join_ns(t[m], 2000000000u)) {
|
||||
printf("prepare-shaders: progress translate %lld %zu\n", (long long)p.done, p.n_ans);
|
||||
fflush(stdout);
|
||||
if (++polls % 5 == 0)
|
||||
rt_log("gfx_shader: prepare: %lld of %zu pairs, %llu translated, %.0f ms", (long long)p.done, p.n_ans,
|
||||
(unsigned long long)((uint64_t)n_xlated - xl0), (double)plat_now_ms() - t0);
|
||||
}
|
||||
printf("prepare-shaders: progress translate %zu %zu\n", p.n_ans, p.n_ans);
|
||||
fflush(stdout);
|
||||
svc_set_limit(svc_runtime_limit());
|
||||
rt_log("gfx_shader: prepare: %zu %s of %zu programs: %llu translated, %lld failed, %lld without a program, the rest "
|
||||
"cached; %lld default variants with refined answers; %.0f ms", p.n_ans, what, p.progs.n,
|
||||
|
||||
@@ -3570,7 +3570,18 @@ static void lb_warm_prepared(const char *dir) {
|
||||
for (int i = 0; i < cpus && i < JR_SLOTS; i++)
|
||||
if ((t[k] = plat_thread_new(lb_warm_worker, (void *)(intptr_t)k, "lib warm"))) k++;
|
||||
if (!k) lb_warm_worker((void *)(intptr_t)0);
|
||||
for (int i = 0; i < k; i++) plat_thread_join(t[i]);
|
||||
// every 2 s a count on stdout for tools/make_game.py (step_shaders), as gfx_shader.c prepare_files prints for the
|
||||
// translation: a driver building for a long time still shows the step moving
|
||||
for (int i = 0; i < k; i++)
|
||||
while (!plat_thread_join_ns(t[i], 2000000000u)) {
|
||||
plat_mutex_lock(&LB.stat);
|
||||
uint64_t d = LB.done;
|
||||
plat_mutex_unlock(&LB.stat);
|
||||
printf("prepare-shaders: progress build %llu %zu\n", (unsigned long long)d, n);
|
||||
fflush(stdout);
|
||||
}
|
||||
printf("prepare-shaders: progress build %zu %zu\n", n, n);
|
||||
fflush(stdout);
|
||||
if (SK.journal) plat_file_close(SK.journal), SK.journal = NULL, plat_file_delete(jp); // every build returned
|
||||
lb_report("preparation done");
|
||||
// with gpl off only compute variants are built: say so, rather than count the skipped ones as built
|
||||
|
||||
@@ -907,6 +907,43 @@ class MakeGameTest(Tmp):
|
||||
steps = json.loads((self.out / 'build-cache' / 'state.json').read_text())['steps']
|
||||
self.assertEqual(set(steps), {'exefs', 'romfs', 'lift', 'compile'})
|
||||
|
||||
def test_shader_progress_reaches_the_launcher(self):
|
||||
"""The shaders step turns nx2navi.exe's counts into progress (translation the first half of the bar, the
|
||||
driver build the second), sends one every few seconds while nothing is printed, says when the counts stop
|
||||
moving, and passes its other lines on to the log."""
|
||||
kit = self.tmp / 'kit'
|
||||
(kit / 'tools').mkdir(parents=True)
|
||||
(kit / 'tools' / 'prepare_shaders.py').write_text(
|
||||
'import time\n'
|
||||
'print("reading the inventory", flush=True)\n'
|
||||
'time.sleep(0.6)\n'
|
||||
'for d in (0, 50, 100):\n'
|
||||
' print(f"prepare-shaders: progress translate {d} 100", flush=True)\n'
|
||||
' time.sleep(0.05)\n'
|
||||
'print("prepare-shaders: progress build 10 100", flush=True)\n'
|
||||
'time.sleep(0.6)\n'
|
||||
'print("prepare-shaders: progress build 100 100", flush=True)\n')
|
||||
for name, value in (('KIT', kit), ('EVENTS', True), ('SHADER_BEAT_S', 0.1), ('SHADER_STALL_S', 0.3)):
|
||||
self.addCleanup(setattr, self.mg, name, getattr(self.mg, name))
|
||||
setattr(self.mg, name, value)
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
self.mg.step_shaders(self.out)
|
||||
lines = buf.getvalue().splitlines()
|
||||
events = [json.loads(ln[2:]) for ln in lines if ln.startswith('@@')]
|
||||
self.assertIn('reading the inventory', lines)
|
||||
self.assertFalse([ln for ln in lines if ln.startswith('prepare-shaders: progress')])
|
||||
self.assertTrue(all(e['ev'] == 'progress' and e['step'] == 'shaders' for e in events))
|
||||
fractions = [e['fraction'] for e in events]
|
||||
self.assertEqual(fractions, sorted(fractions))
|
||||
self.assertEqual(fractions[-1], 1.0)
|
||||
details = [e['detail'] for e in events]
|
||||
self.assertTrue(any(d.startswith("Reading the game's shaders") for d in details))
|
||||
self.assertIn('Translated 50 of 100 shaders', details)
|
||||
self.assertGreaterEqual(sum(1 for e in events if e['fraction'] == 0.55), 2) # heartbeats while the build waits
|
||||
self.assertTrue(any(d.startswith('Built 10 of 100') and 'still running' in d for d in details))
|
||||
self.assertEqual([e['fraction'] for e in events if e['detail'] == 'Translated 50 of 100 shaders'][0], 0.25)
|
||||
|
||||
def test_romfs_failure_stops_the_build(self):
|
||||
ran, lifting, failed = [], threading.Event(), threading.Event()
|
||||
|
||||
@@ -1310,7 +1347,7 @@ class WindowTest(Tmp):
|
||||
r = subprocess.run([sys.executable, '-m', 'launcher', '--root', str(self.tmp / 'w'), '--selftest'],
|
||||
cwd=ROOT, capture_output=True, text=True, timeout=180)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn('9 screens drawn', r.stdout)
|
||||
self.assertIn('10 screens drawn', r.stdout)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -983,11 +983,78 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None):
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
|
||||
SHADER_PROGRESS = re.compile(r'^prepare-shaders: progress (translate|build) (\d+) (\d+)\s*$')
|
||||
SHADER_STALL_S = 180 # no count moved for this long: the launcher says so (the step still runs)
|
||||
SHADER_BEAT_S = 10 # a progress event at least this often while the step runs
|
||||
|
||||
|
||||
def shader_progress(phase, done, total, quiet_s, elapsed_s):
|
||||
"""The shaders step's progress event fields: the translation fills the first half of the bar, the build into the
|
||||
driver's cache the second (nx2navi.exe prints both counts, runtime/gfx_shader.c prepare_files and
|
||||
runtime/vk_pipeline.c lb_warm_prepared); before either count, reading the game's shaders. quiet_s: seconds since a
|
||||
count last moved, said once it passes SHADER_STALL_S, so a slow step never looks frozen without a word. The
|
||||
detail ends without a full stop: the launcher adds one (launcher/ui/app.py)."""
|
||||
clock = f'{int(elapsed_s) // 60}:{int(elapsed_s) % 60:02d}'
|
||||
if phase is None:
|
||||
fraction, detail = 0.0, f'Reading the game\'s shaders ({clock} so far)'
|
||||
elif phase == 'translate':
|
||||
fraction, detail = 0.5 * done / max(total, 1), f'Translated {done:,} of {total:,} shaders'
|
||||
else:
|
||||
fraction, detail = 0.5 + 0.5 * done / max(total, 1), f'Built {done:,} of {total:,} shaders for your graphics card'
|
||||
if quiet_s >= SHADER_STALL_S:
|
||||
detail += (f'. No change for {int(quiet_s) // 60} min, but it is still running: one large shader can take '
|
||||
'a while')
|
||||
return {'step': 'shaders', 'fraction': round(fraction, 4), 'detail': detail}
|
||||
|
||||
|
||||
def step_shaders(out):
|
||||
"""Every shader of the game files into the game's shader cache and the driver's own cache, before the first start."""
|
||||
cmd = [sys.executable, str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
|
||||
"""Every shader of the game files into the game's shader cache and the driver's own cache, before the first start.
|
||||
Its output is read as it comes: the counts nx2navi.exe prints become progress events, the other lines go on to
|
||||
the log, and while nothing moves an event every few seconds keeps saying the step is running and for how long."""
|
||||
cmd = [sys.executable, '-u', str(KIT / 'tools' / 'prepare_shaders.py'), str(out), '--romfs', str(out / 'romfs'),
|
||||
'--exefs', str(out / 'exefs'), '--no-list', '--exe', 'ocarina.exe']
|
||||
if subprocess.run(cmd).returncode:
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8',
|
||||
errors='replace', bufsize=1)
|
||||
t0 = time.monotonic()
|
||||
state = {'phase': None, 'done': 0, 'total': 0, 'moved': t0}
|
||||
lock = threading.Lock()
|
||||
stop = threading.Event()
|
||||
|
||||
def send():
|
||||
now = time.monotonic()
|
||||
emit('progress', **shader_progress(state['phase'], state['done'], state['total'], now - state['moved'],
|
||||
now - t0))
|
||||
|
||||
def heartbeat():
|
||||
while not stop.wait(SHADER_BEAT_S):
|
||||
with lock:
|
||||
send()
|
||||
|
||||
beat = threading.Thread(target=heartbeat, daemon=True)
|
||||
beat.start()
|
||||
last_log = 0.0
|
||||
try:
|
||||
for line in proc.stdout:
|
||||
m = SHADER_PROGRESS.match(line)
|
||||
if not m:
|
||||
say(line.rstrip('\n'))
|
||||
continue
|
||||
phase, done, total = m.group(1), int(m.group(2)), int(m.group(3))
|
||||
with lock:
|
||||
if (phase, done) != (state['phase'], state['done']):
|
||||
state['moved'] = time.monotonic()
|
||||
state.update(phase=phase, done=done, total=total)
|
||||
send()
|
||||
if time.monotonic() - last_log >= 30 or done == total: # the log gets a count now and then, not each one
|
||||
last_log = time.monotonic()
|
||||
say(f' shaders: {"translated" if phase == "translate" else "built"} {done} of {total}, '
|
||||
f'{time.monotonic() - t0:.0f} s')
|
||||
finally:
|
||||
rc = proc.wait()
|
||||
proc.stdout.close()
|
||||
stop.set()
|
||||
beat.join()
|
||||
if rc:
|
||||
fail('preparing the shaders failed (the lines above say why); the game still runs, compiling shaders as it '
|
||||
'meets them. Run the build again to retry this step')
|
||||
for seed in ('seed-programs.bin', 'seed-answers.bin'): # translated now: the shader cache holds them
|
||||
|
||||
Reference in New Issue
Block a user