Files
NX2navi/launcher/jobs.py
chrg6 3d25f00cd2 Estimate the time left from each step's recent pace, so Prepare the Shaders stops saying about 1 minute with 20 to go.
The launcher took the job's weighted fraction as its progress over time. Compile Changed Files carries over half of an
update's weight and can take seconds, so once the shader step began the job read as most of the way done, and the
estimate came out at a fifth of the time spent so far. The step running now is estimated from its progress over the
last minute (the shader step's translation and its slower build into the driver's cache each fill half of its bar),
and the steps still to come from the seconds per weight of the steps already done. On the shader build of a 42,142
shader update the estimate matches the time left.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-09 08:09:26 -05:00

400 lines
17 KiB
Python

"""The launcher's long tasks, run on a worker thread: install, update, rebuild and the update check. Each job is a
list of steps the window shows as rows; the window reads the job's state on a timer (rev changes on every change)
and never blocks on it. The same jobs run headless from launcher/cli.py."""
import collections
import threading
import time
import traceback
from . import build, feed, gamefiles, net, toolchain, updater, versions
PENDING, RUNNING, DONE, SKIPPED, FAILED = 'pending', 'running', 'done', 'skipped', 'failed'
STEPS = {
'tools': ('Gather Build Tools', 'Downloading the compiler and libraries the build uses. This happens once and '
'takes about 170 MB.'),
'tools_update': ('Update Build Tools', 'Bringing the compiler and libraries to the versions this release was '
'tested with.'),
'check': ('Inspect Your Game', 'Making sure the dump is Ocarina of Time 1.1.1.'),
'exefs': ('Unpack the Program', "Unpacking the game's program files."),
'romfs': ('Pack the Game Data', 'Checking every game file against the dump and packing them into one image.'),
'lift': ('Translate the Code', "Each function of your game's program becomes C code for the compiler."),
'compile': ('Compile', 'Compiling the translated code and the port into a Windows program.'),
'compile_changed': ('Compile Changed Files', 'Compiling the files this version changed. Files that did not '
'change are kept from the last build.'),
'shaders': ('Prepare the Shaders', "Building the game's known shaders for your graphics card ahead of time, so "
'play stops for them far less often.'),
'ready': ('Ready to Play', 'The game is built.'),
'download': ('Download {ver}', 'Downloading the new version of NX2navi.'),
'verify': ('Verify the Download', 'Checking the download against its checksum.'),
'swap': ('Swap In the New Build', 'Linking the new build. It replaces the old one only once it is complete.'),
}
WEIGHTS = {'tools': 12, 'tools_update': 4, 'check': 1, 'exefs': 2, 'romfs': 14, 'lift': 14, 'compile': 55,
'compile_changed': 55, 'shaders': 18, 'ready': 0, 'download': 8, 'verify': 1, 'swap': 6}
def mmss(seconds):
s = int(round(seconds))
return f'{s // 3600}:{s // 60 % 60:02d}:{s % 60:02d}' if s >= 3600 else f'{s // 60}:{s % 60:02d}'
class Step:
def __init__(self, sid, ver=''):
label, desc = STEPS[sid]
self.id, self.label, self.desc = sid, label.format(ver=ver), desc
self.state, self.value, self.fraction, self.detail = PENDING, 'Waiting', 0.0, ''
self.t0, self.seconds = None, 0.0
self.pace = collections.deque(maxlen=400) # (time, fraction) of recent progress, for Job.eta
class Job:
kind = 'job'
title = ''
def __init__(self, ctx, step_ids, ver=''):
self.ctx = ctx
self.steps = [Step(s, ver) for s in step_ids]
self.rev = 0
self.cancel_flag = threading.Event()
self.proc = None
self.result = None # None while running, then 'ok', 'failed' or 'cancelled'
self.error = ''
self.lines = collections.deque(maxlen=500)
self.started = time.time()
self.finished = None
self.after = None # what the window does next ('restart', ...)
self._lock = threading.Lock()
# ---- state the window reads
def step(self, sid):
for s in self.steps:
if s.id == sid:
return s
return None
def current(self):
for s in self.steps:
if s.state in (RUNNING, FAILED):
return s
return next((s for s in self.steps if s.state == PENDING), self.steps[-1])
def overall(self):
total = sum(WEIGHTS[s.id] for s in self.steps) or 1
got = 0.0
for s in self.steps:
w = WEIGHTS[s.id]
got += w if s.state in (DONE, SKIPPED) else w * s.fraction if s.state == RUNNING else 0
return min(1.0, got / total)
def eta(self):
# The step running now from its recent pace (its progress over the last minute), and the steps still to come
# from the seconds per weight of the steps already done. A job-wide fraction does not work: a step can take
# far more or less time than its weight says (Compile Changed Files is over half the weight of an update and
# can take seconds), and the shader step then read "about 1 minute left" with 20 minutes to go. A step's own
# average does not either: Prepare the Shaders fills the first half of its bar with the translation and the
# second with the much slower build into the driver's cache.
now = time.time()
if now - self.started < 20:
return None
cur = next((s for s in self.steps if s.state == RUNNING), None)
left = 0.0
if cur:
recent = [p for p in cur.pace if now - p[0] <= 60]
if len(recent) < 2 or recent[-1][0] - recent[0][0] < 10 or recent[-1][1] <= recent[0][1]:
return None
rate = (recent[-1][1] - recent[0][1]) / (recent[-1][0] - recent[0][0]) # fraction per second
left = (1 - cur.fraction) / rate
done_w = sum(WEIGHTS[s.id] for s in self.steps if s.state == DONE and s.seconds > 0)
done_s = sum(s.seconds for s in self.steps if s.state == DONE and s.seconds > 0)
todo_w = sum(WEIGHTS[s.id] for s in self.steps if s.state == PENDING)
if todo_w:
if not done_w:
return None
left += todo_w * done_s / done_w
return left
def elapsed(self):
return (self.finished or time.time()) - self.started
# ---- changes, from the worker
def _touch(self):
self.rev += 1
def log(self, line):
self.lines.append(line)
self.ctx.log(line)
self._touch()
def begin(self, sid, detail=''):
s = self.step(sid)
if s and s.state != RUNNING:
s.state, s.fraction, s.detail, s.t0, s.value = RUNNING, 0.0, detail, time.time(), '0%'
s.pace.clear()
self._touch()
def progress(self, sid, fraction, detail=''):
s = self.step(sid)
if s:
if s.state == PENDING:
self.begin(sid)
s.fraction = max(0.0, min(1.0, fraction))
s.pace.append((time.time(), s.fraction))
s.value = f'{int(s.fraction * 100)}%'
if detail:
s.detail = detail
self._touch()
def finish(self, sid, value=None):
s = self.step(sid)
if s:
s.seconds = time.time() - s.t0 if s.t0 else 0.0
s.state, s.fraction = DONE, 1.0
s.value = value or f'Done · {mmss(s.seconds)}'
self._touch()
def skip(self, sid, value='Unchanged'):
s = self.step(sid)
if s:
s.state, s.value = SKIPPED, value
self._touch()
def cancelled(self):
return self.cancel_flag.is_set()
def cancel(self):
self.cancel_flag.set()
if self.proc:
self.proc.cancel()
self._touch()
# ---- running
def start(self):
threading.Thread(target=self._main, name=f'job-{self.kind}', daemon=True).start()
return self
def run_inline(self):
self._main()
return self
def _main(self):
try:
self.run()
self.result = 'ok'
except net.Cancelled:
self.result = 'cancelled'
except Exception as e: # every failure ends the job with a message; the log has the traceback
if self.cancelled():
self.result = 'cancelled'
else:
self.result, self.error = 'failed', str(e) or type(e).__name__
self.ctx.log(traceback.format_exc())
finally:
for s in self.steps:
if s.state == RUNNING:
s.state = FAILED if self.result == 'failed' else PENDING
s.value = 'Stopped' if self.result == 'failed' else 'Cancelled'
self.finished = time.time()
self._touch()
def run(self):
raise NotImplementedError
# ---- shared pieces
def ensure_tools(self, sid):
layout = self.ctx.layout
self.begin(sid)
lock = toolchain.load_lock(layout, self.ctx.version)
if toolchain.is_current(layout, lock):
have_tools = True
else:
have_tools = False
def prog(phase, done, total):
if phase == 'download':
self.progress(sid, 0.9 * done / max(total, 1),
f'{done / 1e6:.0f} of {total / 1e6:.0f} MB downloaded')
else:
self.progress(sid, 0.9 + 0.1 * done / max(total, 1), f'Unpacking {done} of {total} packages')
toolchain.install(layout, lock, progress=prog, cancel=self.cancelled, log=self.log,
version=self.ctx.version)
toolchain.ensure_zbic(layout, cancel=self.cancelled, log=self.log, version=self.ctx.version)
if have_tools:
self.skip(sid, 'Unchanged' if sid == 'tools_update' else 'Ready')
else:
self.finish(sid)
def make_game(self, args, step_map, link_step=None):
"""Run make_game.py; its steps drive ours through step_map (make_game step -> our step id or None)."""
def on_event(ev):
kind = ev.get('ev')
sid = step_map.get(ev.get('step'))
if kind == 'step' and sid:
st = ev.get('state')
if st == 'start':
self.begin(sid)
elif st == 'done':
if sid == 'check':
self.finish(sid, 'Version 1.1.1')
elif link_step and sid in ('compile', 'compile_changed'):
self.finish(sid)
self.finish(link_step)
else:
self.finish(sid)
elif st == 'skip':
self.skip(sid, 'Kept' if sid in ('exefs', 'romfs') else 'Unchanged')
if link_step and sid in ('compile', 'compile_changed'):
self.skip(link_step, 'Unchanged')
elif kind == 'progress' and sid and 'fraction' in ev: # a step that words its own progress (shaders)
self.progress(sid, float(ev.get('fraction') or 0), ev.get('detail', ''))
elif kind == 'progress' and sid:
done, total, unit = ev.get('done', 0), max(ev.get('total', 1), 1), ev.get('unit', '')
if unit == 'bytes':
detail = f'{done / 1e9:.1f} of {total / 1e9:.1f} GB'
else:
detail = f'{done:,} of {total:,} {unit}'
self.progress(sid, done / total, detail)
elif kind == 'phase' and ev.get('phase') == 'link':
if link_step:
s = self.step(step_map.get('compile'))
if s and s.state == RUNNING:
self.finish(s.id)
self.begin(link_step, 'Linking')
else:
self.progress(sid or 'compile', 0.99, 'Linking')
elif kind == 'error':
self.error = ev.get('message', '')
# Cancel pressed before the build started (while the tools were checked) found no process to stop: it stops
# here, and one pressed while the process starts stops it at once
if self.cancelled():
raise net.Cancelled()
self.proc = build.Run(build.command(self.ctx.layout, args), on_event, self.log)
if self.cancelled():
self.proc.cancel()
code = self.proc.wait()
err = self.proc.error
self.proc = None
if self.cancelled():
raise net.Cancelled()
if code:
raise build.BuildError(err or f'the build stopped (exit code {code})')
class InstallJob(Job):
kind = 'install'
title = 'Installing'
def __init__(self, ctx, dump, game_dir):
super().__init__(ctx, ['tools', 'check', 'exefs', 'romfs', 'lift', 'compile', 'shaders', 'ready'])
self.dump, self.game_dir = str(dump), str(game_dir)
def run(self):
self.ensure_tools('tools')
self.make_game(build.install_args(self.ctx.layout, self.dump, self.game_dir),
{s: s for s in ('check', 'exefs', 'romfs', 'lift', 'compile', 'shaders')})
if self.step('shaders').state == PENDING:
self.skip('shaders')
self.finish('ready', 'Ready')
try:
gamefiles.extract_fonts(self.ctx.layout, self.game_dir)
except Exception as e:
self.log(f'fonts: {e}')
class RebuildJob(Job):
kind = 'rebuild'
title = 'Rebuilding'
def __init__(self, ctx, game_dir, dump=''):
super().__init__(ctx, ['tools_update', 'lift', 'compile_changed', 'swap', 'shaders'])
self.game_dir, self.dump = str(game_dir), dump
def run(self):
self.ensure_tools('tools_update')
layout = self.ctx.layout
if build.game_running(self.game_dir):
raise build.BuildError('the game is running: close it and try again')
if build.can_rebuild(self.game_dir):
args = build.rebuild_args(layout, self.game_dir)
elif self.dump:
args = build.install_args(layout, self.dump, self.game_dir)
else:
raise build.BuildError('the game folder has no earlier build: install the game again')
self.make_game(args, {'lift': 'lift', 'compile': 'compile_changed', 'shaders': 'shaders'}, link_step='swap')
for sid in ('lift', 'compile_changed', 'swap', 'shaders'):
if self.step(sid).state == PENDING:
self.skip(sid)
class UpdateDownloadJob(Job):
"""Before the restart: download, check and stage the release, then switch app/current.txt to it."""
kind = 'update'
def __init__(self, ctx, release):
ver = versions.display(release.version)
super().__init__(ctx, ['download', 'verify', 'tools_update', 'lift', 'compile_changed', 'swap', 'shaders'], ver)
self.release, self.ver = release, ver
self.title = f'Updating to {ver}'
def run(self):
layout = self.ctx.layout
self.begin('download')
prog = lambda d, t: self.progress('download', d / max(t, 1), f'{d / 1e6:.1f} of {t / 1e6:.1f} MB') # noqa: E731
folder = updater.stage_app(layout, self.release, progress=prog, cancel=self.cancelled,
version=self.ctx.version)
if updater.runtime_needed(layout, self.release):
updater.stage_runtime(layout, self.release, progress=prog, cancel=self.cancelled,
version=self.ctx.version)
self.finish('download')
self.begin('verify')
if not (folder / 'launcher' / 'start.pyw').is_file():
raise updater.UpdateError(f'{folder} is incomplete')
self.finish('verify', 'Checksum OK')
if layout.dev: # a checkout runs from the repository: nothing to switch to
self.after = 'staged'
return
updater.switch_to(layout, self.ver)
self.after = 'restart'
class ContinueUpdateJob(RebuildJob):
"""After the restart, in the new version: bring the tools to its lock and rebuild the game."""
kind = 'update'
def __init__(self, ctx, game_dir, old, dump=''):
Job.__init__(self, ctx, ['download', 'verify', 'tools_update', 'lift', 'compile_changed', 'swap', 'shaders'],
ctx.layout.version)
self.game_dir, self.dump, self.old = str(game_dir), dump, old
self.title = f'Updating to {versions.display(ctx.layout.version)}'
for sid, value in (('download', 'Done'), ('verify', 'Checksum OK')):
self.step(sid).state, self.step(sid).value, self.step(sid).fraction = DONE, value, 1.0
def run(self):
if self.game_dir and build.can_rebuild(self.game_dir):
RebuildJob.run(self)
else:
self.ensure_tools('tools_update')
for sid in ('lift', 'compile_changed', 'swap', 'shaders'):
self.skip(sid, 'No game yet')
updater.prune(self.ctx.layout)
class CheckJob(Job):
kind = 'check'
def __init__(self, ctx):
super().__init__(ctx, [])
self.release = None
def run(self):
s = self.ctx.settings
self.release = feed.latest(feed.load(self.ctx.layout, s), s.get('channel', 'stable'),
s.get('github_token', ''), self.ctx.version)
def overall(self):
return 0.0
def update_available(ctx, release):
return bool(release) and versions.newer(release.version, ctx.layout.version)