diff --git a/.gitignore b/.gitignore index 0754b129..eaf35e32 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,10 @@ __pycache__/ # The shared pipeline list a release carries (tools/pipeline_list.py merge; tools/prepare_shaders.py reads it here) /tools/data/pipeline_list.bin /data/pipeline_list.bin +# Execution profiles of the game's code (tools/pgo.py merge writes them under build/; a release carries one in data/) +*.profdata +*.profraw +/data/pgo-*.json *.xci *.xcz *.nsp diff --git a/tests/pgo_build_test.py b/tests/pgo_build_test.py new file mode 100644 index 00000000..4d181c22 --- /dev/null +++ b/tests/pgo_build_test.py @@ -0,0 +1,456 @@ +"""The profile-guided build of tools/make_game.py and tools/pgo.py, with a stand-in for the compiler process. + +Which flags a build compiles with: a kit without an execution profile (the flags and keys it always had), a kit with one +that fits (the lifted code at -O2 and every compile with the profile), a record that names another recomp/, a record +that is missing or names none, a profile the compiler refuses in the trial compile, NX2NAVI_PGO=off, and an +instrumented build (NX2NAVI_PGO=gen). The object keys: the same profile in a kit at another place keeps them, another +profile changes them. tools/pgo.py: the record a merge writes names the recomp/ a kit compares, a game built from +another recomp/ stops the merge, and a release refuses a profile of another tree. tools/check_content.py refuses +profiles in a commit. + +The last test uses the real compiler when MSYS2 UCRT64's clang is installed (and an llvm-profdata, when one is +found): a file that is no profile is refused by the trial compile, a real one is taken. No game files. + +Run: python tests/pgo_build_test.py +""" +import contextlib +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'tools')) +import check_content # noqa: E402 +import make_game as mg # noqa: E402 +import pgo # noqa: E402 + +REFUSAL = ('clang: error: Error in reading profile pgo-v111.profdata: unsupported instrumentation profile format ' + 'version') + + +class Compiler: + """Stands in for subprocess.run in make_game.py: records every command, answers --version, writes the object + and the dependency file of a compile, and refuses a command with -fprofile-use= when refuse is set.""" + + def __init__(self, refuse=''): + self.refuse = refuse + self.cmds = [] + + def __call__(self, cmd, **kw): + cmd = [str(c) for c in cmd] + self.cmds.append(cmd) + if '--version' in cmd: + return mock.Mock(returncode=0, stdout='clang version 23.1.5\nTarget: x86_64-w64-windows-gnu\n', stderr='') + if self.refuse and any(c.startswith('-fprofile-use=') for c in cmd): + return mock.Mock(returncode=1, stdout='', stderr=self.refuse + '\n') + if '-o' in cmd: + Path(cmd[cmd.index('-o') + 1]).write_bytes(b'object') + if '-MF' in cmd: + Path(cmd[cmd.index('-MF') + 1]).write_text( + cmd[cmd.index('-o') + 1].replace('\\', '/') + ': ' + cmd[-1].replace('\\', '/') + '\n') + return mock.Mock(returncode=0, stdout='', stderr='') + + def trials(self): + return [c for c in self.cmds if any(a.startswith('-fprofile-use=') for a in c)] + + +class PgoBuildTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-pgo-')) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.kit = self.make_kit('kit') + self.clang, self.clangxx = self.tmp / 'msys' / 'bin' / 'clang.exe', self.tmp / 'msys' / 'bin' / 'clang++.exe' + self.inc = ['-I', str(self.kit / 'runtime')] + self.cc = Compiler() + env = {k: v for k, v in os.environ.items() if k not in ('NX2NAVI_PGO', 'NX2NAVI_GUEST_OPT')} + for p in (mock.patch.object(mg, 'KIT', self.kit), mock.patch.object(mg, 'WIN', True), + mock.patch.object(mg, 'PGO_MODE', ''), mock.patch.object(mg, 'PGO_IDS', {}), + mock.patch.dict(os.environ, env, clear=True)): + p.start() + self.addCleanup(p.stop) + + def make_kit(self, name, lifter='def lift(): return 1\n'): + kit = self.tmp / name + (kit / 'recomp').mkdir(parents=True) + (kit / 'recomp' / 'gen.py').write_text(lifter) + (kit / 'recomp' / 'patches-v111.txt').write_text('# no patches\n') + (kit / 'runtime').mkdir() + return kit + + def add_profile(self, kit=None, folder='data', content=b'\xfflprofi\x81 counts', record='fits'): + """A profile in the kit's data/ with a record that names the kit's recomp/ ('fits'), another one ('other'), + none ('empty'), or without a record (None).""" + kit = kit or self.kit + d = kit.joinpath(*folder.split('/')) + d.mkdir(parents=True, exist_ok=True) + (d / 'pgo-v111.profdata').write_bytes(content) + if record: + digest = {'fits': mg.recomp_digest(kit), 'other': 'f' * 64}.get(record) + (d / 'pgo-v111.json').write_text(json.dumps({'version': 'v111', **({'recomp_digest': digest} if digest else {})})) + return d / 'pgo-v111.profdata' + + def choose(self, fast=None): + with mock.patch.object(mg.subprocess, 'run', self.cc): + return mg.pgo_profile(self.clang, {}, self.tmp / 'trial', fast, self.tmp / 'msys') + + def commands(self, pgo=()): + """(the lifted code's command, a runtime file's) with the flags pgo, without the compiler and the paths.""" + cmd_for = mg.runtime_cmd(self.clang, self.clangxx, self.inc, level=3, pgo=pgo) + guest = cmd_for(self.tmp / 'main_0001.c', self.tmp / 'main_main_0001.o') + runtime = cmd_for(self.kit / 'runtime' / 'perf.c', self.tmp / 'rt_perf.o') + return guest[1:-5], runtime[1:-5] # up to -I -o + + PLAIN = (['-c', '-std=gnu11', '-O1', '-march=x86-64-v3', '-fno-strict-aliasing', '-w'], + ['-c', '-std=gnu11', '-O2', '-fno-strict-aliasing', '-w']) + + def test_a_kit_without_a_profile_builds_as_before(self): + profile, why = self.choose() + self.assertIsNone(profile) + self.assertIn('no execution profile', why) + self.assertEqual(self.cc.cmds, []) # nothing to try + self.assertEqual(self.commands(), self.PLAIN) + self.assertEqual(mg.pgo_tag(), '') # the compile step's key is what it was + self.assertEqual(mg.pgo_salt('compiler', None), 'compiler') # and so are the object keys + with mock.patch.dict(os.environ, {'NX2NAVI_PGO': 'off'}): + self.assertEqual(mg.pgo_tag(), '') + + def test_a_fitting_profile_goes_into_every_compile(self): + path = self.add_profile() + fast = self.tmp / 'msys' / 'clang64' / 'bin' / 'clang.exe' + profile, why = self.choose(fast) + self.assertEqual((profile, why), (path, '')) + # one trial compile by each compiler that will read the file, the fast one with its sysroot + trials = self.cc.trials() + self.assertEqual([t[0] for t in trials], [str(self.clang), str(fast)]) + self.assertEqual(trials[1][1], f'--sysroot={self.tmp / "msys"}') + self.assertFalse((self.tmp / 'trial').exists()) + use = [f'-fprofile-use={path}', '-mllvm', '-static-func-full-module-prefix=false'] + self.assertEqual(mg.pgo_use_flags(profile), use) + guest, runtime = self.commands(mg.pgo_use_flags(profile)) + self.assertEqual(guest, ['-c', '-std=gnu11', '-O2', '-march=x86-64-v3', '-fno-strict-aliasing', '-w', *use]) + self.assertEqual(runtime, [*self.PLAIN[1], *use]) + self.assertEqual(mg.pgo_tag(), '|pgo ' + mg.pgo_id(path)) + self.assertNotEqual(mg.pgo_salt('compiler', profile), 'compiler') + + def test_a_source_checkout_takes_the_profile_from_build_pgo(self): + path = self.add_profile(folder='build/pgo') + self.assertEqual(self.choose(), (path, '')) + release = self.add_profile() # a release's data/ comes first + self.assertEqual(self.choose(), (release, '')) + + def test_a_record_of_another_tree_leaves_the_profile_out(self): + self.add_profile(record='other') + profile, why = self.choose() + self.assertIsNone(profile) + self.assertIn('trained on another recomp/', why) + self.assertEqual(self.cc.cmds, []) # decided before any compiler runs + self.assertEqual(self.commands(), self.PLAIN) + # the same record fits once the kit's recomp/ is the one it names, and not after the lifter changed again + self.add_profile(record='fits') + self.assertIsNotNone(self.choose()[0]) + (self.kit / 'recomp' / 'gen.py').write_text('def lift(): return 2\n') + self.assertIsNone(self.choose()[0]) + (self.kit / 'recomp' / 'patches-v111.txt').write_text('# a patch\n') + self.assertIsNone(self.choose()[0]) + + def test_a_missing_or_empty_record_leaves_the_profile_out(self): + for record, said in ((None, 'is missing'), ('empty', 'does not name the recomp/')): + shutil.rmtree(self.kit / 'data', ignore_errors=True) + self.add_profile(record=record) + profile, why = self.choose() + self.assertIsNone(profile, record) + self.assertIn(said, why) + (self.kit / 'data' / 'pgo-v111.json').write_text('{ cut short') + self.assertIn('does not name the recomp/', self.choose()[1]) + self.assertEqual(self.cc.cmds, []) + + def test_a_profile_the_compiler_refuses_is_left_out(self): + self.add_profile() + self.cc = Compiler(refuse=REFUSAL) + profile, why = self.choose() + self.assertIsNone(profile) + self.assertIn('the compiler cannot read it', why) + self.assertIn('unsupported instrumentation profile format version', why) + self.assertEqual(len(self.cc.trials()), 1) # one compile says so, not every compile of the build + self.assertEqual(self.commands(), self.PLAIN) + # the second compiler's refusal counts too + fast = self.tmp / 'msys' / 'clang64' / 'bin' / 'clang.exe' + + class OnlyFast(Compiler): + def __call__(self, cmd, **kw): + self.refuse = REFUSAL if str(cmd[0]) == str(fast) else '' + return super().__call__(cmd, **kw) + self.cc = OnlyFast() + self.assertIsNone(self.choose(fast)[0]) + self.assertEqual(len(self.cc.trials()), 2) + + def test_a_compiler_that_does_not_start_leaves_the_profile_out(self): + self.add_profile() + + def gone(cmd, **kw): + raise FileNotFoundError(2, 'No such file or directory', str(cmd[0])) + with mock.patch.object(mg.subprocess, 'run', gone): + profile, why = mg.pgo_profile(self.clang, {}, self.tmp / 'trial') + self.assertIsNone(profile) + self.assertIn('does not start', why) + + def test_off_leaves_a_fitting_profile_out(self): + self.add_profile() + for mode in ({'NX2NAVI_PGO': 'off'}, {'NX2NAVI_PGO': ' OFF '}): + with mock.patch.dict(os.environ, mode): + profile, why = self.choose() + self.assertIsNone(profile) + self.assertIn('NX2NAVI_PGO=off', why) + self.assertEqual(mg.pgo_tag(), '|pgo off') + with mock.patch.object(mg, 'PGO_MODE', 'off'): # --pgo off + self.assertIsNone(self.choose()[0]) + self.assertEqual(self.cc.cmds, []) + with mock.patch.dict(os.environ, {'NX2NAVI_PGO': 'something'}): # no mode: as without it + self.assertIsNotNone(self.choose()[0]) + + def test_only_the_windows_build_takes_the_profile(self): + self.add_profile() + with mock.patch.object(mg, 'WIN', False): + profile, why = self.choose() + self.assertIsNone(profile) + self.assertIn('Windows', why) + + def test_an_instrumented_build_counts_into_the_game_folder(self): + out = self.tmp / 'game out' + with mock.patch.dict(os.environ, {'NX2NAVI_PGO': 'gen'}): + self.assertEqual(mg.pgo_mode(), 'gen') + self.assertEqual(mg.pgo_tag(), '|pgo gen') + flags = mg.pgo_gen_flags(out) + self.assertEqual(flags, [f'-fprofile-generate={(out / "pgo").as_posix()}', '-mllvm', + '-static-func-full-module-prefix=false', '-DNX2NAVI_PGO_GEN']) + guest, runtime = self.commands(flags) + self.assertEqual(guest, ['-c', '-std=gnu11', '-O2', '-march=x86-64-v3', '-fno-strict-aliasing', '-w', *flags]) + self.assertEqual(runtime, [*self.PLAIN[1], *flags]) + # what the game was built from goes beside the counts, and tools/pgo.py reads it back + self.make_real_recomp() + with mock.patch.object(mg.subprocess, 'run', self.cc): + mg.pgo_gen_record(out, self.clang, {}) + made = json.loads((out / 'pgo' / 'training.json').read_text()) + self.assertEqual(made['recomp_digest'], mg.recomp_digest()) + self.assertEqual(made['compiler'], 'clang version 23.1.5') + self.assertEqual(pgo.training(out, 'v111'), made) + + def make_real_recomp(self): + """The test kit's recomp/ becomes this checkout's, so that what make_game.py records in the kit is what + tools/pgo.py, which reads the checkout, compares it with.""" + shutil.rmtree(self.kit / 'recomp') + (self.kit / 'recomp').mkdir() + for p in (ROOT / 'recomp').iterdir(): + if p.suffix == '.py' or p.name == 'patches-v111.txt': + shutil.copy2(p, self.kit / 'recomp' / p.name) + + def test_build_tools_without_the_profile_runtime_stop_an_instrumented_build(self): + def no_runtime(cmd, **kw): + self.assertIn('-fprofile-generate', cmd) + return mock.Mock(returncode=1, stdout='', stderr='ld: cannot find libclang_rt.profile.a: No such file\n' + 'clang++: error: linker command failed\n') + msys = self.tmp / 'msys64' / 'ucrt64' + err = io.StringIO() + with mock.patch.object(mg.subprocess, 'run', no_runtime), contextlib.redirect_stderr(err), \ + self.assertRaises(SystemExit): + mg.pgo_gen_check(self.clangxx, {}, self.tmp / 'trial', msys, ['-fuse-ld=lld']) + self.assertIn('mingw-w64-ucrt-x86_64-compiler-rt', err.getvalue()) + self.assertIn('cannot find libclang_rt.profile.a', err.getvalue()) + self.assertFalse((self.tmp / 'trial').exists()) + with mock.patch.object(mg.subprocess, 'run', self.cc): # with the runtime: nothing to say + mg.pgo_gen_check(self.clangxx, {}, self.tmp / 'trial', msys, []) + + def test_the_object_keys_follow_the_profile_and_not_the_kits_folder(self): + """A launcher update puts the kit into another folder: the same profile there keeps every object. Another + profile, or a build that leaves it out, compiles again.""" + src = self.tmp / 'lift' / 'main_0001.c' + src.parent.mkdir() + src.write_text('int f;\n') + obj = self.tmp / 'obj' + obj.mkdir() + + def key(kit, profile): + cache = mg.ObjectCache(obj / 'objects.json', {'': kit}) + inc = ['-I', str(kit / 'runtime')] + cmd_for = mg.runtime_cmd(self.clang, self.clangxx, inc, level=3, + pgo=mg.pgo_use_flags(profile) if profile else ()) + dst = obj / 'main_main_0001.o' + return mg.object_key(cmd_for(src, dst), src, dst, cache, mg.pgo_salt('compiler', profile)) + here = self.add_profile() + moved = self.make_kit('kit 2') + there = self.add_profile(moved) + self.assertEqual(key(self.kit, here), key(moved, there)) + self.assertNotEqual(key(self.kit, here), key(self.kit, None)) + other = self.add_profile(moved, content=b'\xfflprofi\x81 other counts') + self.assertNotEqual(key(self.kit, here), key(moved, other)) + # without a profile the key is the one a kit before profiles gave: the flags and the compiler's id alone + cache = mg.ObjectCache(obj / 'objects.json', {'': self.kit}) + dst = obj / 'main_main_0001.o' + plain = [str(self.clang), *self.PLAIN[0], '-I', str(self.kit / 'runtime'), '-o', str(dst), str(src)] + self.assertEqual(key(self.kit, None), mg.object_key(plain, src, dst, cache, 'compiler')) + + def test_a_new_profile_compiles_the_objects_again(self): + src, obj = self.tmp / 'src', self.tmp / 'obj' + src.mkdir() + obj.mkdir() + srcs = [] + for i in range(4): + (src / f'main_{i}.c').write_text(f'int f{i};\n') + srcs.append((src / f'main_{i}.c', obj / f'main_main_{i}.o')) + + def build(): + profile, _ = mg.pgo_profile(self.clang, {}, self.tmp / 'trial') + cache = mg.ObjectCache(obj / 'objects.json', {'': self.kit}) + cmd_for = mg.runtime_cmd(self.clang, self.clangxx, self.inc, level=3, + pgo=mg.pgo_use_flags(profile) if profile else ()) + return mg.compile_all(srcs, cmd_for, cache, 2, {}, salt=mg.pgo_salt('compiler', profile))[:2] + with mock.patch.object(mg.subprocess, 'run', self.cc): + self.assertEqual(build(), (4, 0)) # no profile + self.assertEqual(build(), (0, 4)) + self.add_profile() + self.assertEqual(build(), (4, 0)) # the kit brought one + self.assertEqual(build(), (0, 4)) + self.add_profile(content=b'\xfflprofi\x81 counts of a longer session') + self.assertEqual(build(), (4, 0)) # and another + self.add_profile(content=b'\xfflprofi\x81 counts of a longer session', record='other') + self.assertEqual(build(), (4, 0)) # left out: the flags of a kit without one + self.assertEqual(build(), (0, 4)) + compiles = [c for c in self.cc.cmds if '-MF' in c] + self.assertTrue(all('-O1' in c for c in compiles[:4] + compiles[-4:])) + self.assertTrue(all('-O2' in c and any(a.startswith('-fprofile-use=') for a in c) for c in compiles[4:12])) + + +class PgoToolTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-pgo-')) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.build = self.tmp / 'game' + (self.build / 'pgo').mkdir(parents=True) + (self.build / 'pgo' / 'default_1.profraw').write_bytes(b'raw counts') + self.tool = self.tmp / 'llvm' / 'bin' / ('llvm-profdata' + pgo.EXE) + self.tool.parent.mkdir(parents=True) + self.tool.write_bytes(b'') + self.ran = [] + p = mock.patch.object(pgo, 'recomp_tree', lambda: 'a' * 40) + p.start() + self.addCleanup(p.stop) + + def profdata(self, cmd, **kw): + """Stands in for llvm-profdata.""" + cmd = [str(c) for c in cmd] + self.ran.append(cmd) + if 'merge' in cmd: + Path(cmd[cmd.index('-o') + 1]).write_bytes(b'merged') + return mock.Mock(returncode=0, stdout='LLVM (http://llvm.org/):\n LLVM version 23.1.5\n', stderr='') + + def merge(self, version='v111'): + out = self.tmp / 'pgo' / f'pgo-{version}.profdata' + with mock.patch.object(pgo.subprocess, 'run', self.profdata), contextlib.redirect_stdout(io.StringIO()): + pgo.merge(self.build, out, version, self.tool.parent.parent) + return out + + def test_the_merge_records_the_recomp_a_kit_compares(self): + out = self.merge() + self.assertEqual(out.read_bytes(), b'merged') + self.assertEqual(self.ran[0][:3], [str(self.tool), 'merge', '--failure-mode=all']) + self.assertEqual(self.ran[0][-1], str(self.build / 'pgo' / 'default_1.profraw')) + record = json.loads(out.with_suffix('.json').read_text()) + self.assertEqual(record['version'], 'v111') + self.assertEqual(record['recomp_tree'], 'a' * 40) + self.assertEqual(record['tool'], 'LLVM version 23.1.5') + # what make_game.py works out for a kit with this checkout's recomp/: the profile fits such a kit + with mock.patch.object(mg, 'KIT', ROOT): + self.assertEqual(record['recomp_digest'], mg.recomp_digest()) + data = self.tmp / 'release' / 'data' + placed = pgo.place(out, data) + self.assertEqual(mg.pgo_record(placed), '') + self.assertEqual(sorted(p.name for p in data.iterdir()), ['pgo-v111.json', 'pgo-v111.profdata']) + + def test_a_game_built_from_another_recomp_stops_the_merge(self): + made = {'version': 'v111', 'recomp_digest': pgo.recomp_digest('v111'), 'compiler': 'clang version 23.1.5'} + (self.build / 'pgo' / 'training.json').write_text(json.dumps(made)) + record = json.loads(self.merge().with_suffix('.json').read_text()) + self.assertEqual(record['compiler'], 'clang version 23.1.5') + (self.build / 'pgo' / 'training.json').write_text(json.dumps(dict(made, recomp_digest='0' * 64))) + with self.assertRaises(SystemExit) as e: + self.merge() + self.assertIn('another recomp/', str(e.exception)) + (self.build / 'pgo' / 'training.json').write_text(json.dumps(dict(made, version='v143'))) + with self.assertRaises(SystemExit): + self.merge() + + def test_a_merge_needs_counts_and_a_clean_recomp(self): + (self.build / 'pgo' / 'default_1.profraw').unlink() + with self.assertRaises(SystemExit) as e: + self.merge() + self.assertIn('no .profraw', str(e.exception)) + (self.build / 'pgo' / 'default_1.profraw').write_bytes(b'raw counts') + with mock.patch.object(pgo, 'recomp_tree', lambda: None), self.assertRaises(SystemExit) as e: + self.merge() + self.assertIn('uncommitted changes', str(e.exception)) + + def test_a_release_refuses_a_profile_of_another_tree(self): + out = self.merge() + rec = out.with_suffix('.json') + record = json.loads(rec.read_text()) + self.assertEqual(pgo.fitting(out), record) + rec.write_text(json.dumps(dict(record, recomp_digest='0' * 64))) + with self.assertRaises(SystemExit) as e: + pgo.place(out, self.tmp / 'release' / 'data') + self.assertIn('another recomp/', str(e.exception)) + self.assertFalse((self.tmp / 'release').exists()) + rec.write_text(json.dumps(dict(record, version='v143'))) + with self.assertRaises(SystemExit): + pgo.fitting(out) + rec.unlink() + with self.assertRaises(SystemExit): + pgo.fitting(out) + + def test_profiles_stay_out_of_a_commit(self): + self.assertTrue(check_content.execution_profile(Path('data/pgo-v111.profdata'))) + self.assertTrue(check_content.execution_profile(Path('game/pgo/default_1.PROFRAW'))) + self.assertFalse(check_content.execution_profile(Path('data/pgo-v111.json'))) + self.assertFalse(check_content.execution_profile(Path('tools/pgo.py'))) + + +MSYS = Path(r'C:\msys64\ucrt64') +TEXT_PROFILE = ':ir\nnx2navi_pgo_trial\n0\n1\n100\n\n' # one function with one counter, as llvm-profdata reads text + + +@unittest.skipUnless((MSYS / 'bin' / 'clang.exe').is_file(), 'needs MSYS2 UCRT64 clang') +class RealCompilerTest(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-pgo-')) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.clang = MSYS / 'bin' / 'clang.exe' + self.env = dict(os.environ, PATH=str(MSYS / 'bin') + os.pathsep + os.environ.get('PATH', '')) + + def test_the_trial_compile_tells_a_profile_from_a_file_that_is_none(self): + bad = self.tmp / 'pgo-v111.profdata' + bad.write_bytes(b'not a profile') + said = mg.pgo_trial(bad, [[self.clang]], self.env, self.tmp / 'trial') + self.assertIn('Error in reading profile', said) + cut = self.tmp / 'cut.profdata' + cut.write_bytes(b'\xfflprofi\x81') # a profile's first bytes and nothing after them + self.assertTrue(mg.pgo_trial(cut, [[self.clang]], self.env, self.tmp / 'trial')) + self.assertTrue(mg.pgo_trial(self.tmp / 'missing.profdata', [[self.clang]], self.env, self.tmp / 'trial')) + tool = shutil.which('llvm-profdata') or next( + (str(p) for p in (MSYS / 'bin' / 'llvm-profdata.exe', Path(r'C:\Program Files\LLVM\bin\llvm-profdata.exe')) + if p.is_file()), None) + if not tool: + self.skipTest('no llvm-profdata to write a real profile with; the refusals above were checked') + text, good = self.tmp / 'trial.proftext', self.tmp / 'good.profdata' + text.write_text(TEXT_PROFILE) + subprocess.run([tool, 'merge', '-o', str(good), str(text)], check=True, capture_output=True) + self.assertEqual(mg.pgo_trial(good, [[self.clang]], self.env, self.tmp / 'trial'), '') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/check_content.py b/tools/check_content.py index de1ab07e..5ffe64dc 100644 --- a/tools/check_content.py +++ b/tools/check_content.py @@ -108,6 +108,14 @@ def recorded_in_play(path: Path, data: bytes | None) -> bool: return data[:8] in RECORD_MAGICS +def execution_profile(path: Path) -> bool: + """The counts an instrumented game wrote (.profraw) or their merged profile (.profdata, tools/pgo.py): they name + the lifted functions and count their branches, so they are derived from the game's code. A launcher release or + an installer release may carry a profile (data/pgo-.profdata), so make_launcher.py and make_release.py, + which call problems_for, are not held to this.""" + return path.suffix.lower() in (".profdata", ".profraw") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--staged", action="store_true", help="check staged files only") @@ -123,6 +131,8 @@ def main() -> int: problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this") if recorded_in_play(path, data): problems.append("recorded from a game folder (shaders, answers, pipeline keys or a pipeline list)") + if execution_profile(path): + problems.append("execution profile of the game's code: it stays in build/ (tools/pgo.py)") for problem in problems: print(f"{path}: {problem}") failed = True diff --git a/tools/make_game.py b/tools/make_game.py index c0a78d56..d81f17f1 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -49,6 +49,14 @@ sets the number), the largest files first. The lifted code is built for this PC's x86-64 level (cpu_level), with MSYS2 CLANG64's clang when it sits beside the UCRT64 folder and reports the same version (fast_clang: the same objects, about a fifth faster). +Profile-guided build (docs/building.md, "Profile-guided build"). A kit with data/pgo-v111.profdata and its record +(tools/pgo.py merge; in a source checkout build/pgo/) compiles the lifted code at -O2 with that execution profile, and +the runtime with it too (pgo_profile); functions the profile never saw run are optimised for size. The profile is +left out, and the build is the one a kit without a profile makes (the lifted code at -O1), when its record names +another recomp/ than this kit's, when the compilers cannot read the file (one trial compile before the others start), +or with NX2NAVI_PGO=off. --pgo gen (NX2NAVI_PGO=gen, which a launcher started with it passes on) builds an +instrumented game instead: it adds its counts to /pgo/*.profraw every minute and at exit, for tools/pgo.py merge. + python tools/make_game.py --make-runtime-lib /lib --msys # the release's prebuilt runtime On Linux the same steps make /ocarina with the clang, clang++ and lld on PATH and the system's libraries @@ -520,11 +528,17 @@ EMITTED = re.compile(r'emitted (\d+)/(\d+) chunks') def lift_digest(): """The lifter's code, the patch table and the bring-up fixes: what the lifted C depends on besides the ExeFS.""" - h = hashlib.sha256(tree_digest(KIT / 'recomp', ('*.py', 'patches-v111.txt')).encode()) + h = hashlib.sha256(recomp_digest().encode()) h.update(repr(LIFT_FIXES).encode()) return h.hexdigest() +def recomp_digest(kit=None): + """The lifter's code and the patch table by content: what names the generator in the lift's key and in an + execution profile's record (pgo_record). kit: another checkout or kit folder than this one.""" + return tree_digest((kit or KIT) / 'recomp', ('*.py', 'patches-v111.txt')) + + # The mods' code (tools/mod_files.py): the tools that make patch sets of it, the patch file each writes, the status # file the runtime reads in build-cache next to the exe (runtime/cheats.c, runtime/mods.c) and what the log calls it MOD_STAGES = (('cheats.py', 'patches-cheats.txt', 'cheats.status', 'code cheats'), @@ -969,13 +983,18 @@ def compiler_id(clang, env): def object_key(cmd, src, dst, cache, salt, sysroot=False): """The cache key of compiling src into dst with cmd: the compiler's id (salt), the command's flags with the cache's - roots standing for their paths, and the source's sha256. --sysroot is left out: only fast_clang's compiler gets it, - and its objects are the UCRT64 compiler's byte for byte, so the object of either serves (sysroot=True keeps it: the - keys up to 0.6.0).""" + roots standing for their paths (the kit's execution profile too: its content is part of the salt, pgo_salt), and + the source's sha256. --sysroot is left out: only fast_clang's compiler gets it, and its objects are the UCRT64 + compiler's byte for byte, so the object of either serves (sysroot=True keeps it: the keys up to 0.6.0).""" flags = [c for c in cmd[1:] if c not in (str(src), str(dst)) and (sysroot or not c.startswith('--sysroot='))] + + def rooted(f, name, root): + # a path under a root, alone or as the value of -fprofile-use= (the kit's profile, pgo_use_flags) + head = PGO_USE if f.startswith(PGO_USE) else '' + path = os.path.normcase(f[len(head):]) + return head + name + path[len(root):].replace(os.sep, '/') if path.startswith(root) else f for name, root in cache.roots.items(): - flags = [name + os.path.normcase(f)[len(root):].replace(os.sep, '/') if os.path.normcase(f).startswith(root) - else f for f in flags] + flags = [rooted(f, name, root) for f in flags] return sha256_bytes(json.dumps([salt, [os.path.basename(cmd[0])] + flags, cache.file_sha(src)]).encode()) @@ -1425,6 +1444,184 @@ def fast_clang(msys, clang, env): return fast if mine and clang_ident(fast, env) == mine else None +# ---- the profile-guided build (docs/building.md, "Profile-guided build") +# +# An execution profile is the counts an instrumented game gathered in a play session, merged into one file +# (tools/pgo.py merge). With it the compiler lays the code that ran out straight, inlines the calls that were hot and +# optimises the functions that never ran for size. The file names the lifted functions by their address and the +# runtime's by their name, each with a hash of its control flow: a function whose C differs from the one that was +# trained gets no counts, and the compiler says nothing about it. +PGO_NAME = 'pgo-v111.profdata' # in the kit's data/ (a release) or build/pgo/ (a source checkout, tools/pgo.py) +PGO_RAW = 'pgo' # in the game folder: where an instrumented game adds its counts, *.profraw +PGO_TRAINING = 'training.json' # beside the counts: what the instrumented game was built from (pgo_gen_record) +PGO_USE = '-fprofile-use=' +# A static function's profile name carries its source's path by default; the file's base name alone keeps the names +# equal between the folder the profile was trained in and this kit's (tools/build_game.py PGO_NAMES). +PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false'] +PGO_MODE = '' # --pgo; without it NX2NAVI_PGO decides (pgo_mode) + + +def pgo_mode(): + """'gen': build an instrumented game; 'off': build without the kit's profile; '': build with it when it fits. + From --pgo, else from NX2NAVI_PGO, which reaches a build the launcher starts through its environment.""" + mode = (PGO_MODE or os.environ.get('NX2NAVI_PGO', '')).strip().lower() + return mode if mode in ('gen', 'off') else '' + + +def pgo_file(): + """The kit's execution profile, or None: data/ of a release, else build/pgo/ of a source checkout, where + tools/pgo.py merge writes it.""" + for d in (KIT / 'data', KIT / 'build' / 'pgo'): + if (d / PGO_NAME).is_file(): + return d / PGO_NAME + return None + + +PGO_IDS = {} # pgo_id's answers by path, size and modification time: the step keys ask several times a build + + +def pgo_id(profile): + """sha256 over the profile and its record, 'unreadable' when the profile cannot be read: what stands for them in + the compile step's key and in the object keys, so a kit that brings another profile compiles again.""" + try: + st = os.stat(profile) + at = (str(profile), st.st_size, st.st_mtime_ns) + if at not in PGO_IDS: + h = hashlib.sha256(sha256_file(profile).encode()) + try: + h.update(profile.with_suffix('.json').read_bytes()) + except OSError: + pass + PGO_IDS[at] = h.hexdigest() + return PGO_IDS[at] + except OSError: + return 'unreadable' + + +def pgo_tag(): + """The profile-guided build's part of the compile step's key (step_key): '' for a kit without a profile that + builds as before, so the key of every game built so far stays what it is.""" + mode, profile = pgo_mode(), pgo_file() + if mode == 'gen': + return '|pgo gen' + if not profile: + return '' + return '|pgo off' if mode == 'off' else f'|pgo {pgo_id(profile)}' + + +def pgo_record(profile): + """Why the record beside a profile (.json, tools/pgo.py merge) does not let this kit use it, '' when it + does: recomp_digest there must be this kit's recomp/. Functions are matched by name and control-flow hash, so a + profile trained on the C of another generator loses the counts of every function that generator writes + differently, without a word from the compiler: an -O2 build with a profile of unknown worth. Such a kit builds + as one without a profile instead.""" + rec = profile.with_suffix('.json') + try: + trained = json.loads(rec.read_text(encoding='utf-8'))['recomp_digest'] + except OSError: + return f'its record {rec.name} is missing or cannot be read' + except (ValueError, KeyError, TypeError): + return f'its record {rec.name} does not name the recomp/ it was trained on' + return '' if trained == recomp_digest() else 'it was trained on another recomp/ than this kit\'s' + + +def pgo_use_flags(profile): + """Every compile's flags for a build with the profile. The warnings about functions the profile lacks or knows + with other control flow are off like every other warning (-w).""" + return [PGO_USE + str(profile), *PGO_NAMES] + + +def pgo_gen_flags(out): + """Every compile's flags for an instrumented build of the game folder out. The folder for the counts is baked in + as an absolute path: left relative, the game would resolve it against whatever folder it was started from. + NX2NAVI_PGO_GEN: the runtime adds the counts to the file every minute and when it ends the process + (runtime/platform/win32.c pgo_flush_thread, plat_exit).""" + return [f'-fprofile-generate={(Path(out) / PGO_RAW).as_posix()}', *PGO_NAMES, '-DNX2NAVI_PGO_GEN'] + + +def pgo_trial(profile, compilers, env, work): + """Why a compiler that will read the profile cannot, '' when each compiles a test file with it. compilers: each + one's argv up to its flags. The profile was written by the LLVM release of the training build and is read by the + clang of this PC's build tools: a compiler older than the file's format refuses it, as it does a file that is cut + short or is no profile, and it would refuse it again in every compile of the build.""" + try: + work.mkdir(parents=True, exist_ok=True) + src, obj = work / 'trial.c', work / 'trial.o' + src.write_text('int nx2navi_pgo_trial(int x) { return x + 1; }\n', encoding='ascii') + for cc in compilers: + cmd = [*map(str, cc), '-c', '-std=gnu11', '-O2', '-w', *pgo_use_flags(profile), '-o', str(obj), str(src)] + try: + p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace') + except OSError as e: + return f'{cc[0]} does not start: {error_text(e)}' + if p.returncode: + return (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:] + return '' + except OSError as e: + return error_text(e) + finally: + shutil.rmtree(work, ignore_errors=True) + + +def pgo_profile(clang, env, work, fast=None, msys=None): + """(the kit's execution profile, '') when this build compiles with it, else (None, why not). It is left out, and + the build is the one a kit without a profile makes, when the kit has none, with NX2NAVI_PGO=off, off Windows + (the profile is trained on the Windows build), when its record names another recomp/ (pgo_record), and when a + compiler of this build cannot read it (pgo_trial; fast: fast_clang's compiler, which compiles the lifted code, + with msys as its sysroot). work: a folder of its own for the trial compile.""" + profile = pgo_file() + if not profile: + return None, 'this kit has no execution profile' + if pgo_mode() == 'off': + return None, f'the execution profile {profile.name} is left out (NX2NAVI_PGO=off)' + if not WIN: + return None, f'the execution profile {profile.name} is for the Windows build' + why = pgo_record(profile) + if not why: + said = pgo_trial(profile, [[clang]] + ([[fast, f'--sysroot={msys}']] if fast else []), env, work) + why = said and f'the compiler cannot read it ({said})' + return (None, f'the execution profile {profile.name} is not used: {why}') if why else (profile, '') + + +def pgo_salt(cid, profile): + """The compiler's id for the keys of objects compiled with the profile: the profile's content with it, as no + dependency file names the profile. Without a profile the id itself, so those keys stay what they were.""" + return sha256_bytes(f'{cid}|pgo {pgo_id(profile)}'.encode()) if profile else cid + + +def pgo_gen_check(clangxx, env, work, msys, ld): + """An instrumented build links the compiler's profile runtime, which MSYS2 packs apart from clang + (mingw-w64-ucrt-x86_64-compiler-rt) and the launcher's build tools do not hold: an empty program is linked with + it now (ld: the link's own linker flags), so that build tools without it stop the build here and not at the + link, after every compile.""" + shutil.rmtree(work, ignore_errors=True) + work.mkdir(parents=True) + try: + src = work / 'trial.c' + src.write_text('int main(void) { return 0; }\n', encoding='ascii') + p = subprocess.run([str(clangxx), *ld, '-fprofile-generate', '-o', str(work / ('trial.exe' if WIN else 'trial')), + '-x', 'c', str(src)], capture_output=True, text=True, env=env, errors='replace') + finally: + shutil.rmtree(work, ignore_errors=True) + if p.returncode: + said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[0][-300:] + fail('an instrumented build (NX2NAVI_PGO=gen) needs the compiler\'s profile runtime, and these build tools do ' + f'not link it ({said}). ' + + (f'Unpack the MSYS2 package mingw-w64-ucrt-x86_64-compiler-rt of this clang\'s version into {msys.parent}' + if WIN else 'Install the compiler-rt package of this clang\'s version')) + + +def pgo_gen_record(out, clang, env): + """/pgo/training.json: the recomp/ and the compiler the instrumented game was built from. tools/pgo.py merge + takes the profile's record from it, so the record names the generator of the game that was played, whatever the + checkout holds when the counts are merged.""" + raw = Path(out) / PGO_RAW + raw.mkdir(parents=True, exist_ok=True) + record = {'version': 'v111', 'recomp_digest': recomp_digest(), 'compiler': clang_version(clang, env), + 'kit': kit_version()} + (raw / PGO_TRAINING).write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8') + + def guest_opt_ini(out): """NX2NAVI_GUEST_OPT from nx2navi.ini next to the exe (the low-end GPU rows of the menu's Game tab write it there) when the environment does not set it: the next build of the game folder out takes it. Only while low_end_gpu = 1 @@ -1447,13 +1644,15 @@ def guest_opt_ini(out): os.environ['NX2NAVI_GUEST_OPT'] = found -def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): +def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=(), pgo=()): """fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers, and its command, which is part of its objects' keys, stays as it was. NX2NAVI_GUEST_OPT (an environment variable, or guest_opt in nx2navi.ini) 2 builds the lifted code at -O2; by default -O1: the game's own threads run it and - bound the frame on a slower CPU; the build takes several times longer.""" - guest_opt = '-O2' if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2' else '-O1' # -O1 unless guest_opt = 2 + bound the frame on a slower CPU; the build takes several times longer. pgo: the flags of a profile-guided build + for every compile (pgo_use_flags, pgo_gen_flags); with them the lifted code is built at -O2 too, the level the + profile's inlining and its layout of hot and cold code work at. Without them every command is what it was.""" + guest_opt = '-O2' if pgo or os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2' else '-O1' # -O1 unless guest_opt = 2 def cmd_for(src, dst): cpp = src.suffix in ('.cpp', '.cxx') # the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict @@ -1466,14 +1665,16 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()): # -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally declspec = ['-fdeclspec'] if cpp and not WIN else [] return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', guest_opt if guest else '-O2', - *march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)] + *march, '-fno-strict-aliasing', '-w', *declspec, *pgo, *([] if guest else first), *inc, '-o', str(dst), + str(src)] return cmd_for def make_runtime_lib(dest, msys, jobs, obj=None): """--make-runtime-lib: compile every runtime source that does not include a per-install header into /libnx2navirt.a, with runtime-lib.json naming the runtime/ tree and the compiler it was built from. The - objects stay in obj (default /obj), which a release keeps out of what it ships.""" + objects stay in obj (default /obj), which a release keeps out of what it ships. A kit with an execution + profile compiles the library with it, under pgo_profile's conditions; runtime-lib.json then names the profile.""" dest = Path(dest).resolve() dest.mkdir(parents=True, exist_ok=True) obj = Path(obj).resolve() if obj else dest / 'obj' @@ -1484,11 +1685,17 @@ def make_runtime_lib(dest, msys, jobs, obj=None): srcs = [(p, obj / name) for p, name in runtime_sources() if not per_install(p, tainted)] inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)] cache = ObjectCache(obj / 'objects.json', {'': KIT}) + if pgo_mode() == 'gen': + fail('the prebuilt runtime is never an instrumented one: an instrumented build (NX2NAVI_PGO=gen) compiles ' + 'runtime/ itself') + profile, why = pgo_profile(clang, env, obj / 'pgo-trial') + say(f' the runtime is compiled with the execution profile {profile.name}' if profile else f' {why}') t0 = time.time() - compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env)), + compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env), + pgo=pgo_use_flags(profile) if profile else ()), cache, jobs, env, lambda d, t: (d % 20 == 0 or d == t) and say(f' {d} of {t}'), - salt=compiler_id(clang, env)) + salt=pgo_salt(compiler_id(clang, env), profile)) if bad: report_failures(bad) fail(f'{len(bad)} runtime files did not compile') @@ -1502,7 +1709,7 @@ def make_runtime_lib(dest, msys, jobs, obj=None): if subprocess.run([ar, 'rcs', str(lib), '@' + str(rsp)], env=env).returncode: fail('the runtime library could not be archived') meta = {'runtime': runtime_digest(), 'compiler': clang_version(clang, env), - 'objects': sorted(o.name for _, o in srcs)} + 'objects': sorted(o.name for _, o in srcs), 'pgo': pgo_id(profile) if profile else ''} (dest / 'runtime-lib.json').write_text(json.dumps(meta, indent=1), encoding='utf-8') rsp.unlink(missing_ok=True) say(f'{lib}: {len(srcs)} runtime files ({compiled} compiled, {reused} kept), {lib.stat().st_size / 1e6:.1f} MB, ' @@ -1723,7 +1930,9 @@ def write_runtime_map(full, dst): def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): """romfs_job: the RomFS step still running beside this one (a first build). Everything that does not include romfs_config.h compiles meanwhile; the RomFS index and the few files that include it follow once it is done. - update: a rebuild after a kit update (lifted_level).""" + update: a rebuild after a kit update (lifted_level). The kit's execution profile, when this build can use it + (pgo_profile), goes into every compile; NX2NAVI_PGO=gen makes an instrumented game instead (pgo_gen_flags), its + runtime compiled here too, as the prebuilt one counts nothing.""" lift, gen, obj = cache_dir / 'lift', cache_dir / 'gen', cache_dir / 'obj' gen.mkdir(parents=True, exist_ok=True) obj.mkdir(parents=True, exist_ok=True) @@ -1733,7 +1942,12 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): if not romfs_job: romfs_index(out, gen) srcs = [(p, obj / f'{mod}_{p.stem}.o') for mod in ('main', 'sdk', 'subsdk0') for p in sorted((lift / mod).glob('*.c'))] - prebuilt = prebuilt_runtime(clang, env) + gen_counts = pgo_mode() == 'gen' + # lld links several times faster than GNU ld + ld = ['-fuse-ld=lld'] if ((msys / 'bin' / 'ld.lld.exe').is_file() if WIN else shutil.which('ld.lld')) else [] + if gen_counts: + pgo_gen_check(clangxx, env, cache_dir / 'pgo-trial', msys, ld) + prebuilt = None if gen_counts else prebuilt_runtime(clang, env) runtime = [(p, obj / name) for p, name in runtime_sources()] if prebuilt: runtime = [(p, o) for p, o in runtime if o.name not in prebuilt[1]] @@ -1761,7 +1975,17 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): vulkan = vulkan_headers(clang, env) if vulkan: say(f' the Vulkan headers come from {vulkan[1]}') - cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan) + profile = None + if gen_counts: + pgo = pgo_gen_flags(out) + say(f' an instrumented build (NX2NAVI_PGO=gen): the lifted code at -O2 and a slower game, which adds its ' + f'counts to {out / PGO_RAW} every minute and at exit') + else: + profile, why = pgo_profile(clang, env, cache_dir / 'pgo-trial', fast, msys) + pgo = pgo_use_flags(profile) if profile else [] + say(f' the lifted code (at -O2) and the runtime are built with the execution profile {profile.name}' + if profile else f' {why}') + cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan, pgo=pgo) # The keys earlier versions gave the same compiles (compile_all alts): 0.6.0's, with the compiler's file time and, # with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths; and # up to 0.7.1 the runtime's without the Vulkan headers' folder ahead of the system's (an object that compiled @@ -1772,6 +1996,8 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): kits = earlier_kits() def alts(src, dst): + if pgo: # no earlier version compiled with a profile or counted: nothing of theirs is the same compile + return [], kits keys = [object_key(plain(src, dst), src, dst, cache, cid_old, sysroot=True)] if vulkan: keys.append(object_key(no_vulkan(src, dst), src, dst, cache, cid)) @@ -1799,8 +2025,9 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): late = [(s, o) for s, o in runtime + srcs[-(2 if WIN else 1):] if per_install(s, tainted)] say(f' {len(late)} files wait for the RomFS index: {", ".join(s.name for s, _ in late)}') early = [j for j in srcs if j not in late] + salt = pgo_salt(cid, profile) compiled, reused, bad = compile_all(early, cmd_for, cache, jobs, env, lambda d, t: progress(d, len(srcs)), - salt=cid, stop=romfs_job.failed if romfs_job else None, alts=alts) + salt=salt, stop=romfs_job.failed if romfs_job else None, alts=alts) if romfs_job: if romfs_job.thread.is_alive(): say(' waiting for the RomFS image') @@ -1808,7 +2035,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): if not bad: romfs_index(out, gen) c2, r2, bad = compile_all(late, cmd_for, cache, jobs, env, - lambda d, t: progress(len(early) + d, len(srcs)), salt=cid, alts=alts) + lambda d, t: progress(len(early) + d, len(srcs)), salt=salt, alts=alts) compiled, reused = compiled + c2, reused + r2 say(f' {compiled} compiled, {reused} unchanged since the last build') if bad: @@ -1816,19 +2043,24 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): fail(f'{len(bad)} files did not compile') if cache.keep_only({o.name for _, o in srcs}): cache.save() - (obj / LIFTED_BUILD).write_text(json.dumps({'level': level}), encoding='utf-8') # lifted_level reads it next update + # lifted_level reads the level at the next update; pgo says what the objects were compiled with: 'gen', the + # profile's id (pgo_id) or nothing + (obj / LIFTED_BUILD).write_text(json.dumps({'level': level, 'pgo': 'gen' if gen_counts else + pgo_id(profile) if profile else ''}), encoding='utf-8') objs = [o for _, o in srcs] + ([compile_resources(msys, rt, obj, env, lld=(msys / 'bin' / 'ld.lld.exe').is_file())] if WIN else []) exe = out / EXE_NAME rsp = cache_dir / 'link.rsp' rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in objs), encoding='utf-8') - # lld links several times faster than GNU ld; the stack matches tools/build_game.py's build + # the stack matches tools/build_game.py's build # 16 MiB for the main thread with either linker (GNU ld takes the same flag): the default 2 MiB is too little if WIN: - ld = (['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []) + ['-Wl,--stack,16777216'] + ld = ld + ['-Wl,--stack,16777216'] else: # tools/build_game.py's Linux flags; the main thread's stack is the process's (ulimit -s), not the linker's - ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie', '-Wl,-z,noexecstack', '-Wl,-rpath,$ORIGIN'] + ld = ld + ['-no-pie', '-Wl,-z,noexecstack', '-Wl,-rpath,$ORIGIN'] + if gen_counts: # the link takes in the profile runtime, which writes the counts + ld.append('-fprofile-generate') # the link map feeds tools/prof_report.py (symbols for an NX2NAVI_PROFILE sampling profile); both linkers write one. # -Xlinker passes the path as one argument; -Wl, would cut it at every comma of the game folder's path (issue #470) ld += ['-Xlinker', '--Map=' + (cache_dir / 'ocarina.map').as_posix()] @@ -1878,6 +2110,8 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False): beside = copy_dlls(exe, msys / 'bin', out) if WIN else [] say(f' {EXE_NAME} {exe.stat().st_size / 1e6:.0f} MB' + (f'; beside it: {", ".join(beside)}' if beside else '')) (out / 'profile').mkdir(exist_ok=True) + if gen_counts: + pgo_gen_record(out, clang, env) # ---------------------------------------------------------------- main @@ -2340,7 +2574,9 @@ def step_key(step, identity, out=None): processor (cpu_tag); shaders: the compile without the mods' code, the preparing script and the translator; prompts: the game files, the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_ lines of nx2navi.ini, and whether every folder it writes is still there; scan: the game files, the check's version - (SCAN_TOOL) and whether its report is still there.""" + (SCAN_TOOL) and whether its report is still there. The compile's key also names the kit's execution profile or + an instrumented build (pgo_tag); the shaders' key does not, as neither changes what the shaders are prepared + from.""" if step in ('exefs', 'romfs'): return f'{step}-1|{identity}' if step == 'scan': @@ -2362,7 +2598,7 @@ def step_key(step, identity, out=None): return (f'compile|{lift}|{tree_digest(KIT / "runtime", ("**/*",))}|{sha256_file(Path(__file__))}' f'|{kit_version()}|cpu {cpu_tag()}') if step == 'compile': - return compile_of(lift) + return compile_of(lift) + pgo_tag() # shaders: the built game without the mods' code (UNAFFECTED) and the translator that prepared them, the one # xlate_source finds (the kit's bin/, or a source checkout's build/xlate/) return (f'shaders-1|{compile_of(base)}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}' @@ -2391,7 +2627,7 @@ def main(): def build_main(): - global EVENTS + global EVENTS, PGO_MODE ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('--nup', help='the game as one .nup file') ap.add_argument('--exefs', help='the game\'s exefs folder (with --romfs, instead of --nup)') @@ -2410,9 +2646,14 @@ def build_main(): ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)') ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR') ap.add_argument('--obj', help='with --make-runtime-lib: the folder for its objects') + ap.add_argument('--pgo', choices=('gen', 'off'), + help='gen: build an instrumented game, which adds its execution counts to /pgo/*.profraw ' + '(for tools/pgo.py merge); off: build without the execution profile of the kit. Default: the ' + 'environment variable NX2NAVI_PGO, else build with the profile of the kit when it fits') ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step a = ap.parse_args() EVENTS = a.events + PGO_MODE = a.pgo or '' jobs_why = '--jobs' if not a.jobs or a.jobs < 1: a.jobs, jobs_why = default_jobs() @@ -2458,6 +2699,10 @@ def build_main(): fail(f'the Python package {pip_name} is missing or does not load ({e}): ' f'pip install numpy lz4 zstandard zbic{extra}') cache.mkdir(parents=True, exist_ok=True) + if pgo_mode() == 'gen': + # The instrumented game also runs inside this build (the shaders step, the driver cache's warm-up): those + # runs count into the build cache, so that the game folder's counts are those of play alone + os.environ['LLVM_PROFILE_FILE'] = str(cache / 'pgo-build' / 'build_%m.profraw') reset = state.get('version') != STATE_VERSION or state.get('input') != src.identity if reset: state = {'version': STATE_VERSION, 'input': src.identity, 'steps': {}} diff --git a/tools/pgo.py b/tools/pgo.py index 2ab6fc86..e84114c6 100644 --- a/tools/pgo.py +++ b/tools/pgo.py @@ -1,36 +1,52 @@ -"""Merge the raw profiles of an instrumented build into the .profdata that build_game.py --pgo FILE reads. +"""Merge the raw profiles of an instrumented build into the execution profile a build compiles with. + python tools/pgo.py merge --version v111 --toolchain C:\\msys64\\ucrt64 python tools/pgo.py merge build/v143-pgogen --version v143 [--out build/pgo/pgo-v143.profdata] -An instrumented build (build_game.py --pgo gen) writes /pgo/default_.profraw while nx2navi.exe runs (every -minute and at exit); successive runs of the same exe add to that file. llvm-profdata comes from the llvm-mingw toolchain -the instrumented build used: its raw format is tied to the compiler version. +An instrumented build (tools/make_game.py --pgo gen for game 1.1.1, tools/build_game.py --pgo gen for the other +versions) writes /pgo/default_.profraw while the game runs (every minute and at exit); successive runs of +the same exe add to that file. llvm-profdata must come from the toolchain the instrumented build was compiled with +(--toolchain DIR, its bin/llvm-profdata): the raw format is tied to the compiler's version. The merged file is read +by the compilers of the builds that use it; one of the same LLVM release reads it, an older one may refuse it, which +tools/make_game.py finds out with a trial compile before it uses the file (pgo_trial). -Beside the profile goes .json, the training record: the game version and the git tree of recomp/ at the merge. -The profile matches the generated C of that generator; tools/make_release.py ships it only while recomp/ is unchanged, -since llvm-profdata matches functions by name and control-flow hash and a regenerated function silently loses its -counts. Merge right after the training runs, from the commit they were built at. +Beside the profile goes .json, the training record: the game version, recomp/ at the merge as its git tree +(recomp_tree) and by content (recomp_digest, which a kit without git compares: tools/make_game.py pgo_record), and +the tool that merged. The profile matches the generated C of that generator, since functions are matched by name and +control-flow hash and a regenerated function silently loses its counts. tools/make_game.py therefore compiles with a +profile only while recomp/ is the recorded one, and tools/make_launcher.py --pgo and tools/make_release.py say so +when it is not. Merge right after the training runs, from the commit they were built at: a game folder made by +make_game.py --pgo gen names the recomp/ it was built from (pgo/training.json), and the merge stops when the +checkout holds another. """ import argparse import datetime import json +import re +import shutil import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent EXE = '.exe' if sys.platform == 'win32' else '' -DEFAULT_DIR = ROOT / 'build' / 'pgo' # tools/make_release.py takes pgo-.profdata from here +DEFAULT_DIR = ROOT / 'build' / 'pgo' # make_game.py, make_launcher.py and make_release.py look here +MSYS_BIN = Path(r'C:\msys64\ucrt64\bin') def llvm_profdata(toolchain): + """llvm-profdata of --toolchain; without one the newest llvm-mingw under toolchain/, else MSYS2 UCRT64's, else + the one on PATH.""" if toolchain: tool = toolchain / 'bin' / f'llvm-profdata{EXE}' else: found = sorted((ROOT / 'toolchain').glob(f'llvm-mingw-*/llvm-mingw-*/bin/llvm-profdata{EXE}')) - tool = found[-1] if found else Path(f'llvm-profdata{EXE}') + on_path = shutil.which('llvm-profdata') + tool = (found[-1] if found else MSYS_BIN / 'llvm-profdata.exe' if (MSYS_BIN / 'llvm-profdata.exe').is_file() + else Path(on_path) if on_path else Path(f'llvm-profdata{EXE}')) if not tool.is_file(): - sys.exit(f'{tool} not found: pass --toolchain') + sys.exit(f'{tool} not found: pass --toolchain with the folder of the compiler that built the instrumented ' + 'game (docs/building.md, "Profile-guided build", says which package holds llvm-profdata)') return tool @@ -44,37 +60,113 @@ def recomp_tree(): text=True).stdout.strip() or None +def recomp_digest(version, root=ROOT): + """recomp/ by content, as a kit without git works it out too: the lifter's code and the version's patch table + (for v111, tools/make_game.py recomp_digest).""" + sys.path.insert(0, str(ROOT / 'tools')) + import make_game + return make_game.tree_digest(Path(root) / 'recomp', ('*.py', f'patches-{version}.txt')) + + +def training(build, version): + """What the instrumented game in build was made from (pgo/training.json, tools/make_game.py pgo_gen_record), {} + for a build that wrote none (tools/build_game.py). A game built from another recomp/ than this checkout's, or for + another game version, stops the merge: the record would name a generator the counts were not gathered on.""" + try: + made = json.loads((build / 'pgo' / 'training.json').read_text(encoding='utf-8')) + except OSError: + return {} + except ValueError: + sys.exit(f'{build / "pgo" / "training.json"} cannot be read') + if made.get('version') != version: + sys.exit(f'{build} was built for game version {made.get("version")}, not {version}') + if made.get('recomp_digest') != recomp_digest(version): + sys.exit(f'{build} was built from another recomp/ than this checkout\'s: merge from the commit the ' + 'instrumented game was built at') + return made + + +def llvm_release(text): + """The LLVM release a --version line names, '' when it names none.""" + m = re.search(r'version (\d+\.\d+\.\d+)', text or '') + return m.group(1) if m else '' + + def merge(build, out, version, toolchain): raw = sorted((build / 'pgo').glob('*.profraw')) if not raw: - sys.exit(f'no .profraw in {build / "pgo"}: run the instrumented nx2navi.exe first') + sys.exit(f'no .profraw in {build / "pgo"}: run the instrumented game first') tree = recomp_tree() if not tree: sys.exit('recomp/ has uncommitted changes: the record could not name the generator the profile matches') + made = training(build, version) + tool = llvm_profdata(toolchain) out.parent.mkdir(parents=True, exist_ok=True) tmp = out.with_suffix('.profdata.tmp') - cmd = [str(llvm_profdata(toolchain)), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)] + cmd = [str(tool), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)] p = subprocess.run(cmd, capture_output=True, text=True) if p.returncode: tmp.unlink(missing_ok=True) sys.exit(f'llvm-profdata merge failed: {(p.stdout + p.stderr).strip()[-2000:]}') tmp.replace(out) - record = {'version': version, 'recomp_tree': tree, 'merged': datetime.date.today().isoformat(), - 'raw_files': len(raw)} + said = subprocess.run([str(tool), '--version'], capture_output=True, text=True).stdout + record = {'version': version, 'recomp_tree': tree, 'recomp_digest': recomp_digest(version), + 'merged': datetime.date.today().isoformat(), 'raw_files': len(raw), + 'tool': next((line.strip() for line in said.splitlines() if 'version' in line), ''), + 'compiler': made.get('compiler', '')} out.with_suffix('.json').write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8') + if made.get('compiler') and llvm_release(made['compiler']) != llvm_release(record['tool']): + print(f'pgo: warning: the game was built with {made["compiler"]} and its counts were merged with ' + f'{record["tool"]}: merge with the llvm-profdata of the same LLVM release (--toolchain)', file=sys.stderr) size = sum(f.stat().st_size for f in raw) print(f'{out}: {len(raw)} raw files ({size / 2**20:.1f} MiB) -> {out.stat().st_size / 2**20:.1f} MiB; ' f'recomp tree {tree[:12]}') +def fitting(profile, version='v111'): + """The record beside a profile, for a release from this checkout. A profile whose record does not name this + checkout's recomp/ stops the release: the builds made from it would leave the profile out (tools/make_game.py + pgo_record), and the release would carry it for nothing.""" + profile = Path(profile) + rec = profile.with_suffix('.json') + if not profile.is_file(): + sys.exit(f'{profile} not found') + try: + record = json.loads(rec.read_text(encoding='utf-8')) + except (OSError, ValueError): + sys.exit(f'{rec} is missing or cannot be read: tools/pgo.py merge writes it beside the profile') + if record.get('version') != version: + sys.exit(f'{profile} was trained on game version {record.get("version")}, not {version}') + if record.get('recomp_digest') != recomp_digest(version): + sys.exit(f'{profile} was trained on another recomp/ than the one in this checkout (its record names the tree ' + f'{str(record.get("recomp_tree", "unknown"))[:12]}): the builds would leave it out. Train a profile ' + 'again (docs/building.md, "Profile-guided build") or release without one') + return record + + +def place(profile, data, version='v111'): + """Copy a profile and its record into a release's data/ as pgo-.profdata and .json, where the builds + made from that release look for them (tools/make_game.py pgo_file). The path of the profile's copy.""" + fitting(profile, version) + data = Path(data) + data.mkdir(parents=True, exist_ok=True) + shutil.copy2(profile, data / f'pgo-{version}.profdata') + shutil.copy2(Path(profile).with_suffix('.json'), data / f'pgo-{version}.json') + return data / f'pgo-{version}.profdata' + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) sub = ap.add_subparsers(dest='cmd', required=True) m = sub.add_parser('merge', help='merge /pgo/*.profraw into one .profdata and its training record') - m.add_argument('build', type=Path, help='the instrumented build directory') - m.add_argument('--version', required=True, help='game version the build was made for (v121, v143)') + m.add_argument('build', type=Path, help='the instrumented build: the game folder of make_game.py --pgo gen, or ' + 'the build directory of build_game.py --pgo gen') + m.add_argument('--version', required=True, help='game version the build was made for (v111, v143)') m.add_argument('--out', type=Path, help=f'default {DEFAULT_DIR.relative_to(ROOT).as_posix()}/pgo-.profdata') - m.add_argument('--toolchain', type=Path, help='llvm-mingw directory (default: the newest under toolchain/)') + m.add_argument('--toolchain', type=Path, + help='the folder of the compiler that built the instrumented game, with bin/llvm-profdata: an ' + 'MSYS2 UCRT64 folder or an llvm-mingw directory (default: the newest llvm-mingw under ' + 'toolchain/, else C:\\msys64\\ucrt64, else PATH)') a = ap.parse_args() out = (a.out or DEFAULT_DIR / f'pgo-{a.version}.profdata').resolve() merge(a.build.resolve(), out, a.version, a.toolchain)