mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-15 19:19:58 -04:00
Build game 1.1.1 with the kit's execution profile, and make its instrumented build.
tools/make_game.py, the build players run through the launcher, read no profile: only tools/build_game.py did, which does not build 1.1.1. A kit with data/pgo-v111.profdata (build/pgo/ in a checkout) now compiles the lifted code at -O2 with it and the runtime too, the prebuilt runtime library included. The profile is left out, and every command and key is what it was, when the kit has none, when its record names another recomp/ than the kit's, when a compiler of the build refuses the file in one trial compile before the others start, and with NX2NAVI_PGO=off. The record holds recomp/ by content, which a kit without git can compare. The object keys hold the profile's content and not the kit's folder, so a launcher update that moves the kit keeps the objects. --pgo gen (NX2NAVI_PGO=gen from a launcher) builds the instrumented game into the game folder: it counts into <out>/pgo, records what it was built from for the merge, and stops at once when the build tools lack the profile runtime. tools/pgo.py merge takes that record, refuses a game built from another recomp/ and finds llvm-profdata in an MSYS2 folder too. tools/check_content.py refuses profiles in a commit. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
4
.gitignore
vendored
4
.gitignore
vendored
@@ -20,6 +20,10 @@ __pycache__/
|
||||
# The shared pipeline list a release carries (tools/pipeline_list.py merge; tools/prepare_shaders.py reads it here)
|
||||
/tools/data/pipeline_list.bin
|
||||
/data/pipeline_list.bin
|
||||
# Execution profiles of the game's code (tools/pgo.py merge writes them under build/; a release carries one in data/)
|
||||
*.profdata
|
||||
*.profraw
|
||||
/data/pgo-*.json
|
||||
*.xci
|
||||
*.xcz
|
||||
*.nsp
|
||||
|
||||
456
tests/pgo_build_test.py
Normal file
456
tests/pgo_build_test.py
Normal file
@@ -0,0 +1,456 @@
|
||||
"""The profile-guided build of tools/make_game.py and tools/pgo.py, with a stand-in for the compiler process.
|
||||
|
||||
Which flags a build compiles with: a kit without an execution profile (the flags and keys it always had), a kit with one
|
||||
that fits (the lifted code at -O2 and every compile with the profile), a record that names another recomp/, a record
|
||||
that is missing or names none, a profile the compiler refuses in the trial compile, NX2NAVI_PGO=off, and an
|
||||
instrumented build (NX2NAVI_PGO=gen). The object keys: the same profile in a kit at another place keeps them, another
|
||||
profile changes them. tools/pgo.py: the record a merge writes names the recomp/ a kit compares, a game built from
|
||||
another recomp/ stops the merge, and a release refuses a profile of another tree. tools/check_content.py refuses
|
||||
profiles in a commit.
|
||||
|
||||
The last test uses the real compiler when MSYS2 UCRT64's clang is installed (and an llvm-profdata, when one is
|
||||
found): a file that is no profile is refused by the trial compile, a real one is taken. No game files.
|
||||
|
||||
Run: python tests/pgo_build_test.py
|
||||
"""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import check_content # noqa: E402
|
||||
import make_game as mg # noqa: E402
|
||||
import pgo # noqa: E402
|
||||
|
||||
REFUSAL = ('clang: error: Error in reading profile pgo-v111.profdata: unsupported instrumentation profile format '
|
||||
'version')
|
||||
|
||||
|
||||
class Compiler:
|
||||
"""Stands in for subprocess.run in make_game.py: records every command, answers --version, writes the object
|
||||
and the dependency file of a compile, and refuses a command with -fprofile-use= when refuse is set."""
|
||||
|
||||
def __init__(self, refuse=''):
|
||||
self.refuse = refuse
|
||||
self.cmds = []
|
||||
|
||||
def __call__(self, cmd, **kw):
|
||||
cmd = [str(c) for c in cmd]
|
||||
self.cmds.append(cmd)
|
||||
if '--version' in cmd:
|
||||
return mock.Mock(returncode=0, stdout='clang version 23.1.5\nTarget: x86_64-w64-windows-gnu\n', stderr='')
|
||||
if self.refuse and any(c.startswith('-fprofile-use=') for c in cmd):
|
||||
return mock.Mock(returncode=1, stdout='', stderr=self.refuse + '\n')
|
||||
if '-o' in cmd:
|
||||
Path(cmd[cmd.index('-o') + 1]).write_bytes(b'object')
|
||||
if '-MF' in cmd:
|
||||
Path(cmd[cmd.index('-MF') + 1]).write_text(
|
||||
cmd[cmd.index('-o') + 1].replace('\\', '/') + ': ' + cmd[-1].replace('\\', '/') + '\n')
|
||||
return mock.Mock(returncode=0, stdout='', stderr='')
|
||||
|
||||
def trials(self):
|
||||
return [c for c in self.cmds if any(a.startswith('-fprofile-use=') for a in c)]
|
||||
|
||||
|
||||
class PgoBuildTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-pgo-'))
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.kit = self.make_kit('kit')
|
||||
self.clang, self.clangxx = self.tmp / 'msys' / 'bin' / 'clang.exe', self.tmp / 'msys' / 'bin' / 'clang++.exe'
|
||||
self.inc = ['-I', str(self.kit / 'runtime')]
|
||||
self.cc = Compiler()
|
||||
env = {k: v for k, v in os.environ.items() if k not in ('NX2NAVI_PGO', 'NX2NAVI_GUEST_OPT')}
|
||||
for p in (mock.patch.object(mg, 'KIT', self.kit), mock.patch.object(mg, 'WIN', True),
|
||||
mock.patch.object(mg, 'PGO_MODE', ''), mock.patch.object(mg, 'PGO_IDS', {}),
|
||||
mock.patch.dict(os.environ, env, clear=True)):
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def make_kit(self, name, lifter='def lift(): return 1\n'):
|
||||
kit = self.tmp / name
|
||||
(kit / 'recomp').mkdir(parents=True)
|
||||
(kit / 'recomp' / 'gen.py').write_text(lifter)
|
||||
(kit / 'recomp' / 'patches-v111.txt').write_text('# no patches\n')
|
||||
(kit / 'runtime').mkdir()
|
||||
return kit
|
||||
|
||||
def add_profile(self, kit=None, folder='data', content=b'\xfflprofi\x81 counts', record='fits'):
|
||||
"""A profile in the kit's data/ with a record that names the kit's recomp/ ('fits'), another one ('other'),
|
||||
none ('empty'), or without a record (None)."""
|
||||
kit = kit or self.kit
|
||||
d = kit.joinpath(*folder.split('/'))
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
(d / 'pgo-v111.profdata').write_bytes(content)
|
||||
if record:
|
||||
digest = {'fits': mg.recomp_digest(kit), 'other': 'f' * 64}.get(record)
|
||||
(d / 'pgo-v111.json').write_text(json.dumps({'version': 'v111', **({'recomp_digest': digest} if digest else {})}))
|
||||
return d / 'pgo-v111.profdata'
|
||||
|
||||
def choose(self, fast=None):
|
||||
with mock.patch.object(mg.subprocess, 'run', self.cc):
|
||||
return mg.pgo_profile(self.clang, {}, self.tmp / 'trial', fast, self.tmp / 'msys')
|
||||
|
||||
def commands(self, pgo=()):
|
||||
"""(the lifted code's command, a runtime file's) with the flags pgo, without the compiler and the paths."""
|
||||
cmd_for = mg.runtime_cmd(self.clang, self.clangxx, self.inc, level=3, pgo=pgo)
|
||||
guest = cmd_for(self.tmp / 'main_0001.c', self.tmp / 'main_main_0001.o')
|
||||
runtime = cmd_for(self.kit / 'runtime' / 'perf.c', self.tmp / 'rt_perf.o')
|
||||
return guest[1:-5], runtime[1:-5] # up to -I <runtime> -o <object> <source>
|
||||
|
||||
PLAIN = (['-c', '-std=gnu11', '-O1', '-march=x86-64-v3', '-fno-strict-aliasing', '-w'],
|
||||
['-c', '-std=gnu11', '-O2', '-fno-strict-aliasing', '-w'])
|
||||
|
||||
def test_a_kit_without_a_profile_builds_as_before(self):
|
||||
profile, why = self.choose()
|
||||
self.assertIsNone(profile)
|
||||
self.assertIn('no execution profile', why)
|
||||
self.assertEqual(self.cc.cmds, []) # nothing to try
|
||||
self.assertEqual(self.commands(), self.PLAIN)
|
||||
self.assertEqual(mg.pgo_tag(), '') # the compile step's key is what it was
|
||||
self.assertEqual(mg.pgo_salt('compiler', None), 'compiler') # and so are the object keys
|
||||
with mock.patch.dict(os.environ, {'NX2NAVI_PGO': 'off'}):
|
||||
self.assertEqual(mg.pgo_tag(), '')
|
||||
|
||||
def test_a_fitting_profile_goes_into_every_compile(self):
|
||||
path = self.add_profile()
|
||||
fast = self.tmp / 'msys' / 'clang64' / 'bin' / 'clang.exe'
|
||||
profile, why = self.choose(fast)
|
||||
self.assertEqual((profile, why), (path, ''))
|
||||
# one trial compile by each compiler that will read the file, the fast one with its sysroot
|
||||
trials = self.cc.trials()
|
||||
self.assertEqual([t[0] for t in trials], [str(self.clang), str(fast)])
|
||||
self.assertEqual(trials[1][1], f'--sysroot={self.tmp / "msys"}')
|
||||
self.assertFalse((self.tmp / 'trial').exists())
|
||||
use = [f'-fprofile-use={path}', '-mllvm', '-static-func-full-module-prefix=false']
|
||||
self.assertEqual(mg.pgo_use_flags(profile), use)
|
||||
guest, runtime = self.commands(mg.pgo_use_flags(profile))
|
||||
self.assertEqual(guest, ['-c', '-std=gnu11', '-O2', '-march=x86-64-v3', '-fno-strict-aliasing', '-w', *use])
|
||||
self.assertEqual(runtime, [*self.PLAIN[1], *use])
|
||||
self.assertEqual(mg.pgo_tag(), '|pgo ' + mg.pgo_id(path))
|
||||
self.assertNotEqual(mg.pgo_salt('compiler', profile), 'compiler')
|
||||
|
||||
def test_a_source_checkout_takes_the_profile_from_build_pgo(self):
|
||||
path = self.add_profile(folder='build/pgo')
|
||||
self.assertEqual(self.choose(), (path, ''))
|
||||
release = self.add_profile() # a release's data/ comes first
|
||||
self.assertEqual(self.choose(), (release, ''))
|
||||
|
||||
def test_a_record_of_another_tree_leaves_the_profile_out(self):
|
||||
self.add_profile(record='other')
|
||||
profile, why = self.choose()
|
||||
self.assertIsNone(profile)
|
||||
self.assertIn('trained on another recomp/', why)
|
||||
self.assertEqual(self.cc.cmds, []) # decided before any compiler runs
|
||||
self.assertEqual(self.commands(), self.PLAIN)
|
||||
# the same record fits once the kit's recomp/ is the one it names, and not after the lifter changed again
|
||||
self.add_profile(record='fits')
|
||||
self.assertIsNotNone(self.choose()[0])
|
||||
(self.kit / 'recomp' / 'gen.py').write_text('def lift(): return 2\n')
|
||||
self.assertIsNone(self.choose()[0])
|
||||
(self.kit / 'recomp' / 'patches-v111.txt').write_text('# a patch\n')
|
||||
self.assertIsNone(self.choose()[0])
|
||||
|
||||
def test_a_missing_or_empty_record_leaves_the_profile_out(self):
|
||||
for record, said in ((None, 'is missing'), ('empty', 'does not name the recomp/')):
|
||||
shutil.rmtree(self.kit / 'data', ignore_errors=True)
|
||||
self.add_profile(record=record)
|
||||
profile, why = self.choose()
|
||||
self.assertIsNone(profile, record)
|
||||
self.assertIn(said, why)
|
||||
(self.kit / 'data' / 'pgo-v111.json').write_text('{ cut short')
|
||||
self.assertIn('does not name the recomp/', self.choose()[1])
|
||||
self.assertEqual(self.cc.cmds, [])
|
||||
|
||||
def test_a_profile_the_compiler_refuses_is_left_out(self):
|
||||
self.add_profile()
|
||||
self.cc = Compiler(refuse=REFUSAL)
|
||||
profile, why = self.choose()
|
||||
self.assertIsNone(profile)
|
||||
self.assertIn('the compiler cannot read it', why)
|
||||
self.assertIn('unsupported instrumentation profile format version', why)
|
||||
self.assertEqual(len(self.cc.trials()), 1) # one compile says so, not every compile of the build
|
||||
self.assertEqual(self.commands(), self.PLAIN)
|
||||
# the second compiler's refusal counts too
|
||||
fast = self.tmp / 'msys' / 'clang64' / 'bin' / 'clang.exe'
|
||||
|
||||
class OnlyFast(Compiler):
|
||||
def __call__(self, cmd, **kw):
|
||||
self.refuse = REFUSAL if str(cmd[0]) == str(fast) else ''
|
||||
return super().__call__(cmd, **kw)
|
||||
self.cc = OnlyFast()
|
||||
self.assertIsNone(self.choose(fast)[0])
|
||||
self.assertEqual(len(self.cc.trials()), 2)
|
||||
|
||||
def test_a_compiler_that_does_not_start_leaves_the_profile_out(self):
|
||||
self.add_profile()
|
||||
|
||||
def gone(cmd, **kw):
|
||||
raise FileNotFoundError(2, 'No such file or directory', str(cmd[0]))
|
||||
with mock.patch.object(mg.subprocess, 'run', gone):
|
||||
profile, why = mg.pgo_profile(self.clang, {}, self.tmp / 'trial')
|
||||
self.assertIsNone(profile)
|
||||
self.assertIn('does not start', why)
|
||||
|
||||
def test_off_leaves_a_fitting_profile_out(self):
|
||||
self.add_profile()
|
||||
for mode in ({'NX2NAVI_PGO': 'off'}, {'NX2NAVI_PGO': ' OFF '}):
|
||||
with mock.patch.dict(os.environ, mode):
|
||||
profile, why = self.choose()
|
||||
self.assertIsNone(profile)
|
||||
self.assertIn('NX2NAVI_PGO=off', why)
|
||||
self.assertEqual(mg.pgo_tag(), '|pgo off')
|
||||
with mock.patch.object(mg, 'PGO_MODE', 'off'): # --pgo off
|
||||
self.assertIsNone(self.choose()[0])
|
||||
self.assertEqual(self.cc.cmds, [])
|
||||
with mock.patch.dict(os.environ, {'NX2NAVI_PGO': 'something'}): # no mode: as without it
|
||||
self.assertIsNotNone(self.choose()[0])
|
||||
|
||||
def test_only_the_windows_build_takes_the_profile(self):
|
||||
self.add_profile()
|
||||
with mock.patch.object(mg, 'WIN', False):
|
||||
profile, why = self.choose()
|
||||
self.assertIsNone(profile)
|
||||
self.assertIn('Windows', why)
|
||||
|
||||
def test_an_instrumented_build_counts_into_the_game_folder(self):
|
||||
out = self.tmp / 'game out'
|
||||
with mock.patch.dict(os.environ, {'NX2NAVI_PGO': 'gen'}):
|
||||
self.assertEqual(mg.pgo_mode(), 'gen')
|
||||
self.assertEqual(mg.pgo_tag(), '|pgo gen')
|
||||
flags = mg.pgo_gen_flags(out)
|
||||
self.assertEqual(flags, [f'-fprofile-generate={(out / "pgo").as_posix()}', '-mllvm',
|
||||
'-static-func-full-module-prefix=false', '-DNX2NAVI_PGO_GEN'])
|
||||
guest, runtime = self.commands(flags)
|
||||
self.assertEqual(guest, ['-c', '-std=gnu11', '-O2', '-march=x86-64-v3', '-fno-strict-aliasing', '-w', *flags])
|
||||
self.assertEqual(runtime, [*self.PLAIN[1], *flags])
|
||||
# what the game was built from goes beside the counts, and tools/pgo.py reads it back
|
||||
self.make_real_recomp()
|
||||
with mock.patch.object(mg.subprocess, 'run', self.cc):
|
||||
mg.pgo_gen_record(out, self.clang, {})
|
||||
made = json.loads((out / 'pgo' / 'training.json').read_text())
|
||||
self.assertEqual(made['recomp_digest'], mg.recomp_digest())
|
||||
self.assertEqual(made['compiler'], 'clang version 23.1.5')
|
||||
self.assertEqual(pgo.training(out, 'v111'), made)
|
||||
|
||||
def make_real_recomp(self):
|
||||
"""The test kit's recomp/ becomes this checkout's, so that what make_game.py records in the kit is what
|
||||
tools/pgo.py, which reads the checkout, compares it with."""
|
||||
shutil.rmtree(self.kit / 'recomp')
|
||||
(self.kit / 'recomp').mkdir()
|
||||
for p in (ROOT / 'recomp').iterdir():
|
||||
if p.suffix == '.py' or p.name == 'patches-v111.txt':
|
||||
shutil.copy2(p, self.kit / 'recomp' / p.name)
|
||||
|
||||
def test_build_tools_without_the_profile_runtime_stop_an_instrumented_build(self):
|
||||
def no_runtime(cmd, **kw):
|
||||
self.assertIn('-fprofile-generate', cmd)
|
||||
return mock.Mock(returncode=1, stdout='', stderr='ld: cannot find libclang_rt.profile.a: No such file\n'
|
||||
'clang++: error: linker command failed\n')
|
||||
msys = self.tmp / 'msys64' / 'ucrt64'
|
||||
err = io.StringIO()
|
||||
with mock.patch.object(mg.subprocess, 'run', no_runtime), contextlib.redirect_stderr(err), \
|
||||
self.assertRaises(SystemExit):
|
||||
mg.pgo_gen_check(self.clangxx, {}, self.tmp / 'trial', msys, ['-fuse-ld=lld'])
|
||||
self.assertIn('mingw-w64-ucrt-x86_64-compiler-rt', err.getvalue())
|
||||
self.assertIn('cannot find libclang_rt.profile.a', err.getvalue())
|
||||
self.assertFalse((self.tmp / 'trial').exists())
|
||||
with mock.patch.object(mg.subprocess, 'run', self.cc): # with the runtime: nothing to say
|
||||
mg.pgo_gen_check(self.clangxx, {}, self.tmp / 'trial', msys, [])
|
||||
|
||||
def test_the_object_keys_follow_the_profile_and_not_the_kits_folder(self):
|
||||
"""A launcher update puts the kit into another folder: the same profile there keeps every object. Another
|
||||
profile, or a build that leaves it out, compiles again."""
|
||||
src = self.tmp / 'lift' / 'main_0001.c'
|
||||
src.parent.mkdir()
|
||||
src.write_text('int f;\n')
|
||||
obj = self.tmp / 'obj'
|
||||
obj.mkdir()
|
||||
|
||||
def key(kit, profile):
|
||||
cache = mg.ObjectCache(obj / 'objects.json', {'<kit>': kit})
|
||||
inc = ['-I', str(kit / 'runtime')]
|
||||
cmd_for = mg.runtime_cmd(self.clang, self.clangxx, inc, level=3,
|
||||
pgo=mg.pgo_use_flags(profile) if profile else ())
|
||||
dst = obj / 'main_main_0001.o'
|
||||
return mg.object_key(cmd_for(src, dst), src, dst, cache, mg.pgo_salt('compiler', profile))
|
||||
here = self.add_profile()
|
||||
moved = self.make_kit('kit 2')
|
||||
there = self.add_profile(moved)
|
||||
self.assertEqual(key(self.kit, here), key(moved, there))
|
||||
self.assertNotEqual(key(self.kit, here), key(self.kit, None))
|
||||
other = self.add_profile(moved, content=b'\xfflprofi\x81 other counts')
|
||||
self.assertNotEqual(key(self.kit, here), key(moved, other))
|
||||
# without a profile the key is the one a kit before profiles gave: the flags and the compiler's id alone
|
||||
cache = mg.ObjectCache(obj / 'objects.json', {'<kit>': self.kit})
|
||||
dst = obj / 'main_main_0001.o'
|
||||
plain = [str(self.clang), *self.PLAIN[0], '-I', str(self.kit / 'runtime'), '-o', str(dst), str(src)]
|
||||
self.assertEqual(key(self.kit, None), mg.object_key(plain, src, dst, cache, 'compiler'))
|
||||
|
||||
def test_a_new_profile_compiles_the_objects_again(self):
|
||||
src, obj = self.tmp / 'src', self.tmp / 'obj'
|
||||
src.mkdir()
|
||||
obj.mkdir()
|
||||
srcs = []
|
||||
for i in range(4):
|
||||
(src / f'main_{i}.c').write_text(f'int f{i};\n')
|
||||
srcs.append((src / f'main_{i}.c', obj / f'main_main_{i}.o'))
|
||||
|
||||
def build():
|
||||
profile, _ = mg.pgo_profile(self.clang, {}, self.tmp / 'trial')
|
||||
cache = mg.ObjectCache(obj / 'objects.json', {'<kit>': self.kit})
|
||||
cmd_for = mg.runtime_cmd(self.clang, self.clangxx, self.inc, level=3,
|
||||
pgo=mg.pgo_use_flags(profile) if profile else ())
|
||||
return mg.compile_all(srcs, cmd_for, cache, 2, {}, salt=mg.pgo_salt('compiler', profile))[:2]
|
||||
with mock.patch.object(mg.subprocess, 'run', self.cc):
|
||||
self.assertEqual(build(), (4, 0)) # no profile
|
||||
self.assertEqual(build(), (0, 4))
|
||||
self.add_profile()
|
||||
self.assertEqual(build(), (4, 0)) # the kit brought one
|
||||
self.assertEqual(build(), (0, 4))
|
||||
self.add_profile(content=b'\xfflprofi\x81 counts of a longer session')
|
||||
self.assertEqual(build(), (4, 0)) # and another
|
||||
self.add_profile(content=b'\xfflprofi\x81 counts of a longer session', record='other')
|
||||
self.assertEqual(build(), (4, 0)) # left out: the flags of a kit without one
|
||||
self.assertEqual(build(), (0, 4))
|
||||
compiles = [c for c in self.cc.cmds if '-MF' in c]
|
||||
self.assertTrue(all('-O1' in c for c in compiles[:4] + compiles[-4:]))
|
||||
self.assertTrue(all('-O2' in c and any(a.startswith('-fprofile-use=') for a in c) for c in compiles[4:12]))
|
||||
|
||||
|
||||
class PgoToolTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-pgo-'))
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.build = self.tmp / 'game'
|
||||
(self.build / 'pgo').mkdir(parents=True)
|
||||
(self.build / 'pgo' / 'default_1.profraw').write_bytes(b'raw counts')
|
||||
self.tool = self.tmp / 'llvm' / 'bin' / ('llvm-profdata' + pgo.EXE)
|
||||
self.tool.parent.mkdir(parents=True)
|
||||
self.tool.write_bytes(b'')
|
||||
self.ran = []
|
||||
p = mock.patch.object(pgo, 'recomp_tree', lambda: 'a' * 40)
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def profdata(self, cmd, **kw):
|
||||
"""Stands in for llvm-profdata."""
|
||||
cmd = [str(c) for c in cmd]
|
||||
self.ran.append(cmd)
|
||||
if 'merge' in cmd:
|
||||
Path(cmd[cmd.index('-o') + 1]).write_bytes(b'merged')
|
||||
return mock.Mock(returncode=0, stdout='LLVM (http://llvm.org/):\n LLVM version 23.1.5\n', stderr='')
|
||||
|
||||
def merge(self, version='v111'):
|
||||
out = self.tmp / 'pgo' / f'pgo-{version}.profdata'
|
||||
with mock.patch.object(pgo.subprocess, 'run', self.profdata), contextlib.redirect_stdout(io.StringIO()):
|
||||
pgo.merge(self.build, out, version, self.tool.parent.parent)
|
||||
return out
|
||||
|
||||
def test_the_merge_records_the_recomp_a_kit_compares(self):
|
||||
out = self.merge()
|
||||
self.assertEqual(out.read_bytes(), b'merged')
|
||||
self.assertEqual(self.ran[0][:3], [str(self.tool), 'merge', '--failure-mode=all'])
|
||||
self.assertEqual(self.ran[0][-1], str(self.build / 'pgo' / 'default_1.profraw'))
|
||||
record = json.loads(out.with_suffix('.json').read_text())
|
||||
self.assertEqual(record['version'], 'v111')
|
||||
self.assertEqual(record['recomp_tree'], 'a' * 40)
|
||||
self.assertEqual(record['tool'], 'LLVM version 23.1.5')
|
||||
# what make_game.py works out for a kit with this checkout's recomp/: the profile fits such a kit
|
||||
with mock.patch.object(mg, 'KIT', ROOT):
|
||||
self.assertEqual(record['recomp_digest'], mg.recomp_digest())
|
||||
data = self.tmp / 'release' / 'data'
|
||||
placed = pgo.place(out, data)
|
||||
self.assertEqual(mg.pgo_record(placed), '')
|
||||
self.assertEqual(sorted(p.name for p in data.iterdir()), ['pgo-v111.json', 'pgo-v111.profdata'])
|
||||
|
||||
def test_a_game_built_from_another_recomp_stops_the_merge(self):
|
||||
made = {'version': 'v111', 'recomp_digest': pgo.recomp_digest('v111'), 'compiler': 'clang version 23.1.5'}
|
||||
(self.build / 'pgo' / 'training.json').write_text(json.dumps(made))
|
||||
record = json.loads(self.merge().with_suffix('.json').read_text())
|
||||
self.assertEqual(record['compiler'], 'clang version 23.1.5')
|
||||
(self.build / 'pgo' / 'training.json').write_text(json.dumps(dict(made, recomp_digest='0' * 64)))
|
||||
with self.assertRaises(SystemExit) as e:
|
||||
self.merge()
|
||||
self.assertIn('another recomp/', str(e.exception))
|
||||
(self.build / 'pgo' / 'training.json').write_text(json.dumps(dict(made, version='v143')))
|
||||
with self.assertRaises(SystemExit):
|
||||
self.merge()
|
||||
|
||||
def test_a_merge_needs_counts_and_a_clean_recomp(self):
|
||||
(self.build / 'pgo' / 'default_1.profraw').unlink()
|
||||
with self.assertRaises(SystemExit) as e:
|
||||
self.merge()
|
||||
self.assertIn('no .profraw', str(e.exception))
|
||||
(self.build / 'pgo' / 'default_1.profraw').write_bytes(b'raw counts')
|
||||
with mock.patch.object(pgo, 'recomp_tree', lambda: None), self.assertRaises(SystemExit) as e:
|
||||
self.merge()
|
||||
self.assertIn('uncommitted changes', str(e.exception))
|
||||
|
||||
def test_a_release_refuses_a_profile_of_another_tree(self):
|
||||
out = self.merge()
|
||||
rec = out.with_suffix('.json')
|
||||
record = json.loads(rec.read_text())
|
||||
self.assertEqual(pgo.fitting(out), record)
|
||||
rec.write_text(json.dumps(dict(record, recomp_digest='0' * 64)))
|
||||
with self.assertRaises(SystemExit) as e:
|
||||
pgo.place(out, self.tmp / 'release' / 'data')
|
||||
self.assertIn('another recomp/', str(e.exception))
|
||||
self.assertFalse((self.tmp / 'release').exists())
|
||||
rec.write_text(json.dumps(dict(record, version='v143')))
|
||||
with self.assertRaises(SystemExit):
|
||||
pgo.fitting(out)
|
||||
rec.unlink()
|
||||
with self.assertRaises(SystemExit):
|
||||
pgo.fitting(out)
|
||||
|
||||
def test_profiles_stay_out_of_a_commit(self):
|
||||
self.assertTrue(check_content.execution_profile(Path('data/pgo-v111.profdata')))
|
||||
self.assertTrue(check_content.execution_profile(Path('game/pgo/default_1.PROFRAW')))
|
||||
self.assertFalse(check_content.execution_profile(Path('data/pgo-v111.json')))
|
||||
self.assertFalse(check_content.execution_profile(Path('tools/pgo.py')))
|
||||
|
||||
|
||||
MSYS = Path(r'C:\msys64\ucrt64')
|
||||
TEXT_PROFILE = ':ir\nnx2navi_pgo_trial\n0\n1\n100\n\n' # one function with one counter, as llvm-profdata reads text
|
||||
|
||||
|
||||
@unittest.skipUnless((MSYS / 'bin' / 'clang.exe').is_file(), 'needs MSYS2 UCRT64 clang')
|
||||
class RealCompilerTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp(prefix='nx2navi-pgo-'))
|
||||
self.addCleanup(shutil.rmtree, self.tmp, True)
|
||||
self.clang = MSYS / 'bin' / 'clang.exe'
|
||||
self.env = dict(os.environ, PATH=str(MSYS / 'bin') + os.pathsep + os.environ.get('PATH', ''))
|
||||
|
||||
def test_the_trial_compile_tells_a_profile_from_a_file_that_is_none(self):
|
||||
bad = self.tmp / 'pgo-v111.profdata'
|
||||
bad.write_bytes(b'not a profile')
|
||||
said = mg.pgo_trial(bad, [[self.clang]], self.env, self.tmp / 'trial')
|
||||
self.assertIn('Error in reading profile', said)
|
||||
cut = self.tmp / 'cut.profdata'
|
||||
cut.write_bytes(b'\xfflprofi\x81') # a profile's first bytes and nothing after them
|
||||
self.assertTrue(mg.pgo_trial(cut, [[self.clang]], self.env, self.tmp / 'trial'))
|
||||
self.assertTrue(mg.pgo_trial(self.tmp / 'missing.profdata', [[self.clang]], self.env, self.tmp / 'trial'))
|
||||
tool = shutil.which('llvm-profdata') or next(
|
||||
(str(p) for p in (MSYS / 'bin' / 'llvm-profdata.exe', Path(r'C:\Program Files\LLVM\bin\llvm-profdata.exe'))
|
||||
if p.is_file()), None)
|
||||
if not tool:
|
||||
self.skipTest('no llvm-profdata to write a real profile with; the refusals above were checked')
|
||||
text, good = self.tmp / 'trial.proftext', self.tmp / 'good.profdata'
|
||||
text.write_text(TEXT_PROFILE)
|
||||
subprocess.run([tool, 'merge', '-o', str(good), str(text)], check=True, capture_output=True)
|
||||
self.assertEqual(mg.pgo_trial(good, [[self.clang]], self.env, self.tmp / 'trial'), '')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -108,6 +108,14 @@ def recorded_in_play(path: Path, data: bytes | None) -> bool:
|
||||
return data[:8] in RECORD_MAGICS
|
||||
|
||||
|
||||
def execution_profile(path: Path) -> bool:
|
||||
"""The counts an instrumented game wrote (.profraw) or their merged profile (.profdata, tools/pgo.py): they name
|
||||
the lifted functions and count their branches, so they are derived from the game's code. A launcher release or
|
||||
an installer release may carry a profile (data/pgo-<version>.profdata), so make_launcher.py and make_release.py,
|
||||
which call problems_for, are not held to this."""
|
||||
return path.suffix.lower() in (".profdata", ".profraw")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--staged", action="store_true", help="check staged files only")
|
||||
@@ -123,6 +131,8 @@ def main() -> int:
|
||||
problems.append("compiled shader: the repository holds the source, tools/build_shaders.py builds this")
|
||||
if recorded_in_play(path, data):
|
||||
problems.append("recorded from a game folder (shaders, answers, pipeline keys or a pipeline list)")
|
||||
if execution_profile(path):
|
||||
problems.append("execution profile of the game's code: it stays in build/ (tools/pgo.py)")
|
||||
for problem in problems:
|
||||
print(f"{path}: {problem}")
|
||||
failed = True
|
||||
|
||||
@@ -49,6 +49,14 @@ sets the number), the largest files first. The lifted code is built for this
|
||||
PC's x86-64 level (cpu_level), with MSYS2 CLANG64's clang when it sits beside the UCRT64 folder and reports the same
|
||||
version (fast_clang: the same objects, about a fifth faster).
|
||||
|
||||
Profile-guided build (docs/building.md, "Profile-guided build"). A kit with data/pgo-v111.profdata and its record
|
||||
(tools/pgo.py merge; in a source checkout build/pgo/) compiles the lifted code at -O2 with that execution profile, and
|
||||
the runtime with it too (pgo_profile); functions the profile never saw run are optimised for size. The profile is
|
||||
left out, and the build is the one a kit without a profile makes (the lifted code at -O1), when its record names
|
||||
another recomp/ than this kit's, when the compilers cannot read the file (one trial compile before the others start),
|
||||
or with NX2NAVI_PGO=off. --pgo gen (NX2NAVI_PGO=gen, which a launcher started with it passes on) builds an
|
||||
instrumented game instead: it adds its counts to <out>/pgo/*.profraw every minute and at exit, for tools/pgo.py merge.
|
||||
|
||||
python tools/make_game.py --make-runtime-lib <kit>/lib --msys <ucrt64> # the release's prebuilt runtime
|
||||
|
||||
On Linux the same steps make <out>/ocarina with the clang, clang++ and lld on PATH and the system's libraries
|
||||
@@ -520,11 +528,17 @@ EMITTED = re.compile(r'emitted (\d+)/(\d+) chunks')
|
||||
|
||||
def lift_digest():
|
||||
"""The lifter's code, the patch table and the bring-up fixes: what the lifted C depends on besides the ExeFS."""
|
||||
h = hashlib.sha256(tree_digest(KIT / 'recomp', ('*.py', 'patches-v111.txt')).encode())
|
||||
h = hashlib.sha256(recomp_digest().encode())
|
||||
h.update(repr(LIFT_FIXES).encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def recomp_digest(kit=None):
|
||||
"""The lifter's code and the patch table by content: what names the generator in the lift's key and in an
|
||||
execution profile's record (pgo_record). kit: another checkout or kit folder than this one."""
|
||||
return tree_digest((kit or KIT) / 'recomp', ('*.py', 'patches-v111.txt'))
|
||||
|
||||
|
||||
# The mods' code (tools/mod_files.py): the tools that make patch sets of it, the patch file each writes, the status
|
||||
# file the runtime reads in build-cache next to the exe (runtime/cheats.c, runtime/mods.c) and what the log calls it
|
||||
MOD_STAGES = (('cheats.py', 'patches-cheats.txt', 'cheats.status', 'code cheats'),
|
||||
@@ -969,13 +983,18 @@ def compiler_id(clang, env):
|
||||
|
||||
def object_key(cmd, src, dst, cache, salt, sysroot=False):
|
||||
"""The cache key of compiling src into dst with cmd: the compiler's id (salt), the command's flags with the cache's
|
||||
roots standing for their paths, and the source's sha256. --sysroot is left out: only fast_clang's compiler gets it,
|
||||
and its objects are the UCRT64 compiler's byte for byte, so the object of either serves (sysroot=True keeps it: the
|
||||
keys up to 0.6.0)."""
|
||||
roots standing for their paths (the kit's execution profile too: its content is part of the salt, pgo_salt), and
|
||||
the source's sha256. --sysroot is left out: only fast_clang's compiler gets it, and its objects are the UCRT64
|
||||
compiler's byte for byte, so the object of either serves (sysroot=True keeps it: the keys up to 0.6.0)."""
|
||||
flags = [c for c in cmd[1:] if c not in (str(src), str(dst)) and (sysroot or not c.startswith('--sysroot='))]
|
||||
|
||||
def rooted(f, name, root):
|
||||
# a path under a root, alone or as the value of -fprofile-use= (the kit's profile, pgo_use_flags)
|
||||
head = PGO_USE if f.startswith(PGO_USE) else ''
|
||||
path = os.path.normcase(f[len(head):])
|
||||
return head + name + path[len(root):].replace(os.sep, '/') if path.startswith(root) else f
|
||||
for name, root in cache.roots.items():
|
||||
flags = [name + os.path.normcase(f)[len(root):].replace(os.sep, '/') if os.path.normcase(f).startswith(root)
|
||||
else f for f in flags]
|
||||
flags = [rooted(f, name, root) for f in flags]
|
||||
return sha256_bytes(json.dumps([salt, [os.path.basename(cmd[0])] + flags, cache.file_sha(src)]).encode())
|
||||
|
||||
|
||||
@@ -1425,6 +1444,184 @@ def fast_clang(msys, clang, env):
|
||||
return fast if mine and clang_ident(fast, env) == mine else None
|
||||
|
||||
|
||||
# ---- the profile-guided build (docs/building.md, "Profile-guided build")
|
||||
#
|
||||
# An execution profile is the counts an instrumented game gathered in a play session, merged into one file
|
||||
# (tools/pgo.py merge). With it the compiler lays the code that ran out straight, inlines the calls that were hot and
|
||||
# optimises the functions that never ran for size. The file names the lifted functions by their address and the
|
||||
# runtime's by their name, each with a hash of its control flow: a function whose C differs from the one that was
|
||||
# trained gets no counts, and the compiler says nothing about it.
|
||||
PGO_NAME = 'pgo-v111.profdata' # in the kit's data/ (a release) or build/pgo/ (a source checkout, tools/pgo.py)
|
||||
PGO_RAW = 'pgo' # in the game folder: where an instrumented game adds its counts, *.profraw
|
||||
PGO_TRAINING = 'training.json' # beside the counts: what the instrumented game was built from (pgo_gen_record)
|
||||
PGO_USE = '-fprofile-use='
|
||||
# A static function's profile name carries its source's path by default; the file's base name alone keeps the names
|
||||
# equal between the folder the profile was trained in and this kit's (tools/build_game.py PGO_NAMES).
|
||||
PGO_NAMES = ['-mllvm', '-static-func-full-module-prefix=false']
|
||||
PGO_MODE = '' # --pgo; without it NX2NAVI_PGO decides (pgo_mode)
|
||||
|
||||
|
||||
def pgo_mode():
|
||||
"""'gen': build an instrumented game; 'off': build without the kit's profile; '': build with it when it fits.
|
||||
From --pgo, else from NX2NAVI_PGO, which reaches a build the launcher starts through its environment."""
|
||||
mode = (PGO_MODE or os.environ.get('NX2NAVI_PGO', '')).strip().lower()
|
||||
return mode if mode in ('gen', 'off') else ''
|
||||
|
||||
|
||||
def pgo_file():
|
||||
"""The kit's execution profile, or None: data/ of a release, else build/pgo/ of a source checkout, where
|
||||
tools/pgo.py merge writes it."""
|
||||
for d in (KIT / 'data', KIT / 'build' / 'pgo'):
|
||||
if (d / PGO_NAME).is_file():
|
||||
return d / PGO_NAME
|
||||
return None
|
||||
|
||||
|
||||
PGO_IDS = {} # pgo_id's answers by path, size and modification time: the step keys ask several times a build
|
||||
|
||||
|
||||
def pgo_id(profile):
|
||||
"""sha256 over the profile and its record, 'unreadable' when the profile cannot be read: what stands for them in
|
||||
the compile step's key and in the object keys, so a kit that brings another profile compiles again."""
|
||||
try:
|
||||
st = os.stat(profile)
|
||||
at = (str(profile), st.st_size, st.st_mtime_ns)
|
||||
if at not in PGO_IDS:
|
||||
h = hashlib.sha256(sha256_file(profile).encode())
|
||||
try:
|
||||
h.update(profile.with_suffix('.json').read_bytes())
|
||||
except OSError:
|
||||
pass
|
||||
PGO_IDS[at] = h.hexdigest()
|
||||
return PGO_IDS[at]
|
||||
except OSError:
|
||||
return 'unreadable'
|
||||
|
||||
|
||||
def pgo_tag():
|
||||
"""The profile-guided build's part of the compile step's key (step_key): '' for a kit without a profile that
|
||||
builds as before, so the key of every game built so far stays what it is."""
|
||||
mode, profile = pgo_mode(), pgo_file()
|
||||
if mode == 'gen':
|
||||
return '|pgo gen'
|
||||
if not profile:
|
||||
return ''
|
||||
return '|pgo off' if mode == 'off' else f'|pgo {pgo_id(profile)}'
|
||||
|
||||
|
||||
def pgo_record(profile):
|
||||
"""Why the record beside a profile (<name>.json, tools/pgo.py merge) does not let this kit use it, '' when it
|
||||
does: recomp_digest there must be this kit's recomp/. Functions are matched by name and control-flow hash, so a
|
||||
profile trained on the C of another generator loses the counts of every function that generator writes
|
||||
differently, without a word from the compiler: an -O2 build with a profile of unknown worth. Such a kit builds
|
||||
as one without a profile instead."""
|
||||
rec = profile.with_suffix('.json')
|
||||
try:
|
||||
trained = json.loads(rec.read_text(encoding='utf-8'))['recomp_digest']
|
||||
except OSError:
|
||||
return f'its record {rec.name} is missing or cannot be read'
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return f'its record {rec.name} does not name the recomp/ it was trained on'
|
||||
return '' if trained == recomp_digest() else 'it was trained on another recomp/ than this kit\'s'
|
||||
|
||||
|
||||
def pgo_use_flags(profile):
|
||||
"""Every compile's flags for a build with the profile. The warnings about functions the profile lacks or knows
|
||||
with other control flow are off like every other warning (-w)."""
|
||||
return [PGO_USE + str(profile), *PGO_NAMES]
|
||||
|
||||
|
||||
def pgo_gen_flags(out):
|
||||
"""Every compile's flags for an instrumented build of the game folder out. The folder for the counts is baked in
|
||||
as an absolute path: left relative, the game would resolve it against whatever folder it was started from.
|
||||
NX2NAVI_PGO_GEN: the runtime adds the counts to the file every minute and when it ends the process
|
||||
(runtime/platform/win32.c pgo_flush_thread, plat_exit)."""
|
||||
return [f'-fprofile-generate={(Path(out) / PGO_RAW).as_posix()}', *PGO_NAMES, '-DNX2NAVI_PGO_GEN']
|
||||
|
||||
|
||||
def pgo_trial(profile, compilers, env, work):
|
||||
"""Why a compiler that will read the profile cannot, '' when each compiles a test file with it. compilers: each
|
||||
one's argv up to its flags. The profile was written by the LLVM release of the training build and is read by the
|
||||
clang of this PC's build tools: a compiler older than the file's format refuses it, as it does a file that is cut
|
||||
short or is no profile, and it would refuse it again in every compile of the build."""
|
||||
try:
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
src, obj = work / 'trial.c', work / 'trial.o'
|
||||
src.write_text('int nx2navi_pgo_trial(int x) { return x + 1; }\n', encoding='ascii')
|
||||
for cc in compilers:
|
||||
cmd = [*map(str, cc), '-c', '-std=gnu11', '-O2', '-w', *pgo_use_flags(profile), '-o', str(obj), str(src)]
|
||||
try:
|
||||
p = subprocess.run(cmd, capture_output=True, text=True, env=env, errors='replace')
|
||||
except OSError as e:
|
||||
return f'{cc[0]} does not start: {error_text(e)}'
|
||||
if p.returncode:
|
||||
return (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[-1][-300:]
|
||||
return ''
|
||||
except OSError as e:
|
||||
return error_text(e)
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
|
||||
|
||||
def pgo_profile(clang, env, work, fast=None, msys=None):
|
||||
"""(the kit's execution profile, '') when this build compiles with it, else (None, why not). It is left out, and
|
||||
the build is the one a kit without a profile makes, when the kit has none, with NX2NAVI_PGO=off, off Windows
|
||||
(the profile is trained on the Windows build), when its record names another recomp/ (pgo_record), and when a
|
||||
compiler of this build cannot read it (pgo_trial; fast: fast_clang's compiler, which compiles the lifted code,
|
||||
with msys as its sysroot). work: a folder of its own for the trial compile."""
|
||||
profile = pgo_file()
|
||||
if not profile:
|
||||
return None, 'this kit has no execution profile'
|
||||
if pgo_mode() == 'off':
|
||||
return None, f'the execution profile {profile.name} is left out (NX2NAVI_PGO=off)'
|
||||
if not WIN:
|
||||
return None, f'the execution profile {profile.name} is for the Windows build'
|
||||
why = pgo_record(profile)
|
||||
if not why:
|
||||
said = pgo_trial(profile, [[clang]] + ([[fast, f'--sysroot={msys}']] if fast else []), env, work)
|
||||
why = said and f'the compiler cannot read it ({said})'
|
||||
return (None, f'the execution profile {profile.name} is not used: {why}') if why else (profile, '')
|
||||
|
||||
|
||||
def pgo_salt(cid, profile):
|
||||
"""The compiler's id for the keys of objects compiled with the profile: the profile's content with it, as no
|
||||
dependency file names the profile. Without a profile the id itself, so those keys stay what they were."""
|
||||
return sha256_bytes(f'{cid}|pgo {pgo_id(profile)}'.encode()) if profile else cid
|
||||
|
||||
|
||||
def pgo_gen_check(clangxx, env, work, msys, ld):
|
||||
"""An instrumented build links the compiler's profile runtime, which MSYS2 packs apart from clang
|
||||
(mingw-w64-ucrt-x86_64-compiler-rt) and the launcher's build tools do not hold: an empty program is linked with
|
||||
it now (ld: the link's own linker flags), so that build tools without it stop the build here and not at the
|
||||
link, after every compile."""
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
work.mkdir(parents=True)
|
||||
try:
|
||||
src = work / 'trial.c'
|
||||
src.write_text('int main(void) { return 0; }\n', encoding='ascii')
|
||||
p = subprocess.run([str(clangxx), *ld, '-fprofile-generate', '-o', str(work / ('trial.exe' if WIN else 'trial')),
|
||||
'-x', 'c', str(src)], capture_output=True, text=True, env=env, errors='replace')
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
if p.returncode:
|
||||
said = (p.stderr.strip().splitlines() or [f'exit code {p.returncode}'])[0][-300:]
|
||||
fail('an instrumented build (NX2NAVI_PGO=gen) needs the compiler\'s profile runtime, and these build tools do '
|
||||
f'not link it ({said}). ' +
|
||||
(f'Unpack the MSYS2 package mingw-w64-ucrt-x86_64-compiler-rt of this clang\'s version into {msys.parent}'
|
||||
if WIN else 'Install the compiler-rt package of this clang\'s version'))
|
||||
|
||||
|
||||
def pgo_gen_record(out, clang, env):
|
||||
"""<out>/pgo/training.json: the recomp/ and the compiler the instrumented game was built from. tools/pgo.py merge
|
||||
takes the profile's record from it, so the record names the generator of the game that was played, whatever the
|
||||
checkout holds when the counts are merged."""
|
||||
raw = Path(out) / PGO_RAW
|
||||
raw.mkdir(parents=True, exist_ok=True)
|
||||
record = {'version': 'v111', 'recomp_digest': recomp_digest(), 'compiler': clang_version(clang, env),
|
||||
'kit': kit_version()}
|
||||
(raw / PGO_TRAINING).write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def guest_opt_ini(out):
|
||||
"""NX2NAVI_GUEST_OPT from nx2navi.ini next to the exe (the low-end GPU rows of the menu's Game tab write it there)
|
||||
when the environment does not set it: the next build of the game folder out takes it. Only while low_end_gpu = 1
|
||||
@@ -1447,13 +1644,15 @@ def guest_opt_ini(out):
|
||||
os.environ['NX2NAVI_GUEST_OPT'] = found
|
||||
|
||||
|
||||
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()):
|
||||
def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=(), pgo=()):
|
||||
"""fast: fast_clang's compiler for the lifted code, with msys as its sysroot. first: include flags that go
|
||||
ahead of inc for the runtime's sources only (vulkan_headers); the lifted code includes none of those headers,
|
||||
and its command, which is part of its objects' keys, stays as it was. NX2NAVI_GUEST_OPT (an environment variable,
|
||||
or guest_opt in nx2navi.ini) 2 builds the lifted code at -O2; by default -O1: the game's own threads run it and
|
||||
bound the frame on a slower CPU; the build takes several times longer."""
|
||||
guest_opt = '-O2' if os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2' else '-O1' # -O1 unless guest_opt = 2
|
||||
bound the frame on a slower CPU; the build takes several times longer. pgo: the flags of a profile-guided build
|
||||
for every compile (pgo_use_flags, pgo_gen_flags); with them the lifted code is built at -O2 too, the level the
|
||||
profile's inlining and its layout of hot and cold code work at. Without them every command is what it was."""
|
||||
guest_opt = '-O2' if pgo or os.environ.get('NX2NAVI_GUEST_OPT', '').strip() == '2' else '-O1' # -O1 unless guest_opt = 2
|
||||
def cmd_for(src, dst):
|
||||
cpp = src.suffix in ('.cpp', '.cxx')
|
||||
# the lifted code at -O1 (its size makes -O2 slow to build), the runtime at -O2; neither may assume strict
|
||||
@@ -1466,14 +1665,16 @@ def runtime_cmd(clang, clangxx, inc, level=0, fast=None, msys=None, first=()):
|
||||
# -fdeclspec: third_party/ffx_api/ffx_api.h declares its entry points __declspec(dllexport) unconditionally
|
||||
declspec = ['-fdeclspec'] if cpp and not WIN else []
|
||||
return [*cc, '-c', '-std=gnu++17' if cpp else '-std=gnu11', guest_opt if guest else '-O2',
|
||||
*march, '-fno-strict-aliasing', '-w', *declspec, *([] if guest else first), *inc, '-o', str(dst), str(src)]
|
||||
*march, '-fno-strict-aliasing', '-w', *declspec, *pgo, *([] if guest else first), *inc, '-o', str(dst),
|
||||
str(src)]
|
||||
return cmd_for
|
||||
|
||||
|
||||
def make_runtime_lib(dest, msys, jobs, obj=None):
|
||||
"""--make-runtime-lib: compile every runtime source that does not include a per-install header into
|
||||
<dest>/libnx2navirt.a, with runtime-lib.json naming the runtime/ tree and the compiler it was built from. The
|
||||
objects stay in obj (default <dest>/obj), which a release keeps out of what it ships."""
|
||||
objects stay in obj (default <dest>/obj), which a release keeps out of what it ships. A kit with an execution
|
||||
profile compiles the library with it, under pgo_profile's conditions; runtime-lib.json then names the profile."""
|
||||
dest = Path(dest).resolve()
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
obj = Path(obj).resolve() if obj else dest / 'obj'
|
||||
@@ -1484,11 +1685,17 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
|
||||
srcs = [(p, obj / name) for p, name in runtime_sources() if not per_install(p, tainted)]
|
||||
inc = ['-I', str(rt), '-I', str(rt / 'third_party' / 'imgui'), '-I', str(vma)]
|
||||
cache = ObjectCache(obj / 'objects.json', {'<kit>': KIT})
|
||||
if pgo_mode() == 'gen':
|
||||
fail('the prebuilt runtime is never an instrumented one: an instrumented build (NX2NAVI_PGO=gen) compiles '
|
||||
'runtime/ itself')
|
||||
profile, why = pgo_profile(clang, env, obj / 'pgo-trial')
|
||||
say(f' the runtime is compiled with the execution profile {profile.name}' if profile else f' {why}')
|
||||
t0 = time.time()
|
||||
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env)),
|
||||
compiled, reused, bad = compile_all(srcs, runtime_cmd(clang, clangxx, inc, first=vulkan_headers(clang, env),
|
||||
pgo=pgo_use_flags(profile) if profile else ()),
|
||||
cache, jobs, env,
|
||||
lambda d, t: (d % 20 == 0 or d == t) and say(f' {d} of {t}'),
|
||||
salt=compiler_id(clang, env))
|
||||
salt=pgo_salt(compiler_id(clang, env), profile))
|
||||
if bad:
|
||||
report_failures(bad)
|
||||
fail(f'{len(bad)} runtime files did not compile')
|
||||
@@ -1502,7 +1709,7 @@ def make_runtime_lib(dest, msys, jobs, obj=None):
|
||||
if subprocess.run([ar, 'rcs', str(lib), '@' + str(rsp)], env=env).returncode:
|
||||
fail('the runtime library could not be archived')
|
||||
meta = {'runtime': runtime_digest(), 'compiler': clang_version(clang, env),
|
||||
'objects': sorted(o.name for _, o in srcs)}
|
||||
'objects': sorted(o.name for _, o in srcs), 'pgo': pgo_id(profile) if profile else ''}
|
||||
(dest / 'runtime-lib.json').write_text(json.dumps(meta, indent=1), encoding='utf-8')
|
||||
rsp.unlink(missing_ok=True)
|
||||
say(f'{lib}: {len(srcs)} runtime files ({compiled} compiled, {reused} kept), {lib.stat().st_size / 1e6:.1f} MB, '
|
||||
@@ -1723,7 +1930,9 @@ def write_runtime_map(full, dst):
|
||||
def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
"""romfs_job: the RomFS step still running beside this one (a first build). Everything that does not include
|
||||
romfs_config.h compiles meanwhile; the RomFS index and the few files that include it follow once it is done.
|
||||
update: a rebuild after a kit update (lifted_level)."""
|
||||
update: a rebuild after a kit update (lifted_level). The kit's execution profile, when this build can use it
|
||||
(pgo_profile), goes into every compile; NX2NAVI_PGO=gen makes an instrumented game instead (pgo_gen_flags), its
|
||||
runtime compiled here too, as the prebuilt one counts nothing."""
|
||||
lift, gen, obj = cache_dir / 'lift', cache_dir / 'gen', cache_dir / 'obj'
|
||||
gen.mkdir(parents=True, exist_ok=True)
|
||||
obj.mkdir(parents=True, exist_ok=True)
|
||||
@@ -1733,7 +1942,12 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
if not romfs_job:
|
||||
romfs_index(out, gen)
|
||||
srcs = [(p, obj / f'{mod}_{p.stem}.o') for mod in ('main', 'sdk', 'subsdk0') for p in sorted((lift / mod).glob('*.c'))]
|
||||
prebuilt = prebuilt_runtime(clang, env)
|
||||
gen_counts = pgo_mode() == 'gen'
|
||||
# lld links several times faster than GNU ld
|
||||
ld = ['-fuse-ld=lld'] if ((msys / 'bin' / 'ld.lld.exe').is_file() if WIN else shutil.which('ld.lld')) else []
|
||||
if gen_counts:
|
||||
pgo_gen_check(clangxx, env, cache_dir / 'pgo-trial', msys, ld)
|
||||
prebuilt = None if gen_counts else prebuilt_runtime(clang, env)
|
||||
runtime = [(p, obj / name) for p, name in runtime_sources()]
|
||||
if prebuilt:
|
||||
runtime = [(p, o) for p, o in runtime if o.name not in prebuilt[1]]
|
||||
@@ -1761,7 +1975,17 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
vulkan = vulkan_headers(clang, env)
|
||||
if vulkan:
|
||||
say(f' the Vulkan headers come from {vulkan[1]}')
|
||||
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan)
|
||||
profile = None
|
||||
if gen_counts:
|
||||
pgo = pgo_gen_flags(out)
|
||||
say(f' an instrumented build (NX2NAVI_PGO=gen): the lifted code at -O2 and a slower game, which adds its '
|
||||
f'counts to {out / PGO_RAW} every minute and at exit')
|
||||
else:
|
||||
profile, why = pgo_profile(clang, env, cache_dir / 'pgo-trial', fast, msys)
|
||||
pgo = pgo_use_flags(profile) if profile else []
|
||||
say(f' the lifted code (at -O2) and the runtime are built with the execution profile {profile.name}'
|
||||
if profile else f' {why}')
|
||||
cmd_for = runtime_cmd(clang, clangxx, inc, level, fast, msys, first=vulkan, pgo=pgo)
|
||||
# The keys earlier versions gave the same compiles (compile_all alts): 0.6.0's, with the compiler's file time and,
|
||||
# with the fast compiler, its id and --sysroot; 0.5.0's and earlier's, with their kit's folder in the paths; and
|
||||
# up to 0.7.1 the runtime's without the Vulkan headers' folder ahead of the system's (an object that compiled
|
||||
@@ -1772,6 +1996,8 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
kits = earlier_kits()
|
||||
|
||||
def alts(src, dst):
|
||||
if pgo: # no earlier version compiled with a profile or counted: nothing of theirs is the same compile
|
||||
return [], kits
|
||||
keys = [object_key(plain(src, dst), src, dst, cache, cid_old, sysroot=True)]
|
||||
if vulkan:
|
||||
keys.append(object_key(no_vulkan(src, dst), src, dst, cache, cid))
|
||||
@@ -1799,8 +2025,9 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
late = [(s, o) for s, o in runtime + srcs[-(2 if WIN else 1):] if per_install(s, tainted)]
|
||||
say(f' {len(late)} files wait for the RomFS index: {", ".join(s.name for s, _ in late)}')
|
||||
early = [j for j in srcs if j not in late]
|
||||
salt = pgo_salt(cid, profile)
|
||||
compiled, reused, bad = compile_all(early, cmd_for, cache, jobs, env, lambda d, t: progress(d, len(srcs)),
|
||||
salt=cid, stop=romfs_job.failed if romfs_job else None, alts=alts)
|
||||
salt=salt, stop=romfs_job.failed if romfs_job else None, alts=alts)
|
||||
if romfs_job:
|
||||
if romfs_job.thread.is_alive():
|
||||
say(' waiting for the RomFS image')
|
||||
@@ -1808,7 +2035,7 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
if not bad:
|
||||
romfs_index(out, gen)
|
||||
c2, r2, bad = compile_all(late, cmd_for, cache, jobs, env,
|
||||
lambda d, t: progress(len(early) + d, len(srcs)), salt=cid, alts=alts)
|
||||
lambda d, t: progress(len(early) + d, len(srcs)), salt=salt, alts=alts)
|
||||
compiled, reused = compiled + c2, reused + r2
|
||||
say(f' {compiled} compiled, {reused} unchanged since the last build')
|
||||
if bad:
|
||||
@@ -1816,19 +2043,24 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
fail(f'{len(bad)} files did not compile')
|
||||
if cache.keep_only({o.name for _, o in srcs}):
|
||||
cache.save()
|
||||
(obj / LIFTED_BUILD).write_text(json.dumps({'level': level}), encoding='utf-8') # lifted_level reads it next update
|
||||
# lifted_level reads the level at the next update; pgo says what the objects were compiled with: 'gen', the
|
||||
# profile's id (pgo_id) or nothing
|
||||
(obj / LIFTED_BUILD).write_text(json.dumps({'level': level, 'pgo': 'gen' if gen_counts else
|
||||
pgo_id(profile) if profile else ''}), encoding='utf-8')
|
||||
|
||||
objs = [o for _, o in srcs] + ([compile_resources(msys, rt, obj, env, lld=(msys / 'bin' / 'ld.lld.exe').is_file())]
|
||||
if WIN else [])
|
||||
exe = out / EXE_NAME
|
||||
rsp = cache_dir / 'link.rsp'
|
||||
rsp.write_text('\n'.join('"%s"' % o.as_posix() for o in objs), encoding='utf-8')
|
||||
# lld links several times faster than GNU ld; the stack matches tools/build_game.py's build
|
||||
# the stack matches tools/build_game.py's build
|
||||
# 16 MiB for the main thread with either linker (GNU ld takes the same flag): the default 2 MiB is too little
|
||||
if WIN:
|
||||
ld = (['-fuse-ld=lld'] if (msys / 'bin' / 'ld.lld.exe').is_file() else []) + ['-Wl,--stack,16777216']
|
||||
ld = ld + ['-Wl,--stack,16777216']
|
||||
else: # tools/build_game.py's Linux flags; the main thread's stack is the process's (ulimit -s), not the linker's
|
||||
ld = (['-fuse-ld=lld'] if shutil.which('ld.lld') else []) + ['-no-pie', '-Wl,-z,noexecstack', '-Wl,-rpath,$ORIGIN']
|
||||
ld = ld + ['-no-pie', '-Wl,-z,noexecstack', '-Wl,-rpath,$ORIGIN']
|
||||
if gen_counts: # the link takes in the profile runtime, which writes the counts
|
||||
ld.append('-fprofile-generate')
|
||||
# the link map feeds tools/prof_report.py (symbols for an NX2NAVI_PROFILE sampling profile); both linkers write one.
|
||||
# -Xlinker passes the path as one argument; -Wl, would cut it at every comma of the game folder's path (issue #470)
|
||||
ld += ['-Xlinker', '--Map=' + (cache_dir / 'ocarina.map').as_posix()]
|
||||
@@ -1878,6 +2110,8 @@ def step_compile(out, cache_dir, msys, jobs, romfs_job=None, update=False):
|
||||
beside = copy_dlls(exe, msys / 'bin', out) if WIN else []
|
||||
say(f' {EXE_NAME} {exe.stat().st_size / 1e6:.0f} MB' + (f'; beside it: {", ".join(beside)}' if beside else ''))
|
||||
(out / 'profile').mkdir(exist_ok=True)
|
||||
if gen_counts:
|
||||
pgo_gen_record(out, clang, env)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- main
|
||||
@@ -2340,7 +2574,9 @@ def step_key(step, identity, out=None):
|
||||
processor (cpu_tag); shaders: the compile without the mods' code, the preparing script and the translator;
|
||||
prompts: the game files, the kit's button art, the version of the tools that draw it (PROMPT_TOOLS), the bind_
|
||||
lines of nx2navi.ini, and whether every folder it writes is still there; scan: the game files, the check's version
|
||||
(SCAN_TOOL) and whether its report is still there."""
|
||||
(SCAN_TOOL) and whether its report is still there. The compile's key also names the kit's execution profile or
|
||||
an instrumented build (pgo_tag); the shaders' key does not, as neither changes what the shaders are prepared
|
||||
from."""
|
||||
if step in ('exefs', 'romfs'):
|
||||
return f'{step}-1|{identity}'
|
||||
if step == 'scan':
|
||||
@@ -2362,7 +2598,7 @@ def step_key(step, identity, out=None):
|
||||
return (f'compile|{lift}|{tree_digest(KIT / "runtime", ("**/*",))}|{sha256_file(Path(__file__))}'
|
||||
f'|{kit_version()}|cpu {cpu_tag()}')
|
||||
if step == 'compile':
|
||||
return compile_of(lift)
|
||||
return compile_of(lift) + pgo_tag()
|
||||
# shaders: the built game without the mods' code (UNAFFECTED) and the translator that prepared them, the one
|
||||
# xlate_source finds (the kit's bin/, or a source checkout's build/xlate/)
|
||||
return (f'shaders-1|{compile_of(base)}|{sha256_file(KIT / "tools" / "prepare_shaders.py")}'
|
||||
@@ -2391,7 +2627,7 @@ def main():
|
||||
|
||||
|
||||
def build_main():
|
||||
global EVENTS
|
||||
global EVENTS, PGO_MODE
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument('--nup', help='the game as one .nup file')
|
||||
ap.add_argument('--exefs', help='the game\'s exefs folder (with --romfs, instead of --nup)')
|
||||
@@ -2410,9 +2646,14 @@ def build_main():
|
||||
ap.add_argument('--events', action='store_true', help='machine-readable progress lines on stdout ("@@" + JSON)')
|
||||
ap.add_argument('--make-runtime-lib', metavar='DIR', help='build the prebuilt runtime library of a release into DIR')
|
||||
ap.add_argument('--obj', help='with --make-runtime-lib: the folder for its objects')
|
||||
ap.add_argument('--pgo', choices=('gen', 'off'),
|
||||
help='gen: build an instrumented game, which adds its execution counts to <out>/pgo/*.profraw '
|
||||
'(for tools/pgo.py merge); off: build without the execution profile of the kit. Default: the '
|
||||
'environment variable NX2NAVI_PGO, else build with the profile of the kit when it fits')
|
||||
ap.add_argument('--until', choices=STEPS, help=argparse.SUPPRESS) # tests: stop after this step
|
||||
a = ap.parse_args()
|
||||
EVENTS = a.events
|
||||
PGO_MODE = a.pgo or ''
|
||||
jobs_why = '--jobs'
|
||||
if not a.jobs or a.jobs < 1:
|
||||
a.jobs, jobs_why = default_jobs()
|
||||
@@ -2458,6 +2699,10 @@ def build_main():
|
||||
fail(f'the Python package {pip_name} is missing or does not load ({e}): '
|
||||
f'pip install numpy lz4 zstandard zbic{extra}')
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
if pgo_mode() == 'gen':
|
||||
# The instrumented game also runs inside this build (the shaders step, the driver cache's warm-up): those
|
||||
# runs count into the build cache, so that the game folder's counts are those of play alone
|
||||
os.environ['LLVM_PROFILE_FILE'] = str(cache / 'pgo-build' / 'build_%m.profraw')
|
||||
reset = state.get('version') != STATE_VERSION or state.get('input') != src.identity
|
||||
if reset:
|
||||
state = {'version': STATE_VERSION, 'input': src.identity, 'steps': {}}
|
||||
|
||||
128
tools/pgo.py
128
tools/pgo.py
@@ -1,36 +1,52 @@
|
||||
"""Merge the raw profiles of an instrumented build into the .profdata that build_game.py --pgo FILE reads.
|
||||
"""Merge the raw profiles of an instrumented build into the execution profile a build compiles with.
|
||||
|
||||
python tools/pgo.py merge <game folder> --version v111 --toolchain C:\\msys64\\ucrt64
|
||||
python tools/pgo.py merge build/v143-pgogen --version v143 [--out build/pgo/pgo-v143.profdata]
|
||||
|
||||
An instrumented build (build_game.py --pgo gen) writes <build>/pgo/default_<id>.profraw while nx2navi.exe runs (every
|
||||
minute and at exit); successive runs of the same exe add to that file. llvm-profdata comes from the llvm-mingw toolchain
|
||||
the instrumented build used: its raw format is tied to the compiler version.
|
||||
An instrumented build (tools/make_game.py --pgo gen for game 1.1.1, tools/build_game.py --pgo gen for the other
|
||||
versions) writes <build>/pgo/default_<id>.profraw while the game runs (every minute and at exit); successive runs of
|
||||
the same exe add to that file. llvm-profdata must come from the toolchain the instrumented build was compiled with
|
||||
(--toolchain DIR, its bin/llvm-profdata): the raw format is tied to the compiler's version. The merged file is read
|
||||
by the compilers of the builds that use it; one of the same LLVM release reads it, an older one may refuse it, which
|
||||
tools/make_game.py finds out with a trial compile before it uses the file (pgo_trial).
|
||||
|
||||
Beside the profile goes <out>.json, the training record: the game version and the git tree of recomp/ at the merge.
|
||||
The profile matches the generated C of that generator; tools/make_release.py ships it only while recomp/ is unchanged,
|
||||
since llvm-profdata matches functions by name and control-flow hash and a regenerated function silently loses its
|
||||
counts. Merge right after the training runs, from the commit they were built at.
|
||||
Beside the profile goes <out>.json, the training record: the game version, recomp/ at the merge as its git tree
|
||||
(recomp_tree) and by content (recomp_digest, which a kit without git compares: tools/make_game.py pgo_record), and
|
||||
the tool that merged. The profile matches the generated C of that generator, since functions are matched by name and
|
||||
control-flow hash and a regenerated function silently loses its counts. tools/make_game.py therefore compiles with a
|
||||
profile only while recomp/ is the recorded one, and tools/make_launcher.py --pgo and tools/make_release.py say so
|
||||
when it is not. Merge right after the training runs, from the commit they were built at: a game folder made by
|
||||
make_game.py --pgo gen names the recomp/ it was built from (pgo/training.json), and the merge stops when the
|
||||
checkout holds another.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
EXE = '.exe' if sys.platform == 'win32' else ''
|
||||
DEFAULT_DIR = ROOT / 'build' / 'pgo' # tools/make_release.py takes pgo-<version>.profdata from here
|
||||
DEFAULT_DIR = ROOT / 'build' / 'pgo' # make_game.py, make_launcher.py and make_release.py look here
|
||||
MSYS_BIN = Path(r'C:\msys64\ucrt64\bin')
|
||||
|
||||
|
||||
def llvm_profdata(toolchain):
|
||||
"""llvm-profdata of --toolchain; without one the newest llvm-mingw under toolchain/, else MSYS2 UCRT64's, else
|
||||
the one on PATH."""
|
||||
if toolchain:
|
||||
tool = toolchain / 'bin' / f'llvm-profdata{EXE}'
|
||||
else:
|
||||
found = sorted((ROOT / 'toolchain').glob(f'llvm-mingw-*/llvm-mingw-*/bin/llvm-profdata{EXE}'))
|
||||
tool = found[-1] if found else Path(f'llvm-profdata{EXE}')
|
||||
on_path = shutil.which('llvm-profdata')
|
||||
tool = (found[-1] if found else MSYS_BIN / 'llvm-profdata.exe' if (MSYS_BIN / 'llvm-profdata.exe').is_file()
|
||||
else Path(on_path) if on_path else Path(f'llvm-profdata{EXE}'))
|
||||
if not tool.is_file():
|
||||
sys.exit(f'{tool} not found: pass --toolchain')
|
||||
sys.exit(f'{tool} not found: pass --toolchain with the folder of the compiler that built the instrumented '
|
||||
'game (docs/building.md, "Profile-guided build", says which package holds llvm-profdata)')
|
||||
return tool
|
||||
|
||||
|
||||
@@ -44,37 +60,113 @@ def recomp_tree():
|
||||
text=True).stdout.strip() or None
|
||||
|
||||
|
||||
def recomp_digest(version, root=ROOT):
|
||||
"""recomp/ by content, as a kit without git works it out too: the lifter's code and the version's patch table
|
||||
(for v111, tools/make_game.py recomp_digest)."""
|
||||
sys.path.insert(0, str(ROOT / 'tools'))
|
||||
import make_game
|
||||
return make_game.tree_digest(Path(root) / 'recomp', ('*.py', f'patches-{version}.txt'))
|
||||
|
||||
|
||||
def training(build, version):
|
||||
"""What the instrumented game in build was made from (pgo/training.json, tools/make_game.py pgo_gen_record), {}
|
||||
for a build that wrote none (tools/build_game.py). A game built from another recomp/ than this checkout's, or for
|
||||
another game version, stops the merge: the record would name a generator the counts were not gathered on."""
|
||||
try:
|
||||
made = json.loads((build / 'pgo' / 'training.json').read_text(encoding='utf-8'))
|
||||
except OSError:
|
||||
return {}
|
||||
except ValueError:
|
||||
sys.exit(f'{build / "pgo" / "training.json"} cannot be read')
|
||||
if made.get('version') != version:
|
||||
sys.exit(f'{build} was built for game version {made.get("version")}, not {version}')
|
||||
if made.get('recomp_digest') != recomp_digest(version):
|
||||
sys.exit(f'{build} was built from another recomp/ than this checkout\'s: merge from the commit the '
|
||||
'instrumented game was built at')
|
||||
return made
|
||||
|
||||
|
||||
def llvm_release(text):
|
||||
"""The LLVM release a --version line names, '' when it names none."""
|
||||
m = re.search(r'version (\d+\.\d+\.\d+)', text or '')
|
||||
return m.group(1) if m else ''
|
||||
|
||||
|
||||
def merge(build, out, version, toolchain):
|
||||
raw = sorted((build / 'pgo').glob('*.profraw'))
|
||||
if not raw:
|
||||
sys.exit(f'no .profraw in {build / "pgo"}: run the instrumented nx2navi.exe first')
|
||||
sys.exit(f'no .profraw in {build / "pgo"}: run the instrumented game first')
|
||||
tree = recomp_tree()
|
||||
if not tree:
|
||||
sys.exit('recomp/ has uncommitted changes: the record could not name the generator the profile matches')
|
||||
made = training(build, version)
|
||||
tool = llvm_profdata(toolchain)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = out.with_suffix('.profdata.tmp')
|
||||
cmd = [str(llvm_profdata(toolchain)), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)]
|
||||
cmd = [str(tool), 'merge', '--failure-mode=all', '-o', str(tmp), *map(str, raw)]
|
||||
p = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if p.returncode:
|
||||
tmp.unlink(missing_ok=True)
|
||||
sys.exit(f'llvm-profdata merge failed: {(p.stdout + p.stderr).strip()[-2000:]}')
|
||||
tmp.replace(out)
|
||||
record = {'version': version, 'recomp_tree': tree, 'merged': datetime.date.today().isoformat(),
|
||||
'raw_files': len(raw)}
|
||||
said = subprocess.run([str(tool), '--version'], capture_output=True, text=True).stdout
|
||||
record = {'version': version, 'recomp_tree': tree, 'recomp_digest': recomp_digest(version),
|
||||
'merged': datetime.date.today().isoformat(), 'raw_files': len(raw),
|
||||
'tool': next((line.strip() for line in said.splitlines() if 'version' in line), ''),
|
||||
'compiler': made.get('compiler', '')}
|
||||
out.with_suffix('.json').write_text(json.dumps(record, indent=1) + '\n', encoding='utf-8')
|
||||
if made.get('compiler') and llvm_release(made['compiler']) != llvm_release(record['tool']):
|
||||
print(f'pgo: warning: the game was built with {made["compiler"]} and its counts were merged with '
|
||||
f'{record["tool"]}: merge with the llvm-profdata of the same LLVM release (--toolchain)', file=sys.stderr)
|
||||
size = sum(f.stat().st_size for f in raw)
|
||||
print(f'{out}: {len(raw)} raw files ({size / 2**20:.1f} MiB) -> {out.stat().st_size / 2**20:.1f} MiB; '
|
||||
f'recomp tree {tree[:12]}')
|
||||
|
||||
|
||||
def fitting(profile, version='v111'):
|
||||
"""The record beside a profile, for a release from this checkout. A profile whose record does not name this
|
||||
checkout's recomp/ stops the release: the builds made from it would leave the profile out (tools/make_game.py
|
||||
pgo_record), and the release would carry it for nothing."""
|
||||
profile = Path(profile)
|
||||
rec = profile.with_suffix('.json')
|
||||
if not profile.is_file():
|
||||
sys.exit(f'{profile} not found')
|
||||
try:
|
||||
record = json.loads(rec.read_text(encoding='utf-8'))
|
||||
except (OSError, ValueError):
|
||||
sys.exit(f'{rec} is missing or cannot be read: tools/pgo.py merge writes it beside the profile')
|
||||
if record.get('version') != version:
|
||||
sys.exit(f'{profile} was trained on game version {record.get("version")}, not {version}')
|
||||
if record.get('recomp_digest') != recomp_digest(version):
|
||||
sys.exit(f'{profile} was trained on another recomp/ than the one in this checkout (its record names the tree '
|
||||
f'{str(record.get("recomp_tree", "unknown"))[:12]}): the builds would leave it out. Train a profile '
|
||||
'again (docs/building.md, "Profile-guided build") or release without one')
|
||||
return record
|
||||
|
||||
|
||||
def place(profile, data, version='v111'):
|
||||
"""Copy a profile and its record into a release's data/ as pgo-<version>.profdata and .json, where the builds
|
||||
made from that release look for them (tools/make_game.py pgo_file). The path of the profile's copy."""
|
||||
fitting(profile, version)
|
||||
data = Path(data)
|
||||
data.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(profile, data / f'pgo-{version}.profdata')
|
||||
shutil.copy2(Path(profile).with_suffix('.json'), data / f'pgo-{version}.json')
|
||||
return data / f'pgo-{version}.profdata'
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = ap.add_subparsers(dest='cmd', required=True)
|
||||
m = sub.add_parser('merge', help='merge <build>/pgo/*.profraw into one .profdata and its training record')
|
||||
m.add_argument('build', type=Path, help='the instrumented build directory')
|
||||
m.add_argument('--version', required=True, help='game version the build was made for (v121, v143)')
|
||||
m.add_argument('build', type=Path, help='the instrumented build: the game folder of make_game.py --pgo gen, or '
|
||||
'the build directory of build_game.py --pgo gen')
|
||||
m.add_argument('--version', required=True, help='game version the build was made for (v111, v143)')
|
||||
m.add_argument('--out', type=Path, help=f'default {DEFAULT_DIR.relative_to(ROOT).as_posix()}/pgo-<version>.profdata')
|
||||
m.add_argument('--toolchain', type=Path, help='llvm-mingw directory (default: the newest under toolchain/)')
|
||||
m.add_argument('--toolchain', type=Path,
|
||||
help='the folder of the compiler that built the instrumented game, with bin/llvm-profdata: an '
|
||||
'MSYS2 UCRT64 folder or an llvm-mingw directory (default: the newest llvm-mingw under '
|
||||
'toolchain/, else C:\\msys64\\ucrt64, else PATH)')
|
||||
a = ap.parse_args()
|
||||
out = (a.out or DEFAULT_DIR / f'pgo-{a.version}.profdata').resolve()
|
||||
merge(a.build.resolve(), out, a.version, a.toolchain)
|
||||
|
||||
Reference in New Issue
Block a user