From a5e7eddcfd26b8c186c6a5c8442faba67e115bd7 Mon Sep 17 00:00:00 2001 From: chrg6 Date: Tue, 10 Nov 2026 15:38:23 -0500 Subject: [PATCH] Write the game folder's paths into modules.c as UTF-8. modules_c wrote the paths of the game folder as C strings with JSON's default escapes, which turn a character outside the Basic Multilingual Plane into two surrogate escapes. C refuses those ("invalid universal character"), so a game folder with an emoji or a rare Han character in its path failed at the end of the compile. The characters now stand as themselves in the UTF-8 source, and a test compiles the generated file. Co-Authored-By: Claude Fable 5.2 --- tests/launcher_test.py | 32 ++++++++++++++++++++++++++++++++ tools/make_game.py | 17 +++++++++++++---- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/launcher_test.py b/tests/launcher_test.py index 98f4df8a..9dcc7a78 100644 --- a/tests/launcher_test.py +++ b/tests/launcher_test.py @@ -2624,6 +2624,38 @@ class MakeGameTest(Tmp): 'No such file or directory: x.bin') self.assertEqual(self.mg.error_text(MemoryError()), 'MemoryError') + def test_a_game_folder_outside_the_basic_plane_compiles(self): + """modules.c holds the game folder's paths as C strings. A character outside the Basic Multilingual Plane + (an emoji, a rare Han character) is written as itself in UTF-8, not as the two surrogate escapes JSON's + default makes of it, which C refuses.""" + fairy, han = chr(0x1F9DA), chr(0x20BB7) + out = self.tmp / f'Games {fairy}' / f'V\u00edctor \u30bc\u30eb\u30c0 {han}ya' + lift, gen = self.tmp / 'lift', self.tmp / 'gen' + for d in (out / 'exefs', lift, gen): + d.mkdir(parents=True) + for name in self.mg.EXEFS_FILES: + (out / 'exefs' / name).write_bytes(b'x') + (lift / 'layout.json').write_text(json.dumps([{'name': 'main'}])) + text = self.mg.modules_c(lift, out, gen).read_text(encoding='utf-8') + for half in ('\\ud83e', '\\ud842'): # the first surrogate halves of the two characters + self.assertNotIn(half, text.lower()) + want = 'const char rt_default_exefs[] = "' + str(out / 'exefs').replace('\\', '\\\\') + '";' + self.assertIn(want, text) + self.assertEqual(self.mg.c_string('a"b\\c\n' + fairy), '"a\\"b\\\\c\\n' + fairy + '"') + cc = shutil.which('cc') or shutil.which('clang') or shutil.which('gcc') + env = dict(os.environ) + if not cc and (Path(r'C:\msys64\ucrt64\bin') / 'clang.exe').is_file(): # make_game.py's default --msys + cc = r'C:\msys64\ucrt64\bin\clang.exe' + env['PATH'] = r'C:\msys64\ucrt64\bin' + os.pathsep + env['PATH'] + if not cc: + self.skipTest('the text is right; compiling it needs a C compiler') + (gen / 'layout.h').write_text('#define MOD_MAIN_BASE 0x7000000000ull\n#define MOD_MAIN_SIZE 0x1000000ull\n') + obj = self.tmp / 'modules.o' + r = subprocess.run([cc, '-c', '-std=gnu11', '-w', '-I', str(ROOT / 'runtime'), '-I', str(gen), '-o', str(obj), + str(gen / 'modules.c')], capture_output=True, text=True, errors='replace', env=env) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + self.assertIn(str(out / 'exefs').encode('utf-8'), obj.read_bytes()) # the path's own bytes are in the object + def main_rebuild(self, lift, compile_, *more): """make_game.main() --rebuild in this process on self.out, with the given functions in place of the lift and the compile. Returns (the SystemExit code or 0, what it printed).""" diff --git a/tools/make_game.py b/tools/make_game.py index a096732e..b9eeb64e 100644 --- a/tools/make_game.py +++ b/tools/make_game.py @@ -527,6 +527,15 @@ def step_lift(out, cache): # ---------------------------------------------------------------- compile +def c_string(text): + """text as a C string literal for a source file in UTF-8, which modules.c is: JSON's escapes for the quote, the + backslash and the control characters are C's too, and every other character stands as itself. With JSON's + default, which writes each character outside ASCII as \\uXXXX, a character outside the Basic Multilingual Plane + came out as two surrogate halves (\\ud83e\\uddda), which C refuses ("invalid universal character"): a game + folder with an emoji or a rare Han character in its path failed at the end of the compile.""" + return json.dumps(text, ensure_ascii=False) + + def modules_c(lift, out, gen): layout = json.loads((lift / 'layout.json').read_text(encoding='utf-8')) L = ['#include "dispatch.h"', '#include "host.h"', '#include "layout.h"', ''] @@ -545,13 +554,13 @@ def modules_c(lift, out, gen): L += [' return n;', '}', '', 'const RtFileHash rt_exefs_sha256[] = {'] L += [f' {{"{name}", "{sha256_file(out / "exefs" / name)}"}},' for name in EXEFS_FILES] L += ['};', f'const int rt_exefs_sha256_count = {len(EXEFS_FILES)};', '', - f'const char rt_default_exefs[] = {json.dumps(str(out / "exefs"))};', - f'const char rt_default_romfs[] = {json.dumps(str(out / "romfs"))};', - f'const char rt_default_profile[] = {json.dumps(str(out / "profile"))};', + f'const char rt_default_exefs[] = {c_string(str(out / "exefs"))};', + f'const char rt_default_romfs[] = {c_string(str(out / "romfs"))};', + f'const char rt_default_profile[] = {c_string(str(out / "profile"))};', 'const int rt_game_version = 111;', 'const unsigned long long rt_app_pool_bytes = 9809ull << 20; /* the Switch 2 application pool */', 'const int rt_nvn_api_version[2] = {231, 40}; /* NVN2: main asks for major 231, minor 40 or later */', - f'const char rt_port_version[] = {json.dumps(kit_version())};'] + f'const char rt_port_version[] = {c_string(kit_version())};'] p = gen / 'modules.c' write_if_changed(p, '\n'.join(L) + '\n') if WIN: # on Linux ui/menu_sdl.cpp defines it