"""Windows paths as UTF-8: the game exe's manifest makes its code page UTF-8 (nx2/win_manifest.py), so a game folder named outside the system's ANSI code page (Japanese on a Western Windows, an emoji anywhere) opens; before, the game closed at start. Everywhere: the manifest resource's bytes, that nx2/build.py links it into hookshot.exe, that host/hos_main.c checks at start, and that host/ and gpu/vk convert no text with the ANSI code page (CP_ACP and the like: UTF-8 is the one code page of the game's char strings). With a compiler for Windows: tests/utf8_paths_test.c (host/hos_main.c's narrow_paths_ok pasted in) built with the manifest, and once without it. On Windows both run from a folder named "ゲーム Zélda 🎮": with the manifest the A functions, fopen, argv, getenv and the log (stderr freopen'd, as hookshot_log.txt) are UTF-8 and the paths round-trip; without it the start-up check refuses the folder (as the game used to fail). Elsewhere the build is checked to carry the manifest resource (HOOKSHOT_MINGW= on Linux). python -I tests/test_utf8_paths.py [--cc ] """ import argparse import os from pathlib import Path import re import shutil import struct import subprocess import sys import tempfile import xml.etree.ElementTree as ET ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "nx2")) import win_manifest # (the game's own sources: not host/posix, the Linux stand-ins, nor the vendored emulator and compiler code) SOURCES = [ROOT / "host", ROOT / "gpu" / "vk"] VENDORED = ("host/posix/", "gpu/vk/eden/", "gpu/vk/sirit/", "gpu/vk/compat/") ANSI = re.compile(r"\b(CP_ACP|CP_OEMCP|CP_THREAD_ACP|CP_MACCP|SetFileApisToOEM|SetFileApisToANSI)\b") def resources(data): """(type, name, language, data) of each entry of a .res file""" out, at = [], 0 while at < len(data): size, header = struct.unpack_from(")") return template = (ROOT / "tests" / "utf8_paths_test.c").read_text(encoding="utf-8") assert template.count("/* ACTUAL_NARROW_PATHS_OK */") == 1 with tempfile.TemporaryDirectory(prefix="hookshot-utf8-") as temp: build = Path(temp) / "build" run_dir = Path(temp) / "ゲーム Zélda 🎮" build.mkdir() run_dir.mkdir() source = build / "probe.c" source.write_text(template.replace("/* ACTUAL_NARROW_PATHS_OK */", narrow_paths_ok_source()), encoding="utf-8") res = win_manifest.write(str(build / "hookshot_manifest.res")) target = [] if os.name == "nt" or "mingw32" in os.path.basename(cc) else ["--target=x86_64-w64-mingw32"] flags = target + ["-std=c11", "-O0", "-Wall", "-Wextra", "-Werror", "-D_CRT_SECURE_NO_WARNINGS", "-fuse-ld=lld"] libs = ["-luser32"] with_manifest, without = build / "probe.exe", build / "probe_ansi.exe" subprocess.run([cc] + flags + [str(source), res, "-o", str(with_manifest)] + libs, check=True) subprocess.run([cc] + flags + [str(source), "-o", str(without)] + libs, check=True) mingw = os.environ.get("HOOKSHOT_MINGW", "") found = has_manifest(with_manifest, mingw) if found is not None: assert found, "the linked exe carries no RT_MANIFEST" assert not has_manifest(without, mingw), "the control build has a manifest of its own" if os.name != "nt": print("probe built for Windows%s; it runs on Windows only" % ("" if found is None else ", manifest linked in")) return for exe, mode in ((with_manifest, "utf8"), (without, "ansi")): copy = run_dir / exe.name shutil.copyfile(exe, copy) env = dict(os.environ, HOS_HEADLESS="1") r = subprocess.run([str(copy), mode, "Hookshot-DLSS ゼルダ Zélda"], capture_output=True, text=True, encoding="utf-8", errors="replace", env=env, cwd=str(run_dir)) print(r.stdout.strip()) assert r.returncode == 0, "%s %s: %s%s" % (exe.name, mode, r.stdout, r.stderr) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--cc", default="") args = parser.parse_args() check_manifest() check_build_links_it() check_start_up() check_no_ansi_code_page() check_probe(args.cc) print("PASS: Windows paths as UTF-8") if __name__ == "__main__": main()