diff --git a/recomp/patches-v111.txt b/recomp/patches-v111.txt index da74879b..8c33c433 100644 --- a/recomp/patches-v111.txt +++ b/recomp/patches-v111.txt @@ -93,9 +93,12 @@ site wide_movie main 3b5c758 a941a269 d100a3b0 bd400a10 bd401211 1e303a32 @page: # The game's zstd decoder (BSD zstd, statically linked into main) through the host's libzstd (runtime/ov_zstd.c): the # three calls of ZSTD_decompressMultiFrame (main+0xeb380: dctx, dst, capacity, src, size, dict, dict size, ddict) go to # the host function on the patch page (+0xff0). main+0xe910c is the file loader's call for a file that fits one read -# chunk. Until a check run has compared the two decoders on this build, the host function passes every call on to the -# game's decoder unless zstd_native = 1 is set (ov_zstd.c trusted). The loading threads' stream feed (main+0xf5c80, -# called at main+0xf5c2c) decodes most of the data and is not replaced yet. +# chunk. The loading threads decode most of the data through the game's stream feed (main+0xf5c80: state, src, size, +# ddict), called once per chunk read at main+0xf5c2c: that call goes to the host's stream decoder on the patch page +# (+0xf80), which takes a stream from its first chunk or leaves it to the game (ov_zstd.c host_stream). Until a check +# run has compared the two decoders on this build, the host functions pass every call on to the game's decoder unless +# zstd_native = 1 is set (ov_zstd.c trusted). site zstd main e910c 9400089d 9536dfb9 # bl ZSTD_decompressMultiFrame -> page+0xff0 site zstd main bf72e0 97d3d028 950aa744 # bl ZSTD_decompressMultiFrame -> page+0xff0 site zstd main f50164 97c66c87 94fd43a3 # bl ZSTD_decompressMultiFrame -> page+0xff0 +site zstd main f5c2c 94000015 9536acd5 # bl the stream feed -> page+0xf80 diff --git a/runtime/host.h b/runtime/host.h index 327f57dc..0d1feb45 100644 --- a/runtime/host.h +++ b/runtime/host.h @@ -370,6 +370,7 @@ int probe_window(void); // gfx_window_probe.c int probe_shader(void); // gfx_shader.c int probe_census_dir(char *dir, size_t n); // gfx_shader_probe.c: build//shaders for the probes int probe_vkmem(void), probe_vkexec(void), probe_astc(void); // tex_probe.c, vk_exec_probe.c, astc_probe.c +int probe_zstd(void); // ov_zstd.c int probe_movie(void); // movie_probe.c: host VP9 movie::Decoder on one RomFS movie int vk_pipeline_prepare_shaders(void); // vk_pipeline.c: --prepare-shaders (tools/prepare_shaders.py) int vk_pipeline_warm_shaders(void); // vk_pipeline.c: --warm-shaders (tools/build_game.py) diff --git a/runtime/main.c b/runtime/main.c index 29624cb3..e5935374 100644 --- a/runtime/main.c +++ b/runtime/main.c @@ -21,6 +21,7 @@ // --probe-gfx-abi NVN object layouts against nvn.h's reserved sizes; needs no game files // --probe-nvn, --probe-nvn-tex, --probe-window, --probe-shader, --probe-vkmem, --probe-vkexec graphics probes // --probe-astc the compute ASTC decoder against astc_decode_ldr; needs no game files +// --probe-zstd host zstd streams (ov_zstd.c) on frames made here; needs no game files // --probe-movie host VP9 movie::Decoder on one RomFS movie; skips without game data // --boot run the game to the first NVN call // --seconds S run the game (implies --boot's start-up) until the world clock (game time since the loaded @@ -492,7 +493,7 @@ static int usage(const char *argv0) { fprintf(stderr, "usage: %s [--exefs DIR] [--romfs DIR] [--profile DIR] [--audio-device VALUE] [--monitor VALUE] [--list-audio-devices | --list-monitors | --selftest | --probe-loader | " "--probe-cfg | --probe-dpi | --probe-mods | --probe-kernel | --probe-ipc | --probe-services | --probe-fs | --probe-gfx-abi | " "--probe-nvn | --probe-nvn-tex | --probe-window | --probe-shader | --probe-vkmem | " - "--probe-vkexec | --probe-astc | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | " + "--probe-vkexec | --probe-astc | --probe-zstd | --probe-movie | --probe-crash-call | --probe-crash-null | --probe-discord | " "--prepare-shaders | --warm-shaders | --boot |--seconds S [--shot PNG]]\n", argv0); return 2; } @@ -628,6 +629,7 @@ int main(int argc, char **argv) { if (!strcmp(mode, "--probe-vkmem")) return probe_vkmem(); if (!strcmp(mode, "--probe-vkexec")) return probe_vkexec(); if (!strcmp(mode, "--probe-astc")) return probe_astc(); + if (!strcmp(mode, "--probe-zstd")) return probe_zstd(); if (!strcmp(mode, "--probe-vkdevice")) return probe_vkdevice(); if (!strcmp(mode, "--probe-movie")) return probe_movie(); if (!strcmp(mode, "--prepare-shaders")) return vk_pipeline_prepare_shaders(); diff --git a/runtime/ov_zstd.c b/runtime/ov_zstd.c index 7febf17c..7b6d7e98 100644 --- a/runtime/ov_zstd.c +++ b/runtime/ov_zstd.c @@ -16,13 +16,15 @@ // ZSTD_decompressMultiFrame, main+0xeb380). Its sites have not been through a check run yet, and 1.4.3 has a caller // whose result the game relies on being the game decoder's own, so on that build the host decodes only when // zstd_native = 1 is set outright or the check runs (trusted); an unset key sends every call on to the game's decoder. -// Its loading threads decode most data through the game's stream feed (main+0xf5c80), which is not replaced yet. +// Its loading threads decode most data through the game's stream feed (main+0xf5c80): host_stream, further down. #include +#include #include #include #define ZSTD_STATIC_LINKING_ONLY #include +#include #include "cfg.h" #include "host.h" @@ -260,12 +262,476 @@ static void decode(Cpu *c, uint64_t dict, uint64_t dict_size, uint64_t ddict, ui count(size, r, kern_now_ns() - t0); } +// ---- the stream feed of Ocarina of Time 1.1.1 (main+0xf5c80), which decodes most of what the loading threads read +// The game's decode job (main+0xf5bf0) hands each chunk it read to feed(S, src, int size, ddict) and looks only at the +// result: 0 the stream is done, bit 31 set an error, anything else go on. The feed drives zstd's bufferless decoder: +// S+0x00 u8 phase: 1 begin, 2 running, 3 failed, 4 done +// S+0x08 u64 where the next output goes S+0x10 i32 output room left S+0x14 u32 content left +// S+0x18 u64 the DCtx workspace S+0x20 u64, +0x28 i32, +0x2c i32 carry buffer, its size, its fill +// S+0x30 u64 the DCtx (set at begin) +// Begin reads the frame header from the first chunk: a header that is not complete in it, a content size that is not +// in it, or a begin that fails gives phase 3 and 0xffffffff. Then each feed decodes what its input completes, moves +// +0x08, +0x10 and +0x14 by the bytes it wrote and returns +0x14; when the frame ends it sets phase 4. Input after the +// frame's end gives phase 3 and 0xfffffffd, a decode error phase 3 and 0xfffffffe (the output is too small) or +// 0xffffffff. A feed in phase 3 returns 0xffffffff and in phase 4 0xfffffffd. +// host_stream owns a stream from its begin or not at all: the guest DCtx never sees the blocks a host stream decoded, +// so there is no falling back half way. It takes a stream whose first chunk holds a whole header of a plain frame with +// its content size, whose output has room for all of that content and does not overlap the chunk, whose dictionary +// (if any) the host can take, and for which a slot is free; every other stream, and every feed of a stream it did not +// begin, goes to the game's feed. For its own streams it keeps the guest's fields and results as above, decoding with +// ZSTD_decompressStream straight into the game's output. That output is one buffer for the whole frame +// (ZSTD_d_stableOutBuffer): the decoder keeps no window of its own and reads earlier output where the game has it, +// and a feed that finds the output anywhere but where the last one left it fails the stream. +// The game's job stops feeding at a result of 0, which a frame gives once its content is out even when its last bytes +// (the checksum) are still to come: such a stream keeps its slot for those bytes but is marked spent, and a new stream +// takes a spent slot when no free one is left. +// With NX2NAVI_ZSTD_CHECK=1 the game's feed decodes every stream as before, the host decodes the same chunks into a +// buffer of its own, and the two outputs are compared when the game ends the stream or has all its content out +// ("zstd check: stream ..."). The check also reports a game output that did not advance through one buffer. +// --probe-zstd (probe_zstd, at the end) runs host streams on frames it makes itself. +#define GUEST_STREAM 0x70000f5c80ull +#define HOST_STREAM (RT_PATCH_PAGE + 0xf80) +#define NSTREAM 16 + +typedef struct { + uint64_t s; // the guest stream state's address; 0: free + ZSTD_DCtx *dctx; // stays with the slot from stream to stream + uint8_t *shadow; // check mode: the host's output + uint64_t content, pos; // the frame's size; bytes the host wrote + uint64_t dst0, cap0; // the game's output at the begin: every feed writes on into this one buffer + uint64_t in; // compressed bytes fed + int spent; // all the content is out: the game feeds no more, and the slot may be taken + int done, bad, moved; // check mode: the host reached the frame's end / failed; the game's output moved +} Stream; +static Stream streams[NSTREAM]; +static PlatMutex stream_lock = PLAT_MUTEX_INIT; +static struct { uint64_t streams, feeds, in, out, ns, guest, checked, differ; } SS; + +static uint32_t ld32(uint64_t a) { uint32_t v; memcpy(&v, gp(a), 4); return v; } +static uint64_t ld64(uint64_t a) { uint64_t v; memcpy(&v, gp(a), 8); return v; } +static void st32(uint64_t a, uint32_t v) { memcpy(gp(a), &v, 4); } +static void st64(uint64_t a, uint64_t v) { memcpy(gp(a), &v, 8); } + +static Stream *stream_find(uint64_t s) { + Stream *st = NULL; + plat_mutex_lock(&stream_lock); + for (int i = 0; i < NSTREAM && !st; i++) + if (streams[i].s == s) st = &streams[i]; + plat_mutex_unlock(&stream_lock); + return st; +} + +// The slot of s, or a free one for it, or one whose stream is spent; NULL when there is none (that stream decodes in +// the game). +static Stream *stream_take(uint64_t s) { + Stream *st = NULL; + plat_mutex_lock(&stream_lock); + for (int i = 0; i < NSTREAM && !st; i++) + if (streams[i].s == s) st = &streams[i]; + for (int i = 0; i < NSTREAM && !st; i++) + if (!streams[i].s) st = &streams[i]; + for (int i = 0; i < NSTREAM && !st; i++) + if (streams[i].spent) st = &streams[i]; + if (st) st->s = s, st->spent = 0; + plat_mutex_unlock(&stream_lock); + return st; +} + +static void stream_drop(Stream *st) { + if (!st) return; + free(st->shadow); + st->shadow = NULL; + plat_mutex_lock(&stream_lock); + st->s = 0, st->spent = 0; + plat_mutex_unlock(&stream_lock); +} + +static void stream_spent(Stream *st) { + plat_mutex_lock(&stream_lock); + st->spent = 1; + plat_mutex_unlock(&stream_lock); +} + +// A context for streams: the output is one buffer for the whole frame, so the decoder allocates no window (a window +// of its own would be as large as the frame's, and a frame may name one as large as its content), and it takes any +// window size, as the game's decoder does. +static ZSTD_DCtx *stream_dctx(void) { + ZSTD_DCtx *d = ZSTD_createDCtx(); + if (d && (ZSTD_isError(ZSTD_DCtx_setParameter(d, ZSTD_d_stableOutBuffer, 1)) || + ZSTD_isError(ZSTD_DCtx_setParameter(d, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_MAX)))) { + ZSTD_freeDCtx(d); + d = NULL; + } + return d; +} + +// Feeds in to the stream's context until the input is used up, the frame ends (0), an error, or nothing moves. Inside +// the thread's host depth and with the core given up for large chunks, as tracked_decode. +static size_t stream_run(Cpu *c, Stream *st, ZSTD_outBuffer *out, ZSTD_inBuffer *in) { + int *d = kern_host_track && c->thread ? kern_host_depth(c->thread) : NULL; + if (d) __atomic_store_n(d, *d + 1, __ATOMIC_RELAXED); + int away = in->size >= AWAY_BYTES ? kern_core_away() : 0; + if (away) __atomic_add_fetch(&S.away, 1, __ATOMIC_RELAXED); + size_t r = 1; + while (in->pos < in->size) { + size_t i0 = in->pos, o0 = out->pos; + r = ZSTD_decompressStream(st->dctx, out, in); + if (ZSTD_isError(r) || !r || (in->pos == i0 && out->pos == o0)) break; + } + kern_core_back(&away); + if (d) { + int n = *d - 1; + __atomic_store_n(d, n, __ATOMIC_RELAXED); + if (!n && kern_preempt_pending(c->thread)) kern_host_left(c->thread); + } + return r; +} + +static void stream_count(const Stream *st, uint64_t out) { + uint64_t n = __atomic_add_fetch(&SS.streams, 1, __ATOMIC_RELAXED); + __atomic_add_fetch(&SS.in, st->in, __ATOMIC_RELAXED); + __atomic_add_fetch(&SS.out, out, __ATOMIC_RELAXED); + if (n == 1 || !(n & 0x3ff)) + rt_log("zstd: %llu streams on the host in %llu feeds, %.1f MiB in, %.1f MiB out, %.0f ms; %llu streams left to " + "the game's decoder; check: %llu compared, %llu differ", (unsigned long long)n, + (unsigned long long)SS.feeds, SS.in / 1048576.0, SS.out / 1048576.0, SS.ns / 1e6, + (unsigned long long)SS.guest, (unsigned long long)SS.checked, (unsigned long long)SS.differ); +} + +// A stream at its begin (phase 1): its slot, the host context ready for the frame, or NULL when the game's feed is to +// decode it (see above). In check mode the guest's state is left alone. +static Stream *stream_begin(uint64_t s, uint64_t src, int32_t size, uint64_t ddict) { + if (size <= 0 || !src || src + (uint64_t)size > RT_ADDRESS_LIMIT) return NULL; + ZSTD_frameHeader h; + if (ZSTD_getFrameHeader(&h, gp(src), (size_t)size) != 0 || h.frameType != ZSTD_frame || !h.frameContentSize || + h.frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN || h.frameContentSize > 0x7fffffffull) + return NULL; + uint64_t dst = ld64(s + 8); + int32_t cap = (int32_t)ld32(s + 0x10); + // an output without room for the content fails part way in the game, with what fits written: left to the game + if (cap < 0 || (uint64_t)cap < h.frameContentSize || !dst || dst + (uint64_t)cap > RT_ADDRESS_LIMIT) return NULL; + if (src < dst + (uint64_t)cap && dst < src + (uint64_t)size) return NULL; // in place: the game's decoder + ZSTD_DDict *dd = NULL; + if (ddict) { + if (ddict >= RT_ADDRESS_LIMIT - 24) return NULL; + uint64_t f[3]; // ZSTD_DDict: dictBuffer, dictContent, dictSize + memcpy(f, gp(ddict), sizeof f); + if (!(dd = dict_for(ddict, f[1], f[2]))) return NULL; + } + Stream *st = stream_take(s); + if (!st) return NULL; + free(st->shadow); + st->shadow = NULL, st->content = h.frameContentSize, st->pos = 0, st->dst0 = dst, st->cap0 = (uint64_t)cap; + st->in = 0, st->done = st->bad = st->moved = 0; + if ((!st->dctx && !(st->dctx = stream_dctx())) || + ZSTD_isError(ZSTD_DCtx_reset(st->dctx, ZSTD_reset_session_only)) || + ZSTD_isError(ZSTD_DCtx_refDDict(st->dctx, dd)) || + (check_mode() && !(st->shadow = malloc((size_t)st->content)))) { + stream_drop(st); + return NULL; + } + if (!check_mode()) { // what the game's begin leaves in its state + st32(s + 0x14, (uint32_t)st->content); + st64(s + 0x30, ld64(s + 0x18)); + *(uint8_t *)gp(s) = 2; + } + return st; +} + +// A feed of a host stream: the guest's fields and result as the game's feed leaves them. +static void stream_feed(Cpu *c, Stream *st, uint64_t s, uint64_t src, int32_t size) { + if (size <= 0) { // nothing to feed: the content left, as the game + c->x[0] = ld32(s + 0x14); + return; + } + uint64_t t0 = kern_now_ns(), dst = ld64(s + 8); + uint32_t cap = ld32(s + 0x10), ret; + if (!src || src + (uint64_t)size > RT_ADDRESS_LIMIT || dst != st->dst0 + st->pos || cap != st->cap0 - st->pos) { + static int logged; + if (logged++ < 8) + rt_log("zstd: a host stream cannot go on after %llu bytes out: input %#llx + %d, output at %#llx with room " + "for %u where %#llx with %llu was left", (unsigned long long)st->pos, (unsigned long long)src, size, + (unsigned long long)dst, cap, (unsigned long long)(st->dst0 + st->pos), + (unsigned long long)(st->cap0 - st->pos)); + *(uint8_t *)gp(s) = 3; + c->x[0] = 0xffffffffu; + stream_drop(st); + return; + } + ZSTD_inBuffer in = {gp(src), (size_t)size, 0}; + ZSTD_outBuffer out = {gp(st->dst0), (size_t)st->cap0, (size_t)st->pos}; + size_t r = stream_run(c, st, &out, &in); + uint32_t wrote = (uint32_t)(out.pos - st->pos); + st->pos = out.pos, st->in += in.pos; + st64(s + 8, dst + wrote); + st32(s + 0x10, cap - wrote); + st32(s + 0x14, ld32(s + 0x14) - wrote); + __atomic_add_fetch(&SS.feeds, 1, __ATOMIC_RELAXED); + __atomic_add_fetch(&SS.ns, kern_now_ns() - t0, __ATOMIC_RELAXED); + int end = 1; + if (ZSTD_isError(r)) { + *(uint8_t *)gp(s) = 3; + ret = ZSTD_getErrorCode(r) == ZSTD_error_dstSize_tooSmall ? 0xfffffffeu : 0xffffffffu; + static int logged; + if (logged++ < 8) + rt_log("zstd: a host stream failed after %llu bytes in: %s", (unsigned long long)st->in, ZSTD_getErrorName(r)); + } else if (!r && in.pos < in.size) { // input after the frame's end + *(uint8_t *)gp(s) = 3; + ret = 0xfffffffdu; + } else if (!r) { + *(uint8_t *)gp(s) = 4; + ret = ld32(s + 0x14); + } else if (in.pos < in.size) { // nothing moved with input left + *(uint8_t *)gp(s) = 3; + ret = 0xfffffffeu; + } else { + ret = ld32(s + 0x14); + end = 0; + if (!ret) stream_spent(st); + } + c->x[0] = ret; + if (end) { + stream_count(st, st->pos); + stream_drop(st); + } +} + +// Check mode: the host decodes the chunk into its own buffer, the game's feed runs as always and its result stands; +// when the game ends the stream, or has all its content out, the two outputs are compared. +static void stream_check(Cpu *c, Stream *st, uint64_t s, uint64_t src, int32_t size, uint64_t ddict) { + if (st->spent) { // compared already: the game was not expected to feed on after a result of 0 + static int logged; + if (logged++ < 4) + rt_log("zstd check: a feed of %d bytes after the content of a stream was out (content %llu, lr %#llx)", size, + (unsigned long long)st->content, (unsigned long long)c->x[30]); + rt_call(c, GUEST_STREAM, 4, s, src, (uint64_t)(uint32_t)size, ddict); + uint8_t ph = *(const uint8_t *)gp(s); + if (ph == 3 || ph == 4) stream_drop(st); + return; + } + if (!st->bad && !st->done && size > 0 && src && src + (uint64_t)size <= RT_ADDRESS_LIMIT) { + ZSTD_inBuffer in = {gp(src), (size_t)size, 0}; + ZSTD_outBuffer out = {st->shadow, (size_t)st->content, (size_t)st->pos}; + size_t r = stream_run(c, st, &out, &in); + st->pos = out.pos, st->in += in.pos; + if (ZSTD_isError(r)) st->bad = 1; + else if (!r) st->done = in.pos == in.size ? 1 : 2; // 2: input after the frame's end + else if (in.pos < in.size) st->bad = 1; + } + uint32_t g = (uint32_t)rt_call(c, GUEST_STREAM, 4, s, src, (uint64_t)(uint32_t)size, ddict); + uint8_t ph = *(const uint8_t *)gp(s); + // a host stream writes into the buffer the stream began with: the game's output has to advance through it alone + if ((ph == 2 || ph == 4) && ld64(s + 8) - st->dst0 != st->content - ld32(s + 0x14)) st->moved = 1; + int out_all = ph == 2 && !g; // the job feeds no more, with or without the frame's last bytes + if (ph != 3 && ph != 4 && !out_all) return; + __atomic_add_fetch(&SS.checked, 1, __ATOMIC_RELAXED); + int same = ph == 3 ? st->bad || st->done != 1 // the game failed: the host must not have a clean frame + : !st->bad && !st->moved && st->done != 2 && (ph != 4 || st->done == 1) && + st->pos == st->content && !memcmp(st->shadow, gp(st->dst0), (size_t)st->content); + if (!same) { + __atomic_add_fetch(&SS.differ, 1, __ATOMIC_RELAXED); + static int logged; + if (logged++ < 12) + rt_log("zstd check: stream MISMATCH: the game ended in phase %d with %#x after %llu bytes in (content %llu, " + "ddict %#llx, lr %#llx)%s; the host wrote %llu bytes, %s", (int)ph, g, (unsigned long long)st->in, + (unsigned long long)st->content, (unsigned long long)ddict, (unsigned long long)c->x[30], + st->moved ? ", its output not advancing through one buffer" : "", (unsigned long long)st->pos, + st->bad ? "failed" : st->done == 1 ? "reached the frame's end" + : st->done ? "found input after the frame's end" : "had not reached the frame's end"); + } + stream_count(st, st->pos); + if (out_all) { // keeps the slot in case the game does feed on; nothing more is compared + free(st->shadow); + st->shadow = NULL; + stream_spent(st); + } else { + stream_drop(st); + } +} + +static void host_stream(Cpu *c) { + uint64_t s = c->x[0], src = c->x[1], ddict = c->x[3]; + int32_t size = (int32_t)c->x[2]; + Stream *st = NULL; + if (trusted() && s && s < RT_ADDRESS_LIMIT - 0x38) { + uint8_t ph = *(const uint8_t *)gp(s); + if (ph == 1) { + stream_drop(stream_find(s)); // a stream the game gave up without ending it + st = stream_begin(s, src, size, ddict); + if (!st) __atomic_add_fetch(&SS.guest, 1, __ATOMIC_RELAXED); + } else if (ph == 2) { + st = stream_find(s); + } + } + if (!st) rt_call(c, GUEST_STREAM, 4, s, src, c->x[2], ddict); + else if (check_mode()) stream_check(c, st, s, src, size, ddict); + else stream_feed(c, st, s, src, size); +} + static void host_multiframe(Cpu *c) { decode(c, c->x[5], c->x[6], c->x[7], GUEST_MULTIFRAME, 8); } static void host_using_ddict(Cpu *c) { decode(c, 0, 0, c->x[5], GUEST_USING_DDICT, 6); } __attribute__((constructor)) static void zstd_register(void) { rt_register_host(HOST_MULTIFRAME, host_multiframe); rt_register_host(HOST_USING_DDICT, host_using_ddict); + rt_register_host(HOST_STREAM, host_stream); +} + +// ---- --probe-zstd: host streams on frames made here, fed through stream_begin and stream_feed with a stream state +// laid out as the game's. Checks the output and the state's fields after every feed, the frame's end, the streams that +// are left to the game, input after the end, a damaged frame, a moved output and the slots. Needs no game files. +static int zchecks, zfails; +#define ZCHECK(cond, ...) do { zchecks++; if (!(cond)) { zfails++; printf("FAIL: " __VA_ARGS__); printf("\n"); } } while (0) + +static void probe_state(uint64_t s, uint64_t dst, uint32_t cap, uint64_t ws) { + memset(gp(s), 0, 0x38); + *(uint8_t *)gp(s) = 1; + st64(s + 8, dst), st32(s + 0x10, cap), st64(s + 0x18, ws); +} + +static int probe_phase(uint64_t s) { return *(const uint8_t *)gp(s); } + +// One feed as host_stream runs it outside check mode: the result, or 1 << 32 when the stream is the game's. +static uint64_t probe_feed(uint64_t s, uint64_t src, int32_t size) { + static Cpu c; + Stream *st = NULL; + if (probe_phase(s) == 1) stream_drop(stream_find(s)), st = stream_begin(s, src, size, 0); + else if (probe_phase(s) == 2) st = stream_find(s); + if (!st) return 1ull << 32; + stream_feed(&c, st, s, src, size); + return c.x[0]; +} + +// Feeds frame[0, n) in chunks of `chunk` bytes after a first one of `first`. +static void probe_stream(const char *what, uint64_t s, uint64_t ws, uint64_t frame, size_t n, uint64_t out, + const uint8_t *raw, size_t rawn, size_t first, size_t chunk) { + uint32_t cap = (uint32_t)rawn + 4096; + probe_state(s, out, cap, ws); + memset(gp(out), 0xA5, rawn); + size_t off = 0, feeds = 0; + uint64_t r = 1; + while (off < n) { + size_t k = off ? chunk : first; + if (k > n - off) k = n - off; + r = probe_feed(s, frame + off, (int32_t)k); + off += k, feeds++; + uint64_t wrote = ld64(s + 8) - out; + int ph = probe_phase(s); + if (r >> 32 || wrote > rawn || ld32(s + 0x10) != cap - wrote || ld32(s + 0x14) != rawn - wrote || + r != rawn - wrote || (ph != 2 && ph != 4) || (ph == 4) != (off == n)) { + ZCHECK(0, "%s: the feed ending at %zu of %zu gave %#llx in phase %d with %llu bytes out, room %u, left %u", + what, off, n, (unsigned long long)r, ph, (unsigned long long)wrote, ld32(s + 0x10), ld32(s + 0x14)); + stream_drop(stream_find(s)); + return; + } + } + ZCHECK(r == 0 && probe_phase(s) == 4, "%s: ended with %#llx in phase %d", what, (unsigned long long)r, probe_phase(s)); + ZCHECK(ld64(s + 0x30) == ws, "%s: the state's DCtx field is not its workspace", what); + ZCHECK(!memcmp(gp(out), raw, rawn), "%s: the output differs from the source", what); + ZCHECK(!stream_find(s), "%s: the slot is still taken after the frame's end", what); + printf("zstd stream: %s: %zu bytes in %zu feeds gave %zu\n", what, n, feeds, rawn); +} + +int probe_zstd(void) { + enum { RAWN = 3 << 20, SMALLN = 100000, ARENA = 12 << 20 }; + const RtGuestLayout *g = mem_layout(); + uint64_t base = g->alias.base + 0x10000; + if (mem_map_physical(base, ARENA)) { + printf("probe-zstd: FAILED (no guest memory)\n"); + return 1; + } + uint64_t compa = base + (3 << 20), outa = base + (7 << 20), s = base + (11 << 20), ws = s + 0x10000; + uint8_t *raw = gp(base); + // random runs and repeats of earlier bytes from up to 2 MiB back: matches that reach far into earlier output + uint64_t x = 0x9E3779B97F4A7C15ull; + for (size_t i = 0; i < RAWN; i += 64) { + x ^= x << 13, x ^= x >> 7, x ^= x << 17; + if (i >= 4096 && (x & 3)) { + size_t far = i < (2u << 20) ? i : 2u << 20; + memcpy(raw + i, raw + i - 64 - (size_t)((x >> 8) % (far - 63)), 64); + } else { + for (int k = 0; k < 64; k += 8) { + x ^= x << 13, x ^= x >> 7, x ^= x << 17; + memcpy(raw + i + k, &x, 8); + } + } + } + size_t cn = ZSTD_compress(gp(compa), (size_t)3 << 20, raw, RAWN, 3); + ZSTD_CCtx *cc = ZSTD_createCCtx(); + uint64_t smalla = compa + (3 << 20), bada = smalla + (256 << 10); + size_t sn = cc ? ZSTD_CCtx_setParameter(cc, ZSTD_c_checksumFlag, 1), ZSTD_compress2(cc, gp(smalla), 256 << 10, raw, SMALLN) + : (size_t)-1; + ZSTD_freeCCtx(cc); + if (ZSTD_isError(cn) || ZSTD_isError(sn) || sn < 2048) { + printf("probe-zstd: FAILED (the frames could not be made)\n"); + return 1; + } + + probe_stream("a 3 MiB frame in chunks of 64 KiB", s, ws, compa, cn, outa, raw, RAWN, 65536, 65536); + probe_stream("the same in chunks of 4099 bytes", s, ws, compa, cn, outa, raw, RAWN, 64, 4099); + probe_stream("the same in one feed", s, ws, compa, cn, outa, raw, RAWN, cn, cn); + probe_stream("a frame with a checksum in chunks of 977 bytes", s, ws, smalla, sn, outa, raw, SMALLN, 32, 977); + probe_stream("the same with the checksum in a feed of its own", s, ws, smalla, sn, outa, raw, SMALLN, sn - 4, 4); + + probe_state(s, outa, RAWN + 4096, ws); + ZCHECK(probe_feed(s, compa, 3) >> 32 && probe_phase(s) == 1, "a first chunk without the whole header was taken"); + probe_state(s, outa, RAWN - 1, ws); + ZCHECK(probe_feed(s, compa, 65536) >> 32 && probe_phase(s) == 1, "an output smaller than the content was taken"); + probe_state(s, compa + 4096, RAWN + 4096, ws); + ZCHECK(probe_feed(s, compa, 65536) >> 32 && probe_phase(s) == 1, "an output that overlaps its input was taken"); + printf("zstd stream: a short first chunk, a small output and an overlapping one are left to the game\n"); + + probe_state(s, outa, SMALLN + 4096, ws); + uint64_t r = probe_feed(s, smalla, (int32_t)sn + 8); + ZCHECK(r == 0xfffffffdu && probe_phase(s) == 3 && !stream_find(s), "input after the frame's end gave %#llx in phase %d", + (unsigned long long)r, probe_phase(s)); + + memcpy(gp(bada), gp(smalla), sn); + ((uint8_t *)gp(bada))[sn / 2] ^= 0x55; + probe_state(s, outa, SMALLN + 4096, ws); + r = 0; + for (size_t off = 0; off < sn && r < 0x80000000u; off += 977) + r = probe_feed(s, bada + off, (int32_t)(sn - off < 977 ? sn - off : 977)); + ZCHECK((r == 0xffffffffu || r == 0xfffffffeu) && probe_phase(s) == 3 && !stream_find(s), + "a damaged frame gave %#llx in phase %d", (unsigned long long)r, probe_phase(s)); + + probe_state(s, outa, SMALLN + 4096, ws); + r = probe_feed(s, smalla, 977); + st64(s + 8, ld64(s + 8) + 16); + r = probe_feed(s, smalla + 977, 977); + ZCHECK(r == 0xffffffffu && probe_phase(s) == 3 && !stream_find(s), "a feed with the output moved gave %#llx in phase %d", + (unsigned long long)r, probe_phase(s)); + printf("zstd stream: input after the end, a damaged frame and a moved output fail the stream\n"); + + // a stream with its content out and its checksum unread is spent: the next stream takes its slot when none is free + probe_state(s, outa, SMALLN + 4096, ws); + r = probe_feed(s, smalla, (int32_t)sn - 4); + ZCHECK(r == 0 && probe_phase(s) == 2 && stream_find(s), "content out before the checksum gave %#llx in phase %d", + (unsigned long long)r, probe_phase(s)); + int took = 0; + for (int i = 1; i <= NSTREAM; i++) { + probe_state(s + 0x40 * i, outa, SMALLN + 4096, ws); + took += !(probe_feed(s + 0x40 * i, smalla, 64) >> 32); + } + ZCHECK(took == NSTREAM && !stream_find(s), "%d of %d streams begun beside a spent one", took, NSTREAM); + probe_state(s + 0x40 * (NSTREAM + 1), outa, SMALLN + 4096, ws); + ZCHECK(probe_feed(s + 0x40 * (NSTREAM + 1), smalla, 64) >> 32, "a stream was taken with every slot in use"); + for (int i = 1; i <= NSTREAM; i++) stream_drop(stream_find(s + 0x40 * i)); + printf("zstd stream: %d streams at once, one of them in a spent slot; one more is left to the game\n", NSTREAM); + + if (zfails) { + printf("probe-zstd: FAILED (%d of %d checks)\n", zfails, zchecks); + return 1; + } + printf("probe-zstd: passed (%d checks)\n", zchecks); + return 0; +} + +#else + +int probe_zstd(void) { + printf("probe-zstd: this build has no patch page, so no host decoder\n"); + return 0; } #endif