diff --git a/docs/releases/v0.7.0.md b/docs/releases/v0.7.0.md index 3b5c000e..37899e08 100644 --- a/docs/releases/v0.7.0.md +++ b/docs/releases/v0.7.0.md @@ -81,6 +81,18 @@ cards). load they are now made at once, before the area fades in. - The performance overlay's VRAM line counted every program on the graphics card as if it were the game. It now shows the game's own share beside it (*game 6.8 GB* of 10.4 GB in use, in one measurement). +- The performance overlay's frame rate, lows and frame-time graph are now those of the pictures as they reach the + screen. They were measured at the game's present calls, which come in uneven steps although the pictures do not: in + a village the calls came 36 and 9 ms apart in turn and the graph drew a sawtooth with a 1% low of 14 fps, while the + pictures went out about 20 ms apart and changed by 4 ms from one to the next. A picture that arrives early now also + waits until seven eighths of the recent interval have passed (`pace_smooth`, on by default; not with frame + generation), which brought that change down to 3 ms at the same frame rate. +- *GPU frame prep on its own thread* (`replay_async`, off by default): the shader constants of a frame are now + copied when the game hands the frame over, not when the other thread gets to it up to a frame later, by which time + the game had sometimes reused that memory (a shadow pass got another block's values about every 364 frames in one + graveyard). The option can still show a dark or flickering frame: vertex, storage and index data are read late in + the same way, and this game rewrites them without waiting for the GPU. The menu says so now. It gave 15 to 18 + percent more frames per second in a graveyard and a village. - Textures that turned dark or blank for a moment: textures out of view were dropped after 11 seconds at 165 fps (30 seconds were meant) and came back as a stand-in for a few frames. The wait is two minutes of real time at any frame rate now, and the scene's textures are kept through a pause. diff --git a/nx2navi.ini.example b/nx2navi.ini.example index 3314b8d9..9b8ac162 100644 --- a/nx2navi.ini.example +++ b/nx2navi.ini.example @@ -54,7 +54,9 @@ # without the interrupt, 0 = off (the old scheduling). # async_preempt = 1 # Replay NVN submissions on their own thread, at most one frame behind the game, 0 or 1. Default: off. Helps where the -# game waits for the GPU; adds up to a frame of latency. +# game waits for the GPU and in heavy scenes (15 to 18 percent more frames per second in a village); adds up to a frame +# of latency, and can show a dark or flickering frame now and then: the other thread reads vertex and buffer memory +# the game has sometimes rewritten by then. # replay_async = 0 # EXPERIMENTAL, THE GAME CRASHES WITH 1. 1 lifts the virtual core gate: guest threads run without the console's limit # of one thread per core, and the game keeps per-core data without locks. With 1 the game crashes or hangs, diff --git a/runtime/cfg.c b/runtime/cfg.c index 52af67b1..cb9ee2c4 100644 --- a/runtime/cfg.c +++ b/runtime/cfg.c @@ -47,7 +47,7 @@ static const char *const known[] = { "HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE", "HDR_LIVE", "HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD", - "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", + "BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST", "MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE", "MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK", "MV_SURVEY", @@ -114,7 +114,7 @@ static const struct { const char *key, *section; } sections[] = { {"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"}, {"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"}, {"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"}, - {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PIPE_PREBUILD", "performance"}, + {"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"}, {"COMPUTE_KEEP", "performance"}, {"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"}, {"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"}, @@ -216,6 +216,7 @@ static const CfgMeta metas[] = { {"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART}, {"PIPE_LINK_MS", "6", "0..100", RESTART}, // vk_exec_draw.c link_budget {"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold + {"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth {"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART}, {"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold {"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART}, diff --git a/runtime/perfmon.c b/runtime/perfmon.c index 0c1065ba..63d141ef 100644 --- a/runtime/perfmon.c +++ b/runtime/perfmon.c @@ -43,9 +43,42 @@ static struct { int64_t up0, up; } g_win; +// The frame times the overlay shows (the rate, the lows, the graph) are those of the pictures as they go out: the +// present thread reports each one (perfmon_shown). The game's present calls come at other times: with the replay on +// the submitting thread, work held for an event is replayed in the next frame's first submission, so the calls came +// 36 and 9 ms apart in turn in a village while the pictures went out every 17 to 27 ms, and the graph drew a sawtooth +// the screen did not show. A frame held back (vk_backend.c frame_hold) is not shown and lengthens its interval. +// Without a present thread reporting (another back end) the game's present calls are taken, as before. +static uint64_t g_shown_last; // when the last picture went out; 0: none yet +static uint64_t g_shown_report; // when perfmon_shown last ran: perfmon_frame leaves the intervals to it for a second + +static void interval_add(uint64_t now, uint64_t interval_ns) { // (lock held) + g_ring[g_ring_pos] = (float)(interval_ns / 1e6); + g_ring_at[g_ring_pos] = now; + g_ring_pos = (g_ring_pos + 1) % PERF_HISTORY; + if (g_ring_n < PERF_HISTORY) g_ring_n++; + if (now - g_low_t0 >= 1000000000ull) { // the next second: its slice starts empty + g_low_cur = (g_low_cur + 1) % LOW_SLICES; + memset(g_low_hist[g_low_cur], 0, sizeof g_low_hist[0]); + g_low_t0 = now; + } + uint64_t bin = interval_ns / 250000ull; + uint16_t *h = &g_low_hist[g_low_cur][bin < LOW_BINS ? bin : LOW_BINS - 1]; + if (*h < 65535) (*h)++; +} + +void perfmon_shown(uint64_t now) { + plat_mutex_lock(&g_lock); + // a gap of a second or more is a load or a pause, not a frame time + if (g_shown_last && now > g_shown_last && now - g_shown_last < 1000000000ull) interval_add(now, now - g_shown_last); + g_shown_last = g_shown_report = now; + plat_mutex_unlock(&g_lock); +} + void perfmon_frame(uint64_t now, uint64_t interval_ns, const uint64_t *wait_ns, uint64_t replay_ns, int64_t tex_uploads) { plat_mutex_lock(&g_lock); + if (g_shown_report && now - g_shown_report < 1000000000ull) interval_ns = 0; // perfmon_shown has them if (interval_ns) { g_ring[g_ring_pos] = (float)(interval_ns / 1e6); g_ring_at[g_ring_pos] = now; diff --git a/runtime/perfmon.h b/runtime/perfmon.h index fd0ac1bb..bb56709b 100644 --- a/runtime/perfmon.h +++ b/runtime/perfmon.h @@ -50,6 +50,8 @@ void perfmon_set_vram_game(float (*used_mb)(void)); // A game frame was presented (perf_present, the render thread, only while the overlay is on): the time since the last // one, the render thread's waits of this frame (PERF_N entries, host.h), its replay time and the running count of // texture uploads. +// A picture went out (the present thread, vk_backend.c): the overlay's frame times are the intervals between these. +void perfmon_shown(uint64_t now_ns); void perfmon_frame(uint64_t now_ns, uint64_t interval_ns, const uint64_t *wait_ns, uint64_t replay_ns, int64_t tex_uploads); // The latest numbers (the overlay, present path). diff --git a/runtime/ui_text.c b/runtime/ui_text.c index 3103ce42..d8721159 100644 --- a/runtime/ui_text.c +++ b/runtime/ui_text.c @@ -401,7 +401,7 @@ static const char *const text_en[UI_TEXT_COUNT] = { [UI_MENU_ASYNC_PREEMPT_DESC] = "Lets a game thread interrupt another one on the same virtual core.", [UI_MENU_PREEMPT_OFF] = "Off", [UI_MENU_PREEMPT_ON] = "On", [UI_MENU_PREEMPT_INHERIT] = "Inheritance only", [UI_MENU_REPLAY_ASYNC] = "GPU frame prep on its own thread", - [UI_MENU_REPLAY_ASYNC_DESC] = "Prepares frames for the GPU on a separate thread instead of the game thread. Adds up to one frame of latency.", + [UI_MENU_REPLAY_ASYNC_DESC] = "Prepares frames for the GPU on a separate thread instead of the game thread. Faster in heavy scenes, but can show a dark or flickering frame now and then, and adds up to one frame of latency.", [UI_MENU_TIMER_1MS] = "1 ms Windows timer", [UI_MENU_TIMER_1MS_DESC] = "Raises the system timer resolution to 1 ms for steadier frame pacing.", [UI_MENU_PIPE_WAIT] = "Waiting draw handling", @@ -949,7 +949,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = { [UI_MENU_ASYNC_PREEMPT_DESC] = "Позволяет потоку игры прервать другой поток на том же виртуальном ядре.", [UI_MENU_PREEMPT_OFF] = "Выкл.", [UI_MENU_PREEMPT_ON] = "Вкл.", [UI_MENU_PREEMPT_INHERIT] = "Только наследование", [UI_MENU_REPLAY_ASYNC] = "Подготовка кадра для GPU в отдельном потоке", - [UI_MENU_REPLAY_ASYNC_DESC] = "Кадр для GPU готовится в отдельном потоке, а не в потоке игры. Добавляет до одного кадра задержки.", + [UI_MENU_REPLAY_ASYNC_DESC] = "Кадр для GPU готовится в отдельном потоке, а не в потоке игры. Быстрее в тяжёлых сценах, но иногда может показать тёмный или мерцающий кадр и добавляет до одного кадра задержки.", [UI_MENU_TIMER_1MS] = "Таймер Windows 1 мс", [UI_MENU_TIMER_1MS_DESC] = "Повышает точность системного таймера до 1 мс для ровного темпа кадров.", [UI_MENU_PIPE_WAIT] = "Обработка ожидающих отрисовок", @@ -1338,7 +1338,7 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = { [UI_MENU_ASYNC_PREEMPT_DESC] = "Deixa uma thread do jogo interromper outra no mesmo núcleo virtual.", [UI_MENU_PREEMPT_OFF] = "Desligado", [UI_MENU_PREEMPT_ON] = "Ligado", [UI_MENU_PREEMPT_INHERIT] = "Só herança", [UI_MENU_REPLAY_ASYNC] = "Preparo de quadro da GPU em thread própria", - [UI_MENU_REPLAY_ASYNC_DESC] = "Prepara os quadros para a GPU numa thread separada em vez da thread do jogo. Adiciona até um quadro de latência.", + [UI_MENU_REPLAY_ASYNC_DESC] = "Prepara os quadros para a GPU numa thread separada em vez da thread do jogo. Mais rápido em cenas pesadas, mas pode mostrar um quadro escuro ou piscando de vez em quando, e adiciona até um quadro de latência.", [UI_MENU_TIMER_1MS] = "Timer do Windows de 1 ms", [UI_MENU_TIMER_1MS_DESC] = "Aumenta a resolução do timer do sistema para 1 ms, para um ritmo de quadros mais estável.", [UI_MENU_PIPE_WAIT] = "Tratamento de desenhos em espera", @@ -1979,7 +1979,7 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = { [UI_MENU_PREEMPT_ON] = "开", [UI_MENU_PREEMPT_INHERIT] = "仅继承", [UI_MENU_REPLAY_ASYNC] = "GPU 帧准备独立线程", - [UI_MENU_REPLAY_ASYNC_DESC] = "在单独的线程而不是游戏线程上为 GPU 准备帧。" + [UI_MENU_REPLAY_ASYNC_DESC] = "在单独的线程而不是游戏线程上为 GPU 准备帧。在复杂场景中更快,但偶尔可能出现变暗或闪烁的画面。" "最多增加一帧延迟。", [UI_MENU_TIMER_1MS] = "1 ms Windows 计时器", [UI_MENU_TIMER_1MS_DESC] = "将系统计时器精度提高到 1 ms,使帧间隔更平稳。", diff --git a/runtime/vk.h b/runtime/vk.h index daab4870..09a1c61a 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -297,6 +297,9 @@ void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size); // Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before // it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch. void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence); +// NX2NAVI_REPLAY_ASYNC: the submitting thread takes the copy before it queues the submission; vk_usnap_begin on the +// replay thread then uses it. +void vk_usnap_take(const uint64_t *handles, int n); void vk_usnap_end(uint64_t fence); int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out); const uint8_t *vk_usnap_map(void); // a host copy of the current arena, at the offsets vk_usnap_find gives diff --git a/runtime/vk_backend.c b/runtime/vk_backend.c index 91c7627d..37d6bec7 100644 --- a/runtime/vk_backend.c +++ b/runtime/vk_backend.c @@ -32,6 +32,7 @@ #include "platform/platform.h" #include "vk_dlaa.h" #include "vk_dlssg.h" +#include "vk_fg.h" #define WEAK __attribute__((weak)) @@ -49,6 +50,28 @@ static void wait_until(uint64_t at) { while (kern_now_ns() < at) plat_yield(); } +// NX2NAVI_PACE_SMOOTH (default 1; 0 = off): a picture goes out no sooner than seven eighths of the recent interval +// after the one before it. The game's frames reach the present thread in uneven steps although each costs about the +// same: with the replay on the submitting thread, work held for an event is replayed with the next frame, so pictures +// came 13 to 31 ms apart around a mean of 22 ms in a village (5th to 95th percentile), which shows as judder at a +// frame rate that would look even. Only the early ones wait, a few milliseconds each, so the rate stays what it was; +// the recent interval is a moving average over about four pictures of intervals under 100 ms, so the floor follows +// a scene that gets lighter within a second. Not with frame generation, which paces its own presents. +static uint64_t pace_avg_ns; // the present thread's + +static uint64_t pace_smooth(uint64_t at, uint64_t last, uint64_t now) { + static int on = -1; + if (on < 0) on = cfg_bool("NX2NAVI_PACE_SMOOTH"); + if (!on || !last || !pace_avg_ns || vk_fg_active() || vk_dlssg_active()) return at; + uint64_t floor = last + pace_avg_ns / 8 * 7; + return floor > at && floor < now + 50000000ull ? floor : at; +} + +static void pace_shown(uint64_t interval_ns) { + if (interval_ns >= 100000000ull) return; // a load or a hitch is not the rate + pace_avg_ns = pace_avg_ns ? pace_avg_ns + (uint64_t)(((int64_t)interval_ns - (int64_t)pace_avg_ns) / 4) : interval_ns; +} + static void present_main(void *arg) { (void)arg; uint64_t last = 0, last_queued = 0; @@ -72,9 +95,11 @@ static void present_main(void *arg) { plat_cond_broadcast(&pcv); plat_mutex_unlock(&pl); uint64_t t0 = kern_now_ns(); - wait_until(j.at); + wait_until(pace_smooth(j.at, last, t0)); uint64_t t1 = kern_now_ns(); + if (last) pace_shown(t1 - last); vk_present_flip(); + perfmon_shown(t1); // the overlay's frame times: the pictures as shown // NX2NAVI_TRACE=present: one line per present: the host present's interval, the interval of the game's present // calls (when the host present went out before back-edge pacing), the wait and how far past its target it went if (rt_trace & RT_TRACE_PRESENT) @@ -588,6 +613,7 @@ static uint64_t vk_be_submit(const uint64_t *handles, int n) { if (!it.h) rt_fatal("vk: out of memory (replay queue)"); memcpy(it.h, handles, sizeof *it.h * (size_t)n); } + vk_usnap_take(handles, n); // the uniform blocks as they are now: the game writes on once this returns return replay_enqueue(&it); } static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); } diff --git a/runtime/vk_usnap.c b/runtime/vk_usnap.c index b83544c8..fadf70cd 100644 --- a/runtime/vk_usnap.c +++ b/runtime/vk_usnap.c @@ -12,8 +12,12 @@ // - A GPU write within the submission (UpdateUniformBuffer, a storage buffer a shader writes, a buffer copy or fill, // an image-to-buffer copy: vk_usnap_drop) drops the copies it touches from then on, so later draws read the result. // - The arena goes back to a free list once the submission's fence is done (completion thread). -// The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later -// with NX2NAVI_REPLAY_ASYNC. NX2NAVI_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (NX2NAVI_AB). +// The copy is taken at the submit call: where vk_exec_record starts when the replay runs on the submitting thread, and +// by the submitting thread itself with NX2NAVI_REPLAY_ASYNC (vk_usnap_take), where the record starts up to a frame +// later on the replay thread. Taken only there, it held whatever the game had put into those bytes by then: in a +// graveyard the block of the shadow pass came out with another block's values in the frame its allocator moved on to +// the next stretch of memory, about every 364 frames, and that frame's sunlight was all in shadow (a dark frame with +// the interface intact). NX2NAVI_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (NX2NAVI_AB). #include #include @@ -60,6 +64,20 @@ static struct { static Arena *free_arenas; static PlatMutex free_lock = PLAT_MUTEX_INIT; +// U.col, sort_col's buffer and collect's neighbour cache: the submitting thread (vk_usnap_take) and the replay thread +// (vk_usnap_begin, for a submission nothing was taken for) both collect. +static PlatMutex gather_lock = PLAT_MUTEX_INIT; + +// A copy taken at the submit call for the replay thread (vk_usnap_take), in submission order. +typedef struct Taken { + struct Taken *next; + Arena *arena; + Reg *reg; + int nreg, n; + uint64_t first, total, refused, ns; // first, n: the submission's first handle and handle count +} Taken; +static Taken *tq_head, *tq_tail; +static PlatMutex tq_lock = PLAT_MUTEX_INIT; static void *grow(void *p, int *cap, size_t elem) { int n = *cap ? *cap * 2 : 1024; @@ -245,6 +263,94 @@ static void summary(void) { U.subs = U.off_subs = U.regions = U.bytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0; } +// The bytes of regions reg[0..nreg) (total bytes in all) into a's device copy, through its host copy with DLSS. +static void copy_regions(Arena *a, const Reg *reg, int nreg, uint64_t total) { + uint8_t *dst = a->map; + if (vk_dlaa_available()) { + if (!a->cpu && !(a->cpu = malloc(a->size))) rt_fatal("vk_usnap: out of memory"); + dst = a->cpu; + } + for (int i = 0; i < nreg; i++) memcpy(dst + reg[i].off, (const void *)(uintptr_t)reg[i].lo, reg[i].hi - reg[i].lo); + if (dst != a->map) memcpy(a->map, dst, total); // one sequential write into the device copy + vmaFlushAllocation(vk_ctx()->vma, a->alloc, 0, total); // a no-op on coherent memory +} + +// NX2NAVI_REPLAY_ASYNC, the submitting thread before the submission is queued for the replay: the uniform ranges it +// binds are copied now, while they hold what the game submitted. vk_usnap_begin takes the copy over when the replay +// reaches the submission, and leaves out there what GPU writes in flight touch (the replay knows them only then). +void vk_usnap_take(const uint64_t *handles, int n) { + if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; + if (n <= 0 || !RT_KNOB_ON(U.mode - 1)) return; + uint64_t t0 = kern_now_ns(); + Taken *t = calloc(1, sizeof *t); + if (!t) rt_fatal("vk_usnap: out of memory"); + t->n = n, t->first = handles[0]; + int creg = 0; + plat_mutex_lock(&gather_lock); + U.ncol = 0; + for (int i = 0; i < n; i++) + if (handles[i]) collect(nvn_op_first(handles[i]), 0); + sort_col(); + for (int i = 0; i < U.ncol;) { // merged, keeping what lies wholly in one imported pool + uint64_t lo = U.col[i].lo, hi = U.col[i].hi; + for (i++; i < U.ncol && U.col[i].lo <= hi; i++) + if (U.col[i].hi > hi) hi = U.col[i].hi; + VkMemSpan sp; + if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo || + (sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) { + t->refused++; + continue; + } + if (t->nreg == creg) t->reg = grow(t->reg, &creg, sizeof *t->reg); + t->reg[t->nreg++] = (Reg){lo, hi, t->total, 0}; + t->total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1); + } + plat_mutex_unlock(&gather_lock); + if (t->nreg && (t->arena = arena_get(t->total))) copy_regions(t->arena, t->reg, t->nreg, t->total); + t->ns = kern_now_ns() - t0; + plat_mutex_lock(&tq_lock); + if (tq_tail) tq_tail->next = t; + else tq_head = t; + tq_tail = t; + plat_mutex_unlock(&tq_lock); +} + +static void taken_free(Taken *t) { + if (t->arena) arena_done(t->arena); + free(t->reg); + free(t); +} + +// The copy vk_usnap_take made for this submission becomes the replay's. 0: none was taken. +static int adopt(const uint64_t *handles, int n) { + plat_mutex_lock(&tq_lock); + Taken *t = tq_head; + // in submission order: an entry ahead of this submission's belongs to one that never reached the replay + while (t && (t->n != n || t->first != (n > 0 ? handles[0] : 0))) { + tq_head = t->next; + taken_free(t); + t = tq_head; + } + if (t) tq_head = t->next; + if (!tq_head) tq_tail = NULL; + plat_mutex_unlock(&tq_lock); + if (!t) return 0; + U.refused += t->refused; + if (t->arena) { + wr_prune(); + while (U.creg < t->nreg) U.reg = grow(U.reg, &U.creg, sizeof *U.reg); + memcpy(U.reg, t->reg, (size_t)t->nreg * sizeof *U.reg); + U.nreg = t->nreg; + for (int i = 0; i < U.nreg; i++) // a GPU write in flight: the mirror holds those bytes, not guest memory + if (wr_hits(U.reg[i].lo, U.reg[i].hi)) U.reg[i].dropped = 1, U.inflight++; + U.arena = t->arena, t->arena = NULL; + U.on = 1; + U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += t->total, U.copy_ns += t->ns; + } + taken_free(t); + return 1; +} + void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; // stored +1: 0 is "not read yet" U.on = 0, U.fence = fence, U.gen++; @@ -253,7 +359,9 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { U.off_subs++; return; } + if (adopt(handles, n)) return; uint64_t t0 = kern_now_ns(); + plat_mutex_lock(&gather_lock); U.ncol = 0; for (int i = 0; i < n; i++) if (handles[i]) collect(nvn_op_first(handles[i]), 0); @@ -280,17 +388,10 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) { U.reg[U.nreg++] = (Reg){lo, hi, total, 0}; total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1); } + plat_mutex_unlock(&gather_lock); if (!U.nreg) return; if (!(U.arena = arena_get(total))) return; - uint8_t *dst = U.arena->map; - if (vk_dlaa_available()) { - if (!U.arena->cpu && !(U.arena->cpu = malloc(U.arena->size))) rt_fatal("vk_usnap: out of memory"); - dst = U.arena->cpu; - } - for (int i = 0; i < U.nreg; i++) - memcpy(dst + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo); - if (dst != U.arena->map) memcpy(U.arena->map, dst, total); // one sequential write into the device copy - vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory + copy_regions(U.arena, U.reg, U.nreg, total); U.on = 1; U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0; }