Merge GitHub main into the 0.7.0 release line: uniform blocks copied at the submit call with the replay on its own thread, the overlay's frame times measured where the pictures go out, and early pictures kept back.

The two commits main gained since the last merge. docs/releases/v0.6.2.md stays removed; the two notes they added to
it are in docs/releases/v0.7.0.md. Checked on a dev build of the merged tree: the settings, self-test, graphics ABI,
input and kernel probes pass, and a scripted run with six DLSS mode changes presents no dark or black frame, makes the
area's textures in one frame at the load and ends without a fatal error.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-10 07:17:01 -05:00
9 changed files with 199 additions and 19 deletions

View File

@@ -81,6 +81,18 @@ cards).
load they are now made at once, before the area fades in.
- The performance overlay's VRAM line counted every program on the graphics card as if it were the game. It now
shows the game's own share beside it (*game 6.8 GB* of 10.4 GB in use, in one measurement).
- The performance overlay's frame rate, lows and frame-time graph are now those of the pictures as they reach the
screen. They were measured at the game's present calls, which come in uneven steps although the pictures do not: in
a village the calls came 36 and 9 ms apart in turn and the graph drew a sawtooth with a 1% low of 14 fps, while the
pictures went out about 20 ms apart and changed by 4 ms from one to the next. A picture that arrives early now also
waits until seven eighths of the recent interval have passed (`pace_smooth`, on by default; not with frame
generation), which brought that change down to 3 ms at the same frame rate.
- *GPU frame prep on its own thread* (`replay_async`, off by default): the shader constants of a frame are now
copied when the game hands the frame over, not when the other thread gets to it up to a frame later, by which time
the game had sometimes reused that memory (a shadow pass got another block's values about every 364 frames in one
graveyard). The option can still show a dark or flickering frame: vertex, storage and index data are read late in
the same way, and this game rewrites them without waiting for the GPU. The menu says so now. It gave 15 to 18
percent more frames per second in a graveyard and a village.
- Textures that turned dark or blank for a moment: textures out of view were dropped after 11 seconds at 165 fps (30
seconds were meant) and came back as a stand-in for a few frames. The wait is two minutes of real time at any frame
rate now, and the scene's textures are kept through a pause.

View File

@@ -54,7 +54,9 @@
# without the interrupt, 0 = off (the old scheduling).
# async_preempt = 1
# Replay NVN submissions on their own thread, at most one frame behind the game, 0 or 1. Default: off. Helps where the
# game waits for the GPU; adds up to a frame of latency.
# game waits for the GPU and in heavy scenes (15 to 18 percent more frames per second in a village); adds up to a frame
# of latency, and can show a dark or flickering frame now and then: the other thread reads vertex and buffer memory
# the game has sometimes rewritten by then.
# replay_async = 0
# EXPERIMENTAL, THE GAME CRASHES WITH 1. 1 lifts the virtual core gate: guest threads run without the console's limit
# of one thread per core, and the game keeps per-core data without locks. With 1 the game crashes or hangs,

View File

@@ -47,7 +47,7 @@ static const char *const known[] = {
"HDR_EXPOSURE", "HDR_FLARE", "HDR_GAMMA", "HDR_GAME_NITS", "HDR_GRADE", "HDR_HIGHLIGHT_SAT", "HDR_HIGHLIGHTS", "HDR_HUE",
"HDR_LIVE",
"HDR_PEAK_NITS", "HDR_SATURATION", "HDR_SHADOWS", "HDR_TONEMAP", "HDR_UI_NITS", "HIGHFPS_FIX", "INPUT", "INPUT_RECORD", "INPUT_SCRIPT", "LANG", "LIB_PREBUILD",
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
"BIG_BUILD_FREE_MB", "BIG_BUILD_MAX_MB", "LIB_PREBUILD_BIG", "LIB_PREBUILD_BIG_BG", "LIB_PREBUILD_KEEP", "LIB_PREBUILD_THREADS", "LIB_WARM", "PIPE_LINK_MS", "FRAME_HOLD_MS", "PACE_SMOOTH", "LOCALE", "LOD_GUARD", "LSTICK_DEADZONE", "MEMORY", "MEM_CENSUS", "MENU_LANG", "MENU_SOUND_VOLUME", "MENU_TEST",
"MENU_CURSOR", "MENU_CURSOR_LOG", "MENU_TEST_CLICK", "MENU_TEST_DIR", "MENU_TEST_SET", "MINIMAL_LAYOUT", "MICROPHONE", "MIRROR_GAP", "MODS_DIR", "MODS_DISABLED", "MODS_ORDER", "MODS_REBUILD", "MONITOR", "MOTION", "MOUSE_MODE", "MOUSE_DIRECT_PATCH", "MOUSE_SENS", "MOVIE_HOST", "MOVIE_PACE",
"MVOBJ_ALL", "MVOBJ_BANKS", "MVOBJ_EQUAL", "MVOBJ_EXACT", "MVOBJ_EXEC", "MVOBJ_NOC4", "MVOBJ_ONLY", "MVOBJ_TIMES", "MVPREV_CHECK", "MVPREV_GROW", "MVPREV_SKIP", "MVPREV_TRUST", "MV_CHECK",
"MV_SURVEY",
@@ -114,7 +114,7 @@ static const struct { const char *key, *section; } sections[] = {
{"POOL_CREATE_MS", "performance"}, {"POOL_LOAD_BACKLOG", "performance"}, {"POOL_LOAD_MS", "performance"},
{"TIMER_1MS", "performance"}, {"PIPE_WAIT", "performance"}, {"PIPE_SKIP", "performance"}, {"ONESHOT_BUDGET", "performance"},
{"CACHE_DIE", "performance"}, {"ONE_PER_CORE", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PIPE_PREBUILD", "performance"},
{"LIB_PREBUILD", "performance"}, {"LIB_PREBUILD_THREADS", "performance"}, {"LIB_WARM", "performance"}, {"PIPE_LINK_MS", "performance"}, {"FRAME_HOLD_MS", "performance"}, {"PACE_SMOOTH", "performance"}, {"PIPE_PREBUILD", "performance"},
{"COMPUTE_KEEP", "performance"},
{"PREWARM_XLATE", "performance"}, {"GPL", "performance"}, {"ASTC", "performance"}, {"MEMORY", "performance"},
{"TEX_ALIAS", "performance"}, {"TEX_DISUSE", "performance"}, {"TEX_GEN_UNDO", "performance"}, {"TEX_REUSE", "performance"}, {"TEX_VRAM_PRESSURE", "performance"}, {"TEX_PROC_COMMIT_MB", "performance"}, {"ASTC_BC", "performance"}, {"LOD_GUARD", "performance"}, {"FIX_C0", "performance"}, {"FIX_ZERO", "performance"},
@@ -216,6 +216,7 @@ static const CfgMeta metas[] = {
{"PIPE_WAIT", "default", "default|draw", RESTART}, {"PIPE_SKIP", "any", "strict|any|fullscreen", RESTART},
{"PIPE_LINK_MS", "6", "0..100", RESTART}, // vk_exec_draw.c link_budget
{"FRAME_HOLD_MS", "120", "0..500", RESTART}, // vk_backend.c frame_hold
{"PACE_SMOOTH", "1", "0|1", RESTART}, // vk_backend.c pace_smooth
{"LIB_PREBUILD", "1", "0|1", RESTART}, {"LIB_PREBUILD_THREADS", "", "1..64", RESTART},
{"LIB_WARM", "1", "0|1|force", RESTART}, // vk_pipeline.c lw_start: the driver's cache of stage libraries, warmed when cold
{"PIPE_PREBUILD", "1", "0|1", RESTART}, {"PREWARM_XLATE", "", "1..64", RESTART}, {"GPL", "1", "0|1", RESTART},

View File

@@ -43,9 +43,42 @@ static struct {
int64_t up0, up;
} g_win;
// The frame times the overlay shows (the rate, the lows, the graph) are those of the pictures as they go out: the
// present thread reports each one (perfmon_shown). The game's present calls come at other times: with the replay on
// the submitting thread, work held for an event is replayed in the next frame's first submission, so the calls came
// 36 and 9 ms apart in turn in a village while the pictures went out every 17 to 27 ms, and the graph drew a sawtooth
// the screen did not show. A frame held back (vk_backend.c frame_hold) is not shown and lengthens its interval.
// Without a present thread reporting (another back end) the game's present calls are taken, as before.
static uint64_t g_shown_last; // when the last picture went out; 0: none yet
static uint64_t g_shown_report; // when perfmon_shown last ran: perfmon_frame leaves the intervals to it for a second
static void interval_add(uint64_t now, uint64_t interval_ns) { // (lock held)
g_ring[g_ring_pos] = (float)(interval_ns / 1e6);
g_ring_at[g_ring_pos] = now;
g_ring_pos = (g_ring_pos + 1) % PERF_HISTORY;
if (g_ring_n < PERF_HISTORY) g_ring_n++;
if (now - g_low_t0 >= 1000000000ull) { // the next second: its slice starts empty
g_low_cur = (g_low_cur + 1) % LOW_SLICES;
memset(g_low_hist[g_low_cur], 0, sizeof g_low_hist[0]);
g_low_t0 = now;
}
uint64_t bin = interval_ns / 250000ull;
uint16_t *h = &g_low_hist[g_low_cur][bin < LOW_BINS ? bin : LOW_BINS - 1];
if (*h < 65535) (*h)++;
}
void perfmon_shown(uint64_t now) {
plat_mutex_lock(&g_lock);
// a gap of a second or more is a load or a pause, not a frame time
if (g_shown_last && now > g_shown_last && now - g_shown_last < 1000000000ull) interval_add(now, now - g_shown_last);
g_shown_last = g_shown_report = now;
plat_mutex_unlock(&g_lock);
}
void perfmon_frame(uint64_t now, uint64_t interval_ns, const uint64_t *wait_ns, uint64_t replay_ns,
int64_t tex_uploads) {
plat_mutex_lock(&g_lock);
if (g_shown_report && now - g_shown_report < 1000000000ull) interval_ns = 0; // perfmon_shown has them
if (interval_ns) {
g_ring[g_ring_pos] = (float)(interval_ns / 1e6);
g_ring_at[g_ring_pos] = now;

View File

@@ -50,6 +50,8 @@ void perfmon_set_vram_game(float (*used_mb)(void));
// A game frame was presented (perf_present, the render thread, only while the overlay is on): the time since the last
// one, the render thread's waits of this frame (PERF_N entries, host.h), its replay time and the running count of
// texture uploads.
// A picture went out (the present thread, vk_backend.c): the overlay's frame times are the intervals between these.
void perfmon_shown(uint64_t now_ns);
void perfmon_frame(uint64_t now_ns, uint64_t interval_ns, const uint64_t *wait_ns, uint64_t replay_ns,
int64_t tex_uploads);
// The latest numbers (the overlay, present path).

View File

@@ -401,7 +401,7 @@ static const char *const text_en[UI_TEXT_COUNT] = {
[UI_MENU_ASYNC_PREEMPT_DESC] = "Lets a game thread interrupt another one on the same virtual core.",
[UI_MENU_PREEMPT_OFF] = "Off", [UI_MENU_PREEMPT_ON] = "On", [UI_MENU_PREEMPT_INHERIT] = "Inheritance only",
[UI_MENU_REPLAY_ASYNC] = "GPU frame prep on its own thread",
[UI_MENU_REPLAY_ASYNC_DESC] = "Prepares frames for the GPU on a separate thread instead of the game thread. Adds up to one frame of latency.",
[UI_MENU_REPLAY_ASYNC_DESC] = "Prepares frames for the GPU on a separate thread instead of the game thread. Faster in heavy scenes, but can show a dark or flickering frame now and then, and adds up to one frame of latency.",
[UI_MENU_TIMER_1MS] = "1 ms Windows timer",
[UI_MENU_TIMER_1MS_DESC] = "Raises the system timer resolution to 1 ms for steadier frame pacing.",
[UI_MENU_PIPE_WAIT] = "Waiting draw handling",
@@ -949,7 +949,7 @@ static const char *const text_ru[UI_TEXT_COUNT] = {
[UI_MENU_ASYNC_PREEMPT_DESC] = "Позволяет потоку игры прервать другой поток на том же виртуальном ядре.",
[UI_MENU_PREEMPT_OFF] = "Выкл.", [UI_MENU_PREEMPT_ON] = "Вкл.", [UI_MENU_PREEMPT_INHERIT] = "Только наследование",
[UI_MENU_REPLAY_ASYNC] = "Подготовка кадра для GPU в отдельном потоке",
[UI_MENU_REPLAY_ASYNC_DESC] = "Кадр для GPU готовится в отдельном потоке, а не в потоке игры. Добавляет до одного кадра задержки.",
[UI_MENU_REPLAY_ASYNC_DESC] = "Кадр для GPU готовится в отдельном потоке, а не в потоке игры. Быстрее в тяжёлых сценах, но иногда может показать тёмный или мерцающий кадр и добавляет до одного кадра задержки.",
[UI_MENU_TIMER_1MS] = "Таймер Windows 1 мс",
[UI_MENU_TIMER_1MS_DESC] = "Повышает точность системного таймера до 1 мс для ровного темпа кадров.",
[UI_MENU_PIPE_WAIT] = "Обработка ожидающих отрисовок",
@@ -1338,7 +1338,7 @@ static const char *const text_pt_br[UI_TEXT_COUNT] = {
[UI_MENU_ASYNC_PREEMPT_DESC] = "Deixa uma thread do jogo interromper outra no mesmo núcleo virtual.",
[UI_MENU_PREEMPT_OFF] = "Desligado", [UI_MENU_PREEMPT_ON] = "Ligado", [UI_MENU_PREEMPT_INHERIT] = "Só herança",
[UI_MENU_REPLAY_ASYNC] = "Preparo de quadro da GPU em thread própria",
[UI_MENU_REPLAY_ASYNC_DESC] = "Prepara os quadros para a GPU numa thread separada em vez da thread do jogo. Adiciona até um quadro de latência.",
[UI_MENU_REPLAY_ASYNC_DESC] = "Prepara os quadros para a GPU numa thread separada em vez da thread do jogo. Mais rápido em cenas pesadas, mas pode mostrar um quadro escuro ou piscando de vez em quando, e adiciona até um quadro de latência.",
[UI_MENU_TIMER_1MS] = "Timer do Windows de 1 ms",
[UI_MENU_TIMER_1MS_DESC] = "Aumenta a resolução do timer do sistema para 1 ms, para um ritmo de quadros mais estável.",
[UI_MENU_PIPE_WAIT] = "Tratamento de desenhos em espera",
@@ -1979,7 +1979,7 @@ static const char *const text_zh_hans[UI_TEXT_COUNT] = {
[UI_MENU_PREEMPT_ON] = "开",
[UI_MENU_PREEMPT_INHERIT] = "仅继承",
[UI_MENU_REPLAY_ASYNC] = "GPU 帧准备独立线程",
[UI_MENU_REPLAY_ASYNC_DESC] = "在单独的线程而不是游戏线程上为 GPU 准备帧。"
[UI_MENU_REPLAY_ASYNC_DESC] = "在单独的线程而不是游戏线程上为 GPU 准备帧。在复杂场景中更快,但偶尔可能出现变暗或闪烁的画面。"
"最多增加一帧延迟。",
[UI_MENU_TIMER_1MS] = "1 ms Windows 计时器",
[UI_MENU_TIMER_1MS_DESC] = "将系统计时器精度提高到 1 ms,使帧间隔更平稳。",

View File

@@ -297,6 +297,9 @@ void vk_mem_gpu_wrote_shader(const VkMemSpan *sp, uint64_t size);
// Uniform blocks as of the submit call (vk_usnap.c): vk_exec_record copies the uniform ranges a submission binds before
// it records, ubo bindings take the copy (0: not copied, bind memory), GPU writes of the replay drop the copies they touch.
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence);
// NX2NAVI_REPLAY_ASYNC: the submitting thread takes the copy before it queues the submission; vk_usnap_begin on the
// replay thread then uses it.
void vk_usnap_take(const uint64_t *handles, int n);
void vk_usnap_end(uint64_t fence);
int vk_usnap_find(uint64_t addr, uint64_t n, uint64_t align, VkDescriptorBufferInfo *out);
const uint8_t *vk_usnap_map(void); // a host copy of the current arena, at the offsets vk_usnap_find gives

View File

@@ -32,6 +32,7 @@
#include "platform/platform.h"
#include "vk_dlaa.h"
#include "vk_dlssg.h"
#include "vk_fg.h"
#define WEAK __attribute__((weak))
@@ -49,6 +50,28 @@ static void wait_until(uint64_t at) {
while (kern_now_ns() < at) plat_yield();
}
// NX2NAVI_PACE_SMOOTH (default 1; 0 = off): a picture goes out no sooner than seven eighths of the recent interval
// after the one before it. The game's frames reach the present thread in uneven steps although each costs about the
// same: with the replay on the submitting thread, work held for an event is replayed with the next frame, so pictures
// came 13 to 31 ms apart around a mean of 22 ms in a village (5th to 95th percentile), which shows as judder at a
// frame rate that would look even. Only the early ones wait, a few milliseconds each, so the rate stays what it was;
// the recent interval is a moving average over about four pictures of intervals under 100 ms, so the floor follows
// a scene that gets lighter within a second. Not with frame generation, which paces its own presents.
static uint64_t pace_avg_ns; // the present thread's
static uint64_t pace_smooth(uint64_t at, uint64_t last, uint64_t now) {
static int on = -1;
if (on < 0) on = cfg_bool("NX2NAVI_PACE_SMOOTH");
if (!on || !last || !pace_avg_ns || vk_fg_active() || vk_dlssg_active()) return at;
uint64_t floor = last + pace_avg_ns / 8 * 7;
return floor > at && floor < now + 50000000ull ? floor : at;
}
static void pace_shown(uint64_t interval_ns) {
if (interval_ns >= 100000000ull) return; // a load or a hitch is not the rate
pace_avg_ns = pace_avg_ns ? pace_avg_ns + (uint64_t)(((int64_t)interval_ns - (int64_t)pace_avg_ns) / 4) : interval_ns;
}
static void present_main(void *arg) {
(void)arg;
uint64_t last = 0, last_queued = 0;
@@ -72,9 +95,11 @@ static void present_main(void *arg) {
plat_cond_broadcast(&pcv);
plat_mutex_unlock(&pl);
uint64_t t0 = kern_now_ns();
wait_until(j.at);
wait_until(pace_smooth(j.at, last, t0));
uint64_t t1 = kern_now_ns();
if (last) pace_shown(t1 - last);
vk_present_flip();
perfmon_shown(t1); // the overlay's frame times: the pictures as shown
// NX2NAVI_TRACE=present: one line per present: the host present's interval, the interval of the game's present
// calls (when the host present went out before back-edge pacing), the wait and how far past its target it went
if (rt_trace & RT_TRACE_PRESENT)
@@ -588,6 +613,7 @@ static uint64_t vk_be_submit(const uint64_t *handles, int n) {
if (!it.h) rt_fatal("vk: out of memory (replay queue)");
memcpy(it.h, handles, sizeof *it.h * (size_t)n);
}
vk_usnap_take(handles, n); // the uniform blocks as they are now: the game writes on once this returns
return replay_enqueue(&it);
}
static void vk_be_finish(void) { vk_sync_start(); replay_drain(DR_FINISH); vk_present_drain(); vk_sync_finish(); }

View File

@@ -12,8 +12,12 @@
// - A GPU write within the submission (UpdateUniformBuffer, a storage buffer a shader writes, a buffer copy or fill,
// an image-to-buffer copy: vk_usnap_drop) drops the copies it touches from then on, so later draws read the result.
// - The arena goes back to a free list once the submission's fence is done (completion thread).
// The copy is taken where vk_exec_record starts: at the submit call when the replay runs on the submitting thread, later
// with NX2NAVI_REPLAY_ASYNC. NX2NAVI_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (NX2NAVI_AB).
// The copy is taken at the submit call: where vk_exec_record starts when the replay runs on the submitting thread, and
// by the submitting thread itself with NX2NAVI_REPLAY_ASYNC (vk_usnap_take), where the record starts up to a frame
// later on the replay thread. Taken only there, it held whatever the game had put into those bytes by then: in a
// graveyard the block of the shadow pass came out with another block's values in the frame its allocator moved on to
// the next stretch of memory, about every 364 frames, and that frame's sunlight was all in shadow (a dark frame with
// the interface intact). NX2NAVI_UBO_SNAPSHOT=0 binds live memory as before; =ab alternates (NX2NAVI_AB).
#include <stdlib.h>
#include <string.h>
@@ -60,6 +64,20 @@ static struct {
static Arena *free_arenas;
static PlatMutex free_lock = PLAT_MUTEX_INIT;
// U.col, sort_col's buffer and collect's neighbour cache: the submitting thread (vk_usnap_take) and the replay thread
// (vk_usnap_begin, for a submission nothing was taken for) both collect.
static PlatMutex gather_lock = PLAT_MUTEX_INIT;
// A copy taken at the submit call for the replay thread (vk_usnap_take), in submission order.
typedef struct Taken {
struct Taken *next;
Arena *arena;
Reg *reg;
int nreg, n;
uint64_t first, total, refused, ns; // first, n: the submission's first handle and handle count
} Taken;
static Taken *tq_head, *tq_tail;
static PlatMutex tq_lock = PLAT_MUTEX_INIT;
static void *grow(void *p, int *cap, size_t elem) {
int n = *cap ? *cap * 2 : 1024;
@@ -245,6 +263,94 @@ static void summary(void) {
U.subs = U.off_subs = U.regions = U.bytes = U.copy_ns = U.hits = U.drops = U.inflight = U.refused = 0;
}
// The bytes of regions reg[0..nreg) (total bytes in all) into a's device copy, through its host copy with DLSS.
static void copy_regions(Arena *a, const Reg *reg, int nreg, uint64_t total) {
uint8_t *dst = a->map;
if (vk_dlaa_available()) {
if (!a->cpu && !(a->cpu = malloc(a->size))) rt_fatal("vk_usnap: out of memory");
dst = a->cpu;
}
for (int i = 0; i < nreg; i++) memcpy(dst + reg[i].off, (const void *)(uintptr_t)reg[i].lo, reg[i].hi - reg[i].lo);
if (dst != a->map) memcpy(a->map, dst, total); // one sequential write into the device copy
vmaFlushAllocation(vk_ctx()->vma, a->alloc, 0, total); // a no-op on coherent memory
}
// NX2NAVI_REPLAY_ASYNC, the submitting thread before the submission is queued for the replay: the uniform ranges it
// binds are copied now, while they hold what the game submitted. vk_usnap_begin takes the copy over when the replay
// reaches the submission, and leaves out there what GPU writes in flight touch (the replay knows them only then).
void vk_usnap_take(const uint64_t *handles, int n) {
if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1;
if (n <= 0 || !RT_KNOB_ON(U.mode - 1)) return;
uint64_t t0 = kern_now_ns();
Taken *t = calloc(1, sizeof *t);
if (!t) rt_fatal("vk_usnap: out of memory");
t->n = n, t->first = handles[0];
int creg = 0;
plat_mutex_lock(&gather_lock);
U.ncol = 0;
for (int i = 0; i < n; i++)
if (handles[i]) collect(nvn_op_first(handles[i]), 0);
sort_col();
for (int i = 0; i < U.ncol;) { // merged, keeping what lies wholly in one imported pool
uint64_t lo = U.col[i].lo, hi = U.col[i].hi;
for (i++; i < U.ncol && U.col[i].lo <= hi; i++)
if (U.col[i].hi > hi) hi = U.col[i].hi;
VkMemSpan sp;
if (!vk_mem_resolve(lo, hi - lo, &sp) || sp.alloc || !sp.pool || sp.avail < hi - lo ||
(sp.cls != VK_MCLS_HOST_CACHED && sp.cls != VK_MCLS_HOST_UNC)) {
t->refused++;
continue;
}
if (t->nreg == creg) t->reg = grow(t->reg, &creg, sizeof *t->reg);
t->reg[t->nreg++] = (Reg){lo, hi, t->total, 0};
t->total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1);
}
plat_mutex_unlock(&gather_lock);
if (t->nreg && (t->arena = arena_get(t->total))) copy_regions(t->arena, t->reg, t->nreg, t->total);
t->ns = kern_now_ns() - t0;
plat_mutex_lock(&tq_lock);
if (tq_tail) tq_tail->next = t;
else tq_head = t;
tq_tail = t;
plat_mutex_unlock(&tq_lock);
}
static void taken_free(Taken *t) {
if (t->arena) arena_done(t->arena);
free(t->reg);
free(t);
}
// The copy vk_usnap_take made for this submission becomes the replay's. 0: none was taken.
static int adopt(const uint64_t *handles, int n) {
plat_mutex_lock(&tq_lock);
Taken *t = tq_head;
// in submission order: an entry ahead of this submission's belongs to one that never reached the replay
while (t && (t->n != n || t->first != (n > 0 ? handles[0] : 0))) {
tq_head = t->next;
taken_free(t);
t = tq_head;
}
if (t) tq_head = t->next;
if (!tq_head) tq_tail = NULL;
plat_mutex_unlock(&tq_lock);
if (!t) return 0;
U.refused += t->refused;
if (t->arena) {
wr_prune();
while (U.creg < t->nreg) U.reg = grow(U.reg, &U.creg, sizeof *U.reg);
memcpy(U.reg, t->reg, (size_t)t->nreg * sizeof *U.reg);
U.nreg = t->nreg;
for (int i = 0; i < U.nreg; i++) // a GPU write in flight: the mirror holds those bytes, not guest memory
if (wr_hits(U.reg[i].lo, U.reg[i].hi)) U.reg[i].dropped = 1, U.inflight++;
U.arena = t->arena, t->arena = NULL;
U.on = 1;
U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += t->total, U.copy_ns += t->ns;
}
taken_free(t);
return 1;
}
void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) {
if (!U.mode) U.mode = rt_knob("UBO_SNAPSHOT", "1") + 1; // stored +1: 0 is "not read yet"
U.on = 0, U.fence = fence, U.gen++;
@@ -253,7 +359,9 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) {
U.off_subs++;
return;
}
if (adopt(handles, n)) return;
uint64_t t0 = kern_now_ns();
plat_mutex_lock(&gather_lock);
U.ncol = 0;
for (int i = 0; i < n; i++)
if (handles[i]) collect(nvn_op_first(handles[i]), 0);
@@ -280,17 +388,10 @@ void vk_usnap_begin(const uint64_t *handles, int n, uint64_t fence) {
U.reg[U.nreg++] = (Reg){lo, hi, total, 0};
total += (hi - lo + ALIGN - 1) & ~(uint64_t)(ALIGN - 1);
}
plat_mutex_unlock(&gather_lock);
if (!U.nreg) return;
if (!(U.arena = arena_get(total))) return;
uint8_t *dst = U.arena->map;
if (vk_dlaa_available()) {
if (!U.arena->cpu && !(U.arena->cpu = malloc(U.arena->size))) rt_fatal("vk_usnap: out of memory");
dst = U.arena->cpu;
}
for (int i = 0; i < U.nreg; i++)
memcpy(dst + U.reg[i].off, (const void *)(uintptr_t)U.reg[i].lo, U.reg[i].hi - U.reg[i].lo);
if (dst != U.arena->map) memcpy(U.arena->map, dst, total); // one sequential write into the device copy
vmaFlushAllocation(vk_ctx()->vma, U.arena->alloc, 0, total); // a no-op on coherent memory
copy_regions(U.arena, U.reg, U.nreg, total);
U.on = 1;
U.subs++, U.regions += (uint64_t)U.nreg, U.bytes += total, U.copy_ns += kern_now_ns() - t0;
}