Files
Triforce-Source/tests/parallel_init_order_test.cpp
ruckus7382sfgsgdf e6f2f248f2 Triforce v0.7.0 "Sage Of Courage"
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
2026-11-22 11:59:06 -06:00

905 lines
50 KiB
C++

// A parallel replay's new images in queue order: the production upload command buffer state, create()'s initialization,
// bc_scratch_layout, the texture segment hooks, replay_handle's segment jobs and submits and the init pools' startup,
// included from the init_*.inc files tests/test_parallel_init_order.py cuts from the source. Vulkan is a spy: commands
// are recorded per command buffer (never into one not begun, ended or submitted), and the submitted ones are played in
// queue order, the image layouts tracked as the GPU would see them. The workers run on threads of their own: one after
// another in the order a case sets (the one that makes an image first need not be the first segment in the queue), or
// all at once, held at fixed points by latches (a worker waits for another to be inside its init recording, or for an
// earlier segment's submit). A command buffer can be made to fail its begin or its end; replay_handle's part after the
// segments (begin_recording) is modelled as a recording of its own.
#define VK_NO_PROTOTYPES
#include <vulkan/vulkan_core.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdarg>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <string>
#include <thread>
#include <vector>
#include "../gpu/vk/helper_spin.h" // (replay_handle's spin for a segment before its wait: in init_jobs.inc)
#include "../gpu/vk/upload_earliest.h" // ((Triforce) the upload's segment: UploadRedirect, in init_upload.inc)
struct Op {
char kind; // F: full barrier, T: UNDEFINED -> GENERAL, B: another image barrier, C: clear, U: a use (copy, dispatch, draw),
// G: a GPU timestamp (a segment's, gpu_time_begin and gpu_time_end); W: an upload of the submission's bytes into
// it (texture_touch's, or the twin), R: a draw that samples it
VkImage image;
};
struct VkCommandBuffer_T {
std::string name;
std::vector<Op> ops;
bool begun = false, ended = false, submitted = false;
};
std::vector<VkCommandBuffer_T*> all_cbs, queue;
std::vector<std::unique_ptr<VkCommandBuffer_T>> owned_cbs;
std::set<int> published_slots; // draw-stream CPU writes published before a worker slot reaches the GPU
std::atomic<int> failures{0};
#define CHECK(v) do { if (!(v)) { std::printf("FAIL:%d %s\n", __LINE__, #v); std::fflush(stdout); std::_Exit(1); } } while (0)
struct Latch {
std::mutex mu;
std::condition_variable cv;
bool set = false;
void post() {
{
std::lock_guard l(mu);
set = true;
}
cv.notify_all();
}
void wait() {
std::unique_lock l(mu);
CHECK(cv.wait_for(l, std::chrono::seconds(10), [&] { return set; }));
}
};
Latch latch[128]; // (by letter. x: a worker is inside the texture's initialization, y: let it go on, z: segment 0 submitted)
const VkImage kTex = reinterpret_cast<VkImage>(uintptr_t(0x10)), kScratch = reinterpret_cast<VkImage>(uintptr_t(0x20)),
kScratch2 = reinterpret_cast<VkImage>(uintptr_t(0x30)), kTex2 = reinterpret_cast<VkImage>(uintptr_t(0x40));
const char* image_name(VkImage i) {
return i == kTex ? "t" : i == kScratch ? "s" : i == kScratch2 ? "s2" : i == kTex2 ? "t2" : "?";
}
bool hold_in_init; // (close_race: the texture's initialization waits there, its lock held, for the other worker)
void record(VkCommandBuffer cb, char kind, VkImage image = VK_NULL_HANDLE) {
CHECK(cb && cb->begun && !cb->ended && !cb->submitted);
cb->ops.push_back({kind, image});
}
void vkCmdPipelineBarrier2(VkCommandBuffer cb, const VkDependencyInfo* di) {
for (uint32_t i = 0; i < di->imageMemoryBarrierCount; i++) {
const VkImageMemoryBarrier2& b = di->pImageMemoryBarriers[i];
const bool first = b.oldLayout == VK_IMAGE_LAYOUT_UNDEFINED && b.newLayout == VK_IMAGE_LAYOUT_GENERAL;
record(cb, first ? 'T' : 'B', b.image);
if (first && b.image == kTex && hold_in_init) {
latch['x'].post();
latch['y'].wait();
std::this_thread::sleep_for(std::chrono::milliseconds(20)); // (the other worker closing its segment meanwhile)
}
}
}
void vkCmdClearColorImage(VkCommandBuffer cb, VkImage i, VkImageLayout, const VkClearColorValue*, uint32_t, const VkImageSubresourceRange*) {
record(cb, 'C', i);
}
void vkCmdClearDepthStencilImage(VkCommandBuffer cb, VkImage i, VkImageLayout, const VkClearDepthStencilValue*, uint32_t,
const VkImageSubresourceRange*) {
record(cb, 'C', i);
}
VkResult vkResetCommandPool(VkDevice, VkCommandPool, VkCommandPoolResetFlags) {
return VK_SUCCESS;
}
std::string fail_begin, fail_end; // (this command buffer cannot be begun / ended)
VkResult vkBeginCommandBuffer(VkCommandBuffer cb, const VkCommandBufferBeginInfo*) {
CHECK(cb);
if (cb->name == fail_begin) return VK_ERROR_OUT_OF_HOST_MEMORY;
cb->ops.clear(), cb->begun = true, cb->ended = false;
return VK_SUCCESS;
}
VkResult vkEndCommandBuffer(VkCommandBuffer cb) {
CHECK(cb && cb->begun && !cb->ended);
if (cb->name == fail_end) return VK_ERROR_OUT_OF_DEVICE_MEMORY;
cb->ended = true;
return VK_SUCCESS;
}
VkResult vkQueueSubmit2(VkQueue, uint32_t n, const VkSubmitInfo2* si, VkFence) {
for (uint32_t s = 0; s < n; s++)
for (uint32_t i = 0; i < si[s].commandBufferInfoCount; i++) {
VkCommandBuffer cb = si[s].pCommandBufferInfos[i].commandBuffer;
CHECK(cb && cb->ended && !cb->submitted);
if (cb->name.rfind("cmd_", 0) == 0)
CHECK(published_slots.count(std::stoi(cb->name.substr(4))) == 1);
cb->submitted = true;
queue.push_back(cb);
if (cb->name == "cmd_0") latch['z'].post();
}
return VK_SUCCESS;
}
std::mutex cbs_mu;
VkCommandBuffer make_cb(const char* what, int k) {
auto cb = std::make_unique<VkCommandBuffer_T>();
cb->name = std::string(what) + "_" + std::to_string(k);
auto* result = cb.get();
std::lock_guard l(cbs_mu);
all_cbs.push_back(result);
owned_cbs.push_back(std::move(cb));
return result;
}
// (the init pools' startup: each pool creation and allocation can be made to fail, its output left as garbage)
const uintptr_t kGarbage = 0xBAD0;
int alloc_calls, fail_at;
std::set<uintptr_t> live_pools;
uintptr_t next_pool = 0x1000;
VkResult vkCreateCommandPool(VkDevice, const VkCommandPoolCreateInfo*, const VkAllocationCallbacks*, VkCommandPool* p) {
if (++alloc_calls == fail_at) {
*p = reinterpret_cast<VkCommandPool>(kGarbage);
return VK_ERROR_OUT_OF_DEVICE_MEMORY;
}
live_pools.insert(next_pool);
*p = reinterpret_cast<VkCommandPool>(next_pool++);
return VK_SUCCESS;
}
VkResult vkAllocateCommandBuffers(VkDevice, const VkCommandBufferAllocateInfo* ai, VkCommandBuffer* out) {
CHECK(ai->commandBufferCount == 1 && live_pools.count(reinterpret_cast<uintptr_t>(ai->commandPool)));
if (++alloc_calls == fail_at) {
*out = reinterpret_cast<VkCommandBuffer>(kGarbage);
return VK_ERROR_OUT_OF_HOST_MEMORY;
}
*out = make_cb("pool", alloc_calls);
return VK_SUCCESS;
}
void vkDestroyCommandPool(VkDevice, VkCommandPool p, const VkAllocationCallbacks*) {
CHECK(p && live_pools.erase(reinterpret_cast<uintptr_t>(p))); // (only a live one, once)
}
namespace mkwvk {
std::mutex log_mu;
std::string log_text;
std::function<void(const std::string&)> on_log; // (init_end_fail: looks at the state when the replay stops)
void logf(const char* f, ...) {
char line[512];
va_list a;
va_start(a, f);
std::vsnprintf(line, sizeof line, f, a);
va_end(a);
{
std::lock_guard l(log_mu);
log_text += std::string(line) + "\n";
}
if (on_log) on_log(line);
}
static const struct FixtureHost { void (*log_flush)(); } fixture_host{}, *host = &fixture_host; // (replay_fatal's)
[[noreturn]] void end_process(int code) { std::_Exit(code); } // (vk_replay.cpp: TerminateProcess on Windows)
#include "init_fatal.inc"
void full_barrier(VkCommandBuffer cmd, const char* = "") {
record(cmd, 'F');
}
void memory_table_begin() {}
bool memory_table_flush() { return false; }
bool memory_rb_refresh(VkCommandBuffer) { return false; }
void memory_rb_recording_begun() {}
void memory_mirror_binds_flush() {} // (the mirrors' pieces: none here, so no bind to wait for)
void checkpoint_mark(VkCommandBuffer, uint64_t, char) {} // (RX_VK_CHECKPOINTS' markers: none here)
bool memory_mirror_bind_wait(VkSemaphoreSubmitInfo*) { return false; }
// ((Triforce, P4) the barrier rules' marks are each command buffer's own, cleared at its begin (gpu/vk/cmd_work.h work_begun,
// checked by tests/test_barrier_rules.py): texture_begin_recording has none of its own to reset)
bool is_bc(VkFormat) { return false; }
bool is_astc(VkFormat) { return false; }
#include "init_upload.inc"
#include "init_hooks.inc"
// ---- create()'s initialization of a new image ----
struct Texture {
VkImage image;
};
std::vector<std::unique_ptr<Texture>> textures;
std::mutex par_lock; // (texture_get's ParLock: an image is made once, by whichever worker gets there first)
Texture* create(VkImage image) {
struct { VkImage image; VkFormat format; VkImageAspectFlags aspect; } x{image, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT};
const bool depth = false;
auto tex = std::make_unique<Texture>(Texture{image});
#include "init_create.inc"
textures.push_back(std::move(tex));
return p;
}
void texture_use() { // (texture_get, and texture_touch's upload again: a copy into it in this thread's uploads)
std::lock_guard lk(par_lock);
if (textures.empty()) create(kTex);
record(upload_cmd(), 'U', kTex);
}
// ---- the BC3 scratch, and the second one (an upload at another size) ----
std::mutex bc_mu;
std::chrono::steady_clock::time_point bc_used_at;
uint64_t bc_last_value = 0;
std::atomic<bool> bc_have{false};
constexpr uint32_t kBcLayers = 8;
uint32_t bc_scratch_layers() { return kBcLayers; } // ((11-21, TEX-10) the scratch's layers: a run's)
VkImage bc_scratch = VK_NULL_HANDLE, bc_scratch2 = VK_NULL_HANDLE;
bool bc_scratch_fresh = false, bc_scratch2_fresh = false;
VkImage bc_small = VK_NULL_HANDLE; // ((11-21, TEX-09) the small scratch: none here)
bool bc_small_fresh = false;
uint32_t bc_small_layers() { return 1; }
bool bc_scratch_make() {
bc_scratch = kScratch, bc_scratch_fresh = true, bc_have = true;
return true;
}
bool bc_scratch2_make() {
bc_scratch2 = kScratch2, bc_scratch2_fresh = true;
return true;
}
#include "init_scratch.inc"
void scratch_use(bool resized) { // (an ASTC/BC3 upload: decoded into the scratch (and blitted into the second), in this thread's uploads)
std::lock_guard lk(par_lock);
CHECK(bc_scratch_layout(upload_cmd(), resized));
record(upload_cmd(), 'U', kScratch);
if (resized) record(upload_cmd(), 'U', kScratch2);
}
// ---- (Triforce, 11-17) texture_touch's look at a texture the CPU wrote before this submission: its upload, or the twin ----
// The steps of vk_texture.cpp texture_touch that decide where the upload goes (tests/test_upload_same.py reads them there): the
// twin when a later segment of this submission uploaded it, nothing more once it was looked at in this submission, else the
// upload (W), into the earliest segment's init command buffer still recorded before this one (UploadRedirect, cut from the
// source with InitRedirect) unless upload_inner reads it on the GPU (upload_reads_live: gpu_route here), and upload_seg the
// segment it went into.
struct Look {
bool uploaded = false;
uint64_t upload_value = 0, upload_seg = 0;
uint32_t checked_epoch = 0;
bool gpu_written = false;
};
Look look;
std::atomic<int> twins{0};
bool hold_in_upload = false; // (upload_close_race: the upload waits, the init lock held, for the other worker to close its segment)
bool nested_upload = false; // (upload_nested: the upload makes an image and both BC3 scratches meanwhile)
bool hold_for_other = false; // (upload_end_other: the upload waits, segment 0's lock held, for segment 1 to end its own)
bool gpu_route = false; // (upload_gpu_route: upload_inner would read the texture on the GPU, upload_reads_live)
void texture_look() {
std::lock_guard lk(par_lock);
if (textures.empty()) create(kTex); // (texture_get)
if (look.uploaded && look.upload_value == rec_value && look.upload_seg > tl_seg) { // (the twin, in this segment's uploads)
record(upload_cmd(), 'W', kTex);
look.upload_seg = tl_seg;
twins++;
return;
}
if (look.uploaded && look.checked_epoch == epoch) return;
look.checked_epoch = epoch;
UploadRedirect early(upload_earliest::wanted(upload_earliest_on(), look.gpu_written, false) && tl_seg_index >= 0 && !gpu_route);
if (nested_upload) { // (upload(): its image made again, and the BC3 encoder's scratches made, inside the redirect)
create(kTex2);
record(upload_cmd(), 'U', kTex2);
CHECK(bc_scratch_layout(upload_cmd(), true));
record(upload_cmd(), 'U', kScratch), record(upload_cmd(), 'U', kScratch2);
}
record(upload_cmd(), 'W', kTex);
if (hold_in_upload) {
latch['x'].post();
latch['y'].wait();
// (until the other worker's end of its segment waits for this one's lock, the upload still recorded)
const auto t0 = std::chrono::steady_clock::now();
while (!st_seg_end_waiting.load() && std::chrono::steady_clock::now() - t0 < std::chrono::seconds(10)) std::this_thread::yield();
if (!st_seg_end_waiting.load()) std::printf(" FAIL: segment 0's end did not wait for the upload's lock\n"), failures++;
}
if (hold_for_other) {
latch['x'].post();
const auto t0 = std::chrono::steady_clock::now();
while (seg_inits[1].open.load() && std::chrono::steady_clock::now() - t0 < std::chrono::seconds(10)) std::this_thread::yield();
if (seg_inits[1].open.load())
std::printf(" FAIL: segment 1's end waited for an upload recorded into segment 0's init command buffer\n"), failures++;
}
const uint64_t up_seg = early.order;
early.done();
look.upload_value = rec_value, look.upload_seg = up_seg, look.uploaded = true;
}
bool tex2_made = false;
void texture2_use() { // (another new texture, made by whichever worker gets there first and copied into in its uploads)
std::lock_guard lk(par_lock);
if (!tex2_made) create(kTex2), tex2_made = true;
record(upload_cmd(), 'U', kTex2);
}
// ---- replay_handle's segments ----
#include "init_recording.inc"
struct Segment {
// t: the texture (made if new, uploaded, drawn with), s: the scratch, S: both scratches, -: nothing,
// x / z: wait for latch x (a worker inside the texture's initialization) / z (segment 0 submitted), d: 100 ms more,
// L: the initialization holds segment 0's init lock (seg_inits[0].mu) while it records; then latch y lets it go on,
// W / w: post latch w / wait for it (one segment waits for another to have made the texture),
// u: a draw that samples the texture the CPU wrote (texture_look, then R), r: 0 to 3 ms more, n: another new texture used
std::string script;
const uint8_t* end = nullptr; // (kStop: to the command buffer's end, every segment's commands in order)
// (vk_replay.cpp's Segment has these too, for a temporal upscaler's tail: none here, as with the upscalers off)
const uint8_t* p = nullptr;
bool tail = false;
void* order = nullptr;
};
const uint8_t* const kStop = reinterpret_cast<const uint8_t*>("stop");
const std::vector<Segment>* cur_segs;
const std::thread::id replay_thread = std::this_thread::get_id(); // (main's: the replay thread's part of submit_segments)
std::mutex recorded_mu;
std::vector<std::pair<std::string, bool>> recorded; // (each segment's script as it was recorded, and whether by the replay thread)
int64_t record_segment(VkCommandBuffer& cmd, const Segment& g) { // (vk_replay.cpp's returns its recording's time)
std::string script = g.script;
if (g.end == kStop)
for (size_t k = 1; k < cur_segs->size(); k++) script += (*cur_segs)[k].script;
{
std::lock_guard l(recorded_mu);
recorded.push_back({script, std::this_thread::get_id() == replay_thread});
}
for (char a : script) {
if (a == 't') texture_use(), record(cmd, 'U', kTex);
if (a == 'u') texture_look(), record(cmd, 'R', kTex);
if (a == 'n') texture2_use(), record(cmd, 'U', kTex2);
if (a == 'r') std::this_thread::sleep_for(std::chrono::microseconds(std::rand() % 3000));
if (a == 's' || a == 'S') scratch_use(a == 'S');
if (a == 'x' || a == 'z' || a == 'w') latch[(int)a].wait();
if (a == 'W') latch['w'].post();
if (a == 'd') std::this_thread::sleep_for(std::chrono::milliseconds(100));
if (a == 'L') {
#if HAVE_SEG_INIT
if (seg_inits[0].mu.try_lock()) { // (segment 0's own: what is recorded into its init command buffer)
seg_inits[0].mu.unlock();
std::printf(" FAIL: the init lock is free while another worker records a new image's initialization\n");
failures++;
}
#endif
latch['y'].post();
}
}
return 1;
}
void draw_begin_recording(int, VkCommandBuffer, uint64_t) {}
// (the segments' GPU timestamps: G, into both command buffers while they are open, as vk_replay.cpp's own record them)
void gpu_time_begin(Recording& r) { record(r.cmd, 'G'), record(r.upload, 'G'); }
void gpu_time_end(Recording& r, bool) { record(r.cmd, 'G'), record(r.upload, 'G'); }
#include "init_parstats.inc"
void draw_flush_recording(int slot) { CHECK(slot >= 0 && slot < 8); published_slots.insert(slot); }
void queue_flush_locked() {}
void checkpoints_report() {}
[[noreturn]] void device_lost() { std::_Exit(3); } // (vk_replay.cpp: the report, then the process ends)
struct {
VkDevice device = VK_NULL_HANDLE;
VkQueue queue = VK_NULL_HANDLE;
std::mutex present_mu;
} dev;
std::atomic<int64_t> par_lock_wait_ns{0}, par_locks{0};
std::atomic<bool> par_active{false};
std::vector<int> run_order; // (the order the workers run in, one after another; empty: all at once)
std::vector<std::thread> workers;
bool jobs_started;
std::atomic<int> running{0}; // (workers still recording)
int helper_late_ms = 0; // (at once: the workers begin this late, so that the replay thread takes the segments it reaches first)
namespace rec {
void segments_begin() {} // (vk_recorder.h's: the recorder's ring run before the segments; no recorder here)
}
bool own_on = true;
bool par_own_on() { return own_on; } // (vk_replay.cpp's: RX_VK_PARALLEL_OWN)
struct RunJobs {
void notify_all();
};
struct {
std::mutex mu;
std::vector<std::function<void()>> jobs;
int pending = 0;
RunJobs cv;
std::condition_variable done;
std::atomic<int> queued{0}; // (vk_replay.cpp's: the jobs' count replay_handle stores for helper threads that spin; none here)
} par_pool;
void RunJobs::notify_all() {
jobs_started = true;
std::vector<std::function<void()>> jobs;
{
std::lock_guard lk(par_pool.mu);
jobs.swap(par_pool.jobs);
}
if (run_order.empty()) { // (at once, as par_worker_main runs them; the replay thread submits meanwhile)
for (auto& job : jobs) {
running++;
workers.emplace_back([job] {
if (helper_late_ms) std::this_thread::sleep_for(std::chrono::milliseconds(helper_late_ms));
job();
running--;
{
std::lock_guard lk(par_pool.mu);
par_pool.pending--;
}
par_pool.done.notify_all();
});
}
return;
}
CHECK(jobs.size() == run_order.size());
for (int k : run_order) std::thread(jobs[k]).join();
std::lock_guard lk(par_pool.mu);
par_pool.pending = 0;
}
// ((Triforce) RX_VK_OCC_QUERY's part of a job's submit: off here (occ_live false), nothing sealed
bool occ_live = false;
VkSemaphore rec_timeline = VK_NULL_HANDLE;
uint64_t rec_seq = 0;
void occ_seal(Recording&, uint64_t) {}
Recording recordings[8];
int recording_count = 0;
uint64_t rec_cur_value = 1;
Recording& begin_segment_recording(uint64_t value) {
const int k = recording_count++;
published_slots.erase(k);
Recording& r = recordings[k];
r.cmd = make_cb("cmd", k), r.upload = make_cb("upload", k);
[k](auto& x) { // (the fixed code's own command buffer for its new images)
if constexpr (requires { x.init; }) x.init = make_cb("init", k);
}(r);
vkBeginCommandBuffer(r.cmd, nullptr); // (as begin_segment_recording does; the job begins the uploads)
r.value = value, r.slot = k; // (the submission's, as vk_replay.cpp's: its segments are one submission, one epoch)
return r;
}
// (replay_handle's part after the segments: the replay thread's next recording, submitted later as [its uploads, it])
Recording post;
Recording* rec_cur = nullptr;
int rec_parts = 0;
Recording& begin_recording(uint64_t value) {
post.cmd = make_cb("post", 0), post.upload = make_cb("postupload", 0);
vkBeginCommandBuffer(post.cmd, nullptr);
vkBeginCommandBuffer(post.upload, nullptr);
post.value = value;
texture_begin_recording(post.upload, value);
draw_begin_recording(post.slot, post.cmd, value);
return post;
}
void submit_post() {
VkCommandBufferSubmitInfo cb[2]{{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}, {VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}};
uint32_t n = 0;
if (texture_uploads_recorded()) cb[n++].commandBuffer = post.upload;
cb[n++].commandBuffer = post.cmd;
vkEndCommandBuffer(post.upload);
vkEndCommandBuffer(post.cmd);
VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
si.commandBufferInfoCount = n, si.pCommandBufferInfos = cb;
vkQueueSubmit2(VK_NULL_HANDLE, 1, &si, VK_NULL_HANDLE);
}
// ((Triforce) the early tail's rest after the segments, from the gameplay upscale's dispatch on (vk_replay.cpp tail_rest): none here)
void tail_rest(VkCommandBuffer&, uint64_t, const uint8_t*) {}
/// replay_handle from its segments on (the dry pass, the part before and the slots' order are the caller's)
void submit_segments(std::vector<Segment> seg, std::vector<int> order) {
run_order = order;
cur_segs = &seg;
VkCommandBuffer cmd = VK_NULL_HANDLE;
const uint8_t* const stop = kStop;
const bool up = false; // ((Triforce) replay_handle's: no gameplay upscale's dispatch found by the dry pass (par_up))
const uint64_t cb = 0; // (replay_handle's: the command buffer's replay, for tail_rest)
uint64_t seq = 10;
const uint64_t base = 1;
const auto t_dry0 = std::chrono::steady_clock::now(), t_set0 = t_dry0; // (replay_handle's: the command buffer's start, the setup's)
[&]() -> const uint8_t* {
#include "init_jobs.inc"
return nullptr;
}();
for (auto& w : workers) w.join();
}
#include "init_pools.inc"
} // namespace mkwvk
std::string ops_text(const VkCommandBuffer_T* cb) {
std::string s;
for (const Op& o : cb->ops) s += std::string(s.empty() ? "" : " ") + o.kind + (o.image ? image_name(o.image) : "");
return s;
}
std::string queue_text() {
std::string order;
for (auto* cb : queue) order += " " + cb->name + "[" + ops_text(cb) + "]";
return order;
}
/// The submitted command buffers in queue order: each image has one UNDEFINED -> GENERAL change, before every use
void play(const char* name) {
std::printf("%s: queue%s\n", name, queue_text().c_str());
std::map<VkImage, int> made, early;
for (auto* cb : queue)
for (const Op& o : cb->ops) {
if (!o.image) continue;
if (o.kind == 'T') {
if (made[o.image]++)
std::printf(" FAIL: image %s made UNDEFINED again in %s (what came before is discarded)\n", image_name(o.image),
cb->name.c_str()), failures++;
} else if (!made[o.image] && !early[o.image]++) {
std::printf(" FAIL: %s uses image %s (%c) while it is still UNDEFINED in the queue (VUID-vkCmdDraw-None-09600)\n",
cb->name.c_str(), image_name(o.image), o.kind), failures++;
}
}
for (auto [image, n] : made)
if (n != 1) std::printf(" FAIL: image %s changed from UNDEFINED %d times\n", image_name(image), n), failures++;
for (auto* cb : all_cbs)
for (const Op& o : cb->ops)
if (o.kind != 'F' && o.kind != 'G' && !cb->submitted) { // (a barrier or a timestamp left unsubmitted does nothing)
std::printf(" FAIL: %s has commands (%c%s) and was never submitted\n", cb->name.c_str(), o.kind, image_name(o.image));
failures++;
break;
}
}
VkCommandBuffer_T* named(const char* name) {
for (auto* cb : all_cbs)
if (cb->name == name) return cb;
return nullptr;
}
/// Which command buffer holds image i's UNDEFINED -> GENERAL change ("": none)
std::string init_of(VkImage i) {
for (auto* cb : all_cbs)
for (const Op& o : cb->ops)
if (o.kind == 'T' && o.image == i) return cb->name;
return "";
}
// ---- (Triforce, 11-17) the uploads of a texture the segments sample (texture_look): what each draw sees ----
/// The queue for the texture, in order: every write to it (an upload W, a clear) after a barrier since its last use, every read
/// after a barrier since its last write, and each draw from queue entry `from` on (this submission's) that samples it (R) after
/// an upload made in this submission. Returns the uploads made in it
int play_content(const char* name, size_t from) {
int uploads = 0;
char last = 0; // ('w': written last, 'r': read last)
bool barrier = true, wrote_now = false;
for (size_t q = 0; q < queue.size(); q++)
for (const Op& o : queue[q]->ops) {
if (o.kind == 'F' || (o.image == kTex && (o.kind == 'T' || o.kind == 'B'))) {
barrier = true;
continue;
}
if (o.image != kTex) continue;
const bool write = o.kind == 'W' || o.kind == 'C';
if (last && (write || last == 'w') && !barrier)
std::printf(" FAIL: %s: %s's %c of the texture with no barrier since its last %s\n", name, queue[q]->name.c_str(), o.kind,
last == 'w' ? "write" : "read"), failures++;
if (o.kind == 'R' && q >= from && !wrote_now)
std::printf(" FAIL: %s: %s samples the texture before this submission's upload in the queue\n", name,
queue[q]->name.c_str()), failures++;
if (o.kind == 'W' && q >= from) uploads++, wrote_now = true;
last = write ? 'w' : 'r', barrier = false;
}
return uploads;
}
/// The command buffers this submission's uploads of the texture are in, in queue order
std::string uploads_in(size_t from) {
std::string s;
for (size_t q = from; q < queue.size(); q++)
for (const Op& o : queue[q]->ops)
if (o.kind == 'W' && o.image == kTex) s += (s.empty() ? "" : " ") + queue[q]->name;
return s;
}
/// Another run from scratch (upload_random): the queue, the command buffers, the texture and the look, and a new submission
void reset_run() {
using namespace mkwvk;
queue.clear(), all_cbs.clear(), recording_count = 0;
textures.clear();
look = Look{};
twins = 0;
tex2_made = false;
{
std::lock_guard l(recorded_mu);
recorded.clear();
}
workers.clear();
rec_cur_value++;
}
/// One run of `scripts` (in `order`, or all at once when empty): the images' layouts, the texture's content, the uploads (one,
/// no twin, when RX_VK_UPLOAD_EARLIEST is on), and the command buffer the upload is in when `want_in` says (empty: not checked)
void upload_run(const char* name, const std::vector<std::string>& scripts, const std::vector<int>& order, const std::string& want_in,
bool quiet = false) {
using namespace mkwvk;
std::vector<Segment> seg;
for (const auto& s : scripts) seg.push_back({s});
const size_t from = queue.size();
submit_segments(seg, order);
if (!quiet) play(name);
else {
const int f0 = failures;
std::map<VkImage, int> made;
for (auto* cb : queue)
for (const Op& o : cb->ops)
if (o.kind == 'T') made[o.image]++;
else if (o.image && !made[o.image]) failures++;
if (failures != f0) std::printf(" FAIL: %s: an image used while UNDEFINED in the queue%s\n", name, queue_text().c_str());
}
const int n = play_content(name, from);
const bool on = upload_earliest_on() && !gpu_route; // (a GPU-read upload: its segment's own, the twin as before)
bool sampled = false;
for (const auto& s : scripts) sampled |= s.find('u') != std::string::npos;
if (!quiet) std::printf("%s: uploads in [%s], %d twins, upload_seg %llu\n", name, uploads_in(from).c_str(), twins.load(),
(unsigned long long)look.upload_seg);
if (sampled && (on ? n != 1 || twins != 0 : n != 1 + twins.load()))
std::printf(" FAIL: %s: %d uploads, %d twins (want %s)%s\n", name, n, twins.load(), on ? "one, no twin" : "one and the twins",
queue_text().c_str()), failures++;
if (!want_in.empty() && uploads_in(from) != want_in)
std::printf(" FAIL: %s: the uploads are in [%s], want [%s]%s\n", name, uploads_in(from).c_str(), want_in.c_str(),
queue_text().c_str()), failures++;
}
int main(int argc, char** argv) {
using namespace mkwvk;
const std::string c = argc > 1 ? argv[1] : "";
if (c.rfind("upload_", 0) == 0) // (a lock taken twice by one thread, or a latch never posted: a failure, not a hang)
std::thread([] {
std::this_thread::sleep_for(std::chrono::seconds(40));
std::printf(" FAIL: still running after 40 s (a deadlock)\n");
std::fflush(stdout);
std::_Exit(1);
}).detach();
if (c == "later_makes") { // (segment 1's worker makes the texture, then segment 0's uploads it again and draws with it)
submit_segments({{"t"}, {"t"}}, {1, 0});
play(c.c_str());
} else if (c == "three") { // (segment 0 is done; 2 makes it while 1 still records, and 1 uses it)
submit_segments({{"-"}, {"t"}, {"t"}}, {0, 2, 1});
play(c.c_str());
} else if (c == "scratch") { // (the BC3 scratch: made by segment 1's worker, used by segment 0's too)
submit_segments({{"s"}, {"s"}}, {1, 0});
play(c.c_str());
} else if (c == "scratch2") { // (segment 1 makes the scratch, then the second one; segment 0 uses both; then both fresh at once)
submit_segments({{"S"}, {"sS"}}, {1, 0});
play(c.c_str());
queue.clear(), all_cbs.clear(), recording_count = 0;
bc_scratch = bc_scratch2 = VK_NULL_HANDLE, bc_have = false;
submit_segments({{"S"}, {"S"}}, {1, 0});
play("scratch2 (both fresh)");
} else if (c == "in_order") { // (made by the first segment, as serial replay would)
submit_segments({{"t"}, {"t"}}, {0, 1});
play(c.c_str());
} else if (c == "serial") { // (no segments: the replay thread's uploads get exactly the calls they always did)
VkCommandBuffer up = make_cb("upload", 0);
vkBeginCommandBuffer(up, nullptr);
texture_begin_recording(up, 1);
texture_use();
scratch_use(false);
scratch_use(true);
CHECK(texture_uploads_recorded());
vkEndCommandBuffer(up);
const std::string got = ops_text(up), want = "F Tt Ct Bt Ut Ts Us Ts2 Us Us2 F";
std::printf("serial: %s[%s]\n", up->name.c_str(), got.c_str());
if (got != want) std::printf(" FAIL: serial uploads changed, want [%s]\n", want.c_str()), failures++;
for (auto* cb : all_cbs)
if (cb != up && !cb->ops.empty()) std::printf(" FAIL: serial replay recorded into %s\n", cb->name.c_str()), failures++;
} else if (c == "close_race") { // (at once: segment 1 records the texture's initialization into segment 0's init command
// buffer, holding the lock, while segment 0's worker finishes and closes it)
hold_in_init = true;
submit_segments({{"xL"}, {"t"}}, {});
play(c.c_str());
if (HAVE_SEG_INIT && init_of(kTex) != "init_0")
std::printf(" FAIL: the texture's initialization is in %s, not init_0\n", init_of(kTex).c_str()), failures++;
} else if (c == "after_submit") { // (at once: segment 1 makes the texture only after segment 0 was submitted)
submit_segments({{"-"}, {"zt"}}, {});
play(c.c_str());
if (named("init_0") && !named("init_0")->ops.empty())
std::printf(" FAIL: init_0 (submitted) was recorded into\n"), failures++;
} else if (c == "init_begin_fail") { // (segment 1's init command buffer cannot be begun: no worker starts, and the
// whole command buffer is recorded in order on the replay thread, after the segments)
#if HAVE_SEG_INIT
fail_begin = "init_1";
submit_segments({{"-"}, {"t"}, {"t"}}, {0, 2, 1});
if (jobs_started) std::printf(" FAIL: a segment's worker started\n"), failures++;
submit_post();
play(c.c_str());
for (auto* cb : queue)
if (cb->name.rfind("post", 0) != 0) std::printf(" FAIL: segment command buffer %s submitted\n", cb->name.c_str()), failures++;
if (init_of(kTex) != "postupload_0")
std::printf(" FAIL: the texture's initialization is in %s, not postupload_0\n", init_of(kTex).c_str()), failures++;
if (!named("post_0") || ops_text(named("post_0")) != "F Ut Ut") std::printf(" FAIL: not every segment's draw was recorded\n"), failures++;
#else
std::printf("init_begin_fail: no init command buffers in this code\n");
#endif
} else if (c == "init_end_fail") { // (at once: segments 1 and 2 wait for segment 0's submit, then make and use the texture,
// whose initialization goes to init_1; init_1 cannot be ended. The replay must stop
// for good (replay_fatal ends the process, exit code 3) after segment 2's worker is done,
// with nothing that uses the texture submitted; later work using it must never run)
#if HAVE_SEG_INIT
fail_end = "init_1";
on_log = [](const std::string& line) {
if (line.find("FATAL") == std::string::npos) return;
std::printf("init_end_fail: stopped: queue%s\n", queue_text().c_str());
if (running) std::printf(" FAIL: the replay stopped with a worker still recording\n");
CHECK(published_slots == std::set<int>{0}); // lost init and later slots were not published
for (auto* cb : queue)
for (const Op& o : cb->ops)
if (o.image == kTex) std::printf(" FAIL: %s, submitted, uses the texture whose initialization was lost\n", cb->name.c_str());
if (init_of(kTex) != "init_1") std::printf(" FAIL: the texture's initialization is in %s, not init_1\n", init_of(kTex).c_str());
std::fflush(stdout);
};
submit_segments({{"-"}, {"zt"}, {"ztd"}}, {});
// (the replay went on: the next recording (and frame) uses the texture again)
Recording& later = begin_recording(rec_cur_value);
texture_use();
record(later.cmd, 'U', kTex);
submit_post();
std::printf("%s: went on: queue%s\n", c.c_str(), queue_text().c_str());
std::printf(" FAIL: the replay went on after a new image's initialization was lost\n"), failures++;
#else
std::printf("init_end_fail: no init command buffers in this code\n");
#endif
} else if (c == "own_uses" || c == "own_all" || c == "own_off") {
// (Triforce: the replay thread records the next segment to submit when no worker has begun it. own_uses: the workers late,
// it records segment 0, which waits there until segment 1's worker has made the texture (into segment 0's init command
// buffer, still open) and then uses it. own_all: the workers later still, it records every segment, each the next to
// submit. own_off: RX_VK_PARALLEL_OWN=0, the workers alone however late)
const bool uses = c == "own_uses", all = c == "own_all";
own_on = c != "own_off";
helper_late_ms = all ? 300 : 50;
if (uses) submit_segments({{"wt"}, {"tW"}}, {});
else if (all) submit_segments({{"t"}, {"s"}, {"t"}}, {});
else submit_segments({{"t"}, {"t"}}, {});
play(c.c_str());
const int64_t want = uses ? 1 : all ? 3 : 0;
int by_replay = 0;
for (const auto& [script, own] : recorded) by_replay += own;
std::printf("%s: %lld segments recorded by the replay thread (ParStats), %d seen\n", c.c_str(), (long long)par_st.own.load(), by_replay);
if (par_st.own.load() != want || by_replay != want) std::printf(" FAIL: want %lld by the replay thread\n", (long long)want), failures++;
if (uses && !(recorded.size() == 2 && recorded[0] == std::pair<std::string, bool>{"wt", true} && !recorded[1].second))
std::printf(" FAIL: segment 0 was not the replay thread's, or segment 1 not a worker's\n"), failures++;
if (uses && init_of(kTex) != "init_0")
std::printf(" FAIL: the texture's initialization is in %s, not init_0\n", init_of(kTex).c_str()), failures++;
if (queue.size() < (size_t)(uses ? 4 : all ? 6 : 4)) std::printf(" FAIL: not every segment was submitted\n"), failures++;
} else if (c == "alloc_fail") { // (init_pools_make: each of its pool creations and allocations failing in turn)
#if HAVE_INIT_POOLS
const int n = (int)(sizeof(recordings) / sizeof(recordings[0]));
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
for (int f = 1; f <= 2 * n + 1; f++) { // (the last: none fails)
alloc_calls = 0, fail_at = f <= 2 * n ? f : 0;
const bool ok = init_pools_make(pi);
if (ok != !fail_at) std::printf(" FAIL: step %d failing, init_pools_make returned %d\n", f, ok), failures++;
for (const Recording& r : recordings)
if (ok ? (!r.init_pool || !r.init) : (r.init_pool || r.init))
std::printf(" FAIL: step %d failing, a recording's init pool/buffer is %p/%p\n", f, (void*)r.init_pool, (void*)r.init),
failures++;
if (!ok && !live_pools.empty())
std::printf(" FAIL: step %d failing, %zu init pools left made\n", f, live_pools.size()), failures++;
}
if (live_pools.size() != (size_t)n) std::printf(" FAIL: %zu init pools made, want %d\n", live_pools.size(), n), failures++;
init_pools_destroy();
if (!live_pools.empty()) std::printf(" FAIL: init pools left after init_pools_destroy\n"), failures++;
std::printf("alloc_fail: each of the %d steps failing gives back every pool made and leaves no handle; none failing makes %d\n",
2 * n, n);
#else
std::printf("alloc_fail: no init pools in this code\n");
#endif
} else if (c == "upload_rules") { // ((Triforce, 11-17) upload_earliest.h's rules)
CHECK(upload_earliest::wanted(true, false, false) && !upload_earliest::wanted(false, false, false) &&
!upload_earliest::wanted(true, true, false) && !upload_earliest::wanted(true, false, true));
const bool open[5] = {false, true, false, true, true};
auto is_open = [&](int k) { return open[k]; };
CHECK(upload_earliest::target(is_open, 5, 4) == 1 && upload_earliest::target(is_open, 5, 3) == 1);
CHECK(upload_earliest::target(is_open, 5, 1) == -1); // (itself the earliest open: its own uploads)
CHECK(upload_earliest::target(is_open, 5, 0) == -1 && upload_earliest::target(is_open, 2, 4) == 1);
const bool none[3] = {false, false, false};
CHECK(upload_earliest::target([&](int k) { return none[k]; }, 3, 2) == -1);
CHECK(upload_earliest::order_of(17, 4, 1) == 14 && upload_earliest::order_of(17, 4, 4) == 17);
std::printf("upload_rules: wanted, target and order_of as upload_earliest.h says\n");
} else if (c == "upload_later_first") { // (segment 1's worker uploads the texture first, then segment 0 samples it: one upload,
// in segment 0's init command buffer, before segment 0's draw; no twin)
upload_run(c.c_str(), {"u", "u"}, {1, 0}, upload_earliest_on() ? "init_0" : "upload_0 upload_1");
if (upload_earliest_on() && look.upload_seg != 1)
std::printf(" FAIL: upload_seg %llu, want segment 0's order 1\n", (unsigned long long)look.upload_seg), failures++;
} else if (c == "upload_three") { // (segment 0 is done (its init ended), 2 uploads while 1 still records: into init_1)
upload_run(c.c_str(), {"-", "u", "u"}, {0, 2, 1}, "init_1");
if (look.upload_seg != 2) std::printf(" FAIL: upload_seg %llu, want segment 1's order 2\n", (unsigned long long)look.upload_seg), failures++;
} else if (c == "upload_own") { // (the earliest segment still recorded uploads in its own uploads, as before)
upload_run(c.c_str(), {"u", "u"}, {0, 1}, "upload_0");
if (named("init_0") && !named("init_0")->ops.empty() && ops_text(named("init_0")) != "F Tt Ct Bt F")
std::printf(" FAIL: init_0 holds [%s]\n", ops_text(named("init_0")).c_str()), failures++;
reset_run();
upload_run("upload_own (after a done segment)", {"-", "u"}, {0, 1}, "upload_1");
} else if (c == "upload_close_race") { // (at once: segment 1 records the upload into segment 0's init command buffer, the init
// lock held, while segment 0's worker finishes and closes it)
hold_in_upload = true;
upload_run(c.c_str(), {"xL", "u"}, {}, "init_0");
const std::string init0 = named("init_0") ? ops_text(named("init_0")) : "";
if (init0.size() < 4 || init0.compare(init0.size() - 4, 4, "Wt F") != 0)
std::printf(" FAIL: init_0 is [%s]: not ended after the upload with its barrier\n", init0.c_str()), failures++;
std::printf("%s: segment ends that waited for the init lock: %lld\n", c.c_str(), (long long)st_seg_end_waits.load());
if (st_seg_end_waits.load() < 1) std::printf(" FAIL: segment 0's end did not wait for the upload\n"), failures++;
} else if (c == "upload_end_other") { // (at once: segment 2 records the upload into segment 0's init command buffer, its lock
// held, and segment 1 ends meanwhile: its end does not wait, only segment 0's may)
hold_for_other = true;
upload_run(c.c_str(), {"x", "x", "u"}, {}, "init_0");
} else if (c == "upload_nested") { // (the upload makes an image and both BC3 scratches inside the redirect: their first layout
// changes there too, before the copies, and no deadlock on the init lock)
nested_upload = true;
upload_run(c.c_str(), {"u", "u"}, {1, 0}, "init_0");
for (VkImage i : {kTex2, kScratch, kScratch2})
if (init_of(i) != "init_0")
std::printf(" FAIL: image %s's first layout change is in %s, not init_0\n", image_name(i), init_of(i).c_str()), failures++;
} else if (c == "upload_own_nested") { // (the earliest segment still recorded uploads in its own uploads, and makes an image and
// both BC3 scratches meanwhile: their first layout changes in its init, no lock held twice)
nested_upload = true;
upload_run(c.c_str(), {"u", "u"}, {0, 1}, "upload_0");
for (VkImage i : {kTex2, kScratch, kScratch2})
if (init_of(i) != "init_0")
std::printf(" FAIL: image %s's first layout change is in %s, not init_0\n", image_name(i), init_of(i).c_str()), failures++;
} else if (c == "upload_gpu_route") { // (a texture upload_inner reads on the GPU (the GPU unswizzle, the all-zero clear) stays in
// the uploads of the segment that looked first, the twin in segment 0's as before; one
// staged from the CPU goes into init_0)
gpu_route = true;
upload_run(c.c_str(), {"u", "u"}, {1, 0}, "upload_0 upload_1");
if (twins != 1) std::printf(" FAIL: the GPU-read upload was not left in segment 1's uploads, with the twin\n"), failures++;
gpu_route = false;
reset_run();
upload_run("upload_gpu_route (staged from the CPU)", {"u", "u"}, {1, 0}, upload_earliest_on() ? "init_0" : "upload_0 upload_1");
} else if (c == "upload_then_make") { // (segment 2's worker uploads the texture into init_0, then makes another texture that
// segment 0 uses too: its first layout change goes to init_0 as well, the redirect over)
upload_run(c.c_str(), {"n", "-", "un"}, {2, 1, 0}, "init_0");
if (init_of(kTex2) != "init_0")
std::printf(" FAIL: the second texture's first layout change is in %s, not init_0\n", init_of(kTex2).c_str()), failures++;
// (the redirect gave the worker its own upload command buffer back, unused: segment 2's copy into the second texture
// after the upload is there, after its leading full barrier)
if (!named("upload_2") || ops_text(named("upload_2")) != "G F Ut2 F G")
std::printf(" FAIL: upload_2 is [%s], want [G F Ut2 F G]\n", named("upload_2") ? ops_text(named("upload_2")).c_str() : "-"),
failures++;
} else if (c == "upload_barrier") { // (a submission before reads the texture; this one's upload is init_0's first command:
// a full barrier before it)
VkCommandBuffer up = make_cb("prevupload", 0), prev = make_cb("prev", 0);
vkBeginCommandBuffer(up, nullptr), vkBeginCommandBuffer(prev, nullptr);
texture_begin_recording(up, rec_cur_value);
texture_look(), record(prev, 'R', kTex);
CHECK(texture_uploads_recorded());
vkEndCommandBuffer(up), vkEndCommandBuffer(prev);
VkCommandBufferSubmitInfo cb[2]{{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}, {VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}};
cb[0].commandBuffer = up, cb[1].commandBuffer = prev;
VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
si.commandBufferInfoCount = 2, si.pCommandBufferInfos = cb;
vkQueueSubmit2(VK_NULL_HANDLE, 1, &si, VK_NULL_HANDLE);
rec_cur_value++; // (the next submission)
upload_run(c.c_str(), {"u", "u"}, {1, 0}, "init_0");
if (ops_text(named("init_0")) != "F Wt F") std::printf(" FAIL: init_0 is [%s], want [F Wt F]\n", ops_text(named("init_0")).c_str()), failures++;
} else if (c == "upload_off") { // (RX_VK_UPLOAD_EARLIEST=0: each segment's uploads its own, the twin in segment 0's: as before)
if (upload_earliest_on()) std::printf(" FAIL: run it with RX_VK_UPLOAD_EARLIEST=0\n"), failures++;
upload_run(c.c_str(), {"u", "u"}, {1, 0}, "upload_0 upload_1");
for (auto [name, want] : {std::pair<const char*, const char*>{"init_0", "F Tt Ct Bt F"}, {"upload_0", "G F Wt F G"},
{"upload_1", "G F Wt F G"}, {"cmd_0", "G Rt G"}, {"cmd_1", "G Rt G"}})
if (!named(name) || ops_text(named(name)) != want)
std::printf(" FAIL: %s is [%s], want [%s] as before\n", name, named(name) ? ops_text(named(name)).c_str() : "-", want), failures++;
if (twins != 1 || look.upload_seg != 1) std::printf(" FAIL: want one twin, by segment 0\n"), failures++;
} else if (c == "upload_random") { // (every order of 4 segments' workers and every set of them sampling the texture, one after
// another; then runs at once with the workers' timing mixed)
const bool on = upload_earliest_on();
int runs = 0;
for (int mask = 1; mask < 16; mask++) {
std::vector<int> ord = {0, 1, 2, 3};
do {
reset_run();
std::vector<std::string> scripts;
for (int k = 0; k < 4; k++) scripts.push_back(mask >> k & 1 ? "u" : "-");
size_t f = 0; // (the first to sample it, in the run order)
while (!(mask >> ord[f] & 1)) f++;
int target = -1; // (the earliest segment not run yet, before it)
for (int k = 0; k < ord[f] && target < 0; k++)
if (std::find(ord.begin(), ord.begin() + (ptrdiff_t)f, k) == ord.begin() + (ptrdiff_t)f) target = k;
const std::string want = !on ? "" : target >= 0 ? "init_" + std::to_string(target) : "upload_" + std::to_string(ord[f]);
char name[64];
std::snprintf(name, sizeof name, "upload_random mask %d order %d%d%d%d", mask, ord[0], ord[1], ord[2], ord[3]);
const int f0 = failures;
upload_run(name, scripts, ord, want, true);
if (on && look.upload_seg != 1 + (uint64_t)(target >= 0 ? target : ord[f]))
std::printf(" FAIL: %s: upload_seg %llu\n", name, (unsigned long long)look.upload_seg), failures++;
if (failures != f0) break;
runs++;
} while (std::next_permutation(ord.begin(), ord.end()));
}
std::srand(7);
const char* pick[] = {"-", "u", "ru", "ur", "r", "rru"};
for (int i = 0; i < 60 && !failures; i++, runs++) {
reset_run();
std::vector<std::string> scripts;
for (int k = 0; k < 4; k++) scripts.push_back(pick[std::rand() % 6]);
upload_run("upload_random (at once)", scripts, {}, "", true);
}
std::printf("%s: %d runs, each draw sampling this submission's upload; %s\n", c.c_str(), runs,
on ? "one upload a run, no twin" : "the twins as before (RX_VK_UPLOAD_EARLIEST=0)");
} else {
std::printf("unknown case: %s\n", c.c_str());
return 2;
}
std::fflush(stdout);
return failures ? 1 : 0;
}