// RX_VK_RECORDER's packets against a fake Vulkan (tests/test_recorder_sync.py builds it with the real vk_work.h and vk_recorder.h). // Every wrapped vkCmd* goes to a fake that logs the call with its arguments fully dereferenced, the way a driver reads them (by the // counts and descriptor types, the pointers themselves never), so two logs are equal exactly when the driver was handed the same // values. A frame of representative calls is made the way the renderer makes them (thread-local descriptor writes reset at every draw, // push constants and vertex arrays on the stack, a rendering info with attachments on the stack, barriers, a reused copy vector, // dynamic state arrays), and with `mutate` the caller poisons (0xCD) all of that right after each call, as the next draw reuses it. // // recorder_sync_test the checks; exits 1 on the first difference // recorder_sync_test --bench N the packet cost per draw on the fake (fakes that do nothing), ns, median of 5 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef _WIN32 #include #endif #include "vk_work.h" #ifdef _WIN32 extern "C" __declspec(dllimport) unsigned int __stdcall SetErrorMode(unsigned int); // (a fault exits, without a dialog) extern "C" __declspec(dllimport) int __stdcall GetThreadTimes(void*, void*, void*, void*, void*); // (--idle: the recorder's CPU) #endif namespace mkwvk { std::atomic work_epoch{0}; std::atomic par_active{false}; void logf(const char* fmt, ...) { // (the renderer's log: printed, for test_recorder_sync.py to count) char b[1024]; va_list a; va_start(a, fmt); std::vsnprintf(b, sizeof b, fmt, a); va_end(a); std::printf("LOG: %s\n", b); } } // namespace mkwvk // ---- the arena's aligned blocks: counted (a block lost is a leak), and the Nth one made to fail (g_fail_nth: an allocation failure; // the throwing form throws std::bad_alloc as the system's does) static std::atomic g_aligned_live{0}, g_aligned_calls{0}, g_fail_nth{0}; static bool aligned_fails() { return g_fail_nth && ++g_aligned_calls == g_fail_nth; } static void* aligned_get(size_t n, std::align_val_t al) { #ifdef _WIN32 void* p = _aligned_malloc(n, (size_t)al); #else void* p = nullptr; if (posix_memalign(&p, (size_t)al, n)) p = nullptr; #endif if (p) g_aligned_live++; return p; } static void aligned_put(void* p) { if (!p) return; g_aligned_live--; #ifdef _WIN32 _aligned_free(p); #else free(p); #endif } void* operator new(size_t n, std::align_val_t al) { void* p = aligned_fails() ? nullptr : aligned_get(n, al); if (!p) throw std::bad_alloc(); return p; } void* operator new[](size_t n, std::align_val_t al) { return operator new(n, al); } void* operator new(size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return aligned_fails() ? nullptr : aligned_get(n, al); } void* operator new[](size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return aligned_fails() ? nullptr : aligned_get(n, al); } void operator delete(void* p, std::align_val_t) noexcept { aligned_put(p); } void operator delete[](void* p, std::align_val_t) noexcept { aligned_put(p); } void operator delete(void* p, size_t, std::align_val_t) noexcept { aligned_put(p); } void operator delete[](void* p, size_t, std::align_val_t) noexcept { aligned_put(p); } void operator delete(void* p, std::align_val_t, const std::nothrow_t&) noexcept { aligned_put(p); } void operator delete[](void* p, std::align_val_t, const std::nothrow_t&) noexcept { aligned_put(p); } using namespace mkwvk; // ---- the fake driver's log (RX_VK_RECORDER=thread: the recorder thread and the producer both call the fake: one lock for the log, // and a command buffer's in-use flag held across each call, so two calls on one command buffer at once are seen, not serialized) struct Call { std::string op, bytes; }; static std::vector g_log; static std::mutex g_log_mu; static bool g_quiet = false; // (bench: the fakes only count) static uint64_t g_calls = 0; static int g_spin_ns = 0; // (bench --overlap: the time a quiet fake vkCmd* takes, as the driver's encoding does) static std::atomic g_delay_ns{0}; // (thread runs: each logged vkCmd* takes this long, so the recorder lags the producer) static std::atomic g_gate{false}; // (B_DIRECT control: vkCmd* on other threads wait until it is let go) // (status reads: vkCmd* on other threads also wait until this steady-clock time, so the recorder cannot catch up before the read // however the runner schedules the threads; a read that waits for the recorder is let through when it passes) static std::atomic g_hold_until_ns{0}; static bool held_until() { const int64_t until = g_hold_until_ns.load(); return until && std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() < until; } // (thread runs of the ordering scenarios: hold() before each point where the producer must wait for the recorder (a recording's // end, a submit while another command buffer records, the replay lock let go, a pool reset) holds the recorder for this long, so its // commands are still in flight there whatever the runner's load: a producer that does not wait is caught every time, not only when // the 1 us a call happened to keep the recorder behind (the flakes of the broken variants on CI). 0: no hold (the reference runs) static int g_hold_ms = 0; static void hold() { if (!g_hold_ms) return; g_hold_until_ns = (int64_t)std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count() + (int64_t)g_hold_ms * 1000000; } static bool g_hash_only = false; // (stress: each call folded into g_hash and dropped) static uint64_t g_hash = 0xcbf29ce484222325ull; static std::thread::id g_main_id; static std::atomic g_cmd_off_main{0}, g_cmd_calls{0}; // (vkCmd* the fake was given, and those on another thread) static std::atomic g_throw{false}; // (--throw: the next vkCmdDraw throws, as a driver call could) static void spin_for(int ns) { const auto t0 = std::chrono::steady_clock::now(); while (std::chrono::steady_clock::now() - t0 < std::chrono::nanoseconds(ns)) {} } static void fold(const Call& c) { for (const std::string* str : {&c.op, &c.bytes}) for (unsigned char ch : *str) g_hash = (g_hash ^ ch) * 0x100000001b3ull; g_hash = (g_hash ^ 0xFF) * 0x100000001b3ull; } static void begin(const char* op) { g_log.push_back({op, {}}); } template static void put(const T& v) { static_assert(std::is_arithmetic_v || std::is_enum_v, "fields one by one: no padding, no pointers"); g_log.back().bytes.append(reinterpret_cast(&v), sizeof v); } template static void handle(T h) { put((uint64_t)(uintptr_t)h); } static void raw(const void* p, size_t n) { g_log.back().bytes.append(static_cast(p), n); } static void tag(const char* s) { g_log.back().bytes.append(s).push_back(0); } // (a pointer or count the caller has since poisoned, which a missed copy hands over: logged as such, never followed) static bool readable(const void* p, uint64_t n) { if ((uintptr_t)p == (uintptr_t)0xCDCDCDCDCDCDCDCDull || n > 4096) { tag("UNREADABLE"); return false; } return true; } template static void each(const T* p, uint64_t n, F f) { put((uint32_t)n); if (!n || !readable(p, n)) return; for (uint64_t i = 0; i < n; i++) f(p[i]); } template static void opt(const T* p, F f) { // (an optional struct: there or not, then what it holds) put((uint8_t)(p != nullptr)); if (p && readable(p, 1)) f(*p); } static void L(const VkImageSubresourceRange& r) { put(r.aspectMask), put(r.baseMipLevel), put(r.levelCount), put(r.baseArrayLayer), put(r.layerCount); } static void L(const VkImageSubresourceLayers& r) { put(r.aspectMask), put(r.mipLevel), put(r.baseArrayLayer), put(r.layerCount); } static void L(const VkOffset3D& o) { put(o.x), put(o.y), put(o.z); } static void L(const VkExtent3D& e) { put(e.width), put(e.height), put(e.depth); } static void L(const VkRect2D& r) { put(r.offset.x), put(r.offset.y), put(r.extent.width), put(r.extent.height); } static void L(const VkViewport& v) { put(v.x), put(v.y), put(v.width), put(v.height), put(v.minDepth), put(v.maxDepth); } static void L(const VkClearValue& v) { raw(&v, sizeof v); } // (a union of 16 bytes, no padding) static void L(const VkClearColorValue& v) { raw(&v, sizeof v); } static void L(const VkClearDepthStencilValue& v) { put(v.depth), put(v.stencil); } static void L(const VkMemoryBarrier2& b) { put(b.sType), put((uint8_t)!!b.pNext), put(b.srcStageMask), put(b.srcAccessMask), put(b.dstStageMask), put(b.dstAccessMask); } static void L(const VkBufferMemoryBarrier2& b) { put(b.sType), put((uint8_t)!!b.pNext), put(b.srcStageMask), put(b.srcAccessMask), put(b.dstStageMask), put(b.dstAccessMask); put(b.srcQueueFamilyIndex), put(b.dstQueueFamilyIndex), handle(b.buffer), put(b.offset), put(b.size); } static void L(const VkImageMemoryBarrier2& b) { put(b.sType), put((uint8_t)!!b.pNext), put(b.srcStageMask), put(b.srcAccessMask), put(b.dstStageMask), put(b.dstAccessMask); put(b.oldLayout), put(b.newLayout), put(b.srcQueueFamilyIndex), put(b.dstQueueFamilyIndex), handle(b.image), L(b.subresourceRange); } static void L(const VkRenderingAttachmentInfo& a) { put(a.sType), put((uint8_t)!!a.pNext), handle(a.imageView), put(a.imageLayout), put(a.resolveMode), handle(a.resolveImageView); put(a.resolveImageLayout), put(a.loadOp), put(a.storeOp), L(a.clearValue); } static void L(const VkBufferCopy& c) { put(c.srcOffset), put(c.dstOffset), put(c.size); } static void L(const VkBufferImageCopy& c) { put(c.bufferOffset), put(c.bufferRowLength), put(c.bufferImageHeight), L(c.imageSubresource), L(c.imageOffset), L(c.imageExtent); } static void L(const VkImageCopy& c) { L(c.srcSubresource), L(c.srcOffset), L(c.dstSubresource), L(c.dstOffset), L(c.extent); } static void L(const VkImageResolve& c) { L(c.srcSubresource), L(c.srcOffset), L(c.dstSubresource), L(c.dstOffset), L(c.extent); } static void L(const VkImageBlit& b) { L(b.srcSubresource), L(b.srcOffsets[0]), L(b.srcOffsets[1]), L(b.dstSubresource), L(b.dstOffsets[0]), L(b.dstOffsets[1]); } static void L(const VkClearAttachment& a) { put(a.aspectMask), put(a.colorAttachment), L(a.clearValue); } static void L(const VkClearRect& r) { L(r.rect), put(r.baseArrayLayer), put(r.layerCount); } static void L(const VkColorBlendEquationEXT& e) { put(e.srcColorBlendFactor), put(e.dstColorBlendFactor), put(e.colorBlendOp), put(e.srcAlphaBlendFactor), put(e.dstAlphaBlendFactor), put(e.alphaBlendOp); } static void L(const VkWriteDescriptorSet& w) { // (as a driver reads a push: by type, one of the three arrays) put(w.sType), put((uint8_t)!!w.pNext), handle(w.dstSet), put(w.dstBinding), put(w.dstArrayElement), put(w.descriptorType); switch (rec::write_kind(w.descriptorType)) { case 1: each(w.pImageInfo, w.descriptorCount, [](const VkDescriptorImageInfo& i) { handle(i.sampler), handle(i.imageView), put(i.imageLayout); }); break; case 2: each(w.pBufferInfo, w.descriptorCount, [](const VkDescriptorBufferInfo& b) { handle(b.buffer), put(b.offset), put(b.range); }); break; case 3: each(w.pTexelBufferView, w.descriptorCount, [](VkBufferView v) { handle(v); }); break; default: put(w.descriptorCount); break; // (not read here: the packer leaves these to the direct call) } } template static void A(const T* p, uint64_t n) { each(p, n, [](const T& v) { L(v); }); } #define QUIET \ if (g_quiet) { g_calls++; if (g_spin_ns) spin_for(g_spin_ns); return; } // ---- the fake lifecycle (g_track): each command buffer's state as a driver tracks it. A vkCmd* on one that is not recording, a begin // of one recording, an end of one not recording or a submit of one not executable is a violation; the boundaries are logged as calls // too, so two logs also compare where the begins, ends, resets, submits, waits, reads and destructions fall among the commands. // Always: a call on a command buffer (or a reset of its pool, a submit of it) while another call on it is under way is a violation enum CbState { INITIAL, RECORDING, EXECUTABLE }; static bool g_track = false; static std::map g_state; static std::map g_pool_of; // (command buffer -> its pool) static std::mutex g_viol_mu; static int g_violations; static std::string g_violation; void violation(const char* op, const void* cb, const char* what) { std::lock_guard lk(g_viol_mu); if (g_violations++) return; char b[240]; std::snprintf(b, sizeof b, "%s on command buffer %#llx %s (driver call %zu)", op, (unsigned long long)(uintptr_t)cb, what, g_log.size()); g_violation = b; } static CbState& state(const void* cb) { return g_state[(uint64_t)(uintptr_t)cb]; } static void track(const void* cb, const char* op) { if (g_track && state(cb) != RECORDING) violation(op, cb, "not recording"); } // ---- the fake GPU (g_track): a command buffer's recorded effects (timestamps, checkpoints, fills) take effect when it is submitted, // as on a device: then the timeline steps and the fence is signalled. The status reads below (fences, semaphores, checkpoints, fault // data, memory invalidation) see only that, and are logged apart (g_reads), unordered against the vkCmd* stream; the ordered stream // keeps every other call. Destroyed buffers are remembered: a recorded command naming one is a violation struct Fx { int kind; // 1 timestamp (a: pool << 16 | query), 2 checkpoint (a: marker), 3 fill (a: buffer, b: data) uint64_t a, b; }; static std::map> g_cb_fx; static std::map g_query, g_mem; static std::set g_fences_done, g_destroyed; static uint64_t g_timeline = 0, g_checkpoint = 0; static std::vector g_reads; // (each status read: the call, what it saw) static std::map g_reads_overlapped; // (status reads made while the producer had packets not yet run) static void gpu_reset() { g_cb_fx.clear(), g_query.clear(), g_mem.clear(), g_fences_done.clear(), g_destroyed.clear(); g_timeline = 0, g_checkpoint = 0, g_reads.clear(), g_reads_overlapped.clear(); } static void fx(const void* cb, int kind, uint64_t a, uint64_t b = 0) { if (g_track) g_cb_fx[(uint64_t)(uintptr_t)cb].push_back({kind, a, b}); } void violation(const char* op, const void* cb, const char* what); static void alive(const void* cb, const char* op, VkBuffer b) { // (a recorded command naming a destroyed buffer) if (g_track && g_destroyed.count((uint64_t)(uintptr_t)b)) violation(op, cb, "names a destroyed buffer"); } static std::atomic g_inuse[256]; static std::atomic& inuse(uint64_t cb) { return g_inuse[(cb >> 8) & 255]; } // (the tests' command buffers: 0x100, 0x200, 0x300) // (Triforce: the recorder thread's own calls while par_active: a parallel replay's segments record directly, and the ring is empty // then, so it makes none; t_is_recorder set by rec::g_thread_start_hook) static thread_local bool t_is_recorder = false; static std::atomic g_rec_calls_par{0}; /// One fake call: its command buffer marked in use for the call (a second call on it meanwhile is a violation), a vkCmd* slowed by /// g_delay_ns, then the log's lock struct Guard { const void* cb; bool held = false; std::unique_lock lk; Guard(const void* c, const char* op, bool cmd) : cb(c) { while (cmd && std::this_thread::get_id() != g_main_id && (g_gate.load() || held_until())) std::this_thread::yield(); // (before the in-use mark) if (cb) { if (inuse((uint64_t)(uintptr_t)cb).exchange(true)) violation(op, cb, "in use by another thread (no external synchronization)"); else held = true; } if (cmd) { g_cmd_calls++; if (std::this_thread::get_id() != g_main_id) g_cmd_off_main++; if (t_is_recorder && par_active.load()) g_rec_calls_par++; if (const int d = g_delay_ns.load(std::memory_order_relaxed)) spin_for(d); } lk = std::unique_lock(g_log_mu); } ~Guard() { if (g_hash_only && !g_log.empty()) fold(g_log.back()), g_log.pop_back(); lk.unlock(); if (held) inuse((uint64_t)(uintptr_t)cb).store(false); } }; /// (a pool reset or a submit: its command buffers must not be in use meanwhile) static void not_in_use(const void* cb, const char* op) { if (inuse((uint64_t)(uintptr_t)cb).load()) violation(op, cb, "in use by another thread (no external synchronization)"); } #define F(name, params, ...) \ static void VKAPI_CALL fake_##name params { QUIET Guard g_(cb, #name, true); begin(#name); track(cb, #name); __VA_ARGS__ } F(vkCmdPipelineBarrier2, (VkCommandBuffer cb, const VkDependencyInfo* d), handle(cb); put(d->sType), put((uint8_t)!!d->pNext), put(d->dependencyFlags); A(d->pMemoryBarriers, d->memoryBarrierCount), A(d->pBufferMemoryBarriers, d->bufferMemoryBarrierCount); A(d->pImageMemoryBarriers, d->imageMemoryBarrierCount);) F(vkCmdBeginRendering, (VkCommandBuffer cb, const VkRenderingInfo* r), handle(cb); put(r->sType), put((uint8_t)!!r->pNext), put(r->flags); L(r->renderArea), put(r->layerCount), put(r->viewMask); A(r->pColorAttachments, r->colorAttachmentCount); opt(r->pDepthAttachment, [](const VkRenderingAttachmentInfo& a) { L(a); }); opt(r->pStencilAttachment, [](const VkRenderingAttachmentInfo& a) { L(a); });) F(vkCmdEndRendering, (VkCommandBuffer cb), handle(cb);) F(vkCmdDraw, (VkCommandBuffer cb, uint32_t a, uint32_t b, uint32_t c, uint32_t d), handle(cb), put(a), put(b), put(c), put(d); if (g_throw.exchange(false)) throw std::runtime_error("a driver call threw (test)");) F(vkCmdDrawIndexed, (VkCommandBuffer cb, uint32_t a, uint32_t b, uint32_t c, int32_t d, uint32_t e), handle(cb), put(a), put(b), put(c), put(d), put(e);) F(vkCmdDrawIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), handle(cb), handle(b), put(o), put(n), put(s);) F(vkCmdDrawIndexedIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), handle(cb), handle(b), put(o), put(n), put(s);) F(vkCmdDrawIndirectCount, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkBuffer cbuf, VkDeviceSize co, uint32_t m, uint32_t s), handle(cb), handle(b), put(o), handle(cbuf), put(co), put(m), put(s);) F(vkCmdDrawIndexedIndirectCount, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkBuffer cbuf, VkDeviceSize co, uint32_t m, uint32_t s), handle(cb), handle(b), put(o), handle(cbuf), put(co), put(m), put(s);) F(vkCmdDrawMeshTasksEXT, (VkCommandBuffer cb, uint32_t x, uint32_t y, uint32_t z), handle(cb), put(x), put(y), put(z);) F(vkCmdDrawMeshTasksIndirectEXT, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, uint32_t n, uint32_t s), handle(cb), handle(b), put(o), put(n), put(s);) F(vkCmdDrawMeshTasksIndirectCountEXT, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkBuffer cbuf, VkDeviceSize co, uint32_t m, uint32_t s), handle(cb), handle(b), put(o), handle(cbuf), put(co), put(m), put(s);) F(vkCmdDispatch, (VkCommandBuffer cb, uint32_t x, uint32_t y, uint32_t z), handle(cb), put(x), put(y), put(z);) F(vkCmdDispatchIndirect, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o), handle(cb), handle(b), put(o);) F(vkCmdCopyBuffer, (VkCommandBuffer cb, VkBuffer s, VkBuffer d, uint32_t n, const VkBufferCopy* r), handle(cb), handle(s), handle(d); A(r, n); alive(cb, "vkCmdCopyBuffer", s), alive(cb, "vkCmdCopyBuffer", d);) F(vkCmdCopyBufferToImage, (VkCommandBuffer cb, VkBuffer s, VkImage d, VkImageLayout l, uint32_t n, const VkBufferImageCopy* r), handle(cb), handle(s), handle(d), put(l); A(r, n);) F(vkCmdCopyImageToBuffer, (VkCommandBuffer cb, VkImage s, VkImageLayout l, VkBuffer d, uint32_t n, const VkBufferImageCopy* r), handle(cb), handle(s), put(l), handle(d); A(r, n);) F(vkCmdCopyImage, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, const VkImageCopy* r), handle(cb), handle(s), put(sl), handle(d), put(dl); A(r, n);) F(vkCmdBlitImage, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, const VkImageBlit* r, VkFilter f), handle(cb), handle(s), put(sl), handle(d), put(dl); A(r, n); put(f);) F(vkCmdResolveImage, (VkCommandBuffer cb, VkImage s, VkImageLayout sl, VkImage d, VkImageLayout dl, uint32_t n, const VkImageResolve* r), handle(cb), handle(s), put(sl), handle(d), put(dl); A(r, n);) F(vkCmdClearColorImage, (VkCommandBuffer cb, VkImage i, VkImageLayout l, const VkClearColorValue* c, uint32_t n, const VkImageSubresourceRange* r), handle(cb), handle(i), put(l); opt(c, [](const VkClearColorValue& v) { L(v); }); A(r, n);) F(vkCmdClearDepthStencilImage, (VkCommandBuffer cb, VkImage i, VkImageLayout l, const VkClearDepthStencilValue* c, uint32_t n, const VkImageSubresourceRange* r), handle(cb), handle(i), put(l); opt(c, [](const VkClearDepthStencilValue& v) { L(v); }); A(r, n);) F(vkCmdClearAttachments, (VkCommandBuffer cb, uint32_t na, const VkClearAttachment* a, uint32_t nr, const VkClearRect* r), handle(cb); A(a, na); A(r, nr);) F(vkCmdFillBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize s, uint32_t d), handle(cb), handle(b), put(o), put(s), put(d); alive(cb, "vkCmdFillBuffer", b); fx(cb, 3, (uint64_t)(uintptr_t)b, d);) F(vkCmdUpdateBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkDeviceSize s, const void* d), handle(cb), handle(b), put(o), put(s); if (readable(d, s / 16)) raw(d, (size_t)s);) F(vkCmdBeginQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q, VkQueryControlFlags f), handle(cb), handle(p), put(q), put(f);) F(vkCmdEndQuery, (VkCommandBuffer cb, VkQueryPool p, uint32_t q), handle(cb), handle(p), put(q);) F(vkCmdBindPipeline, (VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipeline p), handle(cb), put(bp), handle(p);) F(vkCmdBindDescriptorSets, (VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipelineLayout l, uint32_t first, uint32_t n, const VkDescriptorSet* s, uint32_t nd, const uint32_t* d), handle(cb), put(bp), handle(l), put(first); each(s, n, [](VkDescriptorSet v) { handle(v); }); each(d, nd, [](uint32_t v) { put(v); });) F(vkCmdBindIndexBuffer, (VkCommandBuffer cb, VkBuffer b, VkDeviceSize o, VkIndexType t), handle(cb), handle(b), put(o), put(t);) F(vkCmdBindVertexBuffers2, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkBuffer* b, const VkDeviceSize* o, const VkDeviceSize* s, const VkDeviceSize* st), handle(cb), put(first); each(b, n, [](VkBuffer v) { handle(v); }); each(o, n, [](VkDeviceSize v) { put(v); }); put((uint8_t)!!s); if (s) each(s, n, [](VkDeviceSize v) { put(v); }); put((uint8_t)!!st); if (st) each(st, n, [](VkDeviceSize v) { put(v); });) F(vkCmdPushConstants, (VkCommandBuffer cb, VkPipelineLayout l, VkShaderStageFlags f, uint32_t o, uint32_t s, const void* v), handle(cb), handle(l), put(f), put(o), put(s); if (readable(v, s / 16)) raw(v, s);) F(vkCmdPushDescriptorSetKHR, (VkCommandBuffer cb, VkPipelineBindPoint bp, VkPipelineLayout l, uint32_t set, uint32_t n, const VkWriteDescriptorSet* w), handle(cb), put(bp), handle(l), put(set); A(w, n);) // ((Triforce) the bytes a push template reads, as the driver knows them from the template: here its handle's low 12 bits, kTplHandle + n) constexpr uint64_t kTplHandle = 0xA000; F(vkCmdPushDescriptorSetWithTemplateKHR, (VkCommandBuffer cb, VkDescriptorUpdateTemplate t, VkPipelineLayout l, uint32_t set, const void* d), handle(cb), handle(t), handle(l), put(set); if (readable(d, ((uintptr_t)t & 0xFFF) / 16)) raw(d, (uintptr_t)t & 0xFFF);) F(vkCmdSetViewport, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkViewport* v), handle(cb), put(first); A(v, n);) F(vkCmdSetScissor, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkRect2D* r), handle(cb), put(first); A(r, n);) F(vkCmdSetViewportWithCount, (VkCommandBuffer cb, uint32_t n, const VkViewport* v), handle(cb); A(v, n);) F(vkCmdSetScissorWithCount, (VkCommandBuffer cb, uint32_t n, const VkRect2D* r), handle(cb); A(r, n);) F(vkCmdSetBlendConstants, (VkCommandBuffer cb, const float b[4]), handle(cb); each(b, 4, [](float v) { put(v); });) F(vkCmdSetStencilReference, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), handle(cb), put(f), put(v);) F(vkCmdSetStencilCompareMask, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), handle(cb), put(f), put(v);) F(vkCmdSetStencilWriteMask, (VkCommandBuffer cb, VkStencilFaceFlags f, uint32_t v), handle(cb), put(f), put(v);) F(vkCmdSetStencilOp, (VkCommandBuffer cb, VkStencilFaceFlags f, VkStencilOp a, VkStencilOp b, VkStencilOp c, VkCompareOp d), handle(cb), put(f), put(a), put(b), put(c), put(d);) F(vkCmdSetDepthBias, (VkCommandBuffer cb, float a, float b, float c), handle(cb), put(a), put(b), put(c);) F(vkCmdSetDepthBiasEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetLineWidth, (VkCommandBuffer cb, float v), handle(cb), put(v);) F(vkCmdSetDepthBoundsTestEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetCullMode, (VkCommandBuffer cb, VkCullModeFlags v), handle(cb), put(v);) F(vkCmdSetFrontFace, (VkCommandBuffer cb, VkFrontFace v), handle(cb), put(v);) F(vkCmdSetPrimitiveTopology, (VkCommandBuffer cb, VkPrimitiveTopology v), handle(cb), put(v);) F(vkCmdSetPrimitiveRestartEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetDepthTestEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetDepthWriteEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetDepthCompareOp, (VkCommandBuffer cb, VkCompareOp v), handle(cb), put(v);) F(vkCmdSetStencilTestEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetRasterizerDiscardEnable, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetPolygonModeEXT, (VkCommandBuffer cb, VkPolygonMode v), handle(cb), put(v);) F(vkCmdSetDepthClampEnableEXT, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetAlphaToCoverageEnableEXT, (VkCommandBuffer cb, VkBool32 v), handle(cb), put(v);) F(vkCmdSetColorBlendEnableEXT, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkBool32* v), handle(cb), put(first); each(v, n, [](VkBool32 x) { put(x); });) F(vkCmdSetColorBlendEquationEXT, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkColorBlendEquationEXT* v), handle(cb), put(first); A(v, n);) F(vkCmdSetColorWriteMaskEXT, (VkCommandBuffer cb, uint32_t first, uint32_t n, const VkColorComponentFlags* v), handle(cb), put(first); each(v, n, [](VkColorComponentFlags x) { put(x); });) F(vkCmdWriteBufferMarkerAMD, (VkCommandBuffer cb, VkPipelineStageFlagBits s, VkBuffer b, VkDeviceSize o, uint32_t m), handle(cb), put(s), handle(b), put(o), put(m);) F(vkCmdSetCheckpointNV, (VkCommandBuffer cb, const void* m), handle(cb), handle(m); fx(cb, 2, (uint64_t)(uintptr_t)m);) // (an opaque value: the pointer is the datum) F(vkCmdWriteTimestamp2, (VkCommandBuffer cb, VkPipelineStageFlags2 s, VkQueryPool p, uint32_t q), handle(cb), put(s), handle(p), put(q); fx(cb, 1, (uint64_t)(uintptr_t)p << 16 | q);) F(vkCmdResetQueryPool, (VkCommandBuffer cb, VkQueryPool p, uint32_t f, uint32_t n), handle(cb), handle(p), put(f), put(n);) static VkResult VKAPI_CALL fake_vkBeginCommandBuffer(VkCommandBuffer cb, const VkCommandBufferBeginInfo* bi) { Guard g_(cb, "vkBeginCommandBuffer", false); begin("vkBeginCommandBuffer"), handle(cb), put(bi->flags); g_cb_fx.erase((uint64_t)(uintptr_t)cb); if (g_track && state(cb) == RECORDING) violation("vkBeginCommandBuffer", cb, "already recording"); state(cb) = RECORDING; return VK_SUCCESS; } static VkResult VKAPI_CALL fake_vkEndCommandBuffer(VkCommandBuffer cb) { Guard g_(cb, "vkEndCommandBuffer", false); begin("vkEndCommandBuffer"), handle(cb); track(cb, "vkEndCommandBuffer"); state(cb) = EXECUTABLE; return VK_SUCCESS; } static VkResult VKAPI_CALL fake_vkResetCommandPool(VkDevice d, VkCommandPool pool, VkCommandPoolResetFlags f) { Guard g_(nullptr, "vkResetCommandPool", false); begin("vkResetCommandPool"), handle(d), handle(pool), put(f); for (const auto& [cb, of] : g_pool_of) if (of == (uint64_t)(uintptr_t)pool) not_in_use((const void*)(uintptr_t)cb, "vkResetCommandPool"), g_state[cb] = INITIAL, g_cb_fx.erase(cb); return VK_SUCCESS; } static VkResult VKAPI_CALL fake_vkQueueSubmit2(VkQueue q, uint32_t n, const VkSubmitInfo2* si, VkFence fence) { Guard g_(nullptr, "vkQueueSubmit2", false); begin("vkQueueSubmit2"), handle(q); for (uint32_t i = 0; i < n; i++) for (uint32_t c = 0; c < si[i].commandBufferInfoCount; c++) { const VkCommandBuffer cb = si[i].pCommandBufferInfos[c].commandBuffer; handle(cb); not_in_use(cb, "vkQueueSubmit2"); if (g_track && state(cb) != EXECUTABLE) violation("vkQueueSubmit2", cb, "not executable"); for (const Fx& e : g_cb_fx[(uint64_t)(uintptr_t)cb]) // (the fake GPU runs it: its effects, now) if (e.kind == 1) g_query[e.a] = g_timeline + 1; else if (e.kind == 2) g_checkpoint = e.a; else g_mem[e.a] = e.b; } handle(fence); g_timeline++; if (fence) g_fences_done.insert((uint64_t)(uintptr_t)fence); return VK_SUCCESS; } /// A status read: logged apart from the ordered stream, with what it saw; counted when it ran with packets not yet run static void read_seen(const char* op, uint64_t v) { g_reads.push_back(std::hash()(op)), g_reads.push_back(v); if (rec::mode() == rec::THREAD && rec::t_prod && rec::g_rec.w > rec::g_rec.R.load()) g_reads_overlapped[op]++; } static VkResult VKAPI_CALL fake_vkWaitForFences(VkDevice, uint32_t n, const VkFence* f, VkBool32, uint64_t) { Guard g_(nullptr, "vkWaitForFences", false); bool done = true; for (uint32_t i = 0; i < n; i++) done = done && g_fences_done.count((uint64_t)(uintptr_t)f[i]); read_seen("vkWaitForFences", done); return done ? VK_SUCCESS : VK_TIMEOUT; } static VkResult VKAPI_CALL fake_vkGetSemaphoreCounterValue(VkDevice, VkSemaphore, uint64_t* v) { Guard g_(nullptr, "vkGetSemaphoreCounterValue", false); *v = g_timeline; read_seen("vkGetSemaphoreCounterValue", *v); return VK_SUCCESS; } static VkResult VKAPI_CALL fake_vkWaitSemaphores(VkDevice, const VkSemaphoreWaitInfo* wi, uint64_t) { Guard g_(nullptr, "vkWaitSemaphores", false); const bool done = g_timeline >= wi->pValues[0]; read_seen("vkWaitSemaphores", done); return done ? VK_SUCCESS : VK_TIMEOUT; } static void VKAPI_CALL fake_vkGetQueueCheckpointDataNV(VkQueue, uint32_t* n, VkCheckpointDataNV* cd) { Guard g_(nullptr, "vkGetQueueCheckpointDataNV", false); if (cd && *n) cd[0].pCheckpointMarker = (void*)(uintptr_t)g_checkpoint, *n = 1; read_seen("vkGetQueueCheckpointDataNV", g_checkpoint); } static VkResult VKAPI_CALL fake_vkGetDeviceFaultInfoEXT(VkDevice, VkDeviceFaultCountsEXT* c, VkDeviceFaultInfoEXT*) { Guard g_(nullptr, "vkGetDeviceFaultInfoEXT", false); c->addressInfoCount = 0, c->vendorInfoCount = 0, c->vendorBinarySize = g_timeline; // (what the device ran: the submits) read_seen("vkGetDeviceFaultInfoEXT", g_timeline); return VK_SUCCESS; } // (VMA's, faked: the allocation is named by the buffer the GPU wrote; what the host then sees) struct VmaAllocator_T; struct VmaAllocation_T; typedef VmaAllocator_T* VmaAllocator; typedef VmaAllocation_T* VmaAllocation; VkResult (vmaInvalidateAllocation)(VmaAllocator, VmaAllocation a, VkDeviceSize, VkDeviceSize) { Guard g_(nullptr, "vmaInvalidateAllocation", false); const uint64_t v = g_mem.count((uint64_t)(uintptr_t)a) ? g_mem[(uint64_t)(uintptr_t)a] : 0; read_seen("vmaInvalidateAllocation", v); return VK_SUCCESS; } static VkResult VKAPI_CALL fake_vkGetQueryPoolResults(VkDevice d, VkQueryPool p, uint32_t first, uint32_t n, size_t size, void* data, VkDeviceSize stride, VkQueryResultFlags flags) { Guard g_(nullptr, "vkGetQueryPoolResults", false); begin("vkGetQueryPoolResults"), handle(d), handle(p), put(first), put(n), put(stride), put(flags); std::memset(data, 0, size); for (uint32_t i = 0; i < n && (i + 1) * stride <= size; i++) { // (the timestamps the submitted command buffers wrote) const auto it = g_query.find((uint64_t)(uintptr_t)p << 16 | (first + i)); const uint64_t v = it == g_query.end() ? 0 : it->second; std::memcpy(static_cast(data) + i * stride, &v, 8), put(v); } return VK_SUCCESS; } static void VKAPI_CALL fake_vkDestroyBuffer(VkDevice d, VkBuffer b, const VkAllocationCallbacks*) { Guard g_(nullptr, "vkDestroyBuffer", false); begin("vkDestroyBuffer"), handle(d), handle(b); g_destroyed.insert((uint64_t)(uintptr_t)b); } static void VKAPI_CALL fake_vkDestroyDevice(VkDevice d, const VkAllocationCallbacks*) { Guard g_(nullptr, "vkDestroyDevice", false); begin("vkDestroyDevice"), handle(d); } namespace vkx { #define RX_FAKE(f) PFN_##f f = fake_##f; RX_REC_CALLS(RX_FAKE, RX_FAKE) RX_FAKE(vkBeginCommandBuffer) RX_FAKE(vkEndCommandBuffer) RX_FAKE(vkResetCommandPool) RX_FAKE(vkQueueSubmit2) RX_FAKE(vkWaitForFences) RX_FAKE(vkGetQueryPoolResults) RX_FAKE(vkDestroyBuffer) RX_FAKE(vkDestroyDevice) RX_FAKE(vkGetSemaphoreCounterValue) RX_FAKE(vkWaitSemaphores) RX_FAKE(vkGetQueueCheckpointDataNV) RX_FAKE(vkGetDeviceFaultInfoEXT) #undef RX_FAKE } // namespace vkx // ---- the renderer's shapes of argument storage template static T H(uint64_t v) { return (T)(uintptr_t)v; } template static void poison(T* p, size_t n = 1) { std::memset((void*)p, 0xCD, sizeof(T) * n); } template static void poison(std::vector& v) { // (the next draw's reset: the storage stays, its contents change) if (!v.empty()) poison(v.data(), v.size()); v.clear(); } static bool g_mutate = false; constexpr uint32_t kPushSize = 96; // (vk_draw.cpp's: MESH_FALLBACK_PUSH_END) struct DescriptorWrites { // (vk_draw.cpp's thread-local dw: reset at every build, the writes point into the info vectors) std::vector writes; std::vector bufs; std::vector imgs; std::vector views; void reset() { writes.clear(), bufs.clear(), imgs.clear(), views.clear(); bufs.reserve(64), imgs.reserve(64), views.reserve(64); } void after_call() { if (g_mutate) poison(writes), poison(bufs), poison(imgs), poison(views); } }; static thread_local DescriptorWrites dw; static VkWriteDescriptorSet write(uint32_t binding, VkDescriptorType t, uint32_t count) { VkWriteDescriptorSet w{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; w.dstBinding = binding, w.descriptorType = t, w.descriptorCount = count; return w; } static void build_graphics(int d) { dw.reset(); for (uint32_t i = 0; i < 3; i++) { dw.bufs.push_back({H(0xB000 + d * 16 + i), 256ull * i + d, 64ull + i}); VkWriteDescriptorSet w = write(i, i == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_BUFFER : VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1); w.pBufferInfo = &dw.bufs.back(); if (i == 1) w.pImageInfo = (const VkDescriptorImageInfo*)(uintptr_t)0xCDCDCDCDCDCDCDCDull; // (ignored for a buffer) dw.writes.push_back(w); } for (uint32_t i = 0; i < 4; i++) { dw.imgs.push_back({H(0x5000 + i), H(0x6000 + d * 16 + i), VK_IMAGE_LAYOUT_GENERAL}); VkWriteDescriptorSet w = write(3 + i, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1); w.pImageInfo = &dw.imgs.back(); dw.writes.push_back(w); } const size_t at = dw.imgs.size(); // (an array binding: two sampled images) dw.imgs.push_back({VK_NULL_HANDLE, H(0x6100 + d), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL}); dw.imgs.push_back({VK_NULL_HANDLE, H(0x6200 + d), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL}); VkWriteDescriptorSet arr = write(7, VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 2); arr.dstArrayElement = 0, arr.pImageInfo = &dw.imgs[at]; dw.writes.push_back(arr); dw.views.push_back(H(0x7000 + d)); VkWriteDescriptorSet tb = write(8, VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 1); tb.pTexelBufferView = &dw.views.back(); dw.writes.push_back(tb); } static void push_constants(VkCommandBuffer cmd, VkPipelineLayout layout, int d) { uint8_t data[kPushSize]{}; for (uint32_t i = 0; i < kPushSize; i++) data[i] = (uint8_t)(i * 7 + d); vkCmdPushConstants(cmd, layout, VK_SHADER_STAGE_ALL_GRAPHICS, 0, kPushSize, data); if (g_mutate) poison(data, kPushSize); const uint64_t firsts = 0x1000ull + d; // (a mesh draw's firsts: 8 bytes at their offset) uint64_t f = firsts; vkCmdPushConstants(cmd, layout, VK_SHADER_STAGE_ALL_GRAPHICS, 80, 8, &f); if (g_mutate) poison(&f); } struct DynState { // (vk_draw.cpp's DynState: the arrays the dynamic state calls point into) VkViewport vp; VkRect2D sc; float blend[4]; VkBool32 en[8]; VkColorBlendEquationEXT eq[8]; VkColorComponentFlags wm[8]; }; static void dynamic_state(VkCommandBuffer cmd, int d) { DynState s{}; s.vp = {0, (float)d, 1280, 720, 0, 1}; s.sc = {{0, d}, {1280, 720}}; for (int i = 0; i < 4; i++) s.blend[i] = 0.25f * i + d; for (uint32_t i = 0; i < 8; i++) { s.en[i] = (i + d) & 1; s.eq[i] = {VK_BLEND_FACTOR_SRC_ALPHA, VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA, VK_BLEND_OP_ADD, VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_ZERO, (VkBlendOp)(i % 3)}; s.wm[i] = 0xF >> (i & 3); } vkCmdSetViewportWithCount(cmd, 1, &s.vp); vkCmdSetScissorWithCount(cmd, 1, &s.sc); vkCmdSetBlendConstants(cmd, s.blend); vkCmdSetStencilReference(cmd, VK_STENCIL_FACE_FRONT_BIT, 1 + d); vkCmdSetStencilReference(cmd, VK_STENCIL_FACE_BACK_BIT, 2 + d); vkCmdSetStencilCompareMask(cmd, VK_STENCIL_FACE_FRONT_BIT, 0xFF); vkCmdSetStencilWriteMask(cmd, VK_STENCIL_FACE_BACK_BIT, 0x0F); vkCmdSetStencilOp(cmd, VK_STENCIL_FACE_FRONT_BIT, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_REPLACE, VK_STENCIL_OP_INCREMENT_AND_CLAMP, VK_COMPARE_OP_LESS_OR_EQUAL); vkCmdSetDepthBias(cmd, 1.5f, 0.0f, -2.0f + d); vkCmdSetDepthBiasEnable(cmd, VK_TRUE); if (d == 0) vkCmdSetLineWidth(cmd, 1.0f), vkCmdSetDepthBoundsTestEnable(cmd, VK_FALSE); vkCmdSetCullMode(cmd, VK_CULL_MODE_BACK_BIT); vkCmdSetFrontFace(cmd, VK_FRONT_FACE_COUNTER_CLOCKWISE); vkCmdSetPrimitiveTopology(cmd, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST); vkCmdSetPrimitiveRestartEnable(cmd, VK_FALSE); vkCmdSetDepthTestEnable(cmd, VK_TRUE); vkCmdSetDepthWriteEnable(cmd, d & 1); vkCmdSetDepthCompareOp(cmd, VK_COMPARE_OP_GREATER); vkCmdSetStencilTestEnable(cmd, VK_FALSE); vkCmdSetRasterizerDiscardEnable(cmd, VK_FALSE); vkCmdSetPolygonModeEXT(cmd, VK_POLYGON_MODE_FILL); vkCmdSetDepthClampEnableEXT(cmd, VK_FALSE); vkCmdSetAlphaToCoverageEnableEXT(cmd, d == 2); vkCmdSetColorBlendEnableEXT(cmd, 0, 3, s.en); vkCmdSetColorBlendEquationEXT(cmd, 0, 3, s.eq); vkCmdSetColorWriteMaskEXT(cmd, 0, 3, s.wm); if (g_mutate) poison(&s); } static void vertex_streams(VkCommandBuffer cmd, int d) { VkBuffer vbufs[16]; VkDeviceSize voffs[16], vsizes[16], vstrides[16]; for (uint32_t i = 0; i < 16; i++) vbufs[i] = H(0x9000 + i), voffs[i] = 64ull * i + d, vsizes[i] = 4096 - i, vstrides[i] = 12 + 4 * (i % 3); vkCmdBindVertexBuffers2(cmd, 0, 16, vbufs, voffs, vsizes, vstrides); if (g_mutate) poison(vbufs, 16), poison(voffs, 16), poison(vsizes, 16), poison(vstrides, 16); VkBuffer one = H(0x9100 + d); VkDeviceSize off = 32; vkCmdBindVertexBuffers2(cmd, 3, 1, &one, &off, nullptr, nullptr); // (sizes and strides left out: null stays null) if (g_mutate) poison(&one), poison(&off); } static void begin_pass(VkCommandBuffer cmd, int pass) { // (vk_draw.cpp's begin_pass: ca[8] and da on the stack) VkRenderingAttachmentInfo ca[8]{}, da{VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO}; for (uint32_t i = 0; i < 8; i++) { ca[i].sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO; ca[i].imageView = H(0xA000 + pass * 16 + i); ca[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL; ca[i].loadOp = i ? VK_ATTACHMENT_LOAD_OP_LOAD : VK_ATTACHMENT_LOAD_OP_CLEAR; ca[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE; ca[i].clearValue.color.float32[0] = 0.5f + i, ca[i].clearValue.color.float32[3] = 1.0f; } da.imageView = H(0xA0FF + pass), da.imageLayout = VK_IMAGE_LAYOUT_GENERAL, da.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; da.storeOp = VK_ATTACHMENT_STORE_OP_STORE, da.clearValue.depthStencil = {0.0f, 7}; VkRenderingInfo ri{VK_STRUCTURE_TYPE_RENDERING_INFO}; ri.renderArea = {{0, 0}, {1280u + pass, 720}}; ri.layerCount = 1; ri.colorAttachmentCount = 3; ri.pColorAttachments = ca; ri.pDepthAttachment = &da; ri.pStencilAttachment = pass ? &da : nullptr; // (the same struct for both, or no stencil) vkCmdBeginRendering(cmd, &ri); if (g_mutate) poison(ca, 8), poison(&da), poison(&ri); } static void barrier(VkCommandBuffer cmd, int k) { // (full_barrier / texture transitions: barriers on the stack) VkMemoryBarrier2 mb{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; mb.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, mb.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT; mb.dstStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, mb.dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | k; VkBufferMemoryBarrier2 bb[2]{{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2}, {VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2}}; for (int i = 0; i < 2; i++) bb[i].buffer = H(0xC000 + i), bb[i].offset = 16ull * k, bb[i].size = VK_WHOLE_SIZE, bb[i].srcQueueFamilyIndex = bb[i].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; VkImageMemoryBarrier2 ib[3]{}; for (int i = 0; i < 3; i++) { ib[i].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2; ib[i].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, ib[i].newLayout = VK_IMAGE_LAYOUT_GENERAL; ib[i].image = H(0xD000 + i + k); ib[i].subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1 + (uint32_t)i, 0, VK_REMAINING_ARRAY_LAYERS}; } VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; di.memoryBarrierCount = 1, di.pMemoryBarriers = &mb; di.bufferMemoryBarrierCount = 2, di.pBufferMemoryBarriers = bb; di.imageMemoryBarrierCount = k ? 3 : 0, di.pImageMemoryBarriers = ib; // (a count of 0 with a pointer: not read) vkCmdPipelineBarrier2(cmd, &di); if (g_mutate) poison(&mb), poison(bb, 2), poison(ib, 3), poison(&di); } static thread_local std::vector copies; // (vk_memory.cpp's mirror copies: a reused thread-local vector) static void copy_runs(VkCommandBuffer up, int k) { copies.clear(); for (int i = 0; i < 5 - k; i++) copies.push_back({4096ull * i, 8192ull * i + k, 256ull + i}); vkCmdCopyBuffer(up, H(0xE000), H(0xE001 + k), (uint32_t)copies.size(), copies.data()); if (g_mutate) poison(copies); } static void frame() { const VkCommandBuffer cmd = H(0x100), up = H(0x200); // (the recording's and its upload one) const VkPipelineLayout layout = H(0x300); vkCmdResetQueryPool(up, H(0x400), 0, 64); vkCmdWriteTimestamp2(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, H(0x400), 0); copy_runs(up, 0); copy_runs(up, 2); { uint8_t data[64]; for (int i = 0; i < 64; i++) data[i] = (uint8_t)(255 - i); vkCmdUpdateBuffer(up, H(0xE100), 128, sizeof data, data); if (g_mutate) poison(data, 64); } vkCmdFillBuffer(cmd, H(0xE200), 0, VK_WHOLE_SIZE, 0); barrier(cmd, 0); for (int pass = 0; pass < 2; pass++) { begin_pass(cmd, pass); for (int d = 0; d < 3; d++) { // (draw(): pipeline, descriptors, push constants, dynamic state, streams, the draw) const int at = pass * 3 + d; vkCmdWriteBufferMarkerAMD(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, H(0xF000), 8ull * at, 100 + at); vkCmdSetCheckpointNV(cmd, (void*)(uintptr_t)(at + 1)); vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, H(0x800 + at)); build_graphics(at); vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, (uint32_t)dw.writes.size(), dw.writes.data()); { // (bind_descriptors' delta: the changed writes copied into another thread-local vector) static thread_local std::vector changed; changed.clear(); changed.push_back(dw.writes[0]), changed.push_back(dw.writes[4]), changed.push_back(dw.writes[8]); vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, (uint32_t)changed.size(), changed.data()); if (g_mutate) poison(changed); } { // ((Triforce) bind_descriptors' whole push through a push template: the descriptors packed into a reused thread-local // array, whose size the call does not carry (push_write_template tells the packer: rec::push_template_bytes)) static thread_local std::vector packed; packed.assign(72 + 24 * (size_t)d, 0); for (size_t i = 0; i < packed.size(); i++) packed[i] = (uint8_t)(i * 5 + at); rec::push_template_bytes(packed.size()); vkCmdPushDescriptorSetWithTemplateKHR(cmd, H(kTplHandle + packed.size()), layout, 0, packed.data()); if (g_mutate) poison(packed.data(), packed.size()); } dw.after_call(); push_constants(cmd, layout, at); dynamic_state(cmd, at); vertex_streams(cmd, at); vkCmdBindIndexBuffer(cmd, H(0x9F00), 4ull * at, d ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); vkCmdBeginQuery(cmd, H(0x401), at, 0); switch ((pass * 3 + d) % 6) { case 0: vkCmdDrawIndexed(cmd, 36, 1, 0, -4, 0); vkCmdDraw(cmd, 3, 1, 0, 0); break; case 1: vkCmdDrawIndirect(cmd, H(0x9E00), 16, 4, 16); vkCmdDrawIndexedIndirect(cmd, H(0x9E00), 64, 2, 20); break; case 2: vkCmdDrawIndirectCount(cmd, H(0x9E00), 0, H(0x9E01), 4, 8, 16); vkCmdDrawIndexedIndirectCount(cmd, H(0x9E00), 0, H(0x9E01), 8, 8, 20); break; case 3: vkCmdDrawMeshTasksEXT(cmd, 4, 1, 1); break; case 4: vkCmdDrawMeshTasksIndirectEXT(cmd, H(0x9E02), 12, 3, 12); break; default: vkCmdDrawMeshTasksIndirectCountEXT(cmd, H(0x9E02), 0, H(0x9E01), 12, 5, 12); break; } vkCmdEndQuery(cmd, H(0x401), at); } { // (a clear inside the pass: a rect and an attachment on the stack; the shader clear's own viewport and scissor) VkClearAttachment ca{VK_IMAGE_ASPECT_COLOR_BIT, 1, {}}; ca.clearValue.color.uint32[2] = 0xABCD + pass; VkClearRect cr{{{0, 0}, {64, 32}}, 0, 1}; vkCmdClearAttachments(cmd, 1, &ca, 1, &cr); if (g_mutate) poison(&ca), poison(&cr); VkViewport v{0, 0, 64, 32, 0, 1}; VkRect2D rect{{0, 0}, {64, 32}}; vkCmdSetViewport(cmd, 0, 1, &v); vkCmdSetScissor(cmd, 0, 1, &rect); if (g_mutate) poison(&v), poison(&rect); } vkCmdEndRendering(cmd); barrier(cmd, 1 + pass); } { // (a compute dispatch: storage image, storage buffer, texel buffer; push constants; descriptor sets with dynamic offsets) vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, H(0x900)); VkDescriptorImageInfo ii{VK_NULL_HANDLE, H(0x6500), VK_IMAGE_LAYOUT_GENERAL}; VkDescriptorBufferInfo bi{H(0xB500), 0, VK_WHOLE_SIZE}; VkBufferView tv = H(0x7500); VkWriteDescriptorSet w[3] = {write(0, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1), write(1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1), write(2, VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 1)}; w[0].pImageInfo = &ii, w[1].pBufferInfo = &bi, w[2].pTexelBufferView = &tv; vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 3, w); if (g_mutate) poison(&ii), poison(&bi), poison(&tv), poison(w, 3); uint32_t size[3] = {640, 480, 1}; vkCmdPushConstants(cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, 12, size); if (g_mutate) poison(size, 3); vkCmdDispatch(cmd, 80, 60, 1); vkCmdDispatchIndirect(cmd, H(0x9D00), 48); VkDescriptorSet sets[2] = {H(0x1100), H(0x1101)}; uint32_t dyn[3] = {256, 512, 768}; vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 2, sets, 3, dyn); if (g_mutate) poison(dyn, 3); vkCmdBindDescriptorSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 2, 1, &sets[1], 0, dyn); // (no dynamic offsets: not read) if (g_mutate) poison(sets, 2); } { // (texture work: uploads, copies, blits, resolves, clears; regions and values on the stack) VkBufferImageCopy bic[2]{}; for (int i = 0; i < 2; i++) bic[i] = {1024ull * i, 0, 0, {VK_IMAGE_ASPECT_COLOR_BIT, (uint32_t)i, 0, 1}, {i, 0, 0}, {64u >> i, 64u >> i, 1}}; vkCmdCopyBufferToImage(up, H(0xE300), H(0xD100), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 2, bic); vkCmdCopyImageToBuffer(cmd, H(0xD101), VK_IMAGE_LAYOUT_GENERAL, H(0xE301), 1, &bic[1]); if (g_mutate) poison(bic, 2); VkImageCopy ic{{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {1, 2, 0}, {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {3, 4, 0}, {16, 16, 1}}; vkCmdCopyImage(cmd, H(0xD102), VK_IMAGE_LAYOUT_GENERAL, H(0xD103), VK_IMAGE_LAYOUT_GENERAL, 1, &ic); if (g_mutate) poison(&ic); VkImageBlit bl{{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {{0, 0, 0}, {128, 64, 1}}, {VK_IMAGE_ASPECT_COLOR_BIT, 1, 0, 1}, {{0, 0, 0}, {64, 32, 1}}}; vkCmdBlitImage(cmd, H(0xD104), VK_IMAGE_LAYOUT_GENERAL, H(0xD105), VK_IMAGE_LAYOUT_GENERAL, 1, &bl, VK_FILTER_LINEAR); if (g_mutate) poison(&bl); VkImageResolve rs{{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {}, {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}, {}, {1280, 720, 1}}; vkCmdResolveImage(cmd, H(0xD106), VK_IMAGE_LAYOUT_GENERAL, H(0xD107), VK_IMAGE_LAYOUT_GENERAL, 1, &rs); if (g_mutate) poison(&rs); VkClearColorValue col{}; col.float32[0] = 0.1f, col.float32[1] = 0.2f, col.float32[2] = 0.3f, col.float32[3] = 1.0f; VkImageSubresourceRange ranges[2] = {{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}, {VK_IMAGE_ASPECT_COLOR_BIT, 1, 2, 0, 6}}; vkCmdClearColorImage(cmd, H(0xD108), VK_IMAGE_LAYOUT_GENERAL, &col, 2, ranges); if (g_mutate) poison(&col); VkClearDepthStencilValue ds{1.0f, 0x55}; VkImageSubresourceRange dr{VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT, 0, 1, 0, 1}; vkCmdClearDepthStencilImage(cmd, H(0xD109), VK_IMAGE_LAYOUT_GENERAL, &ds, 1, &dr); if (g_mutate) poison(&ds), poison(&dr), poison(ranges, 2); } vkCmdWriteTimestamp2(cmd, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, H(0x400), 1); } // (a call the packer does not take: a pNext chain, a descriptor type it does not know. Held packets go first, then the direct call) static void fallbacks() { const VkCommandBuffer cmd = H(0x100); vkCmdDraw(cmd, 1, 2, 3, 4); VkMemoryBarrier2 next{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; VkDependencyInfo di{VK_STRUCTURE_TYPE_DEPENDENCY_INFO}; di.pNext = &next; vkCmdPipelineBarrier2(cmd, &di); vkCmdDraw(cmd, 5, 6, 7, 8); VkWriteDescriptorSet w = write(0, VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK, 16); vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, H(0x300), 0, 1, &w); vkCmdDraw(cmd, 9, 10, 11, 12); uint8_t packed[48]; // ((Triforce) a push template's data whose size the packer was not told: made directly, after what is held) for (int i = 0; i < 48; i++) packed[i] = (uint8_t)(200 - i); vkCmdPushDescriptorSetWithTemplateKHR(cmd, H(kTplHandle + sizeof packed), H(0x300), 0, packed); vkCmdDispatch(cmd, 1, 1, 1); } // ---- RX_VK_RECORDER=defer's lifecycle: the recording's two command buffers (its own and the upload one, in pools of their own, as // vk_replay.cpp's begin_recording / submit_recording), reset, begun, recorded into interleaved, ended, submitted, waited for, twice; // a query read and a destruction after. Every held command must reach the fake while its command buffer records, none after its end static const VkDevice kDev = H(0x50); static const VkCommandPool kPool = H(0x10), kUpPool = H(0x20); static const VkCommandBuffer kCmd = H(0x100), kUp = H(0x200); static void lifecycle_reset() { g_state.clear(), g_violations = 0, g_violation.clear(); gpu_reset(); g_pool_of = {{0x100, 0x10}, {0x200, 0x20}, {0x300, 0x60}, {0x400, 0x70}, {0x500, 0x80}, {0x600, 0x90}}; // (and the segments') } static void recording_begin() { vkResetCommandPool(kDev, kPool, 0); vkResetCommandPool(kDev, kUpPool, 0); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; vkBeginCommandBuffer(kCmd, &bi); vkBeginCommandBuffer(kUp, &bi); } static void recording_end(int k) { hold(); // (the recording's commands in flight at its end) vkEndCommandBuffer(kUp); vkEndCommandBuffer(kCmd); VkCommandBufferSubmitInfo cbi[2]{{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}, {VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}}; cbi[0].commandBuffer = kUp, cbi[1].commandBuffer = kCmd; VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2}; si.commandBufferInfoCount = 2, si.pCommandBufferInfos = cbi; const VkFence fence = H(0x30 + k); vkQueueSubmit2(H(0x40), 1, &si, fence); if (g_mutate) poison(cbi, 2), poison(&si); vkWaitForFences(kDev, 1, &fence, VK_TRUE, UINT64_MAX); } static void lifecycle() { for (int k = 0; k < 2; k++) { recording_begin(); frame(); fallbacks(); // (FIFO around the calls not packed) frame(); recording_end(k); } uint64_t q[2]; vkGetQueryPoolResults(kDev, H(0x400), 0, 2, sizeof q, q, 8, VK_QUERY_RESULT_64_BIT); vkDestroyBuffer(kDev, H(0xE000), nullptr); } /// (one recording of 40 frames: past 1 MB of packets held, for the budgets) static void big() { recording_begin(); for (int i = 0; i < 40; i++) frame(); recording_end(7); } /// A recording handed between two threads under the replay's lock (vk_replay.cpp's replay_mu, a DrainingMutex): one begins it and /// records, lets the lock go and stays alive; the other records more, ends and submits it static rec::DrainingMutex g_replay_mu; static void handoff() { std::atomic step{0}; std::thread other([&] { { std::lock_guard lk(g_replay_mu); recording_begin(); hold(); // (its commands in flight when it lets the lock go) frame(); } step = 1; while (step.load() != 2) std::this_thread::yield(); }); while (step.load() != 1) std::this_thread::yield(); { std::lock_guard lk(g_replay_mu); frame(); recording_end(9); } step = 2; other.join(); } // ---- RX_VK_RECORDER=thread's own (each also run direct, for the reference log) static const VkCommandPool kPoolB = H(0x60); static const VkCommandBuffer kCmdB = H(0x300); static void draws_into(VkCommandBuffer cb, int n, int k) { // (draws with push constants on the stack) for (int i = 0; i < n; i++) { uint32_t pc[4] = {(uint32_t)i, (uint32_t)k, 0xABCD, 7}; vkCmdPushConstants(cb, H(0x300), VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof pc, pc); if (g_mutate) poison(pc, 4); vkCmdDraw(cb, 3, 1, (uint32_t)i, (uint32_t)k); } } static void submit(const VkCommandBuffer* cbs, uint32_t n, VkFence fence) { VkCommandBufferSubmitInfo cbi[2]{{VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}, {VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO}}; for (uint32_t i = 0; i < n; i++) cbi[i].commandBuffer = cbs[i]; VkSubmitInfo2 si{VK_STRUCTURE_TYPE_SUBMIT_INFO_2}; si.commandBufferInfoCount = n, si.pCommandBufferInfos = cbi; vkQueueSubmit2(H(0x40), 1, &si, fence); if (g_mutate) poison(cbi, 2), poison(&si); vkWaitForFences(kDev, 1, &fence, VK_TRUE, UINT64_MAX); } /// Two recordings at once: A ended and submitted while B, still recording, has commands in flight (A's submit must not pass them) static void two_recordings() { recording_begin(); vkResetCommandPool(kDev, kPoolB, 0); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; vkBeginCommandBuffer(kCmdB, &bi); frame(); draws_into(kCmdB, 40, 0); vkEndCommandBuffer(kUp); vkEndCommandBuffer(kCmd); hold(); // (B's commands in flight at A's submit) draws_into(kCmdB, 40, 1); const VkCommandBuffer a[2] = {kUp, kCmd}; submit(a, 2, H(0x31)); hold(); draws_into(kCmdB, 40, 2); vkEndCommandBuffer(kCmdB); submit(&kCmdB, 1, H(0x32)); } /// A recording given up: its upload pool reset while recording (valid: nothing of it is pending), begun again, recorded, ended static void reset_while_recording() { recording_begin(); copy_runs(kUp, 0); frame(); hold(); // (its commands in flight at the pool's reset) copy_runs(kUp, 2); vkResetCommandPool(kDev, kUpPool, 0); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; vkBeginCommandBuffer(kUp, &bi); copy_runs(kUp, 1); recording_end(5); } /// The replay lock let go by a thread that recorded, then another thread, without the lock, ends and submits the recording: the /// recorder must be idle once the lock is let go static void unlock_then_end() { std::atomic step{0}; std::thread other([&] { { std::lock_guard lk(g_replay_mu); recording_begin(); hold(); // (its commands in flight when it lets the lock go) frame(); } step = 1; while (step.load() != 2) std::this_thread::yield(); }); while (step.load() != 1) std::this_thread::yield(); recording_end(11); step = 2; other.join(); } /// A packet bigger than the room it is given (with the 2 KB room of the run): made directly, in its place among the others static void oversize() { recording_begin(); draws_into(kCmd, 20, 0); std::vector big(4096); for (size_t i = 0; i < big.size(); i++) big[i] = (uint8_t)(i * 13); vkCmdUpdateBuffer(kUp, H(0xE100), 0, big.size(), big.data()); if (g_mutate) poison(big.data(), big.size()); draws_into(kCmd, 20, 1); recording_end(6); } /// The same packet first, the ring empty (the recorder just started, at the ring's start): the ring has room for it, the packet's /// room has not, so it is still made directly (the room alone decides, not how far the recorder has got) static void oversize_first() { recording_begin(); std::vector big(4096); for (size_t i = 0; i < big.size(); i++) big[i] = (uint8_t)(i * 7); vkCmdUpdateBuffer(kUp, H(0xE200), 0, big.size(), big.data()); if (g_mutate) poison(big.data(), big.size()); draws_into(kCmd, 20, 2); recording_end(7); } /// Teardown with work queued: g_at_stop runs with frames in flight (the recorder stopped and joined), then more is recorded (the /// recorder started again), then vkDestroyDevice (stopped again: its boundary) static void (*g_at_stop)() = nullptr; static void teardown() { recording_begin(); for (int i = 0; i < 4; i++) frame(); g_at_stop(); frame(); recording_end(12); vkDestroyDevice(kDev, nullptr); } /// The status reads (the calls the producer makes without waiting: unordered()) while a recording's commands are in flight, and after /// its submit: each must see what the direct calls see, the submitted work and nothing recorded since static void status_reads_at(VkFence prev, bool recording) { // (recording: a few commands put in flight before each read, so that one read's wait cannot empty the ring for the next. The // recorder is held for 50 ms from there: a read made without waiting finds them still in flight on a loaded runner too, and // one that waits is let through once the hold ends) const auto busy = [&] { if (!recording) return; g_hold_until_ns = (int64_t)std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()).count() + 50000000; draws_into(kCmd, 3, 7); }; uint64_t v = 0; busy(); vkGetSemaphoreCounterValue(kDev, H(0x70), &v); const VkSemaphore sem = H(0x70); const uint64_t want = v + 1; // (not reached until the next submit) VkSemaphoreWaitInfo wi{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; wi.semaphoreCount = 1, wi.pSemaphores = &sem, wi.pValues = &want; busy(); vkWaitSemaphores(kDev, &wi, 0); wi.pValues = &v; busy(); vkWaitSemaphores(kDev, &wi, 0); busy(); vkWaitForFences(kDev, 1, &prev, VK_TRUE, 0); busy(); vmaInvalidateAllocation(nullptr, (VmaAllocation)(uintptr_t)0xE400, 0, 16); uint32_t n = 1; VkCheckpointDataNV cd{VK_STRUCTURE_TYPE_CHECKPOINT_DATA_NV}; busy(); vkGetQueueCheckpointDataNV(H(0x40), &n, &cd); VkDeviceFaultCountsEXT fc{VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; busy(); vkGetDeviceFaultInfoEXT(kDev, &fc, nullptr); g_hold_until_ns = 0; } static void status_reads() { for (int k = 0; k < 3; k++) { recording_begin(); vkCmdFillBuffer(kCmd, H(0xE400), 0, 16, 0x100 + k); vkCmdSetCheckpointNV(kCmd, (void*)(uintptr_t)(0x200 + k)); frame(); { // (a query read mid-recording, the frame's commands in flight: kept ordered, so in the stream after them) uint64_t q[2]; vkGetQueryPoolResults(kDev, H(0x400), 0, 2, sizeof q, q, 8, VK_QUERY_RESULT_64_BIT); } frame(); status_reads_at(H(0x30 + 20 + k - 1), true); vkCmdFillBuffer(kCmd, H(0xE400), 0, 16, 0x180 + k); vkCmdSetCheckpointNV(kCmd, (void*)(uintptr_t)(0x280 + k)); status_reads_at(H(0x30 + 20 + k), true); recording_end(20 + k); status_reads_at(H(0x30 + 20 + k), false); uint64_t q[2]; vkGetQueryPoolResults(kDev, H(0x400), 0, 2, sizeof q, q, 8, VK_QUERY_RESULT_64_BIT); } } /// A buffer destroyed while a recording that names it is open (valid: that recording is reset, never submitted); every command /// naming it must have reached the driver before the destruction static void destroy_recorded() { recording_begin(); frame(); copy_runs(kUp, 0); // (names 0xE001: in flight behind the frame when it is destroyed) vkDestroyBuffer(kDev, H(0xE001), nullptr); vkResetCommandPool(kDev, kUpPool, 0); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; vkBeginCommandBuffer(kUp, &bi); copy_runs(kUp, 2); recording_end(14); } static int g_stress_frames = 0; static void stress() { recording_begin(); for (int i = 0; i < g_stress_frames; i++) frame(); recording_end(13); } // ---- (Triforce, 11-15) beside a parallel replay's segments (vk_replay.cpp replay_handle; RX_VK_RECORDER is no longer refused with // RX_VK_PARALLEL): the part before them through the ring (recorded, ended, submitted); the segments' command buffers reset and // begun and given their first timestamps through the ring (gpu_time_begin), held in flight (hold()); rec::segments_begin; // par_active; two helper threads recording theirs directly, then the producer its own (directly too); par_active let go; each // segment ended and submitted in order; the part after through the ring again. g_seg_at_once: the helpers at the same time as each // other and as the producer (the log's order across command buffers is then the threads': compared command buffer by command // buffer). The helpers have no replay lock, as vk_replay.cpp's par_worker_main threads have none static const VkCommandPool kSegPool[3] = {H(0x70), H(0x80), H(0x90)}; static const VkCommandBuffer kSeg[3] = {H(0x400), H(0x500), H(0x600)}; static const VkQueryPool kTsPool = H(0x700); static bool g_seg_at_once = false; static void segment_into(int k) { // (a job: its draws, then its last timestamp: gpu_time_end, by whoever records it) draws_into(kSeg[k], 30, 10 + k); vkCmdWriteTimestamp2(kSeg[k], VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, kTsPool, 2 * k + 1); } static void segments() { recording_begin(); frame(); recording_end(20); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; for (int k = 0; k < 3; k++) { vkResetCommandPool(kDev, kSegPool[k], 0); vkBeginCommandBuffer(kSeg[k], &bi); vkCmdResetQueryPool(kSeg[k], kTsPool, 2 * k, 2); vkCmdWriteTimestamp2(kSeg[k], VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, kTsPool, 2 * k); } hold(); // (those still in flight when the segments start) rec::segments_begin(); par_active = true; if (g_seg_at_once) { std::thread h1([] { segment_into(1); }), h2([] { segment_into(2); }); segment_into(0); h1.join(), h2.join(); } else { std::thread([] { segment_into(1); }).join(); std::thread([] { segment_into(2); }).join(); segment_into(0); } par_active = false; for (int k = 0; k < 3; k++) { vkEndCommandBuffer(kSeg[k]); const VkCommandBuffer one = kSeg[k]; submit(&one, 1, H(0x70 + k)); } recording_begin(); frame(); recording_end(21); } /// A log split by command buffer (a vkCmd*'s, a begin's or an end's first argument), the rest (resets, submits, waits) in order apart static std::map> by_command_buffer(const std::vector& log) { std::map> out; for (const Call& c : log) { const bool on_cb = c.op.rfind("vkCmd", 0) == 0 || c.op == "vkBeginCommandBuffer" || c.op == "vkEndCommandBuffer"; uint64_t cb = 0; if (on_cb && c.bytes.size() >= 8) std::memcpy(&cb, c.bytes.data(), 8); out[on_cb ? cb : 0].push_back(c); } return out; } /// --throw: a driver call throws on the recorder thread: the process must end loudly (exit code 3, "FATAL"), never go on static int throw_test() { rec::set_mode(rec::THREAD); std::lock_guard lk(g_replay_mu); vkCmdDraw(kCmd, 1, 1, 0, 0); g_throw = true; vkCmdDraw(kCmd, 2, 1, 0, 0); vkCmdDraw(kCmd, 3, 1, 0, 0); rec::boundary(rec::B_vkEndCommandBuffer); std::printf("the recorder's exception was not reported\n"); return 0; } // ---- checks static int failures = 0; static std::vector run(void (*scenario)(), int mode, bool mutate) { rec::set_mode(mode); g_mutate = mutate; g_log.clear(); if (mode == rec::THREAD) { // (the producer: the replay lock's holder; its unlock waits for the recorder) std::lock_guard lk(g_replay_mu); scenario(); } else { scenario(); } if (mode == rec::DEFER && !g_log.empty() && scenario == frame) { std::printf("FAIL deferred: %zu calls reached the driver before the drain\n", g_log.size()); failures++; } rec::drain(); rec::set_mode(rec::OFF); return g_log; } static void same(const char* what, const std::vector& ref, const std::vector& got) { for (size_t i = 0; i < std::max(ref.size(), got.size()); i++) { if (i < ref.size() && i < got.size() && ref[i].op == got[i].op && ref[i].bytes == got[i].bytes) continue; size_t at = 0; if (i < ref.size() && i < got.size()) while (at < ref[i].bytes.size() && at < got[i].bytes.size() && ref[i].bytes[at] == got[i].bytes[at]) at++; std::printf("MISMATCH %s: call %zu of %zu/%zu: expected %s, got %s (bytes differ from %zu)\n", what, i, ref.size(), got.size(), i < ref.size() ? ref[i].op.c_str() : "(end)", i < got.size() ? got[i].op.c_str() : "(end)", at); failures++; return; } size_t bytes = 0; for (const Call& c : got) bytes += c.bytes.size(); std::printf("PASS %s: %zu calls, %zu argument bytes identical\n", what, got.size(), bytes); } /// A scenario with the lifecycle tracked: its log, the violations, and the arena's counters for the run (own_thread: on a thread of /// its own, an arena that starts empty: its aligned blocks are counted against its chunks) struct Out { std::vector log; std::vector reads; std::map overlapped; int violations = 0; std::string violation; bool escaped = false; uint64_t failures = 0, budget_drains = 0, fallbacks = 0; size_t capacity = 0; long leaked = 0; }; static Out run_tracked(void (*scenario)(), int mode, bool mutate, bool own_thread = false, bool lock = true) { Out o; const auto body = [&] { rec::Arena& a = rec::t_arena; const uint64_t f0 = a.failures, b0 = a.budget_drains, fb0 = a.fallbacks; const long live0 = g_aligned_live.load() - (long)a.nchunks; rec::set_mode(mode); g_mutate = mutate, g_track = true; lifecycle_reset(); g_log.clear(); try { if (mode == rec::THREAD && lock) { std::lock_guard lk(g_replay_mu); scenario(); } else { scenario(); } } catch (const std::bad_alloc&) { o.escaped = true; } rec::drain(); g_track = false; o.log = g_log, o.violations = g_violations, o.violation = g_violation, o.reads = g_reads, o.overlapped = g_reads_overlapped; o.failures = a.failures - f0, o.budget_drains = a.budget_drains - b0, o.fallbacks = a.fallbacks - fb0, o.capacity = a.capacity; if (own_thread) o.leaked = g_aligned_live.load() - live0 - (long)a.nchunks; }; if (own_thread) { std::thread t(body); t.join(); } else { body(); } rec::set_mode(rec::OFF); return o; } static void check_run(const char* what, const std::vector& ref, const Out& o, const std::vector* ref_reads = nullptr) { if (ref_reads && *ref_reads != o.reads) std::printf("MISMATCH %s: the status reads (fences, semaphores, checkpoints, fault data, invalidated memory) saw other " "values: %zu/%zu\n", what, ref_reads->size(), o.reads.size()), failures++; if (o.escaped) std::printf("FAIL %s: an allocation failure escaped a vkCmd* (the call lost)\n", what), failures++; if (o.violations) std::printf("FAIL %s: %d lifecycle violation(s), the first: %s\n", what, o.violations, o.violation.c_str()), failures++; same(what, ref, o.log); } static double now_ns() { return (double)std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } // ---- bench: a typical draw (~4 calls: the descriptor delta, push constants, streams, the draw) and a cold one (the whole state) static void draw_typical(VkCommandBuffer cmd, VkPipelineLayout layout, int d) { static thread_local std::vector changed; changed.clear(); changed.push_back(dw.writes[0]), changed.push_back(dw.writes[3]), changed.push_back(dw.writes[4]); vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, 3, changed.data()); uint8_t data[kPushSize]; std::memset(data, d, sizeof data); vkCmdPushConstants(cmd, layout, VK_SHADER_STAGE_ALL_GRAPHICS, 0, kPushSize, data); VkBuffer vb[16]; VkDeviceSize vo[16], vs[16], vst[16]; for (int i = 0; i < 16; i++) vb[i] = H(0x9000 + i), vo[i] = i + d, vs[i] = 4096, vst[i] = 16; vkCmdBindVertexBuffers2(cmd, 0, 16, vb, vo, vs, vst); vkCmdDrawIndexed(cmd, 36, 1, 0, d & 7, 0); } static void draw_cold(VkCommandBuffer cmd, VkPipelineLayout layout, int d) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, H(0x800 + (d & 3))); vkCmdPushDescriptorSetKHR(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, (uint32_t)dw.writes.size(), dw.writes.data()); DynState s{}; s.vp = {0, 0, 1280, 720, 0, 1}, s.sc = {{0, 0}, {1280, 720}}, s.en[0] = d & 1, s.wm[0] = 0xF; vkCmdSetViewportWithCount(cmd, 1, &s.vp); vkCmdSetScissorWithCount(cmd, 1, &s.sc); vkCmdSetCullMode(cmd, VK_CULL_MODE_NONE); vkCmdSetDepthTestEnable(cmd, VK_TRUE); vkCmdSetDepthWriteEnable(cmd, d & 1); vkCmdSetDepthCompareOp(cmd, VK_COMPARE_OP_GREATER); vkCmdSetColorBlendEnableEXT(cmd, 0, 1, s.en); vkCmdSetColorBlendEquationEXT(cmd, 0, 1, s.eq); vkCmdSetColorWriteMaskEXT(cmd, 0, 1, s.wm); vkCmdBindIndexBuffer(cmd, H(0x9F00), 0, VK_INDEX_TYPE_UINT16); draw_typical(cmd, layout, d); } static void draw_typical_direct(VkCommandBuffer cmd, VkPipelineLayout layout, int d) { // (no wrapper: the driver's functions, as before) static thread_local std::vector changed; changed.clear(); changed.push_back(dw.writes[0]), changed.push_back(dw.writes[3]), changed.push_back(dw.writes[4]); (::vkx::vkCmdPushDescriptorSetKHR)(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, 3, changed.data()); uint8_t data[kPushSize]; std::memset(data, d, sizeof data); (::vkx::vkCmdPushConstants)(cmd, layout, VK_SHADER_STAGE_ALL_GRAPHICS, 0, kPushSize, data); VkBuffer vb[16]; VkDeviceSize vo[16], vs[16], vst[16]; for (int i = 0; i < 16; i++) vb[i] = H(0x9000 + i), vo[i] = i + d, vs[i] = 4096, vst[i] = 16; (::vkx::vkCmdBindVertexBuffers2)(cmd, 0, 16, vb, vo, vs, vst); (::vkx::vkCmdDrawIndexed)(cmd, 36, 1, 0, d & 7, 0); } static int bench(int n) { if (const char* e = getenv("BENCH_PUBLISH_BYTES")) rec::g_publish_bytes = (size_t)atoll(e); // (the publication batch to time) g_quiet = true; build_graphics(0); const VkCommandBuffer cmd = H(0x100); const VkPipelineLayout layout = H(0x300); struct Shape { const char* name; void (*draw)(VkCommandBuffer, VkPipelineLayout, int); void (*direct)(VkCommandBuffer, VkPipelineLayout, int); int calls; } shapes[] = {{"typical draw (push descriptors x3 writes, push constants 96 B, vertex buffers x16, draw)", draw_typical, draw_typical_direct, 4}, {"cold draw (+ pipeline, 9 writes, 9 dynamic states, index buffer)", draw_cold, nullptr, 16}}; rec::Arena& a = rec::t_arena; for (const Shape& sh : shapes) { // (modes: 0 direct (no wrapper), 1 off (the wrapper's branch), 2 sync, 3 deferred: encode and drain timed apart, 4 thread: // the producer's encode and publish, and its wait at the boundary, timed apart; the recorder's busy time a packet) double med[7] = {}; double bytes_per_packet = 0, rec_ns = 0; for (int m = 0; m < 5; m++) { if (m == 0 && !sh.direct) continue; std::vector reps[3]; for (int rep = 0; rep < 5; rep++) { rec::set_mode(m <= 1 ? rec::OFF : m == 2 ? rec::SYNC : m == 3 ? rec::DEFER : rec::THREAD); const uint64_t p0 = a.packets, b0 = a.bytes; const uint64_t rp0 = rec::g_rec.packets.load(), rb0 = rec::g_rec.busy_ns.load(); double enc = 0, drn = 0; { std::unique_lock lk(g_replay_mu, std::defer_lock); if (m == 4) lk.lock(); for (int i = 0; i < n; i += 256) { // (deferred: drained every 256 draws, about a recording's draws; thread: waited) const double t0 = now_ns(); for (int j = 0; j < 256; j++) (m == 0 ? sh.direct : sh.draw)(cmd, layout, i + j); const double t1 = now_ns(); if (m == 4) rec::boundary(rec::B_vkEndCommandBuffer); else rec::drain(); enc += t1 - t0, drn += now_ns() - t1; } } reps[0].push_back(enc / n), reps[1].push_back(drn / n); if (m == 3) bytes_per_packet = (double)(a.bytes - b0) / (double)(a.packets - p0); if (m == 4) { rec::set_mode(rec::OFF); // (the recorder idle: its counters are in) reps[2].push_back((double)(rec::g_rec.busy_ns.load() - rb0) / (double)(rec::g_rec.packets.load() - rp0)); } } for (auto& r : reps) std::sort(r.begin(), r.end()); med[m] = reps[0][2]; if (m == 3) med[5] = reps[1][2]; if (m == 4) med[6] = reps[1][2], rec_ns = reps[2][2]; } std::printf("BENCH %s, %d calls: ", sh.name, sh.calls); if (sh.direct) std::printf("direct %.1f ns, ", med[0]); std::printf("off %.1f ns, sync %.1f ns (+%.1f over off, +%.1f a call), deferred encode %.1f ns + drain %.1f ns; " "%.0f bytes a packet (median of 5, %d draws)\n", med[1], med[2], med[2] - med[1], (med[2] - med[1]) / sh.calls, med[3], med[5], bytes_per_packet, n); std::printf("BENCH thread: the replay thread %.1f ns a draw (+%.1f over off, %.1f a call), its wait at the boundary every 256 " "draws %.1f ns a draw; the recorder %.1f ns a packet busy (%.1f M packets/s)\n", med[4], med[4] - med[1], med[4] / sh.calls, med[6], rec_ns, rec_ns > 0 ? 1e3 / rec_ns : 0.0); } rec::set_mode(rec::OFF); std::printf("BENCH %llu fake calls, arena %zu chunk(s)\n", (unsigned long long)g_calls, a.nchunks); g_quiet = false; return 0; } /// --overlap F: the critical thread's time a frame, off vs thread, with a driver that takes D ns a vkCmd* (4 calls a draw), translation /// of W ns a draw, 2000 draws a frame, and every K draws a recording's end (the producer waits for the recorder) plus 30 us that stays /// on the replay thread in both (vkQueueSubmit2, the next recording's begin) static void spin_ns_busy(int ns) { spin_for(ns); } static int overlap(int frames) { g_quiet = true; build_graphics(0); const VkCommandBuffer cmd = H(0x100); const VkPipelineLayout layout = H(0x300); const int draws = 2000, D = 300, W = 4000; g_spin_ns = D; std::printf("OVERLAP %d draws a frame, driver %d ns a call (%.2f ms a frame), translation %d ns a draw, 30 us a recording end " "(median of 5, %d frames)\n", draws, D, draws * 4 * D / 1e6, W, frames); for (int K : {120, 30, 8, 1}) { double med[2]; uint64_t waits = 0; double waited = 0; for (int m = 0; m < 2; m++) { std::vector reps; for (int rep = 0; rep < 5; rep++) { rec::set_mode(m ? rec::THREAD : rec::OFF); const uint64_t n0 = rec::g_rec.wait_n[rec::B_vkEndCommandBuffer].load(), ns0 = rec::g_rec.wait_ns[rec::B_vkEndCommandBuffer].load(); const double t0 = now_ns(); for (int f = 0; f < frames; f++) { std::lock_guard lk(g_replay_mu); for (int d = 0; d < draws; d++) { spin_ns_busy(W); draw_typical(cmd, layout, d); if ((d + 1) % K == 0 || d + 1 == draws) { rec::boundary(rec::B_vkEndCommandBuffer); spin_ns_busy(30000); } } } reps.push_back((now_ns() - t0) / frames / 1e6); if (m) waits = rec::g_rec.wait_n[rec::B_vkEndCommandBuffer].load() - n0, waited = (double)(rec::g_rec.wait_ns[rec::B_vkEndCommandBuffer].load() - ns0) / frames / 1e6; } std::sort(reps.begin(), reps.end()); med[m] = reps[2]; } std::printf("OVERLAP every %4d draws (%4d waits a frame): off %.2f ms, thread %.2f ms a frame: %.2f ms saved of %.2f ms of " "driver time (%.0f%%); waited %.3f ms a frame, %llu of the last run's waits found the recorder busy\n", K, (draws + K - 1) / K, med[0], med[1], med[0] - med[1], draws * 4 * D / 1e6, 100 * (med[0] - med[1]) / (draws * 4 * D / 1e6), waited, (unsigned long long)waits); } rec::set_mode(rec::OFF); g_spin_ns = 0; g_quiet = false; return 0; } /// --idle N: the recorder's idle spin (RX_VK_RECORDER_SPIN_US) against its CPU and the replay thread's waits. (a) bursts of one /// typical draw after a gap of G us of translation, each followed by a boundary: the replay thread's wait there (it includes the /// recorder's wake-up when it slept in the gap) and its encode time (it includes the wake-up call); (b) replay-like frames: 2000 /// draws of 4 us of translation, a recording end every 120, a 5 ms gap between frames, the fake driver 300 ns a call: the /// recorder's CPU (its thread's times, and its own spun/busy count) against the wall time, and the replay thread's frame time // (native_handle() is a Windows HANDLE with MSVC's and libc++'s std::thread, an integer pthread id with MinGW's // libstdc++ (winpthreads): the thread's CPU only where it is a HANDLE. A template, so the other branch is not compiled) template static bool thread_cpu_ticks(H h, uint64_t* kernel, uint64_t* user) { #ifdef _WIN32 if constexpr (std::is_pointer_v) { uint64_t c, e; return GetThreadTimes(h, &c, &e, kernel, user) != 0; } #endif (void)h, (void)kernel, (void)user; return false; } static double recorder_cpu_s() { uint64_t k, u; if (rec::g_rec.th.joinable() && thread_cpu_ticks(rec::g_rec.th.native_handle(), &k, &u)) return (double)(k + u) / 1e7; return -1; // (not measured) } static int idle_bench(int iters) { g_quiet = true; build_graphics(0); rec::g_stats_log = false; const VkCommandBuffer cmd = H(0x100); const VkPipelineLayout layout = H(0x300); for (const uint64_t spin_us : {200ull, 20ull, 0ull}) { // (0: adaptive, the default) rec::g_rec.stop(); rec::g_spin_ns = spin_us * 1000; rec::set_mode(rec::THREAD); { // (start it, idle) std::lock_guard lk(g_replay_mu); draw_typical(cmd, layout, 0); } for (const int gap : {2, 10, 30, 100, 1000}) { std::vector waits, encs; const uint64_t sl0 = rec::g_rec.sleeps.load(), sp0 = rec::g_rec.spin_ns.load(), bu0 = rec::g_rec.busy_ns.load(); const int n = std::max(iters, 300000 / gap); // (at least ~0.3 s) const double c0 = recorder_cpu_s(), t0 = now_ns(); { std::lock_guard lk(g_replay_mu); for (int i = 0; i < n; i++) { spin_for(gap * 1000); const double a = now_ns(); draw_typical(cmd, layout, i); const double b = now_ns(); rec::boundary(rec::B_vkEndCommandBuffer); encs.push_back(b - a), waits.push_back(now_ns() - b); } } const double wall = (now_ns() - t0) / 1e9, cpu = recorder_cpu_s() - c0; std::sort(waits.begin(), waits.end()), std::sort(encs.begin(), encs.end()); double wsum = 0, esum = 0; for (double w : waits) wsum += w; for (double e : encs) esum += e; std::printf("IDLE spin %s us, gap %4d us: boundary wait mean %.2f us, p99 %.2f, max %.1f; encode (4 calls) mean %.0f ns, " "p99 %.0f; recorder slept %.2f times a burst, CPU %.2f of a core (its own count; thread times %.2f)\n", spin_us ? std::to_string(spin_us).c_str() : "adaptive", gap, wsum / n / 1e3, waits[n * 99 / 100] / 1e3, waits.back() / 1e3, esum / n, encs[n * 99 / 100], (double)(rec::g_rec.sleeps.load() - sl0) / n, (double)(rec::g_rec.spin_ns.load() - sp0 + rec::g_rec.busy_ns.load() - bu0) / 1e9 / wall, cpu / wall); } { // (b) frames const int frames = 30, draws = 2000; g_spin_ns = 300; const uint64_t sp0 = rec::g_rec.spin_ns.load(), bu0 = rec::g_rec.busy_ns.load(), sl0 = rec::g_rec.sleeps.load(); const uint64_t wn0 = rec::g_rec.wait_n[rec::B_vkEndCommandBuffer].load(), wt0 = rec::g_rec.wait_ns[rec::B_vkEndCommandBuffer].load(); rec::g_rec.wait_max[rec::B_vkEndCommandBuffer] = 0; const double c0 = recorder_cpu_s(), t0 = now_ns(); double replay = 0; for (int f = 0; f < frames; f++) { const double a = now_ns(); { std::lock_guard lk(g_replay_mu); for (int d = 0; d < draws; d++) { spin_for(d % 50 == 49 ? 64000 : 4000); // (and a 60 us stretch of translation every 50 draws) draw_typical(cmd, layout, d); if ((d + 1) % 120 == 0 || d + 1 == draws) rec::boundary(rec::B_vkEndCommandBuffer); } } replay += now_ns() - a; std::this_thread::sleep_for(std::chrono::milliseconds(5)); } g_spin_ns = 0; const double wall = (now_ns() - t0) / 1e9, cpu = recorder_cpu_s() - c0; std::printf("IDLE spin %s us, frames (2000 draws x 4 us, 60 us more every 50, an end every 120, 5 ms between): replay %.2f ms a frame, waited " "%.3f ms a frame (max %.1f us); recorder CPU %.2f of a core by thread times (busy %.2f, spun %.2f s of %.2f s), " "slept %llu times a frame\n", spin_us ? std::to_string(spin_us).c_str() : "adaptive", replay / frames / 1e6, (double)(rec::g_rec.wait_ns[rec::B_vkEndCommandBuffer].load() - wt0) / frames / 1e6, rec::g_rec.wait_max[rec::B_vkEndCommandBuffer].load() / 1e3, cpu / wall, (double)(rec::g_rec.busy_ns.load() - bu0) / 1e9, (double)(rec::g_rec.spin_ns.load() - sp0) / 1e9, wall, (unsigned long long)(rec::g_rec.sleeps.load() - sl0) / frames); (void)wn0; } rec::set_mode(rec::OFF); } rec::g_rec.stop(); g_quiet = false; return 0; } /// RX_VK_RECORDER=thread: the recorder thread's checks (also run alone, --only thread, for the broken variants) static void thread_checks(const std::vector& ref) { rec::g_stats_log = false; const bool was_poison = rec::g_poison.load(); rec::g_poison = true; // (each packet filled with 0xCD once run: a pointer kept into a run packet would show) // the frame, every vkCmd* run by the recorder thread { const uint64_t c0 = g_cmd_calls, off0 = g_cmd_off_main; same("thread (packed, run by the recorder thread in order), caller memory poisoned", ref, run(frame, rec::THREAD, true)); const uint64_t c = g_cmd_calls - c0, off = g_cmd_off_main - off0; if (c != ref.size() || off != c) std::printf("FAIL thread: %llu of %llu vkCmd* on the recorder thread\n", (unsigned long long)off, (unsigned long long)c), failures++; else std::printf("PASS thread: all %llu vkCmd* made on the recorder thread\n", (unsigned long long)off); } g_delay_ns = 1000; // (the recorder behind the producer: a missed wait shows as a call out of place or on a busy command buffer) const Out lref = run_tracked(lifecycle, rec::OFF, false); const uint64_t fb0 = rec::t_arena.fallbacks; g_hold_ms = 50; // (and held at each point the producer must wait: hold()) const Out lt = run_tracked(lifecycle, rec::THREAD, true); g_hold_ms = 0, g_hold_until_ns = 0; check_run("thread lifecycle (2 command buffers interleaved; reset, begin, end, submit, wait x2; query read; destruction; caller " "memory poisoned; driver 1 us a call)", lref.log, lt); if (rec::t_arena.fallbacks - fb0 != 6) std::printf("FAIL thread lifecycle: %llu fallbacks, 6 expected\n", (unsigned long long)(rec::t_arena.fallbacks - fb0)), failures++; // (each thread run with the recorder held at the points the producer must wait: hold(); the direct references without) const auto held = [](void (*scenario)(), bool lock) { g_hold_ms = 50; Out o = run_tracked(scenario, rec::THREAD, true, false, lock); g_hold_ms = 0, g_hold_until_ns = 0; return o; }; check_run("thread handoff (a recording across two threads under the replay lock)", run_tracked(handoff, rec::OFF, false).log, held(handoff, false)); check_run("thread: a submit while another command buffer records (its commands in flight)", run_tracked(two_recordings, rec::OFF, false).log, held(two_recordings, true)); check_run("thread: a pool reset while its command buffer records (re-begun)", run_tracked(reset_while_recording, rec::OFF, false).log, held(reset_while_recording, true)); check_run("thread: the replay lock let go, then the recording ended on another thread without it", run_tracked(unlock_then_end, rec::OFF, false).log, held(unlock_then_end, false)); { // (the status reads made without waiting: the same values as direct, each made while packets were in flight) const Out sref = run_tracked(status_reads, rec::OFF, false); const Out st = run_tracked(status_reads, rec::THREAD, true); check_run("thread: status reads mid-recording (semaphore value and wait, fence wait, invalidate, checkpoints, fault info) " "without waiting for the recorder", sref.log, st, &sref.reads); // (each made without waiting exactly when rec::unordered() lists it: then with packets in flight; else always after them) const std::pair calls[] = { {"vkGetSemaphoreCounterValue", rec::B_vkGetSemaphoreCounterValue}, {"vkWaitSemaphores", rec::B_vkWaitSemaphores}, {"vkWaitForFences", rec::B_vkWaitForFences}, {"vmaInvalidateAllocation", rec::B_vmaInvalidateAllocation}, {"vkGetQueueCheckpointDataNV", rec::B_vkGetQueueCheckpointDataNV}, {"vkGetDeviceFaultInfoEXT", rec::B_vkGetDeviceFaultInfoEXT}}; std::string ran, waited; for (const auto& [c, b] : calls) { const auto it = st.overlapped.find(c); const uint64_t n = it == st.overlapped.end() ? 0 : it->second; if (rec::unordered(b) && !n) std::printf("FAIL thread status reads: %s (in unordered()) always waited for the recorder\n", c), failures++; if (!rec::unordered(b) && n) std::printf("FAIL thread status reads: %s made %llu times with packets in flight: it must wait\n", c, (unsigned long long)n), failures++; std::string& to = rec::unordered(b) ? ran : waited; to += std::string(to.empty() ? "" : ", ") + c + (rec::unordered(b) ? " " + std::to_string(n) : std::string()); } if (sref.reads.size() < 60) std::printf("FAIL thread status reads: only %zu values read\n", sref.reads.size() / 2), failures++; std::printf("PASS thread status reads: %zu values identical to direct; made with packets in flight: %s; waited: %s\n", st.reads.size() / 2, ran.empty() ? "(none)" : ran.c_str(), waited.empty() ? "(none)" : waited.c_str()); check_run("thread: a buffer destroyed while a recording names it (the commands naming it run first)", run_tracked(destroy_recorded, rec::OFF, false).log, run_tracked(destroy_recorded, rec::THREAD, true)); } { // (teardown with work queued: stopped, joined, started again; vkDestroyDevice stops it too) static size_t at_stop; static bool joined_at_stop; g_at_stop = [] { at_stop = g_log.size(); }; const Out tref = run_tracked(teardown, rec::OFF, false); const size_t ref_at = at_stop; g_at_stop = [] { rec::g_rec.stop(); std::lock_guard lk(g_log_mu); at_stop = g_log.size(), joined_at_stop = !rec::g_rec.th.joinable(); if (!joined_at_stop) std::printf("FAIL thread teardown: stop() returned with the recorder thread not joined\n"), failures++; }; const uint64_t want_starts = rec::g_rec.ready.load() ? 1 : 2; // (running: started again after the stop; else at the first packet too) const uint64_t starts0 = rec::g_rec.starts.load(); const Out t = run_tracked(teardown, rec::THREAD, true); check_run("thread teardown (stopped with 4 frames queued, started again, stopped at vkDestroyDevice)", tref.log, t); const uint64_t starts = rec::g_rec.starts.load() - starts0; if (at_stop != ref_at || !joined_at_stop || rec::g_rec.th.joinable() || starts != want_starts) std::printf("FAIL thread teardown: %zu of %zu calls in when stop() returned, joined %d, joinable after vkDestroyDevice %d, " "started %llu times (%llu expected)\n", at_stop, ref_at, (int)joined_at_stop, (int)rec::g_rec.th.joinable(), (unsigned long long)starts, (unsigned long long)want_starts), failures++; else std::printf("PASS thread teardown: all %zu queued calls run when stop() returned, the thread joined; started again lazily; " "joined at vkDestroyDevice\n", at_stop); } { // (backpressure: an 8 KB ring, 2 KB of room a packet) rec::g_rec.stop(); const size_t bytes = rec::g_ring_bytes, room = rec::g_ring_room; rec::g_ring_bytes = 8192, rec::g_ring_room = 2048; const uint64_t full0 = rec::g_rec.wait_n[rec::B_RING_FULL].load(), d0 = rec::g_rec.direct.load(); check_run("thread, an 8 KB ring (the producer waits for room)", lref.log, run_tracked(lifecycle, rec::THREAD, true)); const uint64_t full = rec::g_rec.wait_n[rec::B_RING_FULL].load() - full0; if (!full) std::printf("FAIL thread 8 KB ring: the producer never waited for room\n"), failures++; else std::printf("PASS thread 8 KB ring: the producer waited for room %llu times\n", (unsigned long long)full); const Out oref = run_tracked(oversize, rec::OFF, false); check_run("thread, a packet bigger than its room (4 KB vkCmdUpdateBuffer, 2 KB room): made directly in order", oref.log, run_tracked(oversize, rec::THREAD, true)); if (rec::g_rec.direct.load() - d0 != 1) std::printf("FAIL thread oversize: %llu calls made directly, 1 expected\n", (unsigned long long)(rec::g_rec.direct.load() - d0)), failures++; // (again with the ring empty: stopped, its counters moved on to the ring's start, so the 8 KB free ahead of the first // packet would hold it; the 2 KB room must still decide) rec::g_rec.stop(); const uint64_t at = (rec::g_rec.W.load() + 8191) & ~uint64_t(8191); rec::g_rec.W.store(at), rec::g_rec.R.store(at); const uint64_t d1 = rec::g_rec.direct.load(); check_run("thread, the same packet first with the ring empty: made directly in order", run_tracked(oversize_first, rec::OFF, false).log, run_tracked(oversize_first, rec::THREAD, true)); if (rec::g_rec.direct.load() - d1 != 1) std::printf("FAIL thread oversize, ring empty: %llu calls made directly, 1 expected\n", (unsigned long long)(rec::g_rec.direct.load() - d1)), failures++; rec::g_rec.stop(); rec::g_ring_bytes = bytes, rec::g_ring_room = room; } { // (no memory for the ring: every call made directly, logged once) rec::g_rec.stop(); g_aligned_calls = 0, g_fail_nth = 1; const Out o = run_tracked(lifecycle, rec::THREAD, true); g_fail_nth = 0; check_run("thread, no memory for the ring (every call made directly)", lref.log, o); if (!rec::g_rec.broken.load()) std::printf("FAIL thread no ring: the recorder ran\n"), failures++; rec::g_rec.broken = false; } { // (a parallel replay records: each packet made at once, after what was published) par_active = true; const std::vector got = run(frame, rec::THREAD, true); par_active = false; same("thread while par_active (each call made directly)", ref, got); } { // (Triforce, 11-15: beside a parallel replay's segments. The recorder never asleep in it (a 2 s spin), so the producer's // packets stay unpublished until it waits (a packet published wakes a sleeping recorder at once): the case segments_begin // is there for, every time) rec::g_rec.stop(); const uint64_t spin = rec::g_spin_ns; rec::g_spin_ns = 2000000000ull; rec::g_thread_start_hook = [] { t_is_recorder = true; }; const Out sref = run_tracked(segments, rec::OFF, false); g_rec_calls_par = 0; const Out st = held(segments, true); check_run("thread beside a parallel replay's segments (their first timestamps in flight at their start; two helper threads, " "then the producer, each recording its segment directly)", sref.log, st); if (g_rec_calls_par.load()) std::printf("FAIL thread segments: the recorder made %llu calls while the segments recorded\n", (unsigned long long)g_rec_calls_par.load()), failures++; g_seg_at_once = true; g_rec_calls_par = 0; const Out sa = held(segments, true); g_seg_at_once = false; if (sa.violations) std::printf("FAIL thread segments at once: %d violation(s), the first: %s\n", sa.violations, sa.violation.c_str()), failures++; if (g_rec_calls_par.load()) std::printf("FAIL thread segments at once: the recorder made calls while they recorded\n"), failures++; const auto want = by_command_buffer(sref.log), got = by_command_buffer(sa.log); if (want.size() != got.size()) std::printf("FAIL thread segments at once: %zu command buffers, %zu expected\n", got.size(), want.size()), failures++; for (const auto& [cb, calls] : want) { char what[160]; std::snprintf(what, sizeof what, "thread beside segments recorded at once, %s %#llx", cb ? "command buffer" : "the calls on none", (unsigned long long)cb); const auto it = got.find(cb); same(what, calls, it == got.end() ? std::vector{} : it->second); } rec::g_rec.stop(); rec::g_spin_ns = spin, rec::g_thread_start_hook = nullptr; } g_delay_ns = 0; { // (stress: >= 1e6 packets through a 16 KB ring, twice; each run's stream hashed, against the direct one's) rec::g_rec.stop(); const size_t bytes = rec::g_ring_bytes, room = rec::g_ring_room; rec::g_ring_bytes = 16384, rec::g_ring_room = 4096; g_stress_frames = (int)(1000000 / ref.size()) + 1; uint64_t h[3]; uint64_t calls = 0, full = 0; for (int i = 0; i < 3; i++) { g_hash_only = true, g_hash = 0xcbf29ce484222325ull; const uint64_t c0 = g_cmd_off_main, f0 = rec::g_rec.wait_n[rec::B_RING_FULL].load(); const Out o = run_tracked(stress, i ? rec::THREAD : rec::OFF, i != 0); g_hash_only = false; h[i] = g_hash; if (o.violations) std::printf("FAIL thread stress: %s\n", o.violation.c_str()), failures++; if (i) calls += g_cmd_off_main - c0, full += rec::g_rec.wait_n[rec::B_RING_FULL].load() - f0; } if (h[1] != h[0] || h[2] != h[0] || calls < 2000000) std::printf("FAIL thread stress: hashes %016llx (direct) %016llx %016llx, %llu calls on the recorder\n", (unsigned long long)h[0], (unsigned long long)h[1], (unsigned long long)h[2], (unsigned long long)calls), failures++; else std::printf("PASS thread stress: 2 runs of %llu packets through a 16 KB ring (%llu waits for room), each stream identical " "to the direct one (hash %016llx)\n", (unsigned long long)(calls / 2), (unsigned long long)full, (unsigned long long)h[0]); rec::g_rec.stop(); rec::g_ring_bytes = bytes, rec::g_ring_room = room; } rec::g_poison = was_poison; } int main(int argc, char** argv) { #ifdef _WIN32 SetErrorMode(0x0002 | 0x0001); // (SEM_NOGPFAULTERRORBOX | SEM_FAILCRITICALERRORS) #endif std::setvbuf(stdout, nullptr, _IONBF, 0); // (a broken variant may end the process: what it printed before is kept) // (the switch as the renderer reads it: first use, from the environment) std::printf("env mode %d\n", rec::mode()); const int env_mode = rec::mode(); g_main_id = std::this_thread::get_id(); if (argc > 2 && !std::strcmp(argv[1], "--bench")) return bench(std::atoi(argv[2])); if (argc > 2 && !std::strcmp(argv[1], "--overlap")) return overlap(std::atoi(argv[2])); if (argc > 2 && !std::strcmp(argv[1], "--idle")) return idle_bench(std::atoi(argv[2])); if (argc > 1 && !std::strcmp(argv[1], "--throw")) return throw_test(); const bool only_thread = argc > 2 && !std::strcmp(argv[1], "--only") && !std::strcmp(argv[2], "thread"); const std::vector ref = run(frame, rec::OFF, false); if (only_thread) { thread_checks(ref); std::printf(failures ? "FAILED (%d)\n" : "ALL PASS\n", failures); return failures ? 1 : 0; } same("direct, caller memory reused after each call", ref, run(frame, rec::OFF, true)); if (env_mode != rec::OFF) same("RX_VK_RECORDER from the environment, caller memory reused", ref, run(frame, env_mode, true)); rec::t_arena.packets = 0; same("sync (packed, run at once), caller memory reused", ref, run(frame, rec::SYNC, true)); if (rec::t_arena.packets != ref.size()) std::printf("FAIL sync: %llu packets for %zu calls\n", (unsigned long long)rec::t_arena.packets, ref.size()), failures++; if (rec::t_arena.ci || rec::t_arena.used) std::printf("FAIL sync: the arena not rewound\n"), failures++; same("deferred (packed, run at the drain), caller memory poisoned in between", ref, run(frame, rec::DEFER, true)); if (rec::t_arena.fallbacks) std::printf("FAIL: %llu calls not packed\n", (unsigned long long)rec::t_arena.fallbacks), failures++; const std::vector fref = run(fallbacks, rec::OFF, false); same("fallback order (pNext chain, inline uniform block, a push template of a size not told), deferred", fref, run(fallbacks, rec::DEFER, false)); if (rec::t_arena.fallbacks != 3) std::printf("FAIL: %llu fallbacks, 3 expected\n", (unsigned long long)rec::t_arena.fallbacks), failures++; // ---- defer (step 1.5): the lifecycle, the replay lock, the budgets, an allocation failure, a parallel replay const Out lref = run_tracked(lifecycle, rec::OFF, false); if (lref.violations) std::printf("FAIL the lifecycle scenario itself: %s\n", lref.violation.c_str()), failures++; const Out ld = run_tracked(lifecycle, rec::DEFER, true); check_run("defer lifecycle (2 command buffers interleaved; reset, begin, end, submit, wait x2; query read; destruction; caller memory " "poisoned)", lref.log, ld); if (ld.fallbacks != 6) std::printf("FAIL defer lifecycle: %llu fallbacks, 6 expected\n", (unsigned long long)ld.fallbacks), failures++; const Out href = run_tracked(handoff, rec::OFF, false); check_run("defer handoff (a recording across two threads under the replay lock)", href.log, run_tracked(handoff, rec::DEFER, true)); { const size_t soft = rec::g_soft_budget; rec::g_soft_budget = 2048; const Out o = run_tracked(lifecycle, rec::DEFER, true); rec::g_soft_budget = soft; check_run("defer soft budget 2 KB (held packets drained first past it)", lref.log, o); if (!o.budget_drains) std::printf("FAIL defer soft budget: no drain at the budget\n"), failures++; else std::printf("PASS defer soft budget: %llu drains at the budget\n", (unsigned long long)o.budget_drains); } const Out bref = run_tracked(big, rec::OFF, false); { const size_t hard = rec::g_hard_budget; rec::g_hard_budget = size_t(1) << 20; const Out o = run_tracked(big, rec::DEFER, true, true); rec::g_hard_budget = hard; check_run("defer hard budget 1 MB (past it: what is held drained, the call made directly)", bref.log, o); if (!o.failures || o.capacity > (size_t(1) << 20)) std::printf("FAIL defer hard budget: %llu calls past it, %zu bytes of chunks\n", (unsigned long long)o.failures, o.capacity), failures++; else std::printf("PASS defer hard budget: %llu calls made directly, %zu bytes of chunks\n", (unsigned long long)o.failures, o.capacity); } { const long live = g_aligned_live.load(); g_aligned_calls = 0, g_fail_nth = 2; const Out o = run_tracked(big, rec::DEFER, true, true); g_fail_nth = 0; check_run("defer allocation failure (the second chunk's allocation fails)", bref.log, o); if (o.failures != 1 || o.leaked || g_aligned_live.load() != live) std::printf("FAIL defer allocation failure: %llu failures (1 expected), %ld blocks lost, %ld left after the thread\n", (unsigned long long)o.failures, o.leaked, g_aligned_live.load() - live), failures++; else std::printf("PASS defer allocation failure: held packets drained, the call made directly, no block lost\n"); } { // (a parallel replay records: never held, each packet run at once) par_active = true; rec::set_mode(rec::DEFER); g_mutate = true, g_log.clear(); frame(); const bool at_once = g_log.size() == ref.size() && !rec::t_arena.head; rec::drain(); const std::vector got = g_log; rec::set_mode(rec::OFF); par_active = false; if (!at_once) std::printf("FAIL defer while par_active: packets held\n"), failures++; same("defer while par_active (each packet run at once)", ref, got); } { // (Triforce, 11-15: beside a parallel replay's segments, their first timestamps held at their start) const Out sref = run_tracked(segments, rec::OFF, false); check_run("defer beside a parallel replay's segments (two helper threads, then the producer, each recording its segment " "directly)", sref.log, run_tracked(segments, rec::DEFER, true)); } thread_checks(ref); { // (B_DIRECT: code that records into the command buffer directly, as Dear ImGui's backend does through its own pointers, // after packed calls: with the boundary the driver sees packed, direct, packed in order; without it (the control) not) const auto order = [](int mode, bool bounded) { const VkCommandBuffer cmd = H(0x300); rec::set_mode(mode); g_log.clear(); // (thread: with the boundary, each packed call takes 0.3 ms so a missing wait would show; without it (the // control), the recorder is held at the fake until the direct call is made, so the wrong order is certain) if (mode == rec::THREAD) bounded ? (void)(g_delay_ns = 300000) : (void)(g_gate = true); { std::lock_guard lk(g_replay_mu); vkCmdSetLineWidth(cmd, 1.0f); vkCmdSetLineWidth(cmd, 2.0f); if (bounded) rec::boundary(rec::B_DIRECT); fake_vkCmdSetLineWidth(cmd, 9.0f); // (opaque: not through vk_work.h) g_gate = false; vkCmdSetLineWidth(cmd, 3.0f); } rec::drain(); g_delay_ns = 0; std::string seq; for (const Call& c : g_log) { float v; std::memcpy(&v, c.bytes.data() + c.bytes.size() - sizeof v, sizeof v); seq += (char)('0' + (int)v); } rec::set_mode(rec::OFF); return seq; }; for (const int mode : {rec::DEFER, rec::THREAD}) { const char* name = mode == rec::DEFER ? "defer" : "thread"; const std::string with = order(mode, true), without = order(mode, false); if (with != "1293") std::printf("FAIL %s B_DIRECT: the driver saw %s, 1293 expected\n", name, with.c_str()), failures++; else if (without == "1293") std::printf("FAIL %s B_DIRECT control: without the boundary the order was still 1293\n", name), failures++; else std::printf("PASS %s B_DIRECT: packed, direct, packed reach the driver in order (%s; without the boundary %s)\n", name, with.c_str(), without.c_str()); } } std::set seen; for (const Call& c : ref) seen.insert(c.op); int missing = 0; for (uint32_t op = 0; op < rec::OP_COUNT; op++) if (!seen.count(rec::op_name(op))) std::printf("NOT COVERED %s\n", rec::op_name(op)), missing++; if (missing) failures++; else std::printf("PASS coverage: all %d packed calls made by the frame\n", (int)rec::OP_COUNT); std::printf(failures ? "FAILED (%d)\n" : "ALL PASS\n", failures); return failures ? 1 : 0; }