mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-22 17:19:58 -04:00
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
458 lines
24 KiB
C++
458 lines
24 KiB
C++
// (Triforce) The lazy write-back's page states (vk_memory.cpp "lazy write-back", docs/lazy-writeback.md), in a header of
|
|
// their own so that tests run them without the game or a GPU (tests/test_wb_lazy_state.py).
|
|
//
|
|
// What a shader writes into a mirrored pool is copied back to guest memory at the end of every submission, in case the CPU
|
|
// reads it. With the lazy write-back a whole page that keeps being written and that the CPU leaves alone is no longer
|
|
// copied: the copy is owed, the page gets no access, and whoever touches it pays first (one copy, mirror to guest memory,
|
|
// of everything written since). A page goes through:
|
|
//
|
|
// none guest memory has all the GPU wrote; read and write. Written back at each submission's end, as ever.
|
|
// WATCH the same, and read-only: a CPU write faults and ends the watch (the page is the CPU's too: not for us).
|
|
// It is still written back at each submission's end. Once every CPU write from before the watch is known
|
|
// (a collection and a capture made after it began have come and named no write to it), the next write makes it
|
|
// OWED no access. Later submissions leave their writes in the mirror (seq: the newest recording that did).
|
|
// PAYING no access; one thread copies mirror to guest memory. Every other toucher waits for that copy, then goes on.
|
|
// PENDING access back, after a payment that came between a recording's end and its submit: what that recording
|
|
// wrote is still owed, and is copied right after it is submitted (as its own write-back would have been).
|
|
//
|
|
// The rules the tests hold it to: protection and state never disagree (no access: OWED or PAYING; read-only: WATCH);
|
|
// every reader of an owed page gets the copy first; a CPU write to a watched page ends the watch without a copy, and one
|
|
// to an owed page pays first and never drops the debt; later writes join the one debt and one copy pays them all; one
|
|
// payer at a time for a page; nothing gets protected while a host call has guest memory in the kernel's hands (io).
|
|
//
|
|
// Locking: one mutex, held for the table's steps and the protection calls, never across a copy or a wait for the GPU.
|
|
// A touch of a page in PAYING waits on the condition, which lets the mutex go.
|
|
#pragma once
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <condition_variable>
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <map>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
namespace wb_lazy {
|
|
|
|
constexpr uint64_t kPageShift = 12;
|
|
constexpr uint64_t kRegionPages = 512, kPages = 1ull << 27; // (2 MB regions over the guest's 512 GB, as write_gen_table.h)
|
|
constexpr uint64_t kHotShift = 4; // (a touch holds its 64 KB block: 16 pages)
|
|
constexpr uint64_t kHoldMs = 1000, kHoldMaxMs = 32000; // (the hold after a touch, doubling while it keeps being touched)
|
|
constexpr size_t kMaxSegs = 1u << 16;
|
|
|
|
enum State : uint8_t { WATCH = 1, OWED, PAYING, PENDING };
|
|
enum Access : uint8_t { RW = 0, RO, NONE };
|
|
/// Who asks for a payment: a fault (the game's or the host's code), a host call about to hand the range to the kernel, a GPU
|
|
/// read through the import, code that gives the mirror up, the recording's own submit (PENDING)
|
|
enum Why : int { BY_FAULT = 0, BY_HOST, BY_GPU, BY_GIVEUP, BY_SUBMIT, WHY_N };
|
|
|
|
struct Src { // (the copy's two buffers and the pool's start: a page's offset in both is its guest address - base)
|
|
uint64_t mirror = 0, import = 0, base = 0;
|
|
bool operator==(const Src&) const = default;
|
|
};
|
|
struct Job { // (pages [lo, hi) to copy from src.mirror to src.import)
|
|
uint64_t lo = 0, hi = 0;
|
|
Src src;
|
|
uint64_t seq = 0;
|
|
};
|
|
/// What the collections have seen so far (vk_memory.cpp): a watch may end in OWED only once one of each, begun after the
|
|
/// watch did, has come: by then every CPU write from before the watch has been named
|
|
struct Epoch {
|
|
uint64_t collects_begun = 0; // (collections of the write watch numbered: one's sweep starts after it has its number)
|
|
uint64_t collects = 0; // (the newest collection whose pages were named)
|
|
uint64_t caps_made = 0; // (captures numbered: a capture's collection starts after it has its number)
|
|
uint64_t caps_taken = 0; // (the newest capture whose pages were named)
|
|
bool captures = false; // (captures are made at all)
|
|
};
|
|
struct Counters {
|
|
uint64_t watched = 0, armed = 0, skipped_pages = 0, watch_ended_by_write = 0, watch_ended_by_host = 0;
|
|
uint64_t paid[WHY_N] = {}, paid_pages[WHY_N] = {};
|
|
uint64_t waited = 0, pending = 0, refused_hot = 0, refused_io = 0, refused_full = 0, foreign = 0, protect_failed = 0;
|
|
uint64_t owed_pages = 0, watch_pages = 0; // (now)
|
|
};
|
|
enum Fault { NOT_OURS, RETRY, PAY };
|
|
|
|
/// Whether it runs (vk_memory.cpp lazy_decide): mode 0 off, 1 on, 2 the self-check, and why, for the log line. env: what
|
|
/// RX_VK_LAZY_WB is set to (nullptr or "": not set), which goes over the row; row: the Debug tab's Lazy write-back row;
|
|
/// windows: a Windows build (the fault handler pays on Windows only); amd: AMD's driver, where the row alone does not turn
|
|
/// it on (NX2navi turned its own off there after reports of the game ending in play); mirrors: there are VRAM mirrors
|
|
struct Choice {
|
|
int mode;
|
|
const char* why;
|
|
};
|
|
inline Choice choose(const char* env, bool row, bool windows, bool amd, bool mirrors) {
|
|
const bool set = env && *env;
|
|
const int asked = set ? std::clamp(atoi(env), 0, 2) : row ? 1 : 0;
|
|
if (!asked) return {0, set ? "RX_VK_LAZY_WB=0 is set" : "the Debug tab's row is off"};
|
|
if (!windows) return {0, "it runs on Windows only (its fault handler has not been made for signals)"};
|
|
if (!mirrors) return {0, "there are no VRAM mirrors to leave anything in"};
|
|
if (set) return {asked, asked == 2 ? "RX_VK_LAZY_WB=2 is set" : "RX_VK_LAZY_WB=1 is set"};
|
|
if (amd)
|
|
return {0, "the Debug tab's row is on, but it stays off on AMD's driver, where NX2navi's ended games in play "
|
|
"(RX_VK_LAZY_WB=1 turns it on there)"};
|
|
return {1, "the Debug tab's row is on"};
|
|
}
|
|
|
|
/// Mem: bool set(uint64_t page_lo, uint64_t page_hi, Access a) (false: the pages could not be changed), and
|
|
/// bool accessible(uint64_t page, bool write) (whether the system lets that access through now)
|
|
template <class Mem>
|
|
class Table {
|
|
public:
|
|
explicit Table(Mem& m) : mem(m), regions(new std::atomic<uint32_t>[kPages / kRegionPages]) {
|
|
for (uint64_t i = 0; i < kPages / kRegionPages; i++) regions[i].store(0, std::memory_order_relaxed);
|
|
}
|
|
|
|
/// Whether a fault at that page can be ours at all (no lock: the fault handler's first question)
|
|
bool tracked(uint64_t page) const {
|
|
return page < kPages && regions[page / kRegionPages].load(std::memory_order_acquire) != 0;
|
|
}
|
|
bool any() const { return protected_pages.load(std::memory_order_acquire) != 0; }
|
|
std::atomic<uint64_t> late_retries{0}; // ((Triforce, 11-21) on_fault: faults on a page whose access was given back meanwhile
|
|
|
|
/// A submission's end, for the whole pages [lo, hi) of a run it is about to write back (seq: this recording). skip gets
|
|
/// the pages that are OWED: their copy is left out, and the debt is this recording's now. The others are copied as ever;
|
|
/// of them, the untracked become WATCH and the watched whose time has come OWED (from the next write on they are skipped)
|
|
void advance(uint64_t lo, uint64_t hi, const Src& src, uint64_t seq, const Epoch& e, uint64_t now_ms, bool may_start,
|
|
std::vector<std::pair<uint64_t, uint64_t>>& skip) {
|
|
if (hi <= lo || hi > kPages) return;
|
|
std::lock_guard lk(mu);
|
|
split(lo), split(hi);
|
|
uint64_t at = lo;
|
|
auto it = segs.lower_bound(lo);
|
|
while (at < hi) {
|
|
const uint64_t next = it != segs.end() && it->first < hi ? it->first : hi;
|
|
if (at < next) { // (untracked pages [at, next))
|
|
if (may_start) start_watch(at, next, src, e, now_ms);
|
|
it = segs.lower_bound(next); // (start_watch added segments before it)
|
|
at = next;
|
|
continue;
|
|
}
|
|
Seg& s = it->second;
|
|
if (!(s.src == src)) {
|
|
c.foreign++; // (another mirror's: never skipped. The code that gave that mirror up should have paid)
|
|
} else if (s.state == OWED) {
|
|
s.seq = seq;
|
|
if (!skip.empty() && skip.back().second == s.lo) skip.back().second = s.hi;
|
|
else skip.push_back({s.lo, s.hi});
|
|
c.skipped_pages += s.hi - s.lo;
|
|
} else if (s.state == WATCH && e.collects > s.collects_at && (!e.captures || e.caps_taken > s.caps_at)) {
|
|
if (mem.set(s.lo, s.hi, NONE)) {
|
|
s.state = OWED, s.seq = seq;
|
|
c.armed += s.hi - s.lo, c.owed_pages += s.hi - s.lo, c.watch_pages -= s.hi - s.lo;
|
|
} else {
|
|
c.protect_failed++;
|
|
}
|
|
}
|
|
at = s.hi;
|
|
++it;
|
|
}
|
|
}
|
|
|
|
/// A page the CPU is known to have written (a collection's or a capture's): a watch on it ends, and it is held like a
|
|
/// touched one. True when the page is OWED or PAYING, which the watch is there to rule out (the caller says so in the log)
|
|
bool cpu_wrote(uint64_t page, uint64_t now_ms) {
|
|
if (!tracked(page)) return false;
|
|
std::lock_guard lk(mu);
|
|
auto it = find(page);
|
|
if (it == segs.end()) return false;
|
|
if (it->second.state != WATCH) return it->second.state == OWED || it->second.state == PAYING;
|
|
split(page), split(page + 1);
|
|
end_watch(segs.find(page));
|
|
hold(page, now_ms);
|
|
c.watch_ended_by_write++;
|
|
return false;
|
|
}
|
|
|
|
/// ((Triforce, 11-21) The fault handler's question (lazy_veh), in place of tracked() then fault(). The table gives pages their
|
|
/// access back (end_pay, end_watch) BEFORE it uncounts them, so a thread that faulted on such a page just before can get here
|
|
/// with tracked() already false and the page readable and writable again: that access runs again (RETRY), where the handler
|
|
/// sent it on as not ours and the host's crash report ended the game (a player's report on 0.6.0: about one in 100k frames).
|
|
/// A page untracked and still without access is not this table's (start_watch counts before it protects). Tracked: fault()
|
|
Fault on_fault(uint64_t page, bool write, uint64_t grow, uint64_t now_ms, std::vector<Job>& jobs) {
|
|
if (page >= kPages) return NOT_OURS;
|
|
if (!tracked(page)) {
|
|
if (!mem.accessible(page, write)) return NOT_OURS;
|
|
late_retries.fetch_add(1, std::memory_order_relaxed);
|
|
return RETRY;
|
|
}
|
|
return fault(page, write, grow, now_ms, jobs);
|
|
}
|
|
|
|
/// A fault at page. NOT_OURS: nothing of this table's. RETRY: the access can run again (a watch ended by this write, a
|
|
/// payment by another thread waited for, or the page has its access back by now). PAY: jobs holds what this thread
|
|
/// has to copy (the owed pages around the address, grow more at most, in PAYING now), then end_pay
|
|
Fault fault(uint64_t page, bool write, uint64_t grow, uint64_t now_ms, std::vector<Job>& jobs) {
|
|
if (page >= kPages) return NOT_OURS;
|
|
std::unique_lock lk(mu);
|
|
for (;;) {
|
|
auto it = find(page);
|
|
// (not this table's, or no longer: the access runs again when the page lets it through by now. Another thread's
|
|
// payment may have given the access back between the fault and this look)
|
|
if (it == segs.end() || it->second.state == PENDING) return mem.accessible(page, write) ? RETRY : NOT_OURS;
|
|
Seg& s = it->second;
|
|
if (s.state == WATCH) {
|
|
if (!write) return RETRY; // (a watched page can be read: the fault is from before the watch, when it was owed)
|
|
split(page & ~((1ull << kHotShift) - 1)), split((page | ((1ull << kHotShift) - 1)) + 1); // (its 64 KB block alone)
|
|
end_watch(find(page));
|
|
hold(page, now_ms);
|
|
c.watch_ended_by_write++;
|
|
return RETRY;
|
|
}
|
|
if (s.state == PAYING) {
|
|
c.waited++;
|
|
cv.wait(lk);
|
|
continue; // (whatever it is now)
|
|
}
|
|
const uint64_t a = grow ? page / (grow + 1) * (grow + 1) : page;
|
|
take(std::max(a, s.lo), std::min(a + grow + 1, s.hi), jobs, &jobs);
|
|
return PAY;
|
|
}
|
|
}
|
|
|
|
/// Everything in [lo, hi) settled before a reader that cannot fault: the owed pages go into jobs (PAYING; the caller
|
|
/// copies, then end_pay), payments by other threads are waited for, and a watch ends unless the reader only reads through
|
|
/// the GPU (BY_GPU: a watched page is whole and readable)
|
|
void settle(uint64_t lo, uint64_t hi, Why why, uint64_t now_ms, std::vector<Job>& jobs) {
|
|
if (hi <= lo) return;
|
|
hi = std::min(hi, kPages);
|
|
std::unique_lock lk(mu);
|
|
for (bool again = true; again;) {
|
|
again = false;
|
|
split(lo), split(hi);
|
|
for (auto it = segs.lower_bound(lo); it != segs.end() && it->first < hi;) {
|
|
Seg& s = it->second;
|
|
if (s.state == PAYING && s.payer != &jobs) { // (another thread's copy: waited for, then looked at again)
|
|
c.waited++;
|
|
cv.wait(lk);
|
|
again = true; // (the table changed while the mutex was away)
|
|
break;
|
|
}
|
|
if (s.state == WATCH && why != BY_GPU) {
|
|
if (why == BY_HOST) {
|
|
for (uint64_t b = s.lo >> kHotShift; b <= (s.hi - 1) >> kHotShift; b++) hold(b << kHotShift, now_ms);
|
|
c.watch_ended_by_host++;
|
|
}
|
|
it = end_watch(it);
|
|
continue;
|
|
}
|
|
if (s.state == OWED) {
|
|
const uint64_t a = s.lo, b = s.hi;
|
|
take(a, b, jobs, &jobs);
|
|
it = segs.upper_bound(a);
|
|
continue;
|
|
}
|
|
++it; // (PENDING: its access is back, and what it lacks no reader could have yet)
|
|
}
|
|
}
|
|
}
|
|
|
|
/// With the queue's lock held, as the copy of jobs goes onto the queue. submitted: the newest recording that is on the
|
|
/// queue before it. A job whose debt is newer keeps the rest of it: that recording's submit, which takes the same lock,
|
|
/// finds it (take_pending), whether this payment has ended by then or not
|
|
void mark_rest(const std::vector<Job>& jobs, uint64_t submitted) {
|
|
std::lock_guard lk(mu);
|
|
for (const Job& j : jobs)
|
|
for (auto it = segs.lower_bound(j.lo); it != segs.end() && it->first < j.hi; ++it) {
|
|
Seg& s = it->second;
|
|
if (s.state != PAYING || s.rest || s.seq <= submitted) continue;
|
|
s.rest = true;
|
|
pendings.fetch_add(1, std::memory_order_release);
|
|
c.pending++;
|
|
}
|
|
}
|
|
|
|
/// The copy of jobs is done and waited for: the pages get their access back, and a touch by the CPU holds them. A
|
|
/// piece with a rest (mark_rest) is PENDING from here
|
|
void end_pay(const std::vector<Job>& jobs, Why why, uint64_t now_ms) {
|
|
{
|
|
std::lock_guard lk(mu);
|
|
for (const Job& j : jobs) {
|
|
c.paid[why]++;
|
|
if (why == BY_FAULT || why == BY_HOST || why == BY_GPU)
|
|
for (uint64_t b = j.lo >> kHotShift; b <= (j.hi - 1) >> kHotShift; b++) hold(b << kHotShift, now_ms);
|
|
// (the job's pages, in as many pieces as other steps have cut them into meanwhile: PAYING there is this job's)
|
|
for (auto it = segs.lower_bound(j.lo); it != segs.end() && it->first < j.hi;) {
|
|
Seg& s = it->second;
|
|
if (s.state != PAYING) { ++it; continue; }
|
|
if (!mem.set(s.lo, s.hi, RW)) c.protect_failed++;
|
|
count(s.lo, s.hi, -1);
|
|
c.owed_pages -= s.hi - s.lo, c.paid_pages[why] += s.hi - s.lo;
|
|
if (s.rest) {
|
|
s.state = PENDING, s.payer = nullptr, s.rest = false;
|
|
++it;
|
|
} else {
|
|
it = segs.erase(it);
|
|
nsegs.fetch_sub(1, std::memory_order_release);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
cv.notify_all();
|
|
}
|
|
|
|
/// With the queue's lock held, right after a recording's submit (submitted: its number): the pages with a rest that is
|
|
/// on the queue now (PENDING, or still in a payment that began before the submit), to copy once more; then end_pending
|
|
void take_pending(uint64_t submitted, std::vector<Job>& jobs) {
|
|
if (!pendings.load(std::memory_order_acquire)) return;
|
|
std::lock_guard lk(mu);
|
|
for (auto& [lo, s] : segs)
|
|
if ((s.state == PENDING || (s.state == PAYING && s.rest)) && s.seq <= submitted) jobs.push_back({s.lo, s.hi, s.src, s.seq});
|
|
}
|
|
/// That copy is done and waited for (the queue's lock still held): nothing of those pages is owed any more
|
|
void end_pending(const std::vector<Job>& jobs) {
|
|
std::lock_guard lk(mu);
|
|
for (const Job& j : jobs) {
|
|
c.paid[BY_SUBMIT]++;
|
|
for (auto it = segs.lower_bound(j.lo); it != segs.end() && it->first < j.hi;) {
|
|
Seg& s = it->second;
|
|
if (s.seq > j.seq || !(s.state == PENDING || (s.state == PAYING && s.rest))) { ++it; continue; }
|
|
c.paid_pages[BY_SUBMIT] += s.hi - s.lo;
|
|
pendings.fetch_sub(1, std::memory_order_release);
|
|
if (s.state == PAYING) { // (its payment is still waiting for its own copy: it ends with nothing left)
|
|
s.rest = false;
|
|
++it;
|
|
} else {
|
|
it = segs.erase(it);
|
|
nsegs.fetch_sub(1, std::memory_order_release);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A host call is about to hand guest memory to the kernel, or to change pages' state: from here to io_end nothing
|
|
/// gets protected (the caller settles its own range after this, BY_HOST)
|
|
void io_begin() {
|
|
std::lock_guard lk(mu);
|
|
io++;
|
|
}
|
|
void io_end() {
|
|
std::lock_guard lk(mu);
|
|
if (io) io--;
|
|
}
|
|
|
|
Counters counters() {
|
|
std::lock_guard lk(mu);
|
|
return c;
|
|
}
|
|
/// Pages made OWED so far (a caller that keeps answers about which buffer a range is read from asks whether it grew)
|
|
uint64_t armed() {
|
|
std::lock_guard lk(mu);
|
|
return c.armed;
|
|
}
|
|
/// (tests) the state of a page, 0 none
|
|
int state_of(uint64_t page) {
|
|
std::lock_guard lk(mu);
|
|
auto it = find(page);
|
|
return it == segs.end() ? 0 : (int)it->second.state;
|
|
}
|
|
|
|
private:
|
|
struct Seg {
|
|
uint64_t lo = 0, hi = 0;
|
|
Src src;
|
|
uint64_t seq = 0;
|
|
uint64_t collects_at = 0, caps_at = 0; // (WATCH: what the collections had seen when it began)
|
|
const void* payer = nullptr; // (PAYING: whose jobs it is in)
|
|
bool rest = false; // (PAYING: its newest recording was not on the queue yet: mark_rest)
|
|
State state = WATCH;
|
|
};
|
|
struct Hot { uint64_t until = 0, back = 0, at = 0; };
|
|
|
|
typename std::map<uint64_t, Seg>::iterator find(uint64_t page) {
|
|
auto it = segs.upper_bound(page);
|
|
if (it == segs.begin()) return segs.end();
|
|
--it;
|
|
return page < it->second.hi ? it : segs.end();
|
|
}
|
|
/// A segment that reaches over page is cut there (both halves keep its state: no protection changes)
|
|
void split(uint64_t page) {
|
|
auto it = find(page);
|
|
if (it == segs.end() || it->second.lo == page) return;
|
|
Seg b = it->second;
|
|
b.lo = page;
|
|
it->second.hi = page;
|
|
segs.emplace(page, b);
|
|
nsegs.fetch_add(1, std::memory_order_release);
|
|
if (b.state == PENDING || b.rest) pendings.fetch_add(1, std::memory_order_release);
|
|
}
|
|
/// (regions' counts of pages without access: tracked())
|
|
void count(uint64_t lo, uint64_t hi, int d) {
|
|
for (uint64_t r = lo / kRegionPages; r <= (hi - 1) / kRegionPages; r++) {
|
|
const uint64_t n = std::min(hi, (r + 1) * kRegionPages) - std::max(lo, r * kRegionPages);
|
|
if (d > 0) regions[r].fetch_add((uint32_t)n, std::memory_order_release);
|
|
else regions[r].fetch_sub((uint32_t)n, std::memory_order_release);
|
|
}
|
|
if (d > 0) protected_pages.fetch_add(hi - lo, std::memory_order_release);
|
|
else protected_pages.fetch_sub(hi - lo, std::memory_order_release);
|
|
}
|
|
bool hot(uint64_t page, uint64_t now_ms) const {
|
|
const auto it = hots.find(page >> kHotShift);
|
|
return it != hots.end() && now_ms < it->second.until;
|
|
}
|
|
void hold(uint64_t page, uint64_t now_ms) {
|
|
if (hots.size() > (1u << 16)) hots.clear(); // (a bound; the blocks come back at their next touch)
|
|
Hot& h = hots[page >> kHotShift];
|
|
if (h.back && h.at == now_ms) return; // (the same touch, told once per piece of it)
|
|
h.back = h.back && now_ms < h.until + h.back ? std::min(h.back * 2, kHoldMaxMs) : kHoldMs;
|
|
h.until = now_ms + h.back, h.at = now_ms;
|
|
}
|
|
/// [lo, hi), untracked: read-only and watched from now, block by block, but for the blocks a touch holds; not while
|
|
/// a host call is in the kernel, nor when the table is full
|
|
void start_watch(uint64_t lo, uint64_t hi, const Src& src, const Epoch& e, uint64_t now_ms) {
|
|
if (io) { c.refused_io++; return; }
|
|
for (uint64_t a = lo; a < hi;) {
|
|
const uint64_t block_end = std::min(hi, ((a >> kHotShift) + 1) << kHotShift);
|
|
if (hot(a, now_ms)) { c.refused_hot++; a = block_end; continue; }
|
|
uint64_t b = block_end;
|
|
while (b < hi && !hot(b, now_ms)) b = std::min(hi, ((b >> kHotShift) + 1) << kHotShift);
|
|
if (segs.size() >= kMaxSegs) { c.refused_full++; return; }
|
|
// (a write fault on a watched page has to find the table: its region counts it, and BEFORE the protection, so
|
|
// that no fault can meet a page without access that tracked() does not know yet. 11-21, a player's report: a
|
|
// write between the two went to the crash handler)
|
|
count(a, b, +1);
|
|
if (mem.set(a, b, RO)) {
|
|
Seg s;
|
|
s.lo = a, s.hi = b, s.src = src, s.state = WATCH, s.collects_at = e.collects_begun, s.caps_at = e.caps_made;
|
|
segs.emplace(a, s);
|
|
nsegs.fetch_add(1, std::memory_order_release);
|
|
c.watched += b - a, c.watch_pages += b - a;
|
|
} else {
|
|
count(a, b, -1);
|
|
c.protect_failed++;
|
|
}
|
|
a = b;
|
|
}
|
|
}
|
|
typename std::map<uint64_t, Seg>::iterator end_watch(typename std::map<uint64_t, Seg>::iterator it) {
|
|
Seg& s = it->second;
|
|
if (!mem.set(s.lo, s.hi, RW)) c.protect_failed++;
|
|
count(s.lo, s.hi, -1);
|
|
c.watch_pages -= s.hi - s.lo;
|
|
nsegs.fetch_sub(1, std::memory_order_release);
|
|
return segs.erase(it);
|
|
}
|
|
/// The OWED pages [a, b) of one segment become a job, PAYING
|
|
void take(uint64_t a, uint64_t b, std::vector<Job>& jobs, const void* payer) {
|
|
split(a), split(b);
|
|
auto it = segs.find(a);
|
|
Seg& s = it->second;
|
|
s.state = PAYING, s.payer = payer;
|
|
jobs.push_back({s.lo, s.hi, s.src, s.seq});
|
|
}
|
|
|
|
Mem& mem;
|
|
std::mutex mu;
|
|
std::condition_variable cv;
|
|
std::map<uint64_t, Seg> segs; // by lo, never overlapping
|
|
std::unordered_map<uint64_t, Hot> hots;
|
|
std::unique_ptr<std::atomic<uint32_t>[]> regions; // (pages of each region that are WATCH, OWED or PAYING)
|
|
std::atomic<uint64_t> protected_pages{0}, nsegs{0}, pendings{0};
|
|
uint32_t io = 0;
|
|
Counters c;
|
|
};
|
|
|
|
} // namespace wb_lazy
|