NX2navi/tests/gpa_check.c

// runtime/vk_gpa.c: the device address table against a model of what a shader reads through it, with the allocator,
// the device and the completion queue replaced by the stand-ins below. Built and run by tests/gpa_check.py; prints each
// failed check and exits 1 on any.
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "cfg.h"
#include "host.h"
#include "vk.h"
#include "vk_gpa.h"
#include "platform/platform.h"

static int fails;
#define CHECK(c, ...) do { if (!(c) && fails++ < 40) { printf("FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } } while (0)

// ---- stand-ins for what vk_gpa.c links against
static VkCtx ctx = {.has_buffer_device_address = 1};
VkCtx *vk_ctx(void) { return &ctx; }
const char *cfg_get(const char *name) { (void)name; return NULL; }
void plat_mutex_lock(PlatMutex *m) { (void)m; }
void plat_mutex_unlock(PlatMutex *m) { (void)m; }

static int out_of_blocks_lines;
void rt_log(const char *fmt, ...) {
    char line[512];
    va_list ap;
    va_start(ap, fmt);
    vsnprintf(line, sizeof line, fmt, ap);
    va_end(ap);
    if (strstr(line, "out of L2 blocks")) out_of_blocks_lines++;
}

static const uint32_t *table;   // the first buffer vk_gpa.c creates
static uintptr_t next_handle = 0x1000;
VkResult vmaCreateBuffer(VmaAllocator a, const VkBufferCreateInfo *bi, const VmaAllocationCreateInfo *ai, VkBuffer *buf,
                         VmaAllocation *alloc, VmaAllocationInfo *info) {
    (void)a, (void)ai;
    void *mem = calloc(1, (size_t)bi->size);
    if (!table) table = mem;
    *buf = (VkBuffer)(next_handle += 0x1000);
    *alloc = (VmaAllocation)mem;
    if (info) info->pMappedData = mem;
    return VK_SUCCESS;
}
void vmaDestroyBuffer(VmaAllocator a, VkBuffer buf, VmaAllocation alloc) { (void)a, (void)buf; free(alloc); }

// A buffer's device address: its handle far above the GPU addresses the tests map.
static uint64_t dev_of(VkBuffer b) { return (uint64_t)(uintptr_t)b << 28; }
VkDeviceAddress vkGetBufferDeviceAddress(VkDevice d, const VkBufferDeviceAddressInfo *i) { (void)d; return dev_of(i->buffer); }

// The completion queue: a callback runs once the GPU is past its submission (gpu_done).
static struct { uint64_t fence; void (*fn)(void *); void *arg; } queue[1 << 16];
static int nqueue;
static uint64_t issued;
uint64_t vk_sync_issued(void) { return issued; }
void vk_completion_add(uint64_t fence, void (*fn)(void *), void *arg) {
    if (nqueue == (int)(sizeof queue / sizeof *queue)) { printf("FAIL: completion queue full\n"); exit(1); }
    queue[nqueue].fence = fence, queue[nqueue].fn = fn, queue[nqueue].arg = arg;
    nqueue++;
}
static void gpu_done(void) {   // the submission being recorded is issued and finishes
    issued++;
    int k = 0;
    for (int i = 0; i < nqueue; i++)
        if (queue[i].fence <= issued) queue[i].fn(queue[i].arg);
        else queue[k++] = queue[i];
    nqueue = k;
}

// ---- the table as a shader reads it (the layout at the top of vk_gpa.c)
enum { L1_WORD = 16, L1_N = 65536, L2_WORD = L1_WORD + L1_N, L2_N = 4096, BLOCKS = 1024 };
static uint32_t block_of(uint64_t addr) { return table[L1_WORD + (addr >> 24)]; }
static uint64_t read_through(uint64_t addr, int twin) {   // the device address addr resolves to, 0 for the zero buffer
    uint32_t blk = block_of(addr);
    uint32_t rec = blk ? table[L2_WORD + blk * L2_N + ((addr >> 12) & (L2_N - 1))] : 0;
    if (!rec) return 0;
    const uint32_t *r = table + table[2] + rec * 4 + (twin ? 2 : 0);
    return ((uint64_t)r[0] | (uint64_t)r[1] << 32) + addr;
}

static VkBuffer new_buf(void) { return (VkBuffer)(next_handle += 0x1000); }
#define MiB (1ull << 20)

// Every page of [lo, hi) reads buf (whose byte 0 is at buf_lo), or the zero buffer when buf is none.
static void expect(uint64_t lo, uint64_t hi, VkBuffer buf, uint64_t buf_lo, const char *what) {
    for (uint64_t a = lo; a < hi; a += 4096) {
        uint64_t got = read_through(a, 0), want = buf ? dev_of(buf) + (a - buf_lo) : 0;
        if (got != want) {
            CHECK(0, "%s: %#llx reads %#llx, expected %#llx", what, (unsigned long long)a, (unsigned long long)got,
                  (unsigned long long)want);
            return;
        }
    }
}

static uint32_t cover_rec;
static uint64_t cover_lo, cover_hi;
static uint32_t cover(uint64_t a) { return a >= cover_lo && a + 4096 <= cover_hi ? cover_rec : 0; }

int main(void) {
    CHECK(vk_gpa_on() && table && table[2], "the table exists");

    // a buffer over three 16 MiB ranges, not aligned to them
    uint64_t a0 = 0x600000000ull + 5 * MiB, a1 = a0 + 40 * MiB;
    VkBuffer b0 = new_buf();
    uint32_t r0 = vk_gpa_map(a0, a1, b0, a0);
    CHECK(r0, "a first buffer is mapped");
    expect(a0, a1, b0, a0, "first buffer");
    expect(a0 - 16 * MiB, a0, VK_NULL_HANDLE, 0, "below the first buffer");
    expect(a1, a1 + 16 * MiB, VK_NULL_HANDLE, 0, "above the first buffer");

    // a mirror takes the pages with the imported buffer as twin, and gives them back
    VkBuffer m0 = new_buf();
    r0 = vk_gpa_remap(a0, a1, r0, m0, a0);
    expect(a0, a1, m0, a0, "mirrored");
    CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "the twin of a mirrored page is the imported buffer");
    r0 = vk_gpa_remap_plain(a0, a1, r0, b0, a0);
    expect(a0, a1, b0, a0, "mirror evicted");
    CHECK(read_through(a0 + 4096, 1) == dev_of(b0) + 4096, "a plain page is its own twin");
    gpu_done();

    // an unmapped range reads the zero buffer at once, and its blocks serve no other range while the GPU may still read
    // the old L1 entries
    uint32_t old_blk[3] = {block_of(a0), block_of(a0 + 16 * MiB), block_of(a0 + 32 * MiB)};
    CHECK(old_blk[0] && old_blk[1] && old_blk[2], "three blocks for three ranges");
    vk_gpa_unmap(a0, a1, r0, NULL);
    expect(a0, a1, VK_NULL_HANDLE, 0, "unmapped");
    CHECK(!block_of(a0) && !block_of(a0 + 16 * MiB) && !block_of(a0 + 32 * MiB), "empty blocks leave L1");
    uint64_t c0 = 0x700000000ull;
    VkBuffer b1 = new_buf();
    uint32_t r1 = vk_gpa_map(c0, c0 + 48 * MiB, b1, c0);
    for (int i = 0; i < 3; i++)
        for (int j = 0; j < 3; j++)
            CHECK(block_of(c0 + (uint64_t)i * 16 * MiB) != old_blk[j], "a block still waiting for the GPU serves another range");
    for (int j = 0; j < 3; j++)
        for (uint32_t k = 0; k < L2_N; k++)
            if (table[L2_WORD + old_blk[j] * L2_N + k]) { CHECK(0, "a waiting block holds an entry"); break; }
    // the same range mapped again before the GPU is done takes its block back
    VkBuffer b2 = new_buf();
    uint32_t r2 = vk_gpa_map(a0, a1, b2, a0);
    CHECK(block_of(a0) == old_blk[0] && block_of(a0 + 16 * MiB) == old_blk[1] && block_of(a0 + 32 * MiB) == old_blk[2],
          "a range mapped again takes its waiting blocks back");
    expect(a0, a1, b2, a0, "mapped again");
    gpu_done();   // the callbacks of the first release find the blocks in use
    expect(a0, a1, b2, a0, "mapped again, after the GPU");
    VkBuffer b3 = new_buf();
    uint32_t r3 = vk_gpa_map(0x780000000ull, 0x780000000ull + 16 * MiB, b3, 0x780000000ull);
    for (int j = 0; j < 3; j++) CHECK(block_of(0x780000000ull) != old_blk[j], "a block taken back was freed by its old callback");
    vk_gpa_unmap(0x780000000ull, 0x780000000ull + 16 * MiB, r3, NULL);
    vk_gpa_unmap(a0, a1, r2, NULL);
    vk_gpa_unmap(c0, c0 + 48 * MiB, r1, NULL);
    gpu_done();

    // released, taken back and released within one submission, many times: one block, whatever the callbacks pending
    for (int i = 0; i < 3000; i++) {
        VkBuffer b = new_buf();
        uint32_t r = vk_gpa_map(a0, a0 + 8 * MiB, b, a0);
        if (i % 500 == 0) expect(a0, a0 + 8 * MiB, b, a0, "freed and made again at its address");
        vk_gpa_unmap(a0, a0 + 8 * MiB, r, NULL);
        if (i % 1000 == 999) gpu_done();
    }
    gpu_done();
    CHECK(!out_of_blocks_lines, "a pool freed and made again at its address ran out of blocks");

    // addresses that climb through 64 GiB, as the game's allocator hands them out: 20 MiB pools 48 MiB apart, eight
    // alive at a time, far more ranges than there are blocks
    enum { ALIVE = 8, POOLS = 1300 };
    struct { uint64_t lo, hi; VkBuffer b; uint32_t rec; } live[ALIVE] = {0};
    for (int i = 0; i < POOLS; i++) {
        int s = i % ALIVE;
        if (live[s].rec) vk_gpa_unmap(live[s].lo, live[s].hi, live[s].rec, NULL);
        live[s].lo = 0x800000000ull + (uint64_t)i * 48 * MiB + 3 * MiB, live[s].hi = live[s].lo + 20 * MiB;
        live[s].b = new_buf();
        live[s].rec = vk_gpa_map(live[s].lo, live[s].hi, live[s].b, live[s].lo);
        CHECK(live[s].rec, "climbing pool %d has a record", i);
        if (i % 3 == 0) gpu_done();
        if (i % 97 == 0 || i == POOLS - 1)
            for (int k = 0; k < ALIVE; k++)
                if (live[k].rec) expect(live[k].lo, live[k].hi, live[k].b, live[k].lo, "climbing pool");
    }
    CHECK(!out_of_blocks_lines, "climbing pool addresses ran out of blocks (%d lines)", out_of_blocks_lines);
    for (int k = 0; k < ALIVE; k++) vk_gpa_unmap(live[k].lo, live[k].hi, live[k].rec, NULL);
    gpu_done();

    // overlapping imports: the pages of the one that goes return to the other, and their block stays
    uint64_t o0 = 0x500000000ull, o1 = o0 + 16 * MiB;
    VkBuffer ba = new_buf(), bb = new_buf();
    uint32_t ra = vk_gpa_map(o0, o1, ba, o0), rb = vk_gpa_map(o0 + 4 * MiB, o0 + 8 * MiB, bb, o0 + 4 * MiB);
    expect(o0 + 4 * MiB, o0 + 8 * MiB, bb, o0 + 4 * MiB, "the later import holds the shared pages");
    cover_rec = ra, cover_lo = o0, cover_hi = o1;
    vk_gpa_unmap(o0 + 4 * MiB, o0 + 8 * MiB, rb, cover);
    expect(o0, o1, ba, o0, "the shared pages return to the other import");
    cover_rec = 0;
    vk_gpa_clear(o0 + 2 * MiB, o0 + 3 * MiB);
    expect(o0 + 2 * MiB, o0 + 3 * MiB, VK_NULL_HANDLE, 0, "cleared pages");
    expect(o0, o0 + 2 * MiB, ba, o0, "pages below the cleared ones");
    CHECK(block_of(o0), "a block that still maps pages stays");
    vk_gpa_clear(o0, o1);
    CHECK(!block_of(o0), "a block cleared whole leaves L1");
    vk_gpa_unmap(o0, o1, ra, NULL);
    gpu_done();

    // every block in use at once: the next range is left out and said so once, the ranges after it and the mapped
    // ones are untouched, and a block given back serves the range that was left out
    static struct { VkBuffer b; uint32_t rec; } all[BLOCKS];
    uint64_t f0 = 0x1000000000ull;
    for (uint32_t i = 0; i < BLOCKS - 1; i++) {
        all[i].b = new_buf();
        all[i].rec = vk_gpa_map(f0 + (uint64_t)i * 32 * MiB, f0 + (uint64_t)i * 32 * MiB + MiB, all[i].b, f0 + (uint64_t)i * 32 * MiB);
    }
    CHECK(!out_of_blocks_lines, "the table holds %u ranges at once", BLOCKS - 1);
    uint64_t x0 = 0x2000000000ull;
    VkBuffer bx = new_buf();
    uint32_t rx = vk_gpa_map(x0, x0 + MiB, bx, x0);
    CHECK(out_of_blocks_lines == 1, "a range past the last block is reported (%d lines)", out_of_blocks_lines);
    expect(x0, x0 + MiB, VK_NULL_HANDLE, 0, "a range past the last block");
    for (uint32_t i = 0; i < BLOCKS - 1; i += 37)
        expect(f0 + (uint64_t)i * 32 * MiB, f0 + (uint64_t)i * 32 * MiB + MiB, all[i].b, f0 + (uint64_t)i * 32 * MiB, "a full table");
    vk_gpa_unmap(x0, x0 + MiB, rx, NULL);
    vk_gpa_unmap(f0, f0 + MiB, all[0].rec, NULL);
    rx = vk_gpa_map(x0, x0 + MiB, bx, x0);
    expect(x0, x0 + MiB, VK_NULL_HANDLE, 0, "a block the GPU may still read");
    vk_gpa_unmap(x0, x0 + MiB, rx, NULL);
    gpu_done();
    rx = vk_gpa_map(x0, x0 + MiB, bx, x0);
    expect(x0, x0 + MiB, bx, x0, "a block given back");

    vk_gpa_report();
    printf("gpa_check: %s\n", fails ? "FAILED" : "ok");
    return fails != 0;
}