diff --git a/runtime/astc_probe.c b/runtime/astc_probe.c index 0cdb06aa..13fae111 100644 --- a/runtime/astc_probe.c +++ b/runtime/astc_probe.c @@ -150,7 +150,7 @@ static int gpu_decode_ar(VkCtx *c, const uint8_t *src, size_t src_bytes, const V vmaDestroyBuffer(c->vma, rb, ra), vmaDestroyBuffer(c->vma, ib, ia); vk_defer_image(im.img, im.alloc); vk_upload_flush(1); - return ok; + return ok > 0; } // The two regions as tight blocks: n0 blocks of level 0, then n1 of level 1. @@ -240,7 +240,7 @@ static int gpu_bc(VkCtx *c, const uint8_t *blk, const size_t *n, const uint32_t vmaDestroyBuffer(c->vma, ib, ia); vk_defer_image(im.img, im.alloc); vk_upload_flush(1); - return ok; + return ok > 0; } // One BC3 block to 16 RGBA8 texels, as the format defines it (565 endpoints widened by bit replication; the colour diff --git a/runtime/tex_probe.c b/runtime/tex_probe.c index 769081eb..9d5249c3 100644 --- a/runtime/tex_probe.c +++ b/runtime/tex_probe.c @@ -857,6 +857,47 @@ static int detail_case(pool_t *pool, uint64_t off, uint64_t room, uint32_t size, return 0; } +// An ASTC upload whose GPU decode gets no scratch memory (vk_astc.c scratch_get, as when the video memory is used up; +// forced here through vk_astc_scratch_fail) does not count as done: the texture stays "not uploaded", is left alone for +// the frames of the wait, and the first lookup after them uploads it, after which the image holds the storage's blocks. +// Skipped (0) where ASTC is not decoded on the GPU: there is no scratch buffer then. +static int detail_retry(pool_t *pool, uint64_t off, uint64_t room) { + if (!vk_astc_gpu()) return 0; + tex_t t = make_tex(pool, off, 0x80, 128, 128, 1, 24); // ASTC 8x8 + GCHECK(t.L.size <= room && off + room <= pool->size, "the retry case's storage does not fit the probe's pool"); + if (detail_fill(&t, 1, 0)) return 1; + uint8_t want[4]; + detail_color(0, 0, 0, want); + uint64_t r0 = vk_texture_upload_retries(); + uint32_t ups = 0; + int ch, pw; + vk_astc_scratch_fail(1); + const VkTexImage *img = vk_texture_get(&t); + vk_astc_scratch_fail(0); + GCHECK(img && img->mirrored, "the retry case's texture image"); + if (img->format != VK_FORMAT_R8G8B8A8_UNORM && img->format != VK_FORMAT_BC3_UNORM_BLOCK) return 0; // native ASTC + vk_texture_debug(img, &ups, &ch, &pw); + GCHECK(vk_texture_upload_retries() == r0 + 1 && ups == 1, + "an ASTC upload whose decode had no memory was not left for another try (%llu tries noted, %u uploads)", + (unsigned long long)(vk_texture_upload_retries() - r0), ups); + img = vk_texture_get(&t); // inside the wait: left alone + vk_texture_debug(img, &ups, &ch, &pw); + GCHECK(ups == 1, "the upload was tried again inside its wait (%u uploads)", ups); + __atomic_fetch_add(&gfx_counters.frames, 30, __ATOMIC_RELAXED); // vk_texture.c UP_RETRY_FRAMES + img = vk_texture_get(&t); + vk_texture_debug(img, &ups, &ch, &pw); + GCHECK(ups == 2 && vk_texture_upload_retries() == r0 + 1, "the upload was not tried again after its wait (%u uploads)", ups); + GCHECK(detail_level_is(img, 0, 0, 128, 128, want), "after the second try the image does not hold the storage's blocks"); + img = vk_texture_get(&t); // uploaded now: nothing more + vk_texture_debug(img, &ups, &ch, &pw); + GCHECK(ups == 2, "an uploaded texture was uploaded again (%u uploads)", ups); + g_gpu_checks += 4; + vk_texture_release(&t); + printf("probe-vkmem: an ASTC upload whose decode had no scratch memory is tried again after its wait and then holds " + "its blocks\n"); + return 0; +} + static int test_detail(pool_t *pool) { const char *v = cfg_value("NX2NAVI_TEXTURE_DETAIL"); int most = !strcmp(v, "full") ? 0 : !strcmp(v, "reduced") ? 1 : !strcmp(v, "low") ? 2 : vk_texture_detail_auto(); @@ -864,7 +905,8 @@ static int test_detail(pool_t *pool) { // ones the rule leaves whole: 128x128, and 512x512 without smaller levels if (detail_case(pool, 0, 0x80000, 1024, 6, 1, 20, most > 1 ? 2 : (uint32_t)most) || detail_case(pool, 0x80000, 0x40000, 256, 4, 3, 21, most ? 1 : 0) || - detail_case(pool, 0xc0000, 0x10000, 128, 4, 1, 22, 0) || detail_case(pool, 0xd0000, 0x20000, 512, 1, 1, 23, 0)) + detail_case(pool, 0xc0000, 0x10000, 128, 4, 1, 22, 0) || detail_case(pool, 0xd0000, 0x20000, 512, 1, 1, 23, 0) || + detail_retry(pool, 0xf0000, 0x10000)) return 1; // the sampler of a texture made without levels: an open LOD clamp is the slot's sampler, a closed one moves down samp_par sp; diff --git a/runtime/vk.h b/runtime/vk.h index 809681b4..52f688d6 100644 --- a/runtime/vk.h +++ b/runtime/vk.h @@ -460,9 +460,12 @@ typedef struct { VkDeviceSize src; uint32_t w, h, slices, level, layer; VkOffset // into img (VK_IMAGE_LAYOUT_GENERAL). The caller records its usual barriers around an image copy before and after the // call. readback (optional, host-visible, TRANSFER_DST): also gets the decoded texels, the regions one after the other. // bc: img is BC3 (TexVkFormat.astc_bc); the texels are encoded to BC3 blocks before the copy, and readback is ignored. -// 0 when the decoder is unavailable or the scratch buffer cannot be allocated (nothing recorded). +// 1 when recorded. 0 when the decoder is unavailable or cannot take these regions, and -1 when its scratch buffer +// cannot be allocated: nothing is recorded in either case, and after -1 the same call can succeed once memory is free. int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, const VkAstcRegion *rg, int n, uint32_t bw, uint32_t bh, int srgb, int bc, VkImage img, VkImageAspectFlags aspect, VkBuffer readback); +void vk_astc_scratch_fail(int n); // a test's hook: the next n scratch buffer requests fail (tex_probe.c) +uint64_t vk_texture_upload_retries(void); // uploads left to be tried again: their decode had no memory (vk_texture.c upload) VkDeviceSize vk_astc_region_bytes(const VkAstcRegion *r, uint32_t bw, uint32_t bh); // source bytes the region reads void vk_astc_stats(uint64_t *dispatches, uint64_t *texels); // since start uint64_t vk_astc_scratch_bytes(void); // the decoder's shared scratch now diff --git a/runtime/vk_astc.c b/runtime/vk_astc.c index c62e9b03..9fa16596 100644 --- a/runtime/vk_astc.c +++ b/runtime/vk_astc.c @@ -466,7 +466,12 @@ static int scratch_keep(void) { // A scratch of at least need bytes for recording into cb; *reused: earlier work (in cb, or with the keep rule in an // earlier batch) may still use it. 0 on failure. +// A test's hook (vk.h): the next n requests for the scratch buffer fail, as when the video memory is used up. +static volatile int32_t scratch_fail_n; +void vk_astc_scratch_fail(int n) { plat_xchg32(&scratch_fail_n, n); } + static VkBuffer scratch_get(VkCtx *c, VkCommandBuffer cb, VkDeviceSize need, int *reused) { + if (plat_cas32(&scratch_fail_n, 0, 0) > 0 && plat_add32(&scratch_fail_n, -1) >= 0) return VK_NULL_HANDLE; int keep = scratch_keep(); plat_mutex_lock(&g_lock); uint64_t now = kern_now_ns(); @@ -573,9 +578,9 @@ int vk_astc_record(VkCommandBuffer cb, VkBuffer src, VkDeviceSize src_size, cons VkBuffer sb = scratch_get(c, cb, cap_in + cap_out + cap_bc, &reused); if (!sb) { if (plat_inc32(&n_fail) <= 16) - rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded", + rt_log("vk: ASTC scratch buffer of %llu bytes failed: texture left undecoded for now", (unsigned long long)(cap_in + cap_out + cap_bc)); - return 0; + return -1; // no memory at this moment: the caller tries again later (vk_texture.c upload) } plat_mutex_lock(&g_lock); int need_tab = !G.tab_uploaded; diff --git a/runtime/vk_texture.c b/runtime/vk_texture.c index 2d71d0d3..7a49de3d 100644 --- a/runtime/vk_texture.c +++ b/runtime/vk_texture.c @@ -57,6 +57,7 @@ typedef struct Tex { uint64_t retire_frame; // gfx_counters.frames when it was retired uint64_t retire_due; // vk_sync_issued once every pool set was taken since (0: not yet); destroyed after it int uploaded, released; + uint64_t up_retry; // gfx_counters.frames from which an upload that had no memory is tried again (upload) uint64_t uid; // names the entry in its bucket's released set (tex_gen_flip); 0 until its first release uint64_t rel_frame; // gfx_counters.frames when it was released (drop_stale) uint32_t back_gap; // most presents between a release and its taking back so far (stale_after) @@ -2361,8 +2362,20 @@ static int astc_arena(void) { // Copies the whole storage into the image (every level and layer the image has). An image made without its top levels // (texture detail, VkTexImage.skip) has the game's level lv + skip at its level lv: the loops below run over the image's // levels and read the storage at the game's. +// How long an upload whose decode had no memory waits for its next try (upload, up_wait): half a second of frames. A +// try costs the copy of the texture's blocks into staging memory, so not every frame. +#define UP_RETRY_FRAMES 30 +static uint64_t n_up_retry; +uint64_t vk_texture_upload_retries(void) { return __atomic_load_n(&n_up_retry, __ATOMIC_RELAXED); } + +// 1 while a texture that is not uploaded waits for its next try (e->up_retry; 0 for one never tried). +static int up_wait(const Tex *e) { + return e->up_retry && __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) < e->up_retry; +} + static void upload(Tex *e) { static volatile int32_t n_bad, n_undecoded, n_split; + int no_mem = 0; // a part's ASTC decode had no memory: the texture is uploaded again later const tex_t *t = &e->t; const nvn_texdesc *d = &t->d; int bw, bh, bpb = nvn_fmt_bpb(d->format, &bw, &bh); @@ -2526,10 +2539,12 @@ static void upload(Tex *e) { e->pub.image, e->pub.aspect, rbuf); perf_end(PERF_ASTC, a0); vk_texstat_time(VK_TS_ASTC, a0); - if (!checked) { - log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was", - (unsigned long long)e->pub.addr, d->format, d->width, d->height); + if (checked <= 0) { + log_limited(&n_undecoded, "ASTC texture %#llx (fmt %#x %ux%u) not decoded on the GPU: left as it was%s", + (unsigned long long)e->pub.addr, d->format, d->width, d->height, + checked < 0 ? " and tried again later (no memory for the decode)" : ""); e->chash_ok = 0; // the image does not hold the bytes hashed above: the next write uploads, equal or not + no_mem |= checked < 0; } } else { for (k = k0; k < k1; k++) rg[k].bufferOffset += soff; @@ -2544,7 +2559,7 @@ static void upload(Tex *e) { parts); if (acheck) { // waits for the GPU vk_upload_flush(1); - if (checked) { + if (checked > 0) { vmaInvalidateAllocation(vk_ctx()->vma, ralloc, 0, VK_WHOLE_SIZE); astc_check(e, raw_copy, ar, (int)n, rmap); } @@ -2570,6 +2585,18 @@ static void upload(Tex *e) { e->wt_n = 0; e->uploaded = 1; e->n_up++; + // The decode had no memory (vk_astc_record: its scratch buffer failed while the video memory was used up), so the + // image does not hold this content: a new image holds nothing, an older one what it held before. Counted as + // uploaded, it stayed that way until the game next wrote the storage, which for most textures is never. It counts + // as not uploaded instead, and make_fresh and make_fresh_snap upload it again UP_RETRY_FRAMES frames on, when + // the evictions under memory pressure have had time to make room; again and again until one succeeds. + if (no_mem) { + e->uploaded = 0; + e->up_retry = __atomic_load_n(&gfx_counters.frames, __ATOMIC_RELAXED) + UP_RETRY_FRAMES; + __atomic_add_fetch(&n_up_retry, 1, __ATOMIC_RELAXED); + } else { + e->up_retry = 0; + } perf_end(PERF_TEX, p0); vk_texstat_time(VK_TS_UPLOAD, u0); vk_texstat_add(VK_TS_UPLOAD, 0, src_bytes); @@ -2623,6 +2650,7 @@ static void make_fresh(Tex *e) { if (!e->pub.mirrored || e->pub.gpu_owned) return; uint64_t size = e->t.L.size; if (!e->uploaded) { + if (up_wait(e)) return; vk_mem_changed(e->pub.addr, size, 0, &e->wgen); // resets the write bits before the storage is read upload(e); return; @@ -2639,6 +2667,7 @@ static void make_fresh(Tex *e) { // after it, during this replay) keeps its write-watch bit and is caught by the next submission's snapshot. static void make_fresh_snap(Tex *e) { if (!e->pub.mirrored || e->pub.gpu_owned) return; + if (!e->uploaded && up_wait(e)) return; uint32_t now; n_checks++; int ch = vk_mem_changed_snap(e->pub.addr, e->t.L.size, e->uploaded ? e->wgen : 0, submit_epoch, &now); @@ -3634,7 +3663,7 @@ static int image_texels(Tex *e, const Region *g, const NVNcopyRegion *r, uint8_t free(ar); } if (!agpu) vkCmdCopyBufferToImage(cb, sb, e->pub.image, VK_IMAGE_LAYOUT_GENERAL, (uint32_t)g->nz, rg); - else if (!done) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was", + else if (done <= 0) log_limited(&n_astc, "WriteTexels of ASTC texture %#llx not decoded on the GPU: left as it was", (unsigned long long)e->pub.addr); barrier(cb, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, ALL_RW); vk_upload_end();