mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-22 17:19:58 -04:00
The 0.6 crashes after warps and area changes fixed (no write-back into memory the capture leaves out), invisible objects fixed (occlusion gate), DLSS frame generation flicker fixed, achievements, the renderer built with a play profile, faster translated game code, helper threads judged on settled scenes, memory, shader preparation, audio effects, launcher, controls and Linux fixes.
844 lines
48 KiB
C
844 lines
48 KiB
C
/* Guest memory map: the system region (our own objects), the heap, address regions.
|
|
The whole 4 GB window is committed up front (hos_main.c), so "mapping" here is only bookkeeping and zeroing. */
|
|
#include "hos.h"
|
|
#include "heap_commit.h" /* ((Triforce, 11-21, MEM-09) the heap committed as it is touched) */
|
|
#include "stall_host.h" /* ((Triforce, 11-22, OS-51) the host's part of a long frame, counted where it is spent) */
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#if defined(_WIN32)
|
|
#define WIN32_LEAN_AND_MEAN
|
|
#include <windows.h>
|
|
static SRWLOCK lock = SRWLOCK_INIT;
|
|
#define LOCK() AcquireSRWLockExclusive(&lock)
|
|
#define UNLOCK() ReleaseSRWLockExclusive(&lock)
|
|
#else
|
|
#include <pthread.h>
|
|
#include <sys/mman.h>
|
|
static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
|
|
#define LOCK() pthread_mutex_lock(&lock)
|
|
#define UNLOCK() pthread_mutex_unlock(&lock)
|
|
#endif
|
|
|
|
uint64_t hos_heap_size, hos_heap_used, hos_total_memory, hos_program_size;
|
|
static uint64_t sys_next = HOS_SYSTEM_BASE;
|
|
|
|
/* (Triforce) the guest memory census's map (guest_census.h; the notes, the counts and the give-back's decommit are at this
|
|
file's end). Its chunks come from the system, not the heap: the first touch's exception handler may make one */
|
|
static void *census_alloc(size_t n) {
|
|
#if defined(_WIN32)
|
|
return VirtualAlloc(NULL, n, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
|
#else
|
|
void *p = mmap(NULL, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
|
return p == MAP_FAILED ? NULL : p;
|
|
#endif
|
|
}
|
|
static void census_free(void *p, size_t n) {
|
|
#if defined(_WIN32)
|
|
(void)n;
|
|
VirtualFree(p, 0, MEM_RELEASE);
|
|
#else
|
|
munmap(p, n);
|
|
#endif
|
|
}
|
|
#define GC_ALLOC(n) census_alloc(n)
|
|
#define GC_FREE(p) census_free((p), sizeof(GcChunk))
|
|
#include "guest_census.h"
|
|
static GcMap census;
|
|
|
|
/* Address regions are handed out top-down from HOS_SPACE_END; the heap grows up from HOS_HEAP_BASE. (Triforce) freeing: a
|
|
region being freed, listed until its pages are discarded (hos_region_free); game: one the game asked for itself
|
|
(nn::os::AllocateAddressRegion), which holds nothing of the game's until it takes pages or a physical map in it */
|
|
typedef struct { uint64_t addr, size; int freeing, game; } Range;
|
|
static Range *regions;
|
|
static int nregions, capregions;
|
|
static uint64_t region_floor = ~0ULL; /* the lowest region (none yet: the window's end) */
|
|
|
|
/* Free address space below the heap that regions may also use (web: the window is tight): above the null guard
|
|
to the system region, and from the image's end to the heap. */
|
|
static uint64_t low_hole[2][2];
|
|
|
|
void hos_mem_init(uint64_t image_end) {
|
|
hos_program_size = image_end - hos_image_base;
|
|
hos_commit(HOS_SYSTEM_BASE, HOS_SYSTEM_END - HOS_SYSTEM_BASE);
|
|
hos_mem_census_start(); /* (Triforce: the census line and the give-back, mkw_glue.c) */
|
|
low_hole[0][0] = 0x00100000ULL, low_hole[0][1] = HOS_SYSTEM_BASE;
|
|
low_hole[1][0] = (image_end + 0xFFFFFULL) & ~0xFFFFFULL, low_hole[1][1] = HOS_HEAP_BASE;
|
|
/* Retail Switch gives an application 3285 MB. HOS_MEMORY_MB overrides it: TOTK sizes its heap from this
|
|
(heap = total - used - 1080 MB), which is the lever for fitting the web build into wasm32. */
|
|
const char *mb = getenv("HOS_MEMORY_MB");
|
|
#ifdef A64_MEM_ZERO
|
|
/* (web: TOTK keeps 1080 MB of the total for address regions (its resource memory grows into them) and its heap
|
|
must stay 2104 MB (a 1902 MB heap: an indirect call into nothing 4 s in), so the window must hold heap +
|
|
regions; the holes below the heap help) */
|
|
hos_total_memory = (mb ? strtoull(mb, NULL, 0) : 3285ULL) << 20;
|
|
hos_log("memory: %llu MB for the game (window %llu MB)", (unsigned long long)(hos_total_memory >> 20),
|
|
(unsigned long long)(HOS_SPACE_END >> 20));
|
|
#else
|
|
/* mkw-recomp: a Switch 2 application gets 9809 MB (Eglantine's KSystemControl, 12 GB arrangement) */
|
|
hos_total_memory = (mb ? strtoull(mb, NULL, 0) : 9809ULL) << 20;
|
|
#endif
|
|
}
|
|
|
|
uint64_t hos_region_floor(void) { return region_floor; }
|
|
|
|
uint64_t hos_sys_alloc(uint64_t size, uint64_t align) {
|
|
LOCK();
|
|
uint64_t a = (sys_next + align - 1) & ~(align - 1);
|
|
if (a + size > HOS_SYSTEM_END) {
|
|
UNLOCK();
|
|
hos_fatal(NULL, "system region exhausted (%#llx bytes asked)", (unsigned long long)size);
|
|
}
|
|
sys_next = a + size;
|
|
UNLOCK();
|
|
memset(GP(a), 0, size);
|
|
HOS_WRITTEN(GP(a), size);
|
|
return a;
|
|
}
|
|
|
|
static uint64_t region_alloc(uint64_t size, uint64_t align, int game) {
|
|
if (align < 0x1000) align = 0x1000;
|
|
size = (size + 0xFFF) & ~0xFFFULL;
|
|
LOCK();
|
|
/* first fit, top-down, between existing regions (kept sorted by address, descending) */
|
|
uint64_t top = HOS_SPACE_END, found = 0;
|
|
int at = nregions;
|
|
for (int i = 0; i <= nregions; i++) {
|
|
uint64_t bottom = i < nregions ? regions[i].addr + regions[i].size : HOS_HEAP_BASE + hos_heap_size;
|
|
if (top >= size) {
|
|
uint64_t a = (top - size) & ~(align - 1);
|
|
if (a >= bottom && a >= HOS_HEAP_BASE + hos_heap_size) { found = a; at = i; break; }
|
|
}
|
|
if (i < nregions) top = regions[i].addr;
|
|
}
|
|
if (!found) /* the holes below the heap, top-down, between the regions there (the list is descending) */
|
|
for (int h = 1; h >= 0 && !found; h--) {
|
|
uint64_t hi = low_hole[h][1];
|
|
int i = 0;
|
|
while (i < nregions && regions[i].addr >= hi) i++;
|
|
for (;; i++) {
|
|
const uint64_t lo = i < nregions && regions[i].addr + regions[i].size > low_hole[h][0]
|
|
? regions[i].addr + regions[i].size : low_hole[h][0];
|
|
if (hi >= size) {
|
|
const uint64_t a = (hi - size) & ~(align - 1);
|
|
if (a >= lo) { found = a; at = i; break; }
|
|
}
|
|
if (i >= nregions || regions[i].addr < low_hole[h][0]) break;
|
|
hi = regions[i].addr;
|
|
}
|
|
}
|
|
if (found) {
|
|
if (nregions == capregions) {
|
|
capregions = capregions ? capregions * 2 : 64;
|
|
regions = (Range *)realloc(regions, capregions * sizeof(Range));
|
|
}
|
|
memmove(®ions[at + 1], ®ions[at], (nregions - at) * sizeof(Range));
|
|
regions[at].addr = found;
|
|
regions[at].size = size;
|
|
regions[at].freeing = 0;
|
|
regions[at].game = game;
|
|
nregions++;
|
|
if (found < region_floor && found >= HOS_HEAP_BASE) region_floor = found; /* (not the holes below) */
|
|
}
|
|
UNLOCK();
|
|
if (found) hos_commit(found, size);
|
|
if (!found) hos_fatal(NULL, "address space exhausted (%#llx bytes asked, heap %#llx)", (unsigned long long)size,
|
|
(unsigned long long)hos_heap_size);
|
|
return found;
|
|
}
|
|
uint64_t hos_region_alloc(uint64_t size, uint64_t align) { return region_alloc(size, align, 0); }
|
|
|
|
/* (Triforce) Whether a freed region's pages are discarded (hos_region_free). A region the game asked for itself holds
|
|
nothing of the game's until it takes pages or a physical map in it, and the census knows which of its pages are the
|
|
game's now (GC_GAME: what it was given, less what it gave back, each give-back discarded or decommitted where it was
|
|
made). With none, there is nothing to discard: the pages were committed with the region and never handed over.
|
|
Held to what it is told it has (HOS_MEM_LIMIT), the game's Resource Memory thread asks for a region, is refused its
|
|
pages, frees the region and asks again: 16,640 times in a lab run of three areas (round 79), each free a
|
|
DiscardVirtualMemory over 0.2 to 37 MB of pages nobody had touched, 61% of that thread's time while an area loaded,
|
|
and the game's main thread waited for it: its own part of the frame 8 to 23 ms for seconds at a time, where 2 ms is
|
|
usual (the stutter the limit was set aside for). Without the census (HOS_MEM_CENSUS=0) every region is discarded, as
|
|
before, and so is one the host asked for itself (the renderer's and the pools' storage: written without pages) */
|
|
int hos_region_discard_needed(int game_region, int census_on, uint64_t game_pages) {
|
|
return !(game_region && census_on && !game_pages);
|
|
}
|
|
|
|
static uint64_t pages_used[(1ULL << 32) / 0x1000 / 64];
|
|
|
|
static void pages_mark_used_locked(uint64_t addr, uint64_t size) {
|
|
for (uint64_t p = addr >> 12; p < (addr + size + 0xFFF) >> 12 && p < (1ULL << 20); p++) pages_used[p >> 6] |= 1ULL << (p & 63);
|
|
}
|
|
|
|
/* ((Triforce, 11-22, the performance audit's MEM-13) HOS_MEM_ZERO_MARK=1: hos_guest_fill (hos.h) marks what it fills as
|
|
written. Read once, off unless set */
|
|
int hos_mem_zero_mark(void) {
|
|
static int on = -1;
|
|
if (on < 0) {
|
|
const char *e = getenv("HOS_MEM_ZERO_MARK");
|
|
const int v = e && *e && atoi(e) != 0;
|
|
if (v) hos_log("mem: freed pages zeroed under an import are marked written for the renderer (HOS_MEM_ZERO_MARK=1)");
|
|
on = v;
|
|
}
|
|
return on;
|
|
}
|
|
|
|
/* (Triforce, 11-22, the performance audit's OS-40, after Source A's) Whether a thread keeps its virtual core while it
|
|
discards. A discard is milliseconds in the kernel for a region of some MB, and before it the lazy write-back's pages
|
|
there are paid and pages a GPU import holds are zeroed; all the while the thread runs no guest code and the game
|
|
threads allowed only on its core wait. Source A gives the core up before a freed region's discard and takes it back
|
|
once, after the region is unlisted: at its Market the core waits of 50 ms and more went from 287 to 19 (275 of them
|
|
loader threads'), the longest from 177 to 45 ms, with the frames unchanged; the freeing thread once waited 124 ms for
|
|
its core back, in a load.
|
|
HOS_DISCARD_KEEP_CORE, read once:
|
|
- not set: as before. The core is given up for the discard's own kernel call, and only with the limit on
|
|
(HOS_MEM_LIMIT), where that was measured; hos_region_free itself gives nothing up;
|
|
- 0: Source A's. hos_region_free gives the core up before the whole of its discard (the write-back's payment and the
|
|
import's zero-fill too) and takes it back once the region is unlisted, never under the regions' lock; the discard's
|
|
own call gives it up as well, limit or not, which is what FreeMemoryPages' discard then has (under a region's
|
|
give-up it finds the core gone already and does nothing);
|
|
- 1 (any other number): the core kept through every discard, with the limit too.
|
|
Not the default: it reverses the choice made here to stay on upstream's path without the limit, it changes when a game
|
|
thread waits, and it was not measured here */
|
|
enum { DISCARD_CORE_AS_BEFORE, DISCARD_CORE_GIVEN_UP, DISCARD_CORE_KEPT };
|
|
static int discard_core_rule(const char *env) {
|
|
if (!env || !*env) return DISCARD_CORE_AS_BEFORE;
|
|
return atoi(env) != 0 ? DISCARD_CORE_KEPT : DISCARD_CORE_GIVEN_UP;
|
|
}
|
|
/* (a freed region's give-up around its whole discard; and the discard call's own) */
|
|
static int discard_core_region_away(int mode) { return mode == DISCARD_CORE_GIVEN_UP; }
|
|
static int discard_core_call_away(int mode, int limit_on) {
|
|
return mode == DISCARD_CORE_GIVEN_UP || (mode == DISCARD_CORE_AS_BEFORE && limit_on);
|
|
}
|
|
static int discard_core(void) {
|
|
static int mode = -1;
|
|
if (mode < 0) {
|
|
mode = discard_core_rule(getenv("HOS_DISCARD_KEEP_CORE"));
|
|
if (mode != DISCARD_CORE_AS_BEFORE)
|
|
hos_log("mem: a thread %s while it discards freed memory (HOS_DISCARD_KEEP_CORE)",
|
|
mode == DISCARD_CORE_GIVEN_UP ? "gives its virtual core up for the whole of a freed region's discard" : "keeps its virtual core");
|
|
}
|
|
return mode;
|
|
}
|
|
|
|
static void discard_pages(uint64_t addr, uint64_t size) {
|
|
if (!hos_guest_range(addr, size)) return; /* (hookshot: guest pages only; Linux's discard zeroes what it is given) */
|
|
const uint64_t lo = (addr + 0xFFF) & ~0xFFFULL, hi = (addr + size) & ~0xFFFULL;
|
|
static int keep = -1;
|
|
if (keep < 0) keep = getenv("HOS_KEEP_FREED_PAGES") != NULL;
|
|
if (hi <= lo || keep) return;
|
|
#if defined(_WIN32) && !defined(__EMSCRIPTEN__)
|
|
extern uint64_t hos_large_lo, hos_large_hi; /* (hos_main.c: large pages cannot be discarded) */
|
|
if (hos_large_hi && lo < hos_large_hi && hi > hos_large_lo) return;
|
|
/* ((Triforce, 11-19) pages a Vulkan import holds (a pool's chunk) are zeroed where they are, as FreeMemoryPages and Linux below
|
|
do: the import keeps them locked in RAM, so a discard gave nothing back, and it was slow there: 3.7 ms a region on average,
|
|
23 ms at most, 42 s on the game's "Resource Memory" thread over a lab tour once the game let areas go
|
|
(QueryMemoryInfo above), with the game's frames waiting on it (Hyrule Field 34 fps, Castle Town 15). Zeroed: the pages
|
|
read as zero, as freed pages; the census keeps them for the give-back once no import holds them) */
|
|
extern int rx_guest_imported(uint64_t lo, uint64_t hi);
|
|
const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */
|
|
{
|
|
const int io = hos_guest_io_begin(GP(lo), hi - lo);
|
|
const int held = rx_guest_imported(lo, hi);
|
|
if (held) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); /* ((Triforce, 11-22, MEM-13) marked with HOS_MEM_ZERO_MARK=1) */
|
|
hos_guest_io_end(io);
|
|
if (held) { /* (lab tour: 5.2 GB zeroed in 0.47 s in all) */
|
|
stall_host_end(STALL_HOST_ZERO, stall0, hi - lo);
|
|
return;
|
|
}
|
|
}
|
|
/* (Triforce, the census: let go and still committed. Not the large pages above, left as they were, nor Linux's below,
|
|
noted zeroed or decommitted there) */
|
|
hos_mem_note(HOS_MEM_DISCARDED, lo, hi - lo);
|
|
typedef DWORD(WINAPI * DiscardFn)(PVOID, SIZE_T);
|
|
static DiscardFn discard;
|
|
static int looked;
|
|
if (!looked) {
|
|
looked = 1;
|
|
discard = (DiscardFn)(void *)GetProcAddress(GetModuleHandleA("kernel32.dll"), "DiscardVirtualMemory");
|
|
}
|
|
const int io = hos_guest_io_begin(GP(lo), hi - lo); /* (Triforce: the pages' state changes under the lazy write-back's: rx_backend.c) */
|
|
/* ((Triforce) the thread's virtual core given up for the discard itself: milliseconds in the kernel for a region of some MB,
|
|
in which the thread runs no guest code and cannot see a better thread's ask for its core (hos_thread.c vc_kick). Held to
|
|
its memory (HOS_MEM_LIMIT) the game frees regions by the hundred while an area loads, on its Resource Memory thread.
|
|
Only with the limit on, where it was measured: without it the game frees few regions and the path is upstream's) */
|
|
/* ((Triforce, 11-22, OS-40) HOS_DISCARD_KEEP_CORE: 0 gives it up without the limit too, 1 never: discard_core above) */
|
|
const int away = discard_core_call_away(discard_core(), hos_mem_limit() != 0) ? hos_vcore_away() : 0;
|
|
if (!discard || discard(GP(lo), (SIZE_T)(hi - lo)) != ERROR_SUCCESS)
|
|
VirtualAlloc(GP(lo), (SIZE_T)(hi - lo), MEM_RESET, PAGE_READWRITE);
|
|
stall_host_end(STALL_HOST_DISCARD, stall0, hi - lo); /* ((Triforce, OS-51) with its write-back wait, not the wait for the core) */
|
|
if (away) hos_vcore_back();
|
|
hos_guest_io_end(io);
|
|
#elif !defined(__EMSCRIPTEN__)
|
|
extern int hos_posix_guest_discard(void *p, size_t n); /* (hos_main.c: MADV_REMOVE on the memfd window) */
|
|
extern int rx_guest_imported(uint64_t lo, uint64_t hi);
|
|
/* (pages a dma-buf import holds stay where they are, zeroed: a hole punched under one would leave the GPU on
|
|
the old pages and the game on new ones) */
|
|
if (rx_guest_imported(lo, hi)) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo); /* ((Triforce, 11-22, MEM-13) as above) */
|
|
else hos_posix_guest_discard(GP(lo), (size_t)(hi - lo)), hos_mem_note(HOS_MEM_DECOMMITTED, lo, hi - lo);
|
|
#endif
|
|
}
|
|
|
|
static int64_t region_frees_bare; /* ((Triforce) regions freed without a discard: hos_region_discard_needed; the refusal line's) */
|
|
/* ((Triforce, 11-22, OS-40) the frees that gave their core up for the discard (HOS_DISCARD_KEEP_CORE=0), and how long the
|
|
longest waited for its core back; a line every 1,000) */
|
|
static int64_t region_frees_away, region_back_longest_ns;
|
|
void hos_region_free(uint64_t addr) {
|
|
uint64_t freed_size = 0;
|
|
int game = 0, away = 0;
|
|
LOCK();
|
|
for (int i = 0; i < nregions; i++) {
|
|
if (regions[i].addr == addr && !regions[i].freeing) {
|
|
pages_mark_used_locked(regions[i].addr, regions[i].size);
|
|
freed_size = regions[i].size;
|
|
game = regions[i].game;
|
|
regions[i].freeing = 1;
|
|
break;
|
|
}
|
|
}
|
|
UNLOCK();
|
|
if (!freed_size) return;
|
|
/* (Triforce) discarded, and noted so in the census, while the region is still listed: a region asked for meanwhile is not
|
|
given these pages, which a discard after it would take from it, and a commit of them marked kept by the late note. Not
|
|
under the lock: the discard may wait for the GPU (the lazy write-back) with the thread's virtual core given up, and a
|
|
thread taking its core back with the lock held could wait on one that holds a core and waits for the lock */
|
|
/* ((Triforce) hos_region_discard_needed: a region of the game's own with none of its pages has nothing to discard. Its
|
|
pages are noted let go all the same, as a discard notes them: committed, and nobody's. Only with the limit on
|
|
(HOS_MEM_LIMIT), where the refused asks make such regions by the thousand and it was measured; without it every
|
|
region is discarded, as upstream does) */
|
|
const uint64_t game_pages = addr + freed_size <= GC_SPACE ? gc_count(&census, GC_GAME, addr, addr + freed_size) : ~0ULL;
|
|
if (hos_region_discard_needed(game && hos_mem_limit(), hos_mem_census_on(), game_pages)) {
|
|
/* ((Triforce, 11-22, OS-40) HOS_DISCARD_KEEP_CORE=0: the core given up from here, with the region still listed and
|
|
marked, and taken back below once it is unlisted. A bare free, which makes no kernel call, keeps it) */
|
|
if (discard_core_region_away(discard_core())) away = hos_vcore_away();
|
|
discard_pages(addr, freed_size);
|
|
} else {
|
|
hos_mem_note(HOS_MEM_DISCARDED, addr, freed_size);
|
|
__atomic_add_fetch(®ion_frees_bare, 1, __ATOMIC_RELAXED);
|
|
}
|
|
LOCK();
|
|
for (int i = 0; i < nregions; i++) {
|
|
if (regions[i].addr == addr && regions[i].freeing) {
|
|
memmove(®ions[i], ®ions[i + 1], (nregions - i - 1) * sizeof(Range));
|
|
nregions--;
|
|
break;
|
|
}
|
|
}
|
|
UNLOCK();
|
|
/* ((Triforce, 11-22, OS-40) the core back, after the lock is let go: a thread that queued for its core with the lock
|
|
held could wait on one that holds a core and waits for the lock) */
|
|
if (away) {
|
|
const int64_t t0 = hos_now_ns();
|
|
hos_vcore_back();
|
|
const int64_t waited = hos_now_ns() - t0;
|
|
int64_t longest = __atomic_load_n(®ion_back_longest_ns, __ATOMIC_RELAXED);
|
|
while (waited > longest &&
|
|
!__atomic_compare_exchange_n(®ion_back_longest_ns, &longest, waited, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {}
|
|
const int64_t n = __atomic_add_fetch(®ion_frees_away, 1, __ATOMIC_RELAXED);
|
|
if (n % 1000 == 0)
|
|
hos_log("mem: %lld freed regions discarded with the thread's virtual core given up; the longest waited %.1f ms for it "
|
|
"back (HOS_DISCARD_KEEP_CORE)", (long long)n, __atomic_exchange_n(®ion_back_longest_ns, 0, __ATOMIC_RELAXED) / 1e6);
|
|
}
|
|
}
|
|
|
|
/* ---- nn::os memory -------------------------------------------------------------------------------------- */
|
|
|
|
#if defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
|
|
/* HOS_WRITTEN natively (the soft write watch, rt/a64.h A64_MARK): host code wrote [p, p + n) of guest memory */
|
|
void hos_mark_written(const void *p, size_t n) {
|
|
const uintptr_t a = (uintptr_t)p - (uintptr_t)A64_MEM;
|
|
if (!n || (uintptr_t)p < (uintptr_t)A64_MEM || a >= HOS_GUEST_SPACE) return;
|
|
for (uintptr_t q = a >> 12; q <= (a + n - 1) >> 12 && q < A64_DIRTY_PAGES; q++)
|
|
__atomic_store_n(&a64_dirty[q], 1, __ATOMIC_RELEASE);
|
|
}
|
|
#endif
|
|
|
|
/* (hookshot) a heap of size bytes ends below the address regions and the guest window's end (no wrap-around: the size
|
|
comes from the game). Also svc SetHeapSize's check (hos_svc.c) */
|
|
int hos_heap_size_fits(uint64_t size) {
|
|
const uint64_t limit = region_floor < HOS_SPACE_END ? region_floor : HOS_SPACE_END;
|
|
return limit >= HOS_HEAP_BASE && size <= limit - HOS_HEAP_BASE;
|
|
}
|
|
/* nn::os::SetMemoryHeapSize(unsigned long) -> Result */
|
|
HLE(_ZN2nn2os17SetMemoryHeapSizeEm) {
|
|
uint64_t size = ARG(0);
|
|
HOS_TRACE_CALL("SetMemoryHeapSize");
|
|
if (!hos_heap_size_fits(size)) {
|
|
hos_log("SetMemoryHeapSize(%#llx): collides with address regions at %#llx", (unsigned long long)size,
|
|
(unsigned long long)region_floor);
|
|
RET(0xCA01); /* svc::ResultOutOfMemory */
|
|
return;
|
|
}
|
|
if (size < hos_heap_size) {
|
|
memset(GP(HOS_HEAP_BASE + size), 0, hos_heap_size - size);
|
|
HOS_WRITTEN(GP(HOS_HEAP_BASE + size), hos_heap_size - size);
|
|
}
|
|
/* HOS_PREFAULT=1 (experiment): the new heap pages touched now (a zero written to fresh, zero memory), so that the
|
|
game's first touches while streaming are not page faults (which may hold up the renderer's write-watch scans) */
|
|
if (size > hos_heap_size && getenv("HOS_PREFAULT")) {
|
|
const int64_t t0 = hos_now_ns();
|
|
for (uint64_t a = HOS_HEAP_BASE + hos_heap_size; a < HOS_HEAP_BASE + size; a += 4096) *(volatile uint8_t *)GP(a) = 0;
|
|
hos_log("heap: %llu MB prefaulted in %.0f ms", (unsigned long long)((size - hos_heap_size) >> 20), (hos_now_ns() - t0) / 1e6);
|
|
}
|
|
/* ((Triforce, 11-21, the performance audit's MEM-09, after Source A's) Windows: the heap is left reserved and committed
|
|
as it is touched (mkw_glue.c commit_on_touch), not all at once: the game asks for ~6.9 GB and touches a part of it, and
|
|
committed, all of it counted against RAM plus the page file. heap_commit.h has the rule and what it needed first.
|
|
HOS_HEAP_COMMIT=1: whole, as before; and in large pages (HOS_LARGE_PAGES), reserved committed) */
|
|
{
|
|
extern uint64_t hos_large_hi;
|
|
#if defined(_WIN32)
|
|
const int windows = 1;
|
|
#else
|
|
const int windows = 0;
|
|
#endif
|
|
const int whole = heap_commit_whole(getenv("HOS_HEAP_COMMIT"), hos_large_hi != 0, windows);
|
|
if (size > hos_heap_size && whole) hos_commit(HOS_HEAP_BASE + hos_heap_size, size - hos_heap_size);
|
|
if (size > hos_heap_size && !whole) {
|
|
hos_heap_on_touch = 1; /* (the kernel calls on guest memory touch their buffers first: hos_guest_touch) */
|
|
hos_log("heap: left reserved, committed as it is touched (HOS_HEAP_COMMIT=1: all of it now, as before)");
|
|
}
|
|
}
|
|
hos_heap_size = size;
|
|
hos_log("heap: %llu MB at %#llx", (unsigned long long)(size >> 20), (unsigned long long)HOS_HEAP_BASE);
|
|
RET(0);
|
|
}
|
|
|
|
/* nn::os::AllocateMemoryBlock(unsigned long*, unsigned long) -> Result: 2 MB-aligned blocks from the heap */
|
|
HLE(_ZN2nn2os19AllocateMemoryBlockEPmm) {
|
|
uint64_t out = ARG(0), size = (ARG(1) + 0x1FFFFF) & ~0x1FFFFFULL;
|
|
HOS_TRACE_CALL("AllocateMemoryBlock");
|
|
LOCK();
|
|
uint64_t a = HOS_HEAP_BASE + hos_heap_used;
|
|
int ok = hos_heap_used + size <= hos_heap_size;
|
|
if (ok) hos_heap_used += size;
|
|
UNLOCK();
|
|
if (!ok) {
|
|
hos_log("AllocateMemoryBlock(%#llx): heap exhausted", (unsigned long long)size);
|
|
RET(0x1A01); /* os::ResultOutOfMemory */
|
|
return;
|
|
}
|
|
G64(out) = a;
|
|
RET(0);
|
|
}
|
|
|
|
/* ((Triforce, 11-19, lab) HOS_MEM_TRACE=1: every address region and page call of the game, with its size and caller, for the
|
|
memory that grows all session and is never let go (the census's "let go 0" through whole tours) */
|
|
int hos_mem_trace(void) {
|
|
static int on = -1;
|
|
if (on < 0) on = getenv("HOS_MEM_TRACE") && atoi(getenv("HOS_MEM_TRACE")) != 0;
|
|
return on;
|
|
}
|
|
#define MEM_TRACE(...) do { if (hos_mem_trace()) hos_log(__VA_ARGS__); } while (0)
|
|
/* (the callers above it: the guest's frame records, x29 up) */
|
|
static const char *mem_trace_stack(const Ctx *c, char *out, size_t n) {
|
|
uint64_t fp = c->x[29];
|
|
size_t at = 0;
|
|
out[0] = 0;
|
|
for (int i = 0; i < 10 && at + 20 < n; i++) {
|
|
if (!fp || (fp & 7) || !hos_guest_range(fp, 16)) break;
|
|
const uint64_t ret = G64(fp + 8), up = G64(fp);
|
|
at += (size_t)snprintf(out + at, n - at, " %#llx", (unsigned long long)ret);
|
|
if (up <= fp) break;
|
|
fp = up;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/* nn::os::AllocateAddressRegion(unsigned long*, unsigned long) -> Result */
|
|
HLE(_ZN2nn2os21AllocateAddressRegionEPmm) {
|
|
HOS_TRACE_CALL("AllocateAddressRegion");
|
|
G64(ARG(0)) = region_alloc(ARG(1), 0x10000, 1); /* ((Triforce) the game's own: hos_region_discard_needed) */
|
|
if (hos_mem_trace()) {
|
|
char st[256];
|
|
hos_log("memtrace: region %#llx size %#llx lr %#llx st%s", (unsigned long long)G64(ARG(0)), (unsigned long long)ARG(1),
|
|
(unsigned long long)c->x[30], mem_trace_stack(c, st, sizeof st));
|
|
}
|
|
RET(0);
|
|
}
|
|
|
|
/* nn::os::FreeAddressRegion(unsigned long) -> Result */
|
|
HLE(_ZN2nn2os17FreeAddressRegionEm) {
|
|
HOS_TRACE_CALL("FreeAddressRegion");
|
|
MEM_TRACE("memtrace: region free %#llx lr %#llx", (unsigned long long)ARG(0), (unsigned long long)c->x[30]);
|
|
hos_region_free(ARG(0));
|
|
RET(0);
|
|
}
|
|
|
|
/* nn::os::AllocateMemoryPages(unsigned long, unsigned long) -> Result: new pages read as zero */
|
|
HLE(_ZN2nn2os19AllocateMemoryPagesEmm) {
|
|
HOS_TRACE_CALL("AllocateMemoryPages");
|
|
const uint64_t a = ARG(0), n = ARG(1), end = a + n;
|
|
MEM_TRACE("memtrace: pages %#llx size %#llx lr %#llx", (unsigned long long)a, (unsigned long long)n, (unsigned long long)c->x[30]);
|
|
if (!hos_guest_range(a, n)) { /* (hookshot: a range that wrapped around was zeroed whole, past the guest window) */
|
|
hos_log("AllocateMemoryPages(%#llx, %#llx): outside the guest window", (unsigned long long)a, (unsigned long long)n);
|
|
RET(0xCA01); /* svc::ResultOutOfMemory */
|
|
return;
|
|
}
|
|
if (!hos_mem_room(a, n)) { /* ((Triforce) HOS_MEM_LIMIT: past what the game is told it has, as the console refuses) */
|
|
RET(0x1A01); /* os::ResultOutOfMemory */
|
|
return;
|
|
}
|
|
static int zero_all = -1;
|
|
if (zero_all < 0) zero_all = getenv("HOS_ZERO_ALL_PAGES") != NULL;
|
|
if (end > (1ULL << 32) && !zero_all) {
|
|
/* (mkw-recomp) above 4 GB the used-page bitmap does not reach, and zeroing the whole range wiped live data
|
|
when the game mapped pages over memory it already used (a heap's list lost its links). Horizon's rule
|
|
instead: only newly mapped pages read as zero. A commit leaves committed pages as they are and gives fresh
|
|
ones zeroed; FreeMemoryPages decommits. */
|
|
hos_commit(a, n);
|
|
hos_mem_note(HOS_MEM_PAGES, a, n); /* (Triforce: the census, the game's own pages) */
|
|
} else if (zero_all || end > (1ULL << 32) || end < a) {
|
|
memset(GP(a), 0, n);
|
|
} else {
|
|
uint64_t runs[64][2];
|
|
int nruns = 0, overflow = 0, in_run = 0;
|
|
LOCK();
|
|
for (uint64_t p = a & ~0xFFFULL; p < end; p += 0x1000) {
|
|
const uint64_t pg = p >> 12, bit = 1ULL << (pg & 63);
|
|
const int used = (pages_used[pg >> 6] & bit) != 0;
|
|
pages_used[pg >> 6] |= bit;
|
|
if (used && !in_run) {
|
|
if (nruns == 64) overflow = 1;
|
|
else runs[nruns][0] = p < a ? a : p;
|
|
in_run = 1;
|
|
} else if (!used && in_run) {
|
|
if (nruns < 64) runs[nruns++][1] = p;
|
|
in_run = 0;
|
|
}
|
|
}
|
|
if (in_run && nruns < 64) runs[nruns++][1] = end;
|
|
UNLOCK();
|
|
if (overflow) memset(GP(a), 0, n);
|
|
else
|
|
for (int i = 0; i < nruns; i++) memset(GP(runs[i][0]), 0, runs[i][1] - runs[i][0]);
|
|
}
|
|
HOS_WRITTEN(GP(a), n);
|
|
RET(0);
|
|
}
|
|
|
|
/* nn::os::FreeMemoryPages(unsigned long, unsigned long) -> Result */
|
|
HLE(_ZN2nn2os15FreeMemoryPagesEmm) {
|
|
HOS_TRACE_CALL("FreeMemoryPages");
|
|
MEM_TRACE("memtrace: pages free %#llx size %#llx lr %#llx", (unsigned long long)ARG(0), (unsigned long long)ARG(1), (unsigned long long)c->x[30]);
|
|
if (!hos_guest_range(ARG(0), ARG(1))) { /* (hookshot: decommitted or discarded: guest pages only) */
|
|
hos_log("FreeMemoryPages(%#llx, %#llx): outside the guest window", (unsigned long long)ARG(0), (unsigned long long)ARG(1));
|
|
RET(0xCA01);
|
|
return;
|
|
}
|
|
if (ARG(0) + ARG(1) > (1ULL << 32)) { /* (mkw-recomp: the pair of AllocateMemoryPages' commit above) */
|
|
const uint64_t lo = (ARG(0) + 0xFFF) & ~0xFFFULL, hi = (ARG(0) + ARG(1)) & ~0xFFFULL;
|
|
/* (pages the Vulkan back end imported, as part of a pool's 16 MB chunk, stay committed and are zeroed: the GPU
|
|
still maps them, and decommitted pages under an import faulted the device. Read as zero, as freed pages) */
|
|
extern int rx_guest_imported(uint64_t lo, uint64_t hi);
|
|
/* (Triforce: zeroed or decommitted: the lazy write-back's pages there first, rx_backend.c) */
|
|
const int64_t stall0 = hi > lo ? stall_host_begin() : 0; /* ((Triforce, OS-51) the host's part of a long frame) */
|
|
const int io = hi > lo ? hos_guest_io_begin(GP(lo), hi - lo) : 0;
|
|
/* ((Triforce) the census: zeroed pages stay committed, kept for the give-back once no import holds them) */
|
|
/* ((Triforce, 11-22, MEM-13) hos_guest_fill: the zeros marked written with HOS_MEM_ZERO_MARK=1) */
|
|
if (hi > lo && rx_guest_imported(lo, hi)) hos_guest_fill(lo, 0, hi - lo), hos_mem_note(HOS_MEM_ZEROED, lo, hi - lo);
|
|
#if defined(_WIN32)
|
|
else if (hi > lo) VirtualFree(GP(lo), (SIZE_T)(hi - lo), MEM_DECOMMIT), hos_mem_note(HOS_MEM_DECOMMITTED, lo, hi - lo);
|
|
#else
|
|
else if (hi > lo) { /* (linux: kept mapped, the pages read as zero; MADV_REMOVE on the memfd window) */
|
|
extern int hos_posix_guest_discard(void *p, size_t n);
|
|
hos_posix_guest_discard(GP(lo), (size_t)(hi - lo));
|
|
hos_mem_note(HOS_MEM_DECOMMITTED, lo, hi - lo);
|
|
}
|
|
#endif
|
|
hos_guest_io_end(io);
|
|
stall_host_end(STALL_HOST_DECOMMIT, stall0, hi > lo ? hi - lo : 0);
|
|
RET(0);
|
|
return;
|
|
}
|
|
discard_pages(ARG(0), ARG(1));
|
|
RET(0);
|
|
}
|
|
|
|
/* ((Triforce, 11-19) the game's pages beyond its heap (game_pages_now below) */
|
|
static uint64_t game_pages_now(void);
|
|
/* ((Triforce, 11-19) What Horizon answers as the memory in use: the program, the system's part, the heap and every page the game
|
|
has mapped beyond it (AllocateMemoryPages, MapPhysicalMemory), at most the total. The game's "Resource Memory" allocator
|
|
(main 0x8080f010 / 0x8080f090, beside its region and page calls) sizes what it may still take as total - used from
|
|
QueryMemoryInfo. It was told program + system only, so it always saw about 9.6 GB free, kept every area's resources it had
|
|
ever loaded, and the game's own memory climbed to 12.5 GB of 9809 MB after four areas (16 GB PCs ran out: #bug-tracker
|
|
"0.5.1 Memory-Retention Regression", "Ton of performance issues after a certain amount of time playing", the Steam Deck's
|
|
"crash after loading save"). Told what it holds, it lets old areas go as on a Switch. HOS_MEM_QUERY_USED=0: program + system
|
|
only, as before (also when the census is off: the pages are not counted then) */
|
|
static uint64_t query_used(void) {
|
|
static int full = -1;
|
|
static uint64_t slack;
|
|
if (full < 0) {
|
|
const char *e = getenv("HOS_MEM_QUERY_USED"), *k = getenv("HOS_MEM_QUERY_SLACK_MB");
|
|
full = !(e && *e == '0' && !e[1]) && hos_mem_census_on();
|
|
slack = (uint64_t)(k && *k ? atoll(k) : 1024) << 20;
|
|
hos_log("mem: QueryMemoryInfo's memory in use: %s, %llu MB less (HOS_MEM_QUERY_USED, HOS_MEM_QUERY_SLACK_MB)",
|
|
full ? "the program, the heap and the game's pages, as Horizon" : "the program only, as before",
|
|
(unsigned long long)(slack >> 20));
|
|
}
|
|
const uint64_t base = hos_program_size + (16ULL << 20);
|
|
if (!full) return base;
|
|
/* (the slack, 1024 MB by default (HOS_MEM_QUERY_SLACK_MB): room the game's cache may take beyond what a Switch leaves it,
|
|
as NX2navi's pool extra; from its pages only, so the heap the game sizes at its start from total - used is the same. Our
|
|
game needs ~3.2 GB of address regions at a save load where a Switch leaves ~2.8: with no slack it evicted and reloaded
|
|
in every scene (lab tour, A/B in one build: Kokiri 52 fps, Lake Hylia 42 against 60 / 59.5 with the old answer); with
|
|
1024 MB 60.0 / 59.9 and its own count held at 10.7 GB instead of 12.5 and climbing) */
|
|
const uint64_t pages = game_pages_now(), used = base + hos_heap_size + (pages > slack ? pages - slack : 0);
|
|
return used < hos_total_memory ? used : hos_total_memory;
|
|
}
|
|
|
|
/* nn::os::QueryMemoryInfo(nn::os::MemoryInfo*). TOTK reads +0 (total available) and +8 (total used):
|
|
heap = align_down(total - used, 2 MB) - 0x43800000 (main+0x1131004). */
|
|
HLE(_ZN2nn2os15QueryMemoryInfoEPNS0_10MemoryInfoE) {
|
|
uint64_t info = ARG(0);
|
|
HOS_TRACE_CALL("QueryMemoryInfo");
|
|
const uint64_t used = query_used();
|
|
MEM_TRACE("memtrace: query used %llu MB, free %llu MB lr %#llx", (unsigned long long)(used >> 20),
|
|
(unsigned long long)((hos_total_memory - used) >> 20), (unsigned long long)c->x[30]);
|
|
G64(info + 0x00) = hos_total_memory; /* total available */
|
|
G64(info + 0x08) = used; /* total used: program + system (+ heap + the game's pages) */
|
|
G64(info + 0x10) = hos_heap_size; /* total heap */
|
|
G64(info + 0x18) = hos_heap_used; /* allocated heap */
|
|
G64(info + 0x20) = hos_program_size; /* program */
|
|
G64(info + 0x28) = 0; /* thread stacks */
|
|
G32(info + 0x30) = 1; /* threads */
|
|
}
|
|
|
|
/* nn::os::SetMemoryAllocatorForThreadLocal runs as the SDK's own code: it stores the game's allocator where
|
|
nnosAllocateMemoryForThreadLocal finds it, which nnmusl uses for each thread's TLS (hos_thread.c thread_main). */
|
|
|
|
/* nn::os::SetDyingMessageRegion(unsigned long, unsigned long) -> Result */
|
|
HLE(_ZN2nn2os21SetDyingMessageRegionEmm) { RET(0); }
|
|
|
|
/* ---- (Triforce) the guest memory census (guest_census.h has why) ----------------------------------------------------------
|
|
The notes: what each place that commits, maps or lets go of guest pages did, into the census's map. Kept while the census
|
|
line or the give-back is on (HOS_MEM_CENSUS, on by default; HOS_MEM_GIVE_BACK: mkw_glue.c hos_mem_census_start); with both
|
|
off nothing is noted. */
|
|
#if defined(_WIN32)
|
|
static SRWLOCK census_lock = SRWLOCK_INIT;
|
|
#define CENSUS_LOCK() AcquireSRWLockExclusive(&census_lock)
|
|
#define CENSUS_UNLOCK() ReleaseSRWLockExclusive(&census_lock)
|
|
#else
|
|
static pthread_mutex_t census_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
#define CENSUS_LOCK() pthread_mutex_lock(&census_lock)
|
|
#define CENSUS_UNLOCK() pthread_mutex_unlock(&census_lock)
|
|
#endif
|
|
static uint64_t census_touched, census_zeroed, census_discarded, census_given, census_given_runs, census_refused;
|
|
static int64_t census_game_bytes; /* ((Triforce, 11-19) the game's pages and physical maps, as they are noted: hos_mem_used) */
|
|
static int census_unsupported; /* (the renderer has no mkwvk_guest_release: no give-back) */
|
|
|
|
int hos_mem_census_on(void) {
|
|
static volatile int on = -1;
|
|
int v = on;
|
|
if (v < 0) { /* (asked first by the program's commit, before any thread runs) */
|
|
const char *c = getenv("HOS_MEM_CENSUS"), *g = getenv("HOS_MEM_GIVE_BACK");
|
|
v = !(c && atoi(c) <= 0) || (g && atoi(g) > 0);
|
|
on = v;
|
|
}
|
|
return v;
|
|
}
|
|
|
|
/* ((Triforce, 11-19) The game's pages beyond its heap: the census's GC_GAME map (AllocateMemoryPages, MapPhysicalMemory, less
|
|
what it let go), counted as the map changes (census_game_bytes, which HOS_MEM_LIMIT's hos_mem_used reads too), so a page mapped
|
|
over one the game has already counts once. For QueryMemoryInfo's answer above. (HOS_MEM_BUDGET_MB, the budget that was here
|
|
on 11-19, is HOS_MEM_LIMIT now: the same refusal, with its stutter fix) */
|
|
static uint64_t game_pages_now(void) {
|
|
const int64_t g = __atomic_load_n(&census_game_bytes, __ATOMIC_RELAXED);
|
|
return g > 0 ? (uint64_t)g : 0;
|
|
}
|
|
|
|
void hos_mem_note(int what, uint64_t addr, uint64_t size) {
|
|
const uint64_t lo = addr, hi = addr + size;
|
|
if (!size || hi < lo || hi > GC_SPACE || !hos_mem_census_on()) return;
|
|
switch (what) {
|
|
case HOS_MEM_COMMITTED: /* (before the commit, under the lock the give-back checks and decommits under) */
|
|
CENSUS_LOCK();
|
|
gc_change(&census, GC_HELD, lo, hi, 1);
|
|
gc_change(&census, GC_KEPT, lo, hi, 0);
|
|
CENSUS_UNLOCK();
|
|
break;
|
|
case HOS_MEM_TOUCHED: /* (the first touch's exception handler: no lock. A page it commits was reserved, never kept) */
|
|
gc_change(&census, GC_HELD, lo, hi, 1);
|
|
gc_change(&census, GC_KEPT, lo, hi, 0);
|
|
__atomic_add_fetch(&census_touched, 1, __ATOMIC_RELAXED);
|
|
break;
|
|
case HOS_MEM_PAGES:
|
|
case HOS_MEM_MAPPED:
|
|
__atomic_add_fetch(&census_game_bytes, (int64_t)gc_change(&census, GC_GAME, lo, hi, 1), __ATOMIC_RELAXED);
|
|
break;
|
|
case HOS_MEM_DECOMMITTED:
|
|
gc_change(&census, GC_HELD, lo, hi, 0);
|
|
__atomic_sub_fetch(&census_game_bytes, (int64_t)gc_change(&census, GC_GAME, lo, hi, 0), __ATOMIC_RELAXED);
|
|
gc_change(&census, GC_KEPT, lo, hi, 0);
|
|
break;
|
|
case HOS_MEM_ZEROED: /* (every page of it committed: the zeroing touched it) */
|
|
gc_change(&census, GC_HELD, lo, hi, 0);
|
|
__atomic_sub_fetch(&census_game_bytes, (int64_t)gc_change(&census, GC_GAME, lo, hi, 0), __ATOMIC_RELAXED);
|
|
__atomic_add_fetch(&census_zeroed, gc_change(&census, GC_KEPT, lo, hi, 1), __ATOMIC_RELAXED);
|
|
break;
|
|
case HOS_MEM_DISCARDED: /* (only what was committed stays so) */
|
|
__atomic_sub_fetch(&census_game_bytes, (int64_t)gc_change(&census, GC_GAME, lo, hi, 0), __ATOMIC_RELAXED);
|
|
__atomic_add_fetch(&census_discarded, gc_move(&census, GC_HELD, GC_KEPT, lo, hi), __ATOMIC_RELAXED);
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* ---- (Triforce, 11-19) The game's memory held to what it is told it has (HOS_MEM_LIMIT, the lab's) ----------------------
|
|
The game is told 9809 MB (hos_total_memory), and on the console that is a limit: its heap, its program and the pages and
|
|
physical maps it takes count against it, and a map past it is refused (LimitReached). Here nothing was refused until the
|
|
PC's own commit ran out, and "used" was told as the program and the heap alone, whatever the game had mapped since. So
|
|
the game holds more than a console lets it: 11.7 GB by its own count standing in a fresh Market, 13.5 GB after a tour of
|
|
ten areas (the census line), its resources never let go for want of room. Its 3.2 GB GPU heap then stays within some
|
|
15 MB of full, and its texture compaction, which finds nothing to do in the fresh Market, runs a step every frame from
|
|
the first area change on (host/oot_tex_compact.c): the after-tour slowdown of the lab's tours.
|
|
|
|
HOS_MEM_LIMIT=1: the limit is kept as the console keeps it. AllocateMemoryPages and MapPhysicalMemory are refused when
|
|
the game's count would pass what it is told it has, and UsedMemorySize is the count (never more than the total).
|
|
HOS_MEM_LIMIT=<MB> (above 1): refused past that many MB instead, to try a limit between the console's and none.
|
|
Unset or 0: as before. Needs the census (HOS_MEM_CENSUS, on by default).
|
|
What the game does when refused (lab round 79): it stays within a few MB of the limit and its Resource Memory thread asks
|
|
again, some 16,000 times in a run of three areas, each ask a region taken, its pages refused, the region freed. That
|
|
free's discard of pages nobody had touched was the stutter the limit was first set aside for (the game's main thread
|
|
waits for that thread): hos_region_discard_needed leaves it out. The regions it frees with pages in them, some hundreds
|
|
while an area loads, are discarded with the thread's virtual core given up (discard_pages). With both, the game's own
|
|
part of the frame is what it is without the limit (over 8 ms in 20 frames of a three-area run either way), and the
|
|
game holds 9.8 GB where it held 11.7. */
|
|
int64_t hos_mem_limit(void) {
|
|
static volatile int64_t v = -1;
|
|
int64_t l = v;
|
|
if (l < 0) {
|
|
const char *e = getenv("HOS_MEM_LIMIT");
|
|
long long n = e && *e ? atoll(e) : 0;
|
|
if (n > (1LL << 24)) n = 1LL << 24; /* ((Triforce) 16 TB at most: a larger size would overflow the shift below) */
|
|
if (n > 0 && !hos_mem_census_on())
|
|
hos_log("mem: HOS_MEM_LIMIT is set but the census is off (HOS_MEM_CENSUS=0): it needs the census's count, so no limit is kept");
|
|
l = n <= 0 || !hos_mem_census_on() ? 0 : n == 1 ? (int64_t)hos_total_memory : (int64_t)n << 20;
|
|
if (l) hos_log("mem: the game's memory is held to %lld MB (HOS_MEM_LIMIT; it is told %llu MB)", (long long)(l >> 20),
|
|
(unsigned long long)(hos_total_memory >> 20));
|
|
/* ((Triforce, 11-20) the console's own size starves this build: its heap takes 6.9 GB of the 9.8, and an area's
|
|
resources need 4.7 GB and more (a fresh Market; 6.5 after a tour). The owner's save stayed on its loading screen
|
|
with the game at 9,807 MB and no longer asking: its load threads idle on their queues, read with the hang watch's
|
|
asked-for stacks. A size in MB leaves room: 12800 loaded and played four areas) */
|
|
if (n == 1 && l)
|
|
hos_log("mem: HOS_MEM_LIMIT=1 is the console's own size, which this build does not fit in: a save's load can stay on "
|
|
"its loading screen for good. Give a size in MB instead (12800 loaded four areas)");
|
|
v = l;
|
|
}
|
|
return l;
|
|
}
|
|
/* What the game holds in Horizon's terms: its program, its heap, its pages and physical maps (the census's "own count") */
|
|
uint64_t hos_mem_used(void) {
|
|
const int64_t g = __atomic_load_n(&census_game_bytes, __ATOMIC_RELAXED);
|
|
return hos_program_size + hos_heap_size + (uint64_t)(g > 0 ? g : 0);
|
|
}
|
|
/* The rule: 1 when taking `fresh` more bytes keeps the count within the limit (limit 0: no limit, always). A range the game
|
|
holds whole already (fresh 0) takes nothing, and fits wherever the count stands */
|
|
int hos_mem_fits(uint64_t used, uint64_t fresh, uint64_t limit) { return !limit || !fresh || used + fresh <= limit; }
|
|
/* Whether the game may take [addr, addr + size) as pages or a physical map: only the pages of it that it does not hold yet
|
|
count. A refusal is counted, and the first 16 and every 256th are logged */
|
|
int hos_mem_room(uint64_t addr, uint64_t size) {
|
|
const int64_t limit = hos_mem_limit();
|
|
if (!limit || !size || addr + size < addr || addr + size > GC_SPACE) return 1;
|
|
const uint64_t held = gc_count(&census, GC_GAME, addr, addr + size);
|
|
const uint64_t fresh = size > held ? size - held : 0, used = hos_mem_used();
|
|
if (hos_mem_fits(used, fresh, (uint64_t)limit)) return 1;
|
|
static int64_t refused;
|
|
const int64_t n = __atomic_add_fetch(&refused, 1, __ATOMIC_RELAXED);
|
|
if (n <= 16 || n % 256 == 0)
|
|
hos_log("mem: refused %llu KB at %#llx: the game holds %llu MB of %lld (HOS_MEM_LIMIT; %lld refusals so far; %lld regions "
|
|
"freed with no page of the game's in them, not discarded)",
|
|
(unsigned long long)(fresh >> 10), (unsigned long long)addr, (unsigned long long)(used >> 20), (long long)(limit >> 20),
|
|
(long long)n, (long long)__atomic_load_n(®ion_frees_bare, __ATOMIC_RELAXED));
|
|
return 0;
|
|
}
|
|
|
|
/* hos_commit's notes (mkw_glue.c), around its commit: held before it, under the lock the give-back checks and decommits under
|
|
(so no give-back takes a page it is committing), and no longer kept only once the commit is made. A commit that failed
|
|
puts back the pages it set held (kept ones kept, never committed ones neither); begin's answer goes to end */
|
|
void *hos_mem_commit_begin(uint64_t addr, uint64_t size) {
|
|
const uint64_t lo = addr, hi = addr + size;
|
|
if (!size || hi < lo || hi > GC_SPACE || !hos_mem_census_on()) return NULL;
|
|
const uint64_t n = gc_words(lo, hi);
|
|
uint64_t *had = n ? (uint64_t *)malloc((size_t)n * sizeof *had) : NULL;
|
|
if (had) memset(had, 0xFF, (size_t)n * sizeof *had);
|
|
CENSUS_LOCK();
|
|
if (had) gc_set_keeping(&census, GC_HELD, lo, hi, had);
|
|
else gc_change(&census, GC_HELD, lo, hi, 1); /* (no room to say which were held: a failed commit leaves them held) */
|
|
CENSUS_UNLOCK();
|
|
return had;
|
|
}
|
|
void hos_mem_commit_end(void *had, uint64_t addr, uint64_t size, int committed) {
|
|
const uint64_t lo = addr, hi = addr + size;
|
|
if (size && hi >= lo && hi <= GC_SPACE && hos_mem_census_on()) {
|
|
CENSUS_LOCK();
|
|
if (committed) gc_change(&census, GC_KEPT, lo, hi, 0);
|
|
else if (had) gc_unset_had(&census, GC_HELD, lo, hi, (const uint64_t *)had);
|
|
CENSUS_UNLOCK();
|
|
}
|
|
free(had);
|
|
}
|
|
|
|
/* The counts (mkw_glue.c's line): the live address regions copied under the regions' lock, then counted without it */
|
|
void hos_mem_census(GcCensus *c) {
|
|
static uint64_t (*list)[2];
|
|
static int cap;
|
|
int n = 0;
|
|
LOCK();
|
|
if (nregions > cap) {
|
|
uint64_t(*grown)[2] = (uint64_t(*)[2])realloc(list, (size_t)nregions * sizeof *list);
|
|
if (grown) list = grown, cap = nregions;
|
|
}
|
|
for (; n < nregions && n < cap; n++) list[n][0] = regions[n].addr, list[n][1] = regions[n].size;
|
|
UNLOCK();
|
|
gc_census_count(&census, (const uint64_t(*)[2])list, n, HOS_HEAP_BASE, hos_heap_size, c);
|
|
c->touched = __atomic_load_n(&census_touched, __ATOMIC_RELAXED);
|
|
c->kept_import = __atomic_load_n(&census_zeroed, __ATOMIC_RELAXED);
|
|
c->kept_discard = __atomic_load_n(&census_discarded, __ATOMIC_RELAXED);
|
|
c->given_back = census_given, c->given_runs = census_given_runs, c->refused = census_refused;
|
|
c->budget = hos_total_memory, c->program = hos_program_size;
|
|
}
|
|
|
|
/* The give-back's decommit of [lo, hi), run by the renderer with its import locks held (rx_backend.c rx_guest_release): no
|
|
import holds the pages, and none can start to meanwhile. Only when every page is still kept and none was handed out again
|
|
since the pass found them (hos_commit notes before it commits, under this lock). 1: given back */
|
|
static int census_release(uint64_t lo, uint64_t hi) {
|
|
extern uint64_t hos_large_lo, hos_large_hi;
|
|
if (hos_large_hi && lo < hos_large_hi && hi > hos_large_lo) return 0; /* (large pages are not decommitted) */
|
|
if (lo < (1ULL << 32)) return 0; /* (below 4 GB pages are handed out without a commit: hos_mem_give_back) */
|
|
CENSUS_LOCK();
|
|
int ok = gc_count(&census, GC_KEPT, lo, hi) == hi - lo && !gc_count(&census, GC_HELD, lo, hi);
|
|
if (ok) {
|
|
#if defined(_WIN32)
|
|
ok = VirtualFree(GP(lo), (SIZE_T)(hi - lo), MEM_DECOMMIT) != 0;
|
|
#else
|
|
extern int hos_posix_guest_discard(void *p, size_t n);
|
|
ok = hos_posix_guest_discard(GP(lo), (size_t)(hi - lo)) == 0;
|
|
#endif
|
|
if (ok) gc_change(&census, GC_KEPT, lo, hi, 0);
|
|
}
|
|
CENSUS_UNLOCK();
|
|
return ok;
|
|
}
|
|
/* One kept run offered: the lazy write-back's pages there paid first (as FreeMemoryPages does), then the renderer asked */
|
|
static int census_offer(void *ctx, uint64_t lo, uint64_t hi) {
|
|
(void)ctx;
|
|
int rx_guest_release(uint64_t lo, uint64_t hi, int (*release)(uint64_t lo, uint64_t hi)); /* (rx_backend.c) */
|
|
const int io = hos_guest_io_begin(GP(lo), hi - lo);
|
|
const int r = rx_guest_release(lo, hi, census_release);
|
|
hos_guest_io_end(io);
|
|
if (r == -2) census_unsupported = 1;
|
|
return r == 1;
|
|
}
|
|
/* A pass of the give-back (the census's thread, mkw_glue.c): kept runs of 16 MB at most, 64 of them at most, until max_bytes
|
|
are given back. The bytes given back; -1 when the renderer cannot (an older one, without mkwvk_guest_release) */
|
|
int64_t hos_mem_give_back(uint64_t max_bytes) {
|
|
static uint64_t cursor;
|
|
if (census_unsupported) return -1;
|
|
if (!hos_mem_census_on()) return 0;
|
|
uint64_t runs = 0, refused = 0;
|
|
/* (from 4 GB up: below it AllocateMemoryPages zeroes pages without committing them, and a host call handed one that was
|
|
given back would fail on it without a fault) */
|
|
const int64_t stall0 = stall_host_begin(); /* ((Triforce, OS-51) the host's part of a long frame: stall_host.h) */
|
|
const uint64_t given = gc_give_back(&census, 1ULL << 32, &cursor, 16ULL << 20, max_bytes, 64, census_offer, NULL, &runs, &refused);
|
|
stall_host_end(STALL_HOST_GIVE_BACK, stall0, given);
|
|
census_given += given, census_given_runs += runs, census_refused += refused;
|
|
return census_unsupported && !given ? -1 : (int64_t)given;
|
|
}
|