mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 11:49:59 -04:00
The commit census of NX2NAVI_TRACE=perf with NX2NAVI_MEM_CENSUS=1 reported the process's committed memory by kind, which lumps every guest region together. memory.c mem_guest_census now walks each reserved region (aslr, stack, heap, alias, kernel, host functions) beside it and reports the bytes committed in each, so a process that has grown to thirty gigabytes (issues #203, #215, #240) says which region to look at. Like the census, it runs only when mem_census asks for it, since the walk holds the address-space lock the game's threads take at every page fault. On the Kokiri route the answer was the aslr region at 28 to 31 GB, the heap at 7.5 GB and the alias window at 2.7 GB while the physical-memory pool itself stayed near its limit, which is what the device-loss crash follows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
738 lines
32 KiB
C
738 lines
32 KiB
C
// Contains code Copyright (c) 2026 MacNativeGames contributors, Copyright (c) 2026 Peiikai (MIT; see
|
|
// THIRD_PARTY_NOTICES.md).
|
|
// Guest address space and the memory SVCs: heap, module data permissions, stack moves,
|
|
// QueryMemory, physical memory in the alias region, GetInfo memory ids, and the kernel-region allocator.
|
|
//
|
|
// Every region is a plain reservation; pages are committed and decommitted at 4 KiB granularity. Decommitted pages
|
|
// come back zeroed, so no stale-page bookkeeping is needed. The stack and heap block and the alias region are
|
|
// reserved write-watched (platform.h plat_mem_reserve): mem_written_pages reports the pages the CPU wrote, which
|
|
// the Vulkan back end uploads again.
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "cfg.h"
|
|
#include "host.h"
|
|
#include "layout.h"
|
|
#include "platform/platform.h"
|
|
|
|
#define PAGE 0x1000ull
|
|
#define GiB (1ull << 30)
|
|
#define LOW_SEARCH_FROM 0x100000000ull
|
|
#define LOW_SIZE (14 * GiB) // stack + heap block: room for a Switch 2 heap (HEAP_CAP)
|
|
#define STACK_OFF 0x10000000ull
|
|
#define STACK_SIZE 0x10000000ull
|
|
#define HEAP_OFF (STACK_OFF + STACK_SIZE + 0x10000000ull)
|
|
#define HEAP_CAP (12 * GiB) // heap region: a Switch 2 application pool is 9809 MiB (the 12 GB console arrangement)
|
|
#define ALIAS_SIZE (64 * GiB)
|
|
#define ALIAS_ALIGN (4 * GiB)
|
|
#define ALIAS_PAGES (ALIAS_SIZE / PAGE)
|
|
#define DEFAULT_POOL rt_app_pool_bytes // retail application pool of the title (generated: NX2NAVI 0xCD500000, Switch 2 9809 MiB)
|
|
#define SYSTEM_RESOURCE_BUDGET (32ull << 20)
|
|
#define MAX_STACK_MAPS 4096
|
|
#define REEXEC_LIMIT 20
|
|
|
|
// Horizon memory states, permissions, attributes and results used here.
|
|
enum { ST_FREE = 0, ST_CODE = 3, ST_CODEDATA = 4, ST_NORMAL = 5, ST_STACK = 0xB, ST_THREADLOCAL = 0xC,
|
|
ST_INACCESSIBLE = 0x10 };
|
|
enum { PERM_NONE = 0, PERM_R = 1, PERM_RW = 3, PERM_RX = 5 };
|
|
enum { ATTR_PERMISSION_LOCKED = 16 };
|
|
#define R_OK 0u
|
|
#define R_INVALID_ADDRESS 0xcc01u // misaligned address
|
|
#define R_INVALID_SIZE 0xca01u
|
|
#define R_INVALID_NEW_PERM 0xd801u
|
|
#define R_INVALID_STATE 0xd401u // InvalidCurrentMemory
|
|
#define R_INVALID_REGION 0xdc01u
|
|
#define R_LIMIT_REACHED 0x10801u
|
|
|
|
static RtGuestLayout L;
|
|
static uint64_t mods_lo, mods_hi; // fixed module window
|
|
static PlatMutex lock = PLAT_MUTEX_INIT;
|
|
static uint64_t heap_size, alias_bytes;
|
|
static uint8_t *alias_pages; // 1 per mapped 4 KiB page of the alias region
|
|
typedef struct { uint64_t dst, src, size; } StackMap;
|
|
static StackMap stack_maps[MAX_STACK_MAPS];
|
|
static unsigned nstack_maps;
|
|
static uint8_t *module_perm[MOD_COUNT]; // per data page of each module: perm | ATTR_PERMISSION_LOCKED
|
|
|
|
static uint64_t align_up(uint64_t v, uint64_t a) { return (v + a - 1) & ~(a - 1); }
|
|
|
|
static int reserve_at(uint64_t at, uint64_t size, int watch) { return plat_mem_reserve(at, size, watch); }
|
|
|
|
// ---- frozen regions
|
|
// A 2 MiB region of the watched reservations that nothing wrote for a while can be made read-only (mem_freeze), so the
|
|
// back end's write-watch queries skip it: no page of a frozen region changes before a thaw. The first write faults and
|
|
// thaw_handler makes the region writable again before the write retries. A kernel write into guest memory (a file
|
|
// read) fails on a read-only page instead of faulting, so file reads thaw their buffer first (mem_thaw), and so does
|
|
// every commit, decommit and protection change here. A region stays thawed for FRZ_HOLD_MS after a mem_thaw: a file
|
|
// read still in flight writes its buffer later. frz_lock is a leaf lock: nothing writes guest memory under it.
|
|
#define FRZ_SHIFT 21
|
|
#define FRZ_SIZE (1ull << FRZ_SHIFT)
|
|
#define FRZ_HOLD_MS 5000u
|
|
static volatile uint8_t frz[RT_ADDRESS_LIMIT >> FRZ_SHIFT];
|
|
static uint32_t thaw_ms[RT_ADDRESS_LIMIT >> FRZ_SHIFT]; // frz_lock: GetTickCount of the last thaw request, 0 never
|
|
static volatile int32_t nfrz;
|
|
static PlatMutex frz_lock = PLAT_MUTEX_INIT;
|
|
static volatile int64_t frz_count[4]; // freezes; thaws by a write fault, by mem_thaw, by a mapping change
|
|
|
|
static int watched(uint64_t addr, uint64_t size) {
|
|
const RtGuestLayout *g = &L;
|
|
return (addr >= g->aslr.base && size <= LOW_SIZE && addr - g->aslr.base <= LOW_SIZE - size) ||
|
|
(addr >= g->alias.base && size <= ALIAS_SIZE && addr - g->alias.base <= ALIAS_SIZE - size);
|
|
}
|
|
|
|
// frz_lock held exclusively. why: 1 a write fault, 2 mem_thaw (holds the region thawed), 3 a mapping change here.
|
|
static void thaw_locked(uint64_t r, int why) {
|
|
if (why == 2) thaw_ms[r] = (uint32_t)plat_now_ms() | 1;
|
|
if (!frz[r]) return;
|
|
// The read-only runs only: freeze_locked left reserved holes (heap pages given back) as they were. Every committed
|
|
// page of a frozen region was read-write at the freeze, and mapping changes thaw before they run, so these are all
|
|
// ours.
|
|
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, b;
|
|
for (; a < end; a = b)
|
|
if (plat_mem_state(a, end, &b) == PLAT_PROT_READ && !plat_mem_protect(a, b - a, PLAT_PROT_RW))
|
|
rt_fatal("memory: thawing %#llx+%#llx failed", (unsigned long long)a, (unsigned long long)(b - a));
|
|
__atomic_store_n(&frz[r], 0, __ATOMIC_RELEASE);
|
|
plat_dec32(&nfrz);
|
|
plat_inc64(&frz_count[why]);
|
|
}
|
|
|
|
// frz_lock held exclusively.
|
|
static void thaw_range_locked(uint64_t a, uint64_t n, int why) {
|
|
if (!n || a >= RT_ADDRESS_LIMIT) return;
|
|
uint64_t r0 = a >> FRZ_SHIFT, r1 = (a + n - 1 < RT_ADDRESS_LIMIT ? a + n - 1 : RT_ADDRESS_LIMIT - 1) >> FRZ_SHIFT;
|
|
for (uint64_t r = r0; r <= r1; r++) thaw_locked(r, why);
|
|
}
|
|
|
|
void mem_thaw(uint64_t a, uint64_t n) {
|
|
plat_mutex_lock(&frz_lock); // also when nothing is frozen: the hold keeps a read's buffer writable
|
|
thaw_range_locked(a, n, 2);
|
|
plat_mutex_unlock(&frz_lock);
|
|
}
|
|
|
|
// frz_lock held exclusively. Makes the committed pages of region r read-only when every one of them is read-write;
|
|
// reserved holes (heap pages given back) stay as they are, and thaw_locked finds the read-only runs again.
|
|
static int freeze_locked(uint64_t r) {
|
|
uint64_t a = r << FRZ_SHIFT, end = a + FRZ_SIZE, run[16][2];
|
|
int n = 0;
|
|
for (uint64_t x = a, y; x < end; x = y) {
|
|
int st = plat_mem_state(x, end, &y);
|
|
if (st >= PLAT_PROT_NONE) {
|
|
if (st != PLAT_PROT_RW || n == 16) return 0;
|
|
run[n][0] = x, run[n][1] = y, n++;
|
|
} else if (st != PLAT_MEM_RESERVED) return 0;
|
|
}
|
|
for (int i = 0; i < n; i++)
|
|
if (!plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_READ)) {
|
|
while (i--) plat_mem_protect(run[i][0], run[i][1] - run[i][0], PLAT_PROT_RW);
|
|
return 0;
|
|
}
|
|
return n > 0;
|
|
}
|
|
|
|
int mem_freeze(uint64_t a) {
|
|
if ((a & (FRZ_SIZE - 1)) || !watched(a, FRZ_SIZE)) return 0;
|
|
uint64_t r = a >> FRZ_SHIFT;
|
|
int ok = 0;
|
|
plat_mutex_lock(&frz_lock);
|
|
if (frz[r]) ok = 1;
|
|
else if (thaw_ms[r] && (int32_t)((uint32_t)plat_now_ms() - thaw_ms[r]) < (int32_t)FRZ_HOLD_MS) ok = 0; // |1 may be ahead
|
|
else if (freeze_locked(r)) {
|
|
__atomic_store_n(&frz[r], 1, __ATOMIC_RELEASE);
|
|
plat_inc32(&nfrz);
|
|
plat_inc64(&frz_count[0]);
|
|
ok = 1;
|
|
}
|
|
plat_mutex_unlock(&frz_lock);
|
|
return ok;
|
|
}
|
|
|
|
int mem_frozen(uint64_t a) { return a < RT_ADDRESS_LIMIT && __atomic_load_n(&frz[a >> FRZ_SHIFT], __ATOMIC_ACQUIRE); }
|
|
|
|
void mem_freeze_stats(uint64_t *frozen, uint64_t *freezes, uint64_t *fault_thaws, uint64_t *other_thaws) {
|
|
*frozen = (uint64_t)__atomic_load_n(&nfrz, __ATOMIC_RELAXED);
|
|
*freezes = (uint64_t)plat_xchg64(&frz_count[0], 0);
|
|
*fault_thaws = (uint64_t)plat_xchg64(&frz_count[1], 0);
|
|
*other_thaws = (uint64_t)plat_xchg64(&frz_count[2], 0) + (uint64_t)plat_xchg64(&frz_count[3], 0);
|
|
}
|
|
|
|
// The first write into a frozen region thaws it and retries. A fault in a region another thread thawed meanwhile
|
|
// retries too; any other fault goes on to the crash handler.
|
|
// The platform's write-fault hook (plat_mem_fault_hook): 1 retries the write.
|
|
static int thaw_fault(uint64_t a) {
|
|
if (a >= RT_ADDRESS_LIMIT || !watched(a, 1)) return 0;
|
|
plat_mutex_lock(&frz_lock);
|
|
int was = frz[a >> FRZ_SHIFT];
|
|
thaw_locked(a >> FRZ_SHIFT, 1);
|
|
plat_mutex_unlock(&frz_lock);
|
|
if (was) return 1;
|
|
uint64_t end;
|
|
return plat_mem_state(a & ~(PAGE - 1), (a & ~(PAGE - 1)) + PAGE, &end) == PLAT_PROT_RW;
|
|
}
|
|
|
|
// Mapping changes thaw the regions they touch and run under frz_lock, so no freeze sees them half done.
|
|
static void commit(uint64_t a, uint64_t n, const char *what) {
|
|
plat_mutex_lock(&frz_lock);
|
|
thaw_range_locked(a, n, 3);
|
|
int ok = !n || plat_mem_commit(a, n);
|
|
plat_mutex_unlock(&frz_lock);
|
|
if (!ok) rt_fatal("memory: commit %s %#llx+%#llx failed", what, (unsigned long long)a, (unsigned long long)n);
|
|
}
|
|
|
|
static void decommit(uint64_t a, uint64_t n, const char *what) {
|
|
plat_mutex_lock(&frz_lock);
|
|
thaw_range_locked(a, n, 3);
|
|
int ok = !n || plat_mem_decommit(a, n);
|
|
plat_mutex_unlock(&frz_lock);
|
|
if (!ok) rt_fatal("memory: decommit %s %#llx+%#llx failed", what, (unsigned long long)a, (unsigned long long)n);
|
|
}
|
|
|
|
static void protect(uint64_t a, uint64_t n, int prot, const char *what) {
|
|
plat_mutex_lock(&frz_lock);
|
|
thaw_range_locked(a, n, 3);
|
|
int ok = !n || plat_mem_protect(a, n, prot);
|
|
plat_mutex_unlock(&frz_lock);
|
|
if (!ok) rt_fatal("memory: protect %s %#llx+%#llx failed", what, (unsigned long long)a, (unsigned long long)n);
|
|
}
|
|
|
|
// The process is re-run when a region is taken: Windows places DLLs, heaps and thread stacks at random addresses,
|
|
// and a fresh process gets a fresh layout.
|
|
static void reexec_or_fail(const char *what) {
|
|
const char *e = cfg_get("NX2NAVI_REEXEC");
|
|
int n = e ? atoi(e) : 0;
|
|
if (n >= REEXEC_LIMIT) rt_fatal("memory: %s is not free after %d attempts", what, n);
|
|
char buf[16];
|
|
snprintf(buf, sizeof buf, "%d", n + 1);
|
|
cfg_setenv("NX2NAVI_REEXEC", buf);
|
|
plat_reexec();
|
|
}
|
|
|
|
void mem_init(void) {
|
|
if (L.pool) return;
|
|
// Fixed window: modules, kernel region, host-function addresses.
|
|
mods_lo = MOD_MAIN_BASE;
|
|
mods_hi = align_up(rt_layout[rt_layout_count - 1].base + rt_layout[rt_layout_count - 1].size, 0x10000);
|
|
if (mods_hi > RT_KERNEL_BASE) rt_fatal("memory: modules end at %#llx, inside the kernel region",
|
|
(unsigned long long)mods_hi);
|
|
if (!reserve_at(mods_lo, mods_hi - mods_lo, 0)) reexec_or_fail("the module window");
|
|
if (!reserve_at(RT_KERNEL_BASE, RT_KERNEL_SIZE, 0)) reexec_or_fail("the kernel region");
|
|
if (!reserve_at(RT_HOSTFN_BASE, RT_HOSTFN_SIZE, 0)) reexec_or_fail("the host-function window");
|
|
|
|
// Stack and heap block, then the alias region above it, both searched below the module window.
|
|
uint64_t low = 0, alias = 0;
|
|
for (uint64_t at = LOW_SEARCH_FROM; at + LOW_SIZE <= mods_lo && !low; at += GiB)
|
|
if (reserve_at(at, LOW_SIZE, 1)) low = at;
|
|
if (!low) reexec_or_fail("a 14 GiB stack and heap block below 2^39");
|
|
for (uint64_t at = align_up(low + LOW_SIZE, ALIAS_ALIGN); at + ALIAS_SIZE <= mods_lo && !alias; at += ALIAS_ALIGN)
|
|
if (reserve_at(at, ALIAS_SIZE, 1)) alias = at;
|
|
if (!alias) reexec_or_fail("a 64 GiB alias region below 2^39");
|
|
|
|
plat_mem_fault_hook(thaw_fault); // before the crash handler
|
|
alias_pages = calloc(ALIAS_PAGES, 1);
|
|
if (!alias_pages) rt_fatal("memory: cannot allocate the alias page map");
|
|
// nx2navi.ini is read here, once the address space is claimed and before the first setting: the pool size below
|
|
// depends on NX2NAVI_APP_POOL and on the resolution settings (resolution.c decides them once, on this first call).
|
|
cfg_init();
|
|
const char *pool = cfg_get("NX2NAVI_APP_POOL");
|
|
L = (RtGuestLayout){
|
|
.aslr = {low, RT_ADDRESS_LIMIT - low},
|
|
.stack = {low + STACK_OFF, STACK_SIZE},
|
|
.heap = {low + HEAP_OFF, HEAP_CAP},
|
|
.alias = {alias, ALIAS_SIZE},
|
|
.kernel = {RT_KERNEL_BASE, RT_KERNEL_SIZE},
|
|
.hostfn = {RT_HOSTFN_BASE, RT_HOSTFN_SIZE},
|
|
.pool = pool ? strtoull(pool, NULL, 0) : DEFAULT_POOL + res_heap_extra_bytes(), // resolution.c
|
|
};
|
|
if (!L.pool) rt_fatal("memory: NX2NAVI_APP_POOL is 0");
|
|
}
|
|
|
|
const RtGuestLayout *mem_layout(void) { return &L; }
|
|
|
|
// A range inside one watched reservation (stack and heap block, or alias region) only.
|
|
static uint64_t written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max, int reset) {
|
|
if (!max || !size) return 0;
|
|
if (!watched(addr, size)) return UINT64_MAX;
|
|
return plat_mem_written(addr, size, pages, max, reset);
|
|
}
|
|
|
|
uint64_t mem_written_pages(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
|
|
return written_pages(addr, size, pages, max, 1);
|
|
}
|
|
|
|
uint64_t mem_written_peek(uint64_t addr, uint64_t size, uint64_t *pages, uint64_t max) {
|
|
return written_pages(addr, size, pages, max, 0);
|
|
}
|
|
|
|
static uint64_t image_bytes(void) {
|
|
uint64_t n = 0;
|
|
for (int i = 0; i < ldr_count(); i++) n += ldr_module(i)->size;
|
|
return align_up(n, PAGE);
|
|
}
|
|
|
|
static int overlaps(uint64_t a, uint64_t n, uint64_t b, uint64_t m) { return a < b + m && b < a + n; }
|
|
static int inside(uint64_t a, uint64_t n, RtRange r) { return a >= r.base && n <= r.size && a - r.base <= r.size - n; }
|
|
|
|
// ---- 0x01 SetHeapSize
|
|
uint32_t mem_set_heap_size(uint64_t size, uint64_t *addr) {
|
|
plat_mutex_lock(&lock);
|
|
if (size > HEAP_CAP || size % 0x200000 || size + image_bytes() > L.pool) {
|
|
plat_mutex_unlock(&lock);
|
|
return R_INVALID_STATE;
|
|
}
|
|
if (size > heap_size) commit(L.heap.base + heap_size, size - heap_size, "heap");
|
|
if (size < heap_size) decommit(L.heap.base + size, heap_size - size, "heap");
|
|
heap_size = size;
|
|
plat_mutex_unlock(&lock);
|
|
if (addr) *addr = L.heap.base;
|
|
return R_OK;
|
|
}
|
|
|
|
// ---- module data permissions (0x02, 0x03) and module pages for QueryMemory
|
|
static int module_index(uint64_t a) {
|
|
for (int i = 0; i < ldr_count(); i++) {
|
|
const RtLoaded *m = ldr_module(i);
|
|
if (a - m->base < m->size) return i;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
static uint8_t *perm_table(int i) { // under lock
|
|
if (!module_perm[i]) {
|
|
const RtLoaded *m = ldr_module(i);
|
|
size_t n = (m->size - m->data_off) / PAGE;
|
|
module_perm[i] = malloc(n);
|
|
if (!module_perm[i]) rt_fatal("memory: module permission table");
|
|
memset(module_perm[i], PERM_RW, n);
|
|
}
|
|
return module_perm[i];
|
|
}
|
|
|
|
static unsigned module_page_perm(int i, uint64_t off) { // under lock
|
|
const RtLoaded *m = ldr_module(i);
|
|
if (off < m->data_off) return off < align_up(m->text_end, PAGE) ? PERM_RX : PERM_R;
|
|
return module_perm[i] ? module_perm[i][(off - m->data_off) / PAGE] : PERM_RW;
|
|
}
|
|
|
|
// The module whose data segment holds [a, a+n), or -1.
|
|
static int module_data(uint64_t a, uint64_t n) {
|
|
int i = module_index(a);
|
|
if (i < 0) return -1;
|
|
const RtLoaded *m = ldr_module(i);
|
|
uint64_t off = a - m->base;
|
|
return off >= m->data_off && n <= m->size - off ? i : -1;
|
|
}
|
|
|
|
uint32_t mem_set_permission(uint64_t a, uint64_t n, uint32_t perm) {
|
|
if (a & (PAGE - 1)) return R_INVALID_ADDRESS;
|
|
if (!n || (n & (PAGE - 1))) return R_INVALID_SIZE;
|
|
if (perm != PERM_NONE && perm != PERM_R && perm != PERM_RW) return R_INVALID_NEW_PERM;
|
|
int i = module_data(a, n);
|
|
if (i < 0) return R_INVALID_STATE;
|
|
const RtLoaded *m = ldr_module(i);
|
|
plat_mutex_lock(&lock);
|
|
uint8_t *t = perm_table(i) + (a - m->base - m->data_off) / PAGE;
|
|
for (uint64_t p = 0; p < n / PAGE; p++)
|
|
if (t[p] & ATTR_PERMISSION_LOCKED) {
|
|
plat_mutex_unlock(&lock);
|
|
return R_INVALID_STATE;
|
|
}
|
|
memset(t, (int)perm, n / PAGE);
|
|
protect(a, n, perm == PERM_RW ? PLAT_PROT_RW : perm == PERM_R ? PLAT_PROT_READ : PLAT_PROT_NONE, "module data");
|
|
plat_mutex_unlock(&lock);
|
|
rt_log("%s data permission +%#llx size %#llx perm %u", m->name, (unsigned long long)(a - m->base),
|
|
(unsigned long long)n, perm);
|
|
return R_OK;
|
|
}
|
|
|
|
// The only module attribute the SDK sets is the irreversible RELRO lock.
|
|
uint32_t mem_set_attribute(uint64_t a, uint64_t n, uint32_t mask, uint32_t at) {
|
|
if (a & (PAGE - 1)) return R_INVALID_ADDRESS;
|
|
if (!n || (n & (PAGE - 1))) return R_INVALID_SIZE;
|
|
if (mask != ATTR_PERMISSION_LOCKED || at != ATTR_PERMISSION_LOCKED) return R_INVALID_NEW_PERM;
|
|
int i = module_data(a, n);
|
|
if (i < 0) return R_INVALID_STATE;
|
|
const RtLoaded *m = ldr_module(i);
|
|
plat_mutex_lock(&lock);
|
|
uint8_t *t = perm_table(i) + (a - m->base - m->data_off) / PAGE;
|
|
for (uint64_t p = 0; p < n / PAGE; p++) t[p] |= ATTR_PERMISSION_LOCKED;
|
|
plat_mutex_unlock(&lock);
|
|
return R_OK;
|
|
}
|
|
|
|
// ---- 0x04/0x05 MapMemory, UnmapMemory: thread stacks move from heap or alias into the stack region by copy; the
|
|
// source is inaccessible while moved and gets the contents back on unmap.
|
|
static int alias_all_mapped(uint64_t a, uint64_t n) { // under lock
|
|
uint64_t first = (a - L.alias.base) / PAGE;
|
|
for (uint64_t i = 0; i < n / PAGE; i++)
|
|
if (!alias_pages[first + i]) return 0;
|
|
return 1;
|
|
}
|
|
|
|
static uint32_t stack_move(int unmap, uint64_t dst, uint64_t src, uint64_t n) {
|
|
if (!n || ((dst | src | n) & (PAGE - 1))) return R_INVALID_ADDRESS;
|
|
if (!inside(dst, n, L.stack)) return R_INVALID_REGION;
|
|
int in_heap = inside(src, n, (RtRange){L.heap.base, heap_size});
|
|
int in_alias = inside(src, n, L.alias);
|
|
if (!in_heap && !in_alias) return R_INVALID_REGION;
|
|
plat_mutex_lock(&lock);
|
|
uint32_t r = R_OK;
|
|
if (in_alias && !alias_all_mapped(src, n)) {
|
|
r = R_INVALID_REGION;
|
|
goto out;
|
|
}
|
|
unsigned found = nstack_maps;
|
|
for (unsigned i = 0; i < nstack_maps; i++)
|
|
if (stack_maps[i].dst == dst && stack_maps[i].src == src && stack_maps[i].size == n) found = i;
|
|
if (unmap) {
|
|
if (found == nstack_maps) {
|
|
r = R_INVALID_STATE;
|
|
goto out;
|
|
}
|
|
protect(src, n, PLAT_PROT_RW, "moved stack source");
|
|
memcpy((void *)(uintptr_t)src, (const void *)(uintptr_t)dst, n);
|
|
decommit(dst, n, "moved stack");
|
|
stack_maps[found] = stack_maps[--nstack_maps];
|
|
} else {
|
|
if (nstack_maps == MAX_STACK_MAPS) {
|
|
r = R_INVALID_STATE;
|
|
goto out;
|
|
}
|
|
for (unsigned i = 0; i < nstack_maps; i++)
|
|
if (overlaps(dst, n, stack_maps[i].dst, stack_maps[i].size) ||
|
|
overlaps(src, n, stack_maps[i].src, stack_maps[i].size)) {
|
|
r = R_INVALID_REGION;
|
|
goto out;
|
|
}
|
|
commit(dst, n, "moved stack");
|
|
memcpy((void *)(uintptr_t)dst, (const void *)(uintptr_t)src, n);
|
|
protect(src, n, PLAT_PROT_NONE, "moved stack source");
|
|
stack_maps[nstack_maps++] = (StackMap){dst, src, n};
|
|
}
|
|
out:
|
|
plat_mutex_unlock(&lock);
|
|
return r;
|
|
}
|
|
|
|
uint32_t mem_map(uint64_t dst, uint64_t src, uint64_t n) { return stack_move(0, dst, src, n); }
|
|
uint32_t mem_unmap(uint64_t dst, uint64_t src, uint64_t n) { return stack_move(1, dst, src, n); }
|
|
|
|
// ---- 0x2C/0x2D MapPhysicalMemory, UnmapPhysicalMemory. Horizon maps only the still-unmapped pages of the range and
|
|
// leaves mapped pages and their contents alone; NX2navi's heaps grow by re-mapping ranges they partly own.
|
|
static void alias_runs(uint64_t first, uint64_t count, uint8_t want, int do_commit) { // under lock
|
|
uint64_t i = 0;
|
|
while (i < count) {
|
|
if (alias_pages[first + i] != want) { i++; continue; }
|
|
uint64_t j = i;
|
|
while (j < count && alias_pages[first + j] == want) j++;
|
|
uint64_t a = L.alias.base + (first + i) * PAGE, n = (j - i) * PAGE;
|
|
if (do_commit) commit(a, n, "physical memory");
|
|
else decommit(a, n, "physical memory");
|
|
memset(alias_pages + first + i, !want, j - i);
|
|
i = j;
|
|
}
|
|
}
|
|
|
|
static uint32_t alias_map(int unmap, uint64_t a, uint64_t n) {
|
|
if (!n) return R_INVALID_ADDRESS;
|
|
n = (n + PAGE - 1) & ~(uint64_t)(PAGE - 1); // the SDK asks for byte ranges; a mapping is whole pages
|
|
if ((a | n) & (PAGE - 1)) return R_INVALID_ADDRESS;
|
|
if (!inside(a, n, L.alias)) return R_INVALID_REGION;
|
|
uint64_t first = (a - L.alias.base) / PAGE, count = n / PAGE;
|
|
plat_mutex_lock(&lock);
|
|
uint32_t r = R_OK;
|
|
if (unmap) {
|
|
for (unsigned i = 0; i < nstack_maps; i++)
|
|
if (overlaps(a, n, stack_maps[i].src, stack_maps[i].size)) {
|
|
r = R_INVALID_REGION;
|
|
goto out;
|
|
}
|
|
uint64_t released = 0;
|
|
for (uint64_t i = 0; i < count; i++) released += alias_pages[first + i];
|
|
alias_runs(first, count, 1, 0);
|
|
alias_bytes -= released * PAGE;
|
|
} else {
|
|
uint64_t fresh = 0;
|
|
for (uint64_t i = 0; i < count; i++) fresh += !alias_pages[first + i];
|
|
if (fresh * PAGE + alias_bytes + heap_size + image_bytes() > L.pool) {
|
|
r = R_LIMIT_REACHED; // the SDK maps LimitReached to its recoverable out-of-memory result
|
|
goto out;
|
|
}
|
|
alias_runs(first, count, 0, 1);
|
|
alias_bytes += fresh * PAGE;
|
|
}
|
|
out:
|
|
plat_mutex_unlock(&lock);
|
|
if (r) rt_log("%sMapPhysicalMemory %#llx+%#llx failed %#x (alias %#llx used, heap %#llx)", unmap ? "Un" : "",
|
|
(unsigned long long)a, (unsigned long long)n, r, (unsigned long long)alias_bytes,
|
|
(unsigned long long)heap_size);
|
|
return r;
|
|
}
|
|
|
|
uint32_t mem_map_physical(uint64_t a, uint64_t n) { return alias_map(0, a, n); }
|
|
uint32_t mem_unmap_physical(uint64_t a, uint64_t n) { return alias_map(1, a, n); }
|
|
|
|
// ---- 0x06 QueryMemory
|
|
// The region around a that no known region covers: everything else below 2^39 belongs to the host and is reported
|
|
// as taken (Code, RX), so the SDK never picks it.
|
|
static void taken_gap(uint64_t a, uint64_t *base, uint64_t *size) {
|
|
uint64_t lo = 0, hi = RT_ADDRESS_LIMIT;
|
|
RtRange known[3 + MOD_COUNT + 2];
|
|
int n = 0;
|
|
known[n++] = (RtRange){L.aslr.base, LOW_SIZE};
|
|
known[n++] = L.alias;
|
|
known[n++] = L.kernel;
|
|
known[n++] = L.hostfn;
|
|
for (int i = 0; i < ldr_count(); i++) known[n++] = (RtRange){ldr_module(i)->base, ldr_module(i)->size};
|
|
for (int i = 0; i < n; i++) {
|
|
uint64_t e = known[i].base + known[i].size;
|
|
if (e <= a && e > lo) lo = e;
|
|
if (known[i].base > a && known[i].base < hi) hi = known[i].base;
|
|
}
|
|
*base = lo;
|
|
*size = hi - lo;
|
|
}
|
|
|
|
static uint64_t run_end(const uint8_t *p, uint64_t i, uint64_t n, uint8_t v) {
|
|
while (i < n && (i & 7)) { if (p[i] != v) return i; i++; }
|
|
uint64_t w = v ? 0x0101010101010101ull : 0;
|
|
while (i + 8 <= n && *(const uint64_t *)(p + i) == w) i += 8;
|
|
while (i < n && p[i] == v) i++;
|
|
return i;
|
|
}
|
|
|
|
static uint64_t run_start(const uint8_t *p, uint64_t i, uint8_t v) { // first index of the run that ends at i
|
|
while (i > 0 && (i & 7)) { if (p[i - 1] != v) return i; i--; }
|
|
uint64_t w = v ? 0x0101010101010101ull : 0;
|
|
while (i >= 8 && *(const uint64_t *)(p + i - 8) == w) i -= 8;
|
|
while (i > 0 && p[i - 1] == v) i--;
|
|
return i;
|
|
}
|
|
|
|
static void query_locked(uint64_t a, RtMemoryInfo *out);
|
|
|
|
static int readable_locked(uint64_t a, uint64_t n) {
|
|
for (uint64_t end = a + n; a < end;) {
|
|
RtMemoryInfo r;
|
|
query_locked(a, &r);
|
|
if (r.state == ST_FREE || r.state == ST_INACCESSIBLE || !(r.perm & PERM_R) || r.base + r.size <= a) return 0;
|
|
a = r.base + r.size;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
// Whether every byte of [a, a + n) can be read: no free, inaccessible or no-access page (mem_query's regions). For
|
|
// host reads of guest memory the guest itself may never make, such as a texture upload of a pool slot no shader
|
|
// samples (vk_exec_draw.c, texture pools: stale slots point at unmapped memory).
|
|
int mem_readable(uint64_t a, uint64_t n) {
|
|
plat_mutex_lock_shared(&lock);
|
|
int ok = readable_locked(a, n);
|
|
plat_mutex_unlock_shared(&lock);
|
|
return ok;
|
|
}
|
|
|
|
// mem_readable for a read that takes a while on another thread than the guest's: with 1 the range stays mapped until
|
|
// mem_read_release, as unmapping it (UnmapPhysicalMemory, a smaller heap, a moved stack) waits for the lock. A pool
|
|
// slot's storage the game has just released could otherwise be decommitted halfway through the copy (a texture
|
|
// upload faulted that way in tex_bl_copy). In between, the caller takes no lock of this file (mem_query, mem_readable
|
|
// and the mapping calls would wait for themselves). 0: not readable, and nothing held.
|
|
int mem_read_hold(uint64_t a, uint64_t n) {
|
|
plat_mutex_lock_shared(&lock);
|
|
if (readable_locked(a, n)) return 1;
|
|
plat_mutex_unlock_shared(&lock);
|
|
return 0;
|
|
}
|
|
|
|
void mem_read_release(void) { plat_mutex_unlock_shared(&lock); }
|
|
|
|
uint32_t mem_query(uint64_t a, RtMemoryInfo *out) {
|
|
plat_mutex_lock_shared(&lock);
|
|
query_locked(a, out);
|
|
plat_mutex_unlock_shared(&lock);
|
|
return R_OK;
|
|
}
|
|
|
|
static void query_locked(uint64_t a, RtMemoryInfo *out) {
|
|
RtMemoryInfo r = {.base = a & ~(PAGE - 1), .size = PAGE};
|
|
uint64_t low = L.aslr.base;
|
|
int mi;
|
|
if (a >= RT_ADDRESS_LIMIT) {
|
|
r.base = RT_ADDRESS_LIMIT;
|
|
r.size = 0 - RT_ADDRESS_LIMIT;
|
|
r.state = ST_INACCESSIBLE;
|
|
} else if (a < low) {
|
|
r.base = 0;
|
|
r.size = low;
|
|
r.state = ST_FREE;
|
|
} else if (a < low + LOW_SIZE) {
|
|
uint64_t heap = L.heap.base;
|
|
if (a >= heap && a < heap + heap_size) {
|
|
r = (RtMemoryInfo){.base = heap, .size = heap_size, .state = ST_NORMAL, .perm = PERM_RW};
|
|
} else if (a < heap) {
|
|
r = (RtMemoryInfo){.base = low, .size = heap - low, .state = ST_FREE};
|
|
} else {
|
|
r = (RtMemoryInfo){.base = heap + heap_size, .size = low + LOW_SIZE - heap - heap_size, .state = ST_FREE};
|
|
}
|
|
} else if (a - L.alias.base < L.alias.size) {
|
|
uint64_t p = (a - L.alias.base) / PAGE;
|
|
uint8_t v = alias_pages[p];
|
|
uint64_t s = run_start(alias_pages, p, v), e = run_end(alias_pages, p, ALIAS_PAGES, v);
|
|
r = (RtMemoryInfo){.base = L.alias.base + s * PAGE, .size = (e - s) * PAGE, .state = v ? ST_NORMAL : ST_FREE,
|
|
.perm = v ? PERM_RW : PERM_NONE};
|
|
} else if ((mi = module_index(a)) >= 0) {
|
|
const RtLoaded *m = ldr_module(mi);
|
|
uint64_t off = (a - m->base) & ~(PAGE - 1), lo = off, hi = off + PAGE;
|
|
unsigned perm = module_page_perm(mi, off);
|
|
int data = off >= m->data_off;
|
|
uint64_t limit = data ? m->data_off : 0, end = data ? m->size : m->data_off;
|
|
while (lo > limit && module_page_perm(mi, lo - PAGE) == perm) lo -= PAGE;
|
|
while (hi < end && module_page_perm(mi, hi) == perm) hi += PAGE;
|
|
r = (RtMemoryInfo){.base = m->base + lo, .size = hi - lo, .state = data ? ST_CODEDATA : ST_CODE,
|
|
.perm = perm & 7, .attr = perm & ATTR_PERMISSION_LOCKED};
|
|
} else if (a - L.kernel.base < L.kernel.size) {
|
|
r = (RtMemoryInfo){.base = L.kernel.base, .size = L.kernel.size, .state = ST_THREADLOCAL, .perm = PERM_RW};
|
|
} else if (a - L.hostfn.base < L.hostfn.size) {
|
|
r = (RtMemoryInfo){.base = L.hostfn.base, .size = L.hostfn.size, .state = ST_INACCESSIBLE};
|
|
} else {
|
|
taken_gap(a, &r.base, &r.size);
|
|
r.state = ST_CODE;
|
|
r.perm = PERM_RX;
|
|
}
|
|
// Moved stacks: the destination is Stack, the source Normal without access; free and normal regions around them
|
|
// are clipped.
|
|
for (unsigned i = 0; i < nstack_maps; i++) {
|
|
const StackMap *m = &stack_maps[i];
|
|
uint64_t end = m->dst + m->size, send = m->src + m->size;
|
|
if (a >= m->dst && a < end) {
|
|
r = (RtMemoryInfo){.base = m->dst, .size = m->size, .state = ST_STACK, .perm = PERM_RW};
|
|
break;
|
|
}
|
|
if (a >= m->src && a < send) {
|
|
r = (RtMemoryInfo){.base = m->src, .size = m->size, .state = ST_NORMAL, .perm = PERM_NONE, .attr = 4};
|
|
break;
|
|
}
|
|
uint64_t lo = r.base, hi = r.base + r.size;
|
|
uint64_t b = r.state == ST_FREE ? m->dst : r.state == ST_NORMAL ? m->src : 0;
|
|
uint64_t e = b + m->size;
|
|
if (!b) continue;
|
|
if (e <= a && e > lo) lo = e;
|
|
if (b > a && b < hi) hi = b;
|
|
r.base = lo;
|
|
r.size = hi - lo;
|
|
}
|
|
if (out) *out = r;
|
|
}
|
|
|
|
// ---- GetInfo memory ids
|
|
int mem_info(uint32_t id, uint64_t *out) {
|
|
uint64_t v;
|
|
switch (id) {
|
|
case 2: v = L.alias.base; break;
|
|
case 3: v = L.alias.size; break;
|
|
case 4: v = L.heap.base; break;
|
|
case 5: v = L.heap.size; break;
|
|
case 6: case 21: v = L.pool; break;
|
|
case 7: case 22:
|
|
plat_mutex_lock_shared(&lock);
|
|
v = image_bytes() + heap_size + alias_bytes;
|
|
plat_mutex_unlock_shared(&lock);
|
|
break;
|
|
case 12: v = L.aslr.base; break;
|
|
case 13: v = L.aslr.size; break;
|
|
case 14: v = L.stack.base; break;
|
|
case 15: v = L.stack.size; break;
|
|
case 16: v = SYSTEM_RESOURCE_BUDGET; break;
|
|
case 17: v = sizeof stack_maps + ALIAS_PAGES; break; // system resource in use: our bookkeeping
|
|
case 28: v = 0; break; // AliasRegionExtraSize: no kernel-owned extension
|
|
default: return 0;
|
|
}
|
|
*out = v;
|
|
return 1;
|
|
}
|
|
|
|
// ---- kernel region: power-of-two blocks, a free list per size, bump allocation with lazy commit.
|
|
#define KCLASSES 29 // 64 B .. 256 MiB
|
|
#define KMIN_SHIFT 6
|
|
#define KCOMMIT 0x10000ull
|
|
typedef struct KFree { struct KFree *next; } KFree;
|
|
static KFree *kfree_list[KCLASSES];
|
|
static uint64_t kbump, kcommitted;
|
|
static PlatMutex klock = PLAT_MUTEX_INIT;
|
|
|
|
static int kclass(uint64_t size) {
|
|
int s = KMIN_SHIFT;
|
|
while ((1ull << s) < size) s++;
|
|
return s - KMIN_SHIFT;
|
|
}
|
|
|
|
void *mem_kernel_alloc(uint64_t size, uint64_t align) {
|
|
if (!L.pool) rt_fatal("mem_kernel_alloc before mem_init");
|
|
if (align > KCOMMIT || (align & (align - 1))) rt_fatal("mem_kernel_alloc: alignment %#llx", (unsigned long long)align);
|
|
uint64_t want = size > align ? size : align;
|
|
int k = kclass(want ? want : 1);
|
|
if (k >= KCLASSES) rt_fatal("mem_kernel_alloc: %#llx bytes", (unsigned long long)size);
|
|
uint64_t bytes = 1ull << (k + KMIN_SHIFT), al = bytes < KCOMMIT ? bytes : KCOMMIT;
|
|
plat_mutex_lock(&klock);
|
|
KFree *f = kfree_list[k];
|
|
void *p;
|
|
if (f) {
|
|
kfree_list[k] = f->next;
|
|
p = f;
|
|
} else {
|
|
uint64_t a = align_up(kbump ? kbump : RT_KERNEL_BASE, al);
|
|
if (a + bytes > RT_KERNEL_BASE + RT_KERNEL_SIZE) rt_fatal("kernel region exhausted (%#llx bytes)",
|
|
(unsigned long long)bytes);
|
|
uint64_t need = align_up(a + bytes, KCOMMIT);
|
|
if (!kcommitted) kcommitted = RT_KERNEL_BASE;
|
|
if (need > kcommitted) {
|
|
commit(kcommitted, need - kcommitted, "kernel region");
|
|
kcommitted = need;
|
|
}
|
|
kbump = a + bytes;
|
|
p = (void *)(uintptr_t)a;
|
|
}
|
|
plat_mutex_unlock(&klock);
|
|
memset(p, 0, size ? size : 1);
|
|
return p;
|
|
}
|
|
|
|
void mem_kernel_free(void *p, uint64_t size) {
|
|
if (!p) return;
|
|
uint64_t a = (uintptr_t)p;
|
|
if (a - RT_KERNEL_BASE >= RT_KERNEL_SIZE) rt_fatal("mem_kernel_free(%p): not in the kernel region", p);
|
|
int k = kclass(size ? size : 1);
|
|
// A block allocated with a larger alignment sits in a larger class; freeing it into the smaller class only
|
|
// wastes the rest.
|
|
plat_mutex_lock(&klock);
|
|
KFree *f = p;
|
|
f->next = kfree_list[k];
|
|
kfree_list[k] = f;
|
|
plat_mutex_unlock(&klock);
|
|
}
|
|
|
|
// NX2NAVI_TRACE=perf: how many bytes of each guest region are committed, so a process that has grown says which region
|
|
// holds the memory. A region is reserved whole at start-up and committed page by page, so the walk reads each run of
|
|
// pages of the same state; the alias window is the largest and costs the most steps when it is full.
|
|
void mem_guest_census(uint64_t frame) {
|
|
const RtGuestLayout *g = mem_layout();
|
|
const struct { const char *name; RtRange r; } reg[] = {
|
|
{"aslr", g->aslr}, {"stack", g->stack}, {"heap", g->heap},
|
|
{"alias", g->alias}, {"kernel", g->kernel}, {"hostfn", g->hostfn},
|
|
};
|
|
char line[400];
|
|
int o = snprintf(line, sizeof line, "mem: guest regions at frame %llu (MiB committed):",
|
|
(unsigned long long)frame);
|
|
uint64_t all = 0;
|
|
for (unsigned i = 0; i < sizeof reg / sizeof *reg && o < (int)sizeof line - 40; i++) {
|
|
uint64_t x = reg[i].r.base, end = reg[i].r.base + reg[i].r.size, committed = 0, b = 0;
|
|
while (x < end) {
|
|
int st = plat_mem_state(x, end, &b);
|
|
if (b <= x) break;
|
|
if (st >= 0) committed += b - x; // a PLAT_PROT_* state: these pages are committed
|
|
x = b;
|
|
}
|
|
all += committed;
|
|
o += snprintf(line + o, sizeof line - o, " %s %.0f", reg[i].name, committed / 1048576.0);
|
|
}
|
|
rt_log("%s; all %.0f", line, all / 1048576.0);
|
|
}
|