mirror of
https://github.com/tangerineturtle1/NX2navi.git
synced 2026-11-16 11:49:59 -04:00
Every log line was two writes on the thread that logged it, to stderr and to the log file. Such a write can stall for tens or hundreds of milliseconds (a busy disk, a virus scanner, a console or a pipe that is slow to take it), and the render thread logs too: a frame it spent in a write was a hitch with nothing else to show for it. In a game run the lines now go into a queue that a log thread writes out in order (runtime/logq.c). A fatal error, the crash report and the end of the process write the queue out on their own thread first and write their own lines directly, so nothing is lost to an exit. A thread that finds the queue's lock held for long, or the queue full, writes its line itself. NX2NAVI_LOG_ASYNC=0 writes every line where it is logged, as before. Every write is timed in both modes: a "log: ... writes of the log took over 2 ms" line counts the slow ones, gives the longest and says whose time they took. tests/logq_check.py checks order and completeness with eight threads, a slow destination, a stopped destination with the queue full, and the synchronous flush. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
142 lines
5.9 KiB
C
142 lines
5.9 KiB
C
// Log writes on a thread of their own.
|
|
//
|
|
// A log line was written by the thread that logged it: to stderr and to the log file (main.c log_sink). A write can
|
|
// stall: a console that scrolls, a pipe whose reader is busy, a virus scanner, a disk that flushes; hundreds of
|
|
// milliseconds at worst. The render thread logs too, and a frame it spends in such a write is a hitch with nothing
|
|
// else to show for it.
|
|
//
|
|
// With the queue on (logq_async) a line is copied into a buffer under a lock that is held for the copy alone, and the
|
|
// "log" thread writes the buffer out, so lines keep the order they were queued in. logq_sync writes what is queued on
|
|
// the calling thread and turns the queue off for the rest of the run: a fatal error, the crash report and the end of
|
|
// the process call it (rt_log_sync), so no line is lost to an exit and their own lines are written as they are made.
|
|
//
|
|
// Every write is timed. One that took LOGQ_SLOW_NS or longer is counted, and a line says so at most every ten
|
|
// seconds: with the queue that time was the log thread's, without it (NX2NAVI_LOG_ASYNC=0) the logging thread's.
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "host.h"
|
|
#include "logq.h"
|
|
#include "platform/platform.h"
|
|
|
|
#define LOGQ_CAP (1u << 20)
|
|
#define LOGQ_SLOW_NS 2000000ll
|
|
#define LOGQ_SAY_NS 10000000000ull
|
|
|
|
static struct {
|
|
PlatMutex q, io; // q: buf and n; io: the queue's writes and out. io is taken first
|
|
PlatCond more;
|
|
char *buf, *out; // LOGQ_CAP bytes each: the queue, and what the log thread is writing
|
|
size_t n;
|
|
LogqSink sink;
|
|
volatile int32_t on, saying;
|
|
volatile int64_t slow, said, worst_ns, direct; // slow writes, those reported, the longest; lines written past a busy queue
|
|
volatile uint64_t said_at;
|
|
} Q = {.q = PLAT_MUTEX_INIT, .io = PLAT_MUTEX_INIT, .more = PLAT_COND_INIT};
|
|
|
|
void logq_init(LogqSink sink) { Q.sink = sink; }
|
|
|
|
// The write itself, timed.
|
|
static void put_out(const char *p, size_t n) {
|
|
uint64_t t0 = plat_now_ns();
|
|
Q.sink(p, n);
|
|
int64_t d = (int64_t)(plat_now_ns() - t0);
|
|
if (d < LOGQ_SLOW_NS) return;
|
|
__atomic_add_fetch(&Q.slow, 1, __ATOMIC_RELAXED);
|
|
int64_t w = __atomic_load_n(&Q.worst_ns, __ATOMIC_RELAXED);
|
|
while (d > w && !__atomic_compare_exchange_n(&Q.worst_ns, &w, d, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {}
|
|
}
|
|
|
|
// The line about slow writes, when there are new ones and the last line is ten seconds old. Called with no lock held:
|
|
// the line goes through logq_write like any other.
|
|
static void say_slow(void) {
|
|
int64_t slow = __atomic_load_n(&Q.slow, __ATOMIC_RELAXED);
|
|
if (slow == __atomic_load_n(&Q.said, __ATOMIC_RELAXED)) return;
|
|
uint64_t now = plat_now_ns(), at = __atomic_load_n(&Q.said_at, __ATOMIC_RELAXED);
|
|
if ((at && now - at < LOGQ_SAY_NS) || plat_xchg32(&Q.saying, 1)) return;
|
|
__atomic_store_n(&Q.said, slow, __ATOMIC_RELAXED);
|
|
__atomic_store_n(&Q.said_at, now ? now : 1, __ATOMIC_RELAXED);
|
|
rt_log("log: %lld writes of the log took over %d ms so far, the longest %.1f ms, %s (NX2NAVI_LOG_ASYNC); %lld lines "
|
|
"were written past a busy queue", (long long)slow, (int)(LOGQ_SLOW_NS / 1000000),
|
|
__atomic_load_n(&Q.worst_ns, __ATOMIC_RELAXED) / 1e6,
|
|
__atomic_load_n(&Q.on, __ATOMIC_ACQUIRE) ? "on the log thread" : "on the threads that logged",
|
|
(long long)__atomic_load_n(&Q.direct, __ATOMIC_RELAXED));
|
|
plat_xchg32(&Q.saying, 0);
|
|
}
|
|
|
|
// Writes out what is queued, in order. Any thread.
|
|
static void drain(void) {
|
|
plat_mutex_lock(&Q.io);
|
|
plat_mutex_lock(&Q.q);
|
|
char *b = Q.buf;
|
|
size_t n = Q.n;
|
|
Q.buf = Q.out, Q.out = b, Q.n = 0;
|
|
plat_mutex_unlock(&Q.q);
|
|
if (n) put_out(b, n);
|
|
plat_mutex_unlock(&Q.io);
|
|
}
|
|
|
|
// 1 when the bytes were queued, 0 when the caller is to write them. The queue's lock is only tried: a thread that logs
|
|
// while the lock's holder cannot run (stopped for a profile sample or a core hand-over) writes its line itself.
|
|
static int put(const char *p, size_t n) {
|
|
if (!__atomic_load_n(&Q.on, __ATOMIC_ACQUIRE) || n > LOGQ_CAP / 4) return 0;
|
|
for (int tries = 0; !plat_mutex_trylock(&Q.q); tries++) {
|
|
if (tries == 4000) {
|
|
__atomic_add_fetch(&Q.direct, 1, __ATOMIC_RELAXED);
|
|
return 0;
|
|
}
|
|
plat_pause();
|
|
}
|
|
int on = __atomic_load_n(&Q.on, __ATOMIC_ACQUIRE);
|
|
if (!on || Q.n + n > LOGQ_CAP) { // full: the log thread is held up in a write. What is queued goes out first
|
|
plat_mutex_unlock(&Q.q);
|
|
if (on) drain();
|
|
return 0;
|
|
}
|
|
int first = !Q.n;
|
|
memcpy(Q.buf + Q.n, p, n);
|
|
Q.n += n;
|
|
plat_mutex_unlock(&Q.q);
|
|
if (first) plat_cond_signal(&Q.more);
|
|
return 1;
|
|
}
|
|
|
|
void logq_write(const char *p, size_t n) {
|
|
if (put(p, n)) return;
|
|
put_out(p, n);
|
|
say_slow();
|
|
}
|
|
|
|
static void log_main(void *arg) {
|
|
(void)arg;
|
|
for (;;) {
|
|
plat_mutex_lock(&Q.q);
|
|
while (!Q.n) plat_cond_wait(&Q.more, &Q.q);
|
|
plat_mutex_unlock(&Q.q);
|
|
drain();
|
|
say_slow();
|
|
}
|
|
}
|
|
|
|
int logq_async(void) {
|
|
if (!Q.sink || __atomic_load_n(&Q.on, __ATOMIC_ACQUIRE)) return 0;
|
|
if (!Q.buf) Q.buf = malloc(LOGQ_CAP), Q.out = malloc(LOGQ_CAP);
|
|
if (!Q.buf || !Q.out || !plat_thread_spawn(log_main, NULL, "log", 0)) return 0;
|
|
__atomic_store_n(&Q.on, 1, __ATOMIC_RELEASE);
|
|
return 1;
|
|
}
|
|
|
|
void logq_sync(void) {
|
|
if (!plat_xchg32(&Q.on, 0)) return;
|
|
// Each lock is tried for half a second: the thread that holds it may be the one that crashed, or be stuck in the
|
|
// write this call is there to get past. Without the lock the queue is written as it stands.
|
|
int io = 0, q = 0;
|
|
for (int i = 0; i < 500 && !(io = plat_mutex_trylock(&Q.io)); i++) plat_sleep_ms(1);
|
|
for (int i = 0; i < 500 && !(q = plat_mutex_trylock(&Q.q)); i++) plat_sleep_ms(1);
|
|
size_t n = Q.n;
|
|
Q.n = 0;
|
|
if (n && n <= LOGQ_CAP) put_out(Q.buf, n);
|
|
if (q) plat_mutex_unlock(&Q.q);
|
|
if (io) plat_mutex_unlock(&Q.io);
|
|
}
|