diff --git a/docs/audio-input.md b/docs/audio-input.md new file mode 100644 index 00000000..ef51d705 --- /dev/null +++ b/docs/audio-input.md @@ -0,0 +1,30 @@ +# Polling audio input + +The runtime connects Ocarina's polling `nn::audio` input calls to SDL3's default recording device. Select the +microphone or audio interface as the operating system's default input and enable the game's microphone control. +Both a hummed note and an instrument received by that input go through the game's own note recognition. + +Audio capture opens when the game calls `OpenInputStream` and starts only on `StartInputStream`. The runtime does +not write recordings to disk. SDL converts the input device's format to the requested PCM format and sample rate. +The capture callback uses a bounded ring: unread overflow drops the oldest complete frames instead of retaining +an unlimited recording while the game is paused. Stop, purge and close are distinct operations. Purging an unopened +or failed stream is safe, including the null-stream path that previously crashed in `InputStreamImpl::Purge`. + +This implements the non-event polling overload used by Ocarina 1.1.1. Event-based streams, per-device selection, +hot-unplug recovery and packed 24-bit PCM are not implemented. Statistics return a neutral 24-byte structure. +Unsupported formats and a denied or missing recording device return an input error rather than reporting a +successfully opened stream. Input currently follows the operating system's default recording device; there is no +new runtime menu setting. + +## Validation + +`python tests/audio_input_check.py --toolchain C:/msys64/ucrt64` builds and runs headless checks using SDL's dummy +audio driver. They require no game files and do not open a physical microphone. They cover sample ordering across +ring wrap and overflow, frame counts, start/stop, repeated open/close, purge, unsupported formats and purge/read +after a failed open. On Linux, omit `--toolchain` and install SDL3's development package and pkg-config. + +A separate local real-device check received mono signed PCM16 at 48 kHz, verified the 4800-frame ring remained +bounded, and opened and closed the device three times. In an Ocarina 1.1.1 Windows session built from the 0.5.0 +runtime, the player enabled the ocarina microphone control and confirmed that it worked without the previous crash. +The game's actual requested format was mono signed PCM16 at 48 kHz, capacity 4800 frames. The experimental session +does not establish compatibility with other game updates, drivers or input devices. diff --git a/runtime/audio_capture.c b/runtime/audio_capture.c new file mode 100644 index 00000000..229bcafe --- /dev/null +++ b/runtime/audio_capture.c @@ -0,0 +1,156 @@ +#include "audio_capture.h" +#include +#include +#include +#include + +struct MicCapture { + SDL_AudioStream *stream; + unsigned char *ring; + int cap, head, used, bpf; +}; + +const char *mic_error(void) { return SDL_GetError(); } +int mic_device_count(void) { + if (!SDL_InitSubSystem(SDL_INIT_AUDIO)) + return 0; + int n = 0; + SDL_AudioDeviceID *ids = SDL_GetAudioRecordingDevices(&n); + SDL_free(ids); + SDL_QuitSubSystem(SDL_INIT_AUDIO); + return n; +} + +/* The SDL stream lock serializes the callback and every ring operation. + Discard the oldest frames on overflow, so a paused game cannot grow RAM. */ +static void SDLCALL received(void *userdata, SDL_AudioStream *stream, int added, int total) { + (void)added; + (void)total; + MicCapture *m = userdata; + unsigned char scratch[8192]; + int chunk = (int)sizeof scratch / m->bpf * m->bpf; + for (;;) { + int n = SDL_GetAudioStreamData(stream, scratch, chunk); + if (n <= 0) + break; + n -= n % m->bpf; + if (!n) + break; + const unsigned char *src = scratch; + if (n > m->cap) { + src += n - m->cap; + n = m->cap; + } + int drop = m->used + n - m->cap; + if (drop > 0) { + m->head = (m->head + drop) % m->cap; + m->used -= drop; + } + int tail = (m->head + m->used) % m->cap; + int first = n < m->cap - tail ? n : m->cap - tail; + memcpy(m->ring + tail, src, first); + memcpy(m->ring, src + first, n - first); + m->used += n; + } +} + +MicCapture *mic_open(int format, int channels, int rate, int frames) { + SDL_AudioFormat sf; + switch (format) { + case 1: + sf = SDL_AUDIO_S8; + break; + case 2: + sf = SDL_AUDIO_S16LE; + break; + case 4: + sf = SDL_AUDIO_S32LE; + break; + case 5: + sf = SDL_AUDIO_F32LE; + break; + default: + SDL_SetError("unsupported guest sample format %d", format); + return NULL; + } + if (channels < 1 || channels > 8 || rate < 8000 || rate > 192000 || frames < 1) { + SDL_SetError("invalid capture format or capacity"); + return NULL; + } + if (!SDL_InitSubSystem(SDL_INIT_AUDIO)) + return NULL; + MicCapture *m = calloc(1, sizeof *m); + if (!m) { + SDL_QuitSubSystem(SDL_INIT_AUDIO); + return NULL; + } + m->bpf = channels * (SDL_AUDIO_BITSIZE(sf) / 8); + if (frames > rate * 2) + frames = rate * 2; + m->cap = frames * m->bpf; + m->ring = malloc(m->cap); + if (!m->ring) { + free(m); + SDL_QuitSubSystem(SDL_INIT_AUDIO); + return NULL; + } + SDL_AudioSpec spec = {.format = sf, .channels = channels, .freq = rate}; + m->stream = SDL_OpenAudioDeviceStream(SDL_AUDIO_DEVICE_DEFAULT_RECORDING, &spec, received, m); + if (!m->stream) { + free(m->ring); + free(m); + SDL_QuitSubSystem(SDL_INIT_AUDIO); + return NULL; + } + return m; +} +int mic_start(MicCapture *m) { return m && SDL_ResumeAudioStreamDevice(m->stream); } +void mic_stop(MicCapture *m) { + if (m) + SDL_PauseAudioStreamDevice(m->stream); +} +void mic_close(MicCapture *m) { + if (!m) + return; + SDL_DestroyAudioStream(m->stream); /* joins capture before freeing userdata */ + free(m->ring); + free(m); + SDL_QuitSubSystem(SDL_INIT_AUDIO); +} +void mic_purge(MicCapture *m) { + if (!m) + return; + SDL_LockAudioStream(m->stream); + SDL_ClearAudioStream(m->stream); + m->head = m->used = 0; + SDL_UnlockAudioStream(m->stream); +} +int mic_available(MicCapture *m) { + if (!m) + return 0; + SDL_LockAudioStream(m->stream); + int frames = m->used / m->bpf; + SDL_UnlockAudioStream(m->stream); + return frames; +} +int mic_capacity(MicCapture *m) { return m ? m->cap / m->bpf : 0; } +int mic_read(MicCapture *m, void *out, int frames) { + if (!m || !out || frames <= 0) + return 0; + SDL_LockAudioStream(m->stream); + int avail = m->used / m->bpf; + if (frames > avail) + frames = avail; + int n = frames * m->bpf; + int first = n < m->cap - m->head ? n : m->cap - m->head; + memcpy(out, m->ring + m->head, first); + memcpy((unsigned char *)out + first, m->ring, n - first); + m->head = (m->head + n) % m->cap; + m->used -= n; + SDL_UnlockAudioStream(m->stream); + return frames; +} +void mic_gain(MicCapture *m, float gain) { + if (m) + SDL_SetAudioStreamGain(m->stream, gain); +} diff --git a/runtime/audio_capture.h b/runtime/audio_capture.h new file mode 100644 index 00000000..0422b085 --- /dev/null +++ b/runtime/audio_capture.h @@ -0,0 +1,14 @@ +#pragma once +#include +typedef struct MicCapture MicCapture; +int mic_device_count(void); +MicCapture *mic_open(int format, int channels, int rate, int frames); +int mic_start(MicCapture *m); +void mic_stop(MicCapture *m); +void mic_close(MicCapture *m); +void mic_purge(MicCapture *m); +int mic_available(MicCapture *m); +int mic_read(MicCapture *m, void *out, int frames); +int mic_capacity(MicCapture *m); +void mic_gain(MicCapture *m, float gain); +const char *mic_error(void); diff --git a/runtime/ov_audio_input.c b/runtime/ov_audio_input.c new file mode 100644 index 00000000..94916f2a --- /dev/null +++ b/runtime/ov_audio_input.c @@ -0,0 +1,180 @@ +/* Polling audio input for the public nn::audio calls Ocarina imports. + SDL converts the default recording device to the format the caller requests. */ +#include "audio_capture.h" +#include "host.h" +#include "ipc.h" +#include "platform/platform.h" +#include +#include +#include + +#define BAD_INPUT 0x70899u +typedef struct { + uint64_t key; + MicCapture *mic; + int format[3], bpf, running; + uint64_t frames_read, next_log; +} Input; +static Input inputs[8]; +static const char input_name[] = "WindowsDefaultMicrophone"; +static Input *find(uint64_t key) { + for (int i = 0; i < 8; i++) + if (inputs[i].key == key && key) + return &inputs[i]; + return NULL; +} +static void name_out(uint64_t addr) { + char *p = ipc_guest(addr, 256); + memset(p, 0, 256); + memcpy(p, input_name, sizeof input_name); +} +static void list_devices(Cpu *c) { + int n = mic_device_count(); + if ((int)c->x[1] > 0 && n > 0) + name_out(c->x[0]); + rt_log("mic: input enumeration: %d Windows recording devices", n); + ret_x(c, (int)c->x[1] > 0 && n > 0 ? 1 : 0); +} +static void default_device(Cpu *c) { + name_out(c->x[0]); + ret_x(c, 0); +} + +static void close_input(Cpu *c) { + Input *s = find(c->x[0]); + if (s) { + rt_log("mic: close, %llu real audio frames delivered", (unsigned long long)s->frames_read); + mic_close(s->mic); + memset(s, 0, sizeof *s); + } + if (c->x[0]) { + uint64_t zero = 0; + memcpy(ipc_guest(c->x[0], 8), &zero, 8); + } + ret_x(c, 0); +} +static void open_input(Cpu *c) { + uint64_t key = c->x[0]; + int f[3]; + memcpy(f, ipc_guest(c->x[4], 12), 12); + int frames = (int)c->x[5]; + rt_log("mic: OpenInputStream format=%d channels=%d rate=%d capacity=%d", f[0], f[1], f[2], frames); + Input *s = find(key); + if (s) { + mic_close(s->mic); + memset(s, 0, sizeof *s); + } + for (int i = 0; !s && i < 8; i++) + if (!inputs[i].key) + s = &inputs[i]; + uint64_t zero = 0; + memcpy(ipc_guest(key, 8), &zero, 8); + if (!s) { + ret_x(c, BAD_INPUT); + return; + } + MicCapture *m = mic_open(f[0], f[1], f[2], frames); + if (!m) { + rt_log("mic: cannot open Windows default input: %s", mic_error()); + ret_x(c, BAD_INPUT); + return; + } + s->key = key; + s->mic = m; + memcpy(s->format, f, 12); + s->bpf = f[1] * (f[0] == 1 ? 1 : f[0] == 2 ? 2 : 4); + s->next_log = (uint64_t)f[2] * 5; + uint64_t impl = key + 8; + memcpy(ipc_guest(key, 8), &impl, 8); + rt_log("mic: Windows default input opened; bounded buffer %d frames", mic_capacity(m)); + ret_x(c, 0); +} +static void start_input(Cpu *c) { + Input *s = find(c->x[0]); + if (!s || !mic_start(s->mic)) { + rt_log("mic: start failed: %s", mic_error()); + ret_x(c, BAD_INPUT); + return; + } + s->running = 1; + rt_log("mic: capture started"); + ret_x(c, 0); +} +static void stop_input(Cpu *c) { + Input *s = find(c->x[0]); + if (s) { + mic_stop(s->mic); + s->running = 0; + } + ret_x(c, 0); +} +static void purge_input(Cpu *c) { + Input *s = find(c->x[0]); + if (s) + mic_purge(s->mic); + ret_x(c, 0); +} +static void filled(Cpu *c) { + Input *s = find(c->x[0]); + ret_x(c, s ? mic_available(s->mic) : 0); +} +static void state(Cpu *c) { + Input *s = find(c->x[0]); + ret_x(c, s ? (s->running ? 2 : 1) : 0); +} +static void read_input(Cpu *c) { + Input *s = find(c->x[0]); + int frames = (int)c->x[2]; + if (!s || frames <= 0) { + ret_x(c, 0); + return; + } + int available = mic_available(s->mic); + if (frames > available) + frames = available; + if (!frames) { + ret_x(c, 0); + return; + } + int n = mic_read(s->mic, ipc_guest(c->x[1], (uint64_t)frames * s->bpf), frames); + s->frames_read += n; + if (s->frames_read >= s->next_log) { + rt_log("mic: delivered %llu real frames (%d Hz)", (unsigned long long)s->frames_read, s->format[2]); + s->next_log = s->frames_read + (uint64_t)s->format[2] * 30; + } + ret_x(c, n); +} +static void statistics(Cpu *c) { memset(ipc_guest(c->x[8], 24), 0, 24); } + +static PlatMutex input_lock = PLAT_MUTEX_INIT; +#define LOCKED_HANDLER(fn) \ + static void fn##_locked(Cpu *c) { \ + plat_mutex_lock(&input_lock); \ + fn(c); \ + plat_mutex_unlock(&input_lock); \ + } +LOCKED_HANDLER(list_devices) +LOCKED_HANDLER(default_device) +LOCKED_HANDLER(open_input) +LOCKED_HANDLER(close_input) +LOCKED_HANDLER(start_input) +LOCKED_HANDLER(stop_input) +LOCKED_HANDLER(purge_input) +LOCKED_HANDLER(filled) +LOCKED_HANDLER(state) +LOCKED_HANDLER(read_input) +LOCKED_HANDLER(statistics) + +HOST_OVERRIDE("_ZN2nn5audio23ListVirtualInputDevicesEPNS0_15AudioDeviceNameEi", list_devices_locked); +HOST_OVERRIDE("_ZN2nn5audio28GetDefaultVirtualInputDeviceEPNS0_15AudioDeviceNameE", default_device_locked); +HOST_OVERRIDE("_ZN2nn5audio26GetTrueRawAudioInputDeviceEPNS0_15AudioDeviceNameE", default_device_locked); +HOST_OVERRIDE("_ZN2nn5audio15OpenInputStreamEPNS0_15InputStreamTypeEPvmRKNS0_15AudioDeviceNameERKNS0_12StreamFormatEi", + open_input_locked); +HOST_OVERRIDE("_ZN2nn5audio16CloseInputStreamEPNS0_15InputStreamTypeE", close_input_locked); +HOST_OVERRIDE("_ZN2nn5audio16StartInputStreamEPNS0_15InputStreamTypeE", start_input_locked); +HOST_OVERRIDE("_ZN2nn5audio15StopInputStreamEPNS0_15InputStreamTypeE", stop_input_locked); +HOST_OVERRIDE("_ZN2nn5audio16PurgeInputStreamEPNS0_15InputStreamTypeE", purge_input_locked); +HOST_OVERRIDE("_ZN2nn5audio30GetInputStreamFilledFrameCountEPKNS0_15InputStreamTypeE", filled_locked); +HOST_OVERRIDE("_ZN2nn5audio19GetInputStreamStateEPKNS0_15InputStreamTypeE", state_locked); +HOST_OVERRIDE("_ZN2nn5audio15ReadInputStreamEPNS0_15InputStreamTypeEPvi", read_input_locked); +HOST_OVERRIDE("_ZN2nn5audio24GetInputStreamStatisticsEPNS0_15InputStreamTypeEb", statistics_locked); diff --git a/tests/audio_input_check.c b/tests/audio_input_check.c new file mode 100644 index 00000000..900f8ce9 --- /dev/null +++ b/tests/audio_input_check.c @@ -0,0 +1,132 @@ +/* No game data or physical microphone: exercise conversion, bounded capture and the polling ABI. */ +#include "../runtime/audio_capture.c" +#include "../runtime/ov_audio_input.c" +#include +#include +#include +#include + +static SDL_Mutex *registry_lock; +void plat_mutex_lock(PlatMutex *m) { + (void)m; + SDL_LockMutex(registry_lock); +} +void plat_mutex_unlock(PlatMutex *m) { + (void)m; + SDL_UnlockMutex(registry_lock); +} +void *ipc_guest(uint64_t addr, uint64_t size) { + assert(addr && size); + return (void *)(uintptr_t)addr; +} +void rt_log(const char *fmt, ...) { (void)fmt; } + +static void ring_check(void) { + assert(SDL_InitSubSystem(SDL_INIT_AUDIO)); + MicCapture *m = calloc(1, sizeof *m); + m->bpf = 2; + m->cap = 16; + m->ring = malloc(m->cap); + SDL_AudioSpec spec = {.format = SDL_AUDIO_S16LE, .channels = 1, .freq = 48000}; + m->stream = SDL_CreateAudioStream(&spec, &spec); + assert(m->stream); + assert(SDL_SetAudioStreamPutCallback(m->stream, received, m)); + int16_t first[] = {1, 2, 3, 4, 5, 6, 7, 8}, second[] = {9, 10, 11, 12, 13, 14, 15, 16}, out[16] = {0}; + assert(SDL_PutAudioStreamData(m->stream, first, sizeof first)); + assert(mic_available(m) == 8); + assert(mic_read(m, out, 3) == 3); + assert(out[0] == 1 && out[2] == 3); + assert(SDL_PutAudioStreamData(m->stream, second, sizeof second)); + assert(mic_available(m) == 8); /* oldest unread samples were dropped */ + assert(mic_read(m, out, 16) == 8); + assert(memcmp(out, second, sizeof second) == 0); + assert(mic_read(m, out, -1) == 0); + assert(SDL_PutAudioStreamData(m->stream, first, sizeof first)); + mic_purge(m); + assert(mic_available(m) == 0); + mic_close(m); +} + +static void polling_check(void) { + uint64_t opaque[128] = {0}; + int format[3] = {2, 1, 48000}; + int16_t samples[4800]; + Cpu c = {0}; + uint64_t key = (uintptr_t)opaque; + for (int repeat = 0; repeat < 3; repeat++) { + c.x[0] = key; + c.x[4] = (uintptr_t)format; + c.x[5] = 4800; + open_input_locked(&c); + assert(c.x[0] == 0); + c.x[0] = key; + state_locked(&c); + assert(c.x[0] == 1); + c.x[0] = key; + start_input_locked(&c); + assert(c.x[0] == 0); + c.x[0] = key; + state_locked(&c); + assert(c.x[0] == 2); + SDL_Delay(350); /* overflow the 100 ms ring, not system memory */ + c.x[0] = key; + filled_locked(&c); + assert(c.x[0] > 0 && c.x[0] <= 4800); + c.x[0] = key; + c.x[1] = (uintptr_t)samples; + c.x[2] = 4800; + read_input_locked(&c); + assert(c.x[0] > 0 && c.x[0] <= 4800); + c.x[0] = key; + stop_input_locked(&c); + c.x[0] = key; + purge_input_locked(&c); + c.x[0] = key; + filled_locked(&c); + assert(c.x[0] == 0); + c.x[0] = key; + close_input_locked(&c); + assert(opaque[0] == 0); + c.x[0] = key; + purge_input_locked(&c); /* regression: Purge after failed/closed Open */ + } + format[0] = 99; + c.x[0] = key; + c.x[4] = (uintptr_t)format; + c.x[5] = 4800; + open_input_locked(&c); + assert(c.x[0] == BAD_INPUT); + c.x[0] = key; + purge_input_locked(&c); + c.x[0] = key; + state_locked(&c); + assert(c.x[0] == 0); + c.x[0] = key; + c.x[1] = (uintptr_t)samples; + c.x[2] = 4800; + read_input_locked(&c); + assert(c.x[0] == 0); + unsigned char stats[32]; + memset(stats, 0xa5, sizeof stats); + c.x[8] = (uintptr_t)stats; + statistics_locked(&c); + for (int i = 0; i < 24; i++) + assert(stats[i] == 0); + for (int i = 24; i < 32; i++) + assert(stats[i] == 0xa5); + mic_stop(NULL); + mic_purge(NULL); + mic_close(NULL); + assert(mic_available(NULL) == 0); +} + +int main(void) { + assert(SDL_SetHint(SDL_HINT_AUDIO_DRIVER, "dummy")); + registry_lock = SDL_CreateMutex(); + assert(registry_lock); + ring_check(); + polling_check(); + SDL_DestroyMutex(registry_lock); + puts("PASS: sample order, bounded overflow, polling frame units, reopen, purge and failed-open regression"); + return 0; +} diff --git a/tests/audio_input_check.py b/tests/audio_input_check.py new file mode 100644 index 00000000..648d0677 --- /dev/null +++ b/tests/audio_input_check.py @@ -0,0 +1,31 @@ +"""Build and run the input-audio regression checks without game files or recording hardware. + +Windows: python tests/audio_input_check.py --toolchain C:/msys64/ucrt64 +Linux: python tests/audio_input_check.py (clang and SDL3 development files via pkg-config) +""" +import argparse +import os +from pathlib import Path +import shlex +import shutil +import subprocess + +root = Path(__file__).resolve().parents[1] +ap = argparse.ArgumentParser(description=__doc__) +ap.add_argument('--toolchain', type=Path) +a = ap.parse_args() +out = root / 'build' / 'audio-input-check' +out.mkdir(parents=True, exist_ok=True) +env = dict(os.environ) +if a.toolchain: + env['PATH'] = str(a.toolchain / 'bin') + os.pathsep + env.get('PATH', '') + cc = str(a.toolchain / 'bin' / 'clang.exe') + flags = ['-L', str(a.toolchain / 'lib'), '-lSDL3'] +else: + cc = shutil.which('clang') or 'cc' + flags = shlex.split(subprocess.check_output(['pkg-config', '--cflags', '--libs', 'sdl3'], text=True)) +exe = out / ('audio-input-check.exe' if os.name == 'nt' else 'audio-input-check') +subprocess.run([cc, '-std=gnu11', '-O2', '-fno-strict-aliasing', '-Wall', '-Wextra', '-Werror', + '-I', str(root / 'runtime'), str(root / 'tests' / 'audio_input_check.c'), + '-o', str(exe), *flags], env=env, check=True) +subprocess.run([str(exe)], env=env, timeout=15, check=True)