Add one anti-aliasing setting: the game's SMAA, off, FXAA, SMAA at its Ultra preset, or TAA.

Ocarina anti-aliases every frame with its own SMAA in three compute passes after the bloom composite. The new aa
setting (nx2navi.ini aa, the Screen tab's Anti-aliasing row) keeps that by default. off, fxaa and smaa leave out the
game's edge and weight passes and replace its composition with a pass of our own that writes the same square-rooted
RGB9E5 words into the scene target: a plain copy, an edge smoothing written here, or the SMAA reference implementation
with its area and search textures (MIT). taa is DLSS or FSR 3.1 at the scene's size, which the row sets through the
upscaler; while either runs, the game's SMAA stays whatever aa says. The game's passes are known by a hash of their
guest code. The shader build now also rebuilds a header when a file its source includes changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chrg6
2026-11-11 13:00:56 -05:00
committed by tangerineturtle1
parent d3e50328bf
commit bfe88e503a
23 changed files with 2168 additions and 10 deletions

View File

@@ -0,0 +1,19 @@
// Shared by the anti-aliasing passes (runtime/vk_aa.c) that stand in for the game's SMAA. The game's composition pass
// stores the post-processed picture into its R11G11B10F scene target as raw words: each channel is the square root of
// its linear value, the three packed together as RGB9E5 (red in bits 0-8, green 9-17, blue 18-26, the shared exponent
// in 27-31, Vulkan's E5B9G9R9 layout). The passes here write the same words through an R32_UINT view of that target.
// RGB9E5 of a non-negative colour: the shared exponent of the largest channel, each channel rounded to 9 bits of it.
uint aa_pack_rgb9e5(vec3 c) {
const float MAXV = 65408.0; // (2^9 - 1) / 2^9 * 2^15: the largest value the format holds
c = clamp(c, vec3(0.0), vec3(MAXV));
float m = max(max(c.r, c.g), c.b);
float e = max(-16.0, floor(log2(max(m, 1e-30)))) + 16.0; // biased by 15, plus one
float scale = exp2(e - 24.0); // 2^(e - 15 - 9)
if (floor(m / scale + 0.5) >= 512.0) e += 1.0, scale *= 2.0;
uvec3 q = uvec3(floor(c / scale + 0.5));
return q.r | (q.g << 9) | (q.b << 18) | (uint(e) << 27);
}
// The stored form of a linear colour: its square root, which is also the space the edges are looked for in.
vec3 aa_encode(vec3 linear) { return sqrt(max(linear, vec3(0.0))); }

View File

@@ -0,0 +1,100 @@
#version 450
// The game's SMAA composition replaced (runtime/vk_aa.c): mode 0 stores the composite as it is (anti-aliasing off),
// mode 1 runs a fast approximate edge smoothing (FXAA) over it first. Input: the game's composite, linear, already
// tonemapped. Output: the scene target's raw RGB9E5 words of the square root of each channel (aa_common.glsl).
//
// The smoothing works on the square-rooted luma: where the local contrast is high enough, it finds whether the edge
// runs across or along the pixel, walks along the edge both ways until the luma pair it follows changes, and moves the
// sample point toward the neighbour across the edge by how far the pixel sits from the nearer end of the edge. A
// blur of the 3x3 neighbourhood's contrast handles edges thinner than a pixel. The sample at the moved point is
// bilinear, so the move blends the two sides.
#extension GL_GOOGLE_include_directive : require
#include "aa_common.glsl"
layout(local_size_x = 8, local_size_y = 8) in;
layout(binding = 0) uniform sampler2D src;
layout(binding = 1, r32ui) uniform writeonly uimage2D dst;
layout(push_constant) uniform Push {
vec2 inv; // 1 / size
ivec2 size; // the scene target's size
int mode; // 0 copy, 1 FXAA
} pc;
vec3 at(vec2 uv) { return aa_encode(textureLod(src, uv, 0.0).rgb); }
float luma(vec3 c) { return dot(c, vec3(0.299, 0.587, 0.114)); }
float luma_at(vec2 uv) { return luma(at(uv)); }
const float EDGE_MIN = 0.0312; // no smoothing below this contrast (dark areas)
const float EDGE_REL = 0.125; // nor below this share of the brightest neighbour
const float SUBPIXEL = 0.75; // how much a thin feature is blurred
const int STEPS = 12;
const float STEP_LEN[STEPS] = float[](1.0, 1.0, 1.0, 1.0, 1.0, 1.5, 2.0, 2.0, 2.0, 2.0, 4.0, 8.0);
vec3 fxaa(vec2 uv) {
vec3 cM = at(uv);
float lM = luma(cM);
vec2 d = pc.inv;
float lN = luma_at(uv + vec2(0.0, -d.y)), lS = luma_at(uv + vec2(0.0, d.y));
float lW = luma_at(uv + vec2(-d.x, 0.0)), lE = luma_at(uv + vec2(d.x, 0.0));
float lo = min(lM, min(min(lN, lS), min(lW, lE)));
float hi = max(lM, max(max(lN, lS), max(lW, lE)));
float range = hi - lo;
if (range < max(EDGE_MIN, hi * EDGE_REL)) return cM;
float lNW = luma_at(uv + vec2(-d.x, -d.y)), lNE = luma_at(uv + vec2(d.x, -d.y));
float lSW = luma_at(uv + vec2(-d.x, d.y)), lSE = luma_at(uv + vec2(d.x, d.y));
// thin features: how far the centre stands out from the neighbourhood's weighted mean
float mean = (2.0 * (lN + lS + lW + lE) + lNW + lNE + lSW + lSE) / 12.0;
float sub = clamp(abs(mean - lM) / range, 0.0, 1.0);
sub = smoothstep(0.0, 1.0, sub);
sub = sub * sub * SUBPIXEL;
// the edge's direction: horizontal when the luma changes more from row to row than from column to column
float across_rows = abs(0.25 * lNW - 0.5 * lN + 0.25 * lNE) + abs(0.5 * lW - lM + 0.5 * lE) +
abs(0.25 * lSW - 0.5 * lS + 0.25 * lSE);
float across_cols = abs(0.25 * lNW - 0.5 * lW + 0.25 * lSW) + abs(0.5 * lN - lM + 0.5 * lS) +
abs(0.25 * lNE - 0.5 * lE + 0.25 * lSE);
bool horizontal = across_cols >= across_rows;
// which side of the pixel the edge is on: the neighbour across it with the larger step
float l1 = horizontal ? lN : lW, l2 = horizontal ? lS : lE;
float g1 = abs(l1 - lM), g2 = abs(l2 - lM);
bool first = g1 >= g2;
float grad = 0.25 * max(g1, g2);
float step_len = horizontal ? d.y : d.x;
float pair; // the mean luma of the pixel and its neighbour across the edge, which the walk follows
if (first) step_len = -step_len, pair = 0.5 * (l1 + lM);
else pair = 0.5 * (l2 + lM);
vec2 edge = uv;
if (horizontal) edge.y += 0.5 * step_len;
else edge.x += 0.5 * step_len;
// walk along the edge both ways until the pair's luma changes by more than a quarter of the step across it
vec2 dir = horizontal ? vec2(d.x, 0.0) : vec2(0.0, d.y);
vec2 uv1 = edge - dir * STEP_LEN[0], uv2 = edge + dir * STEP_LEN[0];
float e1 = luma_at(uv1) - pair, e2 = luma_at(uv2) - pair;
bool done1 = abs(e1) >= grad, done2 = abs(e2) >= grad;
for (int i = 1; i < STEPS && !(done1 && done2); i++) {
if (!done1) uv1 -= dir * STEP_LEN[i], e1 = luma_at(uv1) - pair, done1 = abs(e1) >= grad;
if (!done2) uv2 += dir * STEP_LEN[i], e2 = luma_at(uv2) - pair, done2 = abs(e2) >= grad;
}
float dist1 = horizontal ? uv.x - uv1.x : uv.y - uv1.y;
float dist2 = horizontal ? uv2.x - uv.x : uv2.y - uv.y;
bool nearer1 = dist1 < dist2;
float offset = 0.5 - min(dist1, dist2) / (dist1 + dist2);
// only when the luma at the nearer end moves the way the centre's does: else the pixel is outside the edge's span
if (((nearer1 ? e1 : e2) < 0.0) == (lM < pair)) offset = 0.0;
offset = max(offset, sub);
vec2 moved = uv;
if (horizontal) moved.y += offset * step_len;
else moved.x += offset * step_len;
return at(moved);
}
void main() {
ivec2 p = ivec2(gl_GlobalInvocationID.xy);
if (p.x >= pc.size.x || p.y >= pc.size.y) return;
vec2 uv = (vec2(p) + 0.5) * pc.inv;
vec3 c = pc.mode == 1 ? fxaa(uv) : aa_encode(texelFetch(src, p, 0).rgb);
imageStore(dst, p, uvec4(aa_pack_rgb9e5(c), 0u, 0u, 0u));
}

1361
runtime/shaders/smaa.glsl Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,19 @@
#version 450
// SMAA pass 3 (runtime/vk_aa.c): neighbourhood blending of the game's composite by the weights, in linear light (as SMAA
// asks of a picture read through an sRGB view), stored into the game's scene target as its raw RGB9E5 words of the
// square root (aa_common.glsl).
#include "smaa_setup.glsl"
#include "aa_common.glsl"
layout(binding = 0) uniform sampler2D color; // the game's composite, linear filtering
layout(binding = 1) uniform sampler2D weights; // linear filtering
layout(binding = 2, r32ui) uniform writeonly uimage2D dst;
void main() {
ivec2 p;
vec2 uv;
if (!smaa_pixel(p, uv)) return;
vec4 offset;
SMAANeighborhoodBlendingVS(uv, offset);
vec3 c = SMAANeighborhoodBlendingPS(uv, offset, color, weights).rgb;
imageStore(dst, p, uvec4(aa_pack_rgb9e5(aa_encode(c)), 0u, 0u, 0u));
}

View File

@@ -0,0 +1,15 @@
#version 450
// SMAA pass 1 (runtime/vk_aa.c): luma edge detection over pass 0's luma into the edges image (red: an edge on the
// pixel's left, green: on its top).
#include "smaa_setup.glsl"
layout(binding = 0) uniform sampler2D luma; // smaa_encode.comp's, linear filtering
layout(binding = 1, rgba8) uniform writeonly image2D edges;
void main() {
ivec2 p;
vec2 uv;
if (!smaa_pixel(p, uv)) return;
vec4 offset[3];
SMAAEdgeDetectionVS(uv, offset);
imageStore(edges, p, vec4(SMAALumaEdgeDetectionPS(uv, offset, luma), 0.0, 0.0));
}

View File

@@ -0,0 +1,18 @@
#version 450
// SMAA pass 0 (runtime/vk_aa.c): the luma SMAA looks for edges in, from the game's linear composite, in the square-root
// space the scene target stores (aa_common.glsl). One channel: SMAA's luma edge detection takes the dot product of the
// colour it reads with its luma weights, and a one-channel image reads as (r, 0, 0), so r holds the luma over the red
// weight.
#extension GL_GOOGLE_include_directive : require
#include "aa_common.glsl"
layout(local_size_x = 8, local_size_y = 8) in;
layout(binding = 0) uniform sampler2D src;
layout(binding = 1, r16f) uniform writeonly image2D dst;
layout(push_constant) uniform Push { vec4 metrics; } pc;
void main() {
ivec2 p = ivec2(gl_GlobalInvocationID.xy);
if (p.x >= int(pc.metrics.z) || p.y >= int(pc.metrics.w)) return;
const vec3 W = vec3(0.2126, 0.7152, 0.0722); // smaa.glsl SMAALumaEdgeDetectionPS's weights
imageStore(dst, p, vec4(dot(aa_encode(texelFetch(src, p, 0).rgb), W) / W.r, 0.0, 0.0, 1.0));
}

View File

@@ -0,0 +1,21 @@
// Settings and resources shared by the SMAA passes (runtime/vk_aa.c; smaa_*.comp). smaa.glsl holds the SMAA reference
// implementation, unmodified; its pixel- and vertex-shader functions run here inside compute shaders, one invocation
// per pixel, with the texture coordinate of the pixel's centre. Its "discard" of a pixel without edges becomes nothing:
// the edge pass then stores the zero edges it found, which is what the next pass reads for such a pixel anyway.
#extension GL_GOOGLE_include_directive : require
layout(local_size_x = 8, local_size_y = 8) in;
layout(push_constant) uniform Push {
vec4 metrics; // 1 / width, 1 / height, width, height
} pc;
#define SMAA_RT_METRICS pc.metrics
#define SMAA_GLSL_4 1
#define SMAA_PRESET_ULTRA 1
#define discard
#include "smaa.glsl"
// the pixel this invocation works on and its centre's texture coordinate; false past the image's edge
bool smaa_pixel(out ivec2 p, out vec2 uv) {
p = ivec2(gl_GlobalInvocationID.xy);
uv = (vec2(p) + 0.5) * pc.metrics.xy;
return p.x < int(pc.metrics.z) && p.y < int(pc.metrics.w);
}

View File

@@ -0,0 +1,17 @@
#version 450
// SMAA pass 2 (runtime/vk_aa.c): blending weights from the edges, with the area and search lookup textures.
#include "smaa_setup.glsl"
layout(binding = 0) uniform sampler2D edges; // linear filtering
layout(binding = 1) uniform sampler2D area; // linear filtering
layout(binding = 2) uniform sampler2D search; // nearest
layout(binding = 3, rgba8) uniform writeonly image2D weights;
void main() {
ivec2 p;
vec2 uv;
if (!smaa_pixel(p, uv)) return;
vec2 pix;
vec4 offset[3];
SMAABlendingWeightCalculationVS(uv, pix, offset);
imageStore(weights, p, SMAABlendingWeightCalculationPS(uv, pix, offset, edges, area, search, vec4(0.0)));
}