Triforce v0.2.0

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ruckus7382sfgsgdf
2026-11-10 19:32:48 -06:00
commit 34b3683441
1730 changed files with 554853 additions and 0 deletions

1760
gpu/render/nvn_api.h Normal file

File diff suppressed because it is too large Load Diff

875
gpu/render/renderdoc_app.h Normal file
View File

@@ -0,0 +1,875 @@
/******************************************************************************
* The MIT License (MIT)
*
* Copyright (c) 2015-2026 Baldur Karlsson
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
******************************************************************************/
#pragma once
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// Documentation for the API is available at https://renderdoc.org/docs/in_application_api.html
//
#if !defined(RENDERDOC_NO_STDINT)
#include <stdint.h>
#endif
#if defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER)
#define RENDERDOC_CC __cdecl
#elif defined(__linux__) || defined(__FreeBSD__) || defined(__sun__) || defined(__OpenBSD__)
#define RENDERDOC_CC
#elif defined(__APPLE__)
#define RENDERDOC_CC
#else
#error "Unknown platform"
#endif
#ifdef __cplusplus
extern "C" {
#endif
//////////////////////////////////////////////////////////////////////////////////////////////////
// Constants not used directly in below API
// This is a GUID/magic value used for when applications pass a path where shader debug
// information can be found to match up with a stripped shader.
// the define can be used like so: const GUID RENDERDOC_ShaderDebugMagicValue =
// RENDERDOC_ShaderDebugMagicValue_value
#define RENDERDOC_ShaderDebugMagicValue_struct \
{ \
0xeab25520, 0x6670, 0x4865, 0x84, 0x29, 0x6c, 0x8, 0x51, 0x54, 0x00, 0xff \
}
// as an alternative when you want a byte array (assuming x86 endianness):
#define RENDERDOC_ShaderDebugMagicValue_bytearray \
{ \
0x20, 0x55, 0xb2, 0xea, 0x70, 0x66, 0x65, 0x48, 0x84, 0x29, 0x6c, 0x8, 0x51, 0x54, 0x00, 0xff \
}
// truncated version when only a uint64_t is available (e.g. Vulkan tags):
#define RENDERDOC_ShaderDebugMagicValue_truncated 0x48656670eab25520ULL
// this is a magic value for vulkan user tags to indicate which dispatchable API objects are which
// for object annotations
#define RENDERDOC_APIObjectAnnotationHelper 0xfbb3b337b664d0adULL
//////////////////////////////////////////////////////////////////////////////////////////////////
// RenderDoc capture options
//
typedef enum RENDERDOC_CaptureOption
{
// Allow the application to enable vsync
//
// Default - enabled
//
// 1 - The application can enable or disable vsync at will
// 0 - vsync is force disabled
eRENDERDOC_Option_AllowVSync = 0,
// Allow the application to enable fullscreen
//
// Default - enabled
//
// 1 - The application can enable or disable fullscreen at will
// 0 - fullscreen is force disabled
eRENDERDOC_Option_AllowFullscreen = 1,
// Record API debugging events and messages
//
// Default - disabled
//
// 1 - Enable built-in API debugging features and records the results into
// the capture, which is matched up with events on replay
// 0 - no API debugging is forcibly enabled
eRENDERDOC_Option_APIValidation = 2,
eRENDERDOC_Option_DebugDeviceMode = 2, // deprecated name of this enum
// Capture CPU callstacks for API events
//
// Default - disabled
//
// 1 - Enables capturing of callstacks
// 0 - no callstacks are captured
eRENDERDOC_Option_CaptureCallstacks = 3,
// When capturing CPU callstacks, only capture them from actions.
// This option does nothing without the above option being enabled
//
// Default - disabled
//
// 1 - Only captures callstacks for actions.
// Ignored if CaptureCallstacks is disabled
// 0 - Callstacks, if enabled, are captured for every event.
eRENDERDOC_Option_CaptureCallstacksOnlyDraws = 4,
eRENDERDOC_Option_CaptureCallstacksOnlyActions = 4,
// Specify a delay in seconds to wait for a debugger to attach, after
// creating or injecting into a process, before continuing to allow it to run.
//
// 0 indicates no delay, and the process will run immediately after injection
//
// Default - 0 seconds
//
eRENDERDOC_Option_DelayForDebugger = 5,
// Verify buffer access. This includes checking the memory returned by a Map() call to
// detect any out-of-bounds modification, as well as initialising buffers with undefined contents
// to a marker value to catch use of uninitialised memory.
//
// NOTE: This option is only valid for OpenGL and D3D11. Explicit APIs such as D3D12 and Vulkan do
// not do the same kind of interception & checking and undefined contents are really undefined.
//
// Default - disabled
//
// 1 - Verify buffer access
// 0 - No verification is performed, and overwriting bounds may cause crashes or corruption in
// RenderDoc.
eRENDERDOC_Option_VerifyBufferAccess = 6,
// The old name for eRENDERDOC_Option_VerifyBufferAccess was eRENDERDOC_Option_VerifyMapWrites.
// This option now controls the filling of uninitialised buffers with 0xdddddddd which was
// previously always enabled
eRENDERDOC_Option_VerifyMapWrites = eRENDERDOC_Option_VerifyBufferAccess,
// Hooks any system API calls that create child processes, and injects
// RenderDoc into them recursively with the same options.
//
// Default - disabled
//
// 1 - Hooks into spawned child processes
// 0 - Child processes are not hooked by RenderDoc
eRENDERDOC_Option_HookIntoChildren = 7,
// By default RenderDoc only includes resources in the final capture necessary
// for that frame, this allows you to override that behaviour.
//
// Default - disabled
//
// 1 - all live resources at the time of capture are included in the capture
// and available for inspection
// 0 - only the resources referenced by the captured frame are included
eRENDERDOC_Option_RefAllResources = 8,
// **NOTE**: As of RenderDoc v1.1 this option has been deprecated. Setting or
// getting it will be ignored, to allow compatibility with older versions.
// In v1.1 the option acts as if it's always enabled.
//
// By default RenderDoc skips saving initial states for resources where the
// previous contents don't appear to be used, assuming that writes before
// reads indicate previous contents aren't used.
//
// Default - disabled
//
// 1 - initial contents at the start of each captured frame are saved, even if
// they are later overwritten or cleared before being used.
// 0 - unless a read is detected, initial contents will not be saved and will
// appear as black or empty data.
eRENDERDOC_Option_SaveAllInitials = 9,
// In APIs that allow for the recording of command lists to be replayed later,
// RenderDoc may choose to not capture command lists before a frame capture is
// triggered, to reduce overheads. This means any command lists recorded once
// and replayed many times will not be available and may cause a failure to
// capture.
//
// NOTE: This is only true for APIs where multithreading is difficult or
// discouraged. Newer APIs like Vulkan and D3D12 will ignore this option
// and always capture all command lists since the API is heavily oriented
// around it and the overheads have been reduced by API design.
//
// 1 - All command lists are captured from the start of the application
// 0 - Command lists are only captured if their recording begins during
// the period when a frame capture is in progress.
eRENDERDOC_Option_CaptureAllCmdLists = 10,
// Mute API debugging output when the API validation mode option is enabled
//
// Default - enabled
//
// 1 - Mute any API debug messages from being displayed or passed through
// 0 - API debugging is displayed as normal
eRENDERDOC_Option_DebugOutputMute = 11,
// Option to allow vendor extensions to be used even when they may be
// incompatible with RenderDoc and cause corrupted replays or crashes.
//
// Default - inactive
//
// No values are documented, this option should only be used when absolutely
// necessary as directed by a RenderDoc developer.
eRENDERDOC_Option_AllowUnsupportedVendorExtensions = 12,
// Define a soft memory limit which some APIs may aim to keep overhead under where
// possible. Anything above this limit will where possible be saved directly to disk during
// capture.
// This will cause increased disk space use (which may cause a capture to fail if disk space is
// exhausted) as well as slower capture times.
//
// Not all memory allocations may be deferred like this so it is not a guarantee of a memory
// limit.
//
// Units are in MBs, suggested values would range from 200MB to 1000MB.
//
// Default - 0 Megabytes
eRENDERDOC_Option_SoftMemoryLimit = 13,
} RENDERDOC_CaptureOption;
// Sets an option that controls how RenderDoc behaves on capture.
//
// Returns 1 if the option and value are valid
// Returns 0 if either is invalid and the option is unchanged
typedef int(RENDERDOC_CC *pRENDERDOC_SetCaptureOptionU32)(RENDERDOC_CaptureOption opt, uint32_t val);
typedef int(RENDERDOC_CC *pRENDERDOC_SetCaptureOptionF32)(RENDERDOC_CaptureOption opt, float val);
// Gets the current value of an option as a uint32_t
//
// If the option is invalid, 0xffffffff is returned
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetCaptureOptionU32)(RENDERDOC_CaptureOption opt);
// Gets the current value of an option as a float
//
// If the option is invalid, -FLT_MAX is returned
typedef float(RENDERDOC_CC *pRENDERDOC_GetCaptureOptionF32)(RENDERDOC_CaptureOption opt);
typedef enum RENDERDOC_InputButton
{
// '0' - '9' matches ASCII values
eRENDERDOC_Key_0 = 0x30,
eRENDERDOC_Key_1 = 0x31,
eRENDERDOC_Key_2 = 0x32,
eRENDERDOC_Key_3 = 0x33,
eRENDERDOC_Key_4 = 0x34,
eRENDERDOC_Key_5 = 0x35,
eRENDERDOC_Key_6 = 0x36,
eRENDERDOC_Key_7 = 0x37,
eRENDERDOC_Key_8 = 0x38,
eRENDERDOC_Key_9 = 0x39,
// 'A' - 'Z' matches ASCII values
eRENDERDOC_Key_A = 0x41,
eRENDERDOC_Key_B = 0x42,
eRENDERDOC_Key_C = 0x43,
eRENDERDOC_Key_D = 0x44,
eRENDERDOC_Key_E = 0x45,
eRENDERDOC_Key_F = 0x46,
eRENDERDOC_Key_G = 0x47,
eRENDERDOC_Key_H = 0x48,
eRENDERDOC_Key_I = 0x49,
eRENDERDOC_Key_J = 0x4A,
eRENDERDOC_Key_K = 0x4B,
eRENDERDOC_Key_L = 0x4C,
eRENDERDOC_Key_M = 0x4D,
eRENDERDOC_Key_N = 0x4E,
eRENDERDOC_Key_O = 0x4F,
eRENDERDOC_Key_P = 0x50,
eRENDERDOC_Key_Q = 0x51,
eRENDERDOC_Key_R = 0x52,
eRENDERDOC_Key_S = 0x53,
eRENDERDOC_Key_T = 0x54,
eRENDERDOC_Key_U = 0x55,
eRENDERDOC_Key_V = 0x56,
eRENDERDOC_Key_W = 0x57,
eRENDERDOC_Key_X = 0x58,
eRENDERDOC_Key_Y = 0x59,
eRENDERDOC_Key_Z = 0x5A,
// leave the rest of the ASCII range free
// in case we want to use it later
eRENDERDOC_Key_NonPrintable = 0x100,
eRENDERDOC_Key_Divide,
eRENDERDOC_Key_Multiply,
eRENDERDOC_Key_Subtract,
eRENDERDOC_Key_Plus,
eRENDERDOC_Key_F1,
eRENDERDOC_Key_F2,
eRENDERDOC_Key_F3,
eRENDERDOC_Key_F4,
eRENDERDOC_Key_F5,
eRENDERDOC_Key_F6,
eRENDERDOC_Key_F7,
eRENDERDOC_Key_F8,
eRENDERDOC_Key_F9,
eRENDERDOC_Key_F10,
eRENDERDOC_Key_F11,
eRENDERDOC_Key_F12,
eRENDERDOC_Key_Home,
eRENDERDOC_Key_End,
eRENDERDOC_Key_Insert,
eRENDERDOC_Key_Delete,
eRENDERDOC_Key_PageUp,
eRENDERDOC_Key_PageDn,
eRENDERDOC_Key_Backspace,
eRENDERDOC_Key_Tab,
eRENDERDOC_Key_PrtScrn,
eRENDERDOC_Key_Pause,
eRENDERDOC_Key_Max,
} RENDERDOC_InputButton;
// Sets which key or keys can be used to toggle focus between multiple windows
//
// If keys is NULL or num is 0, toggle keys will be disabled
typedef void(RENDERDOC_CC *pRENDERDOC_SetFocusToggleKeys)(RENDERDOC_InputButton *keys, int num);
// Sets which key or keys can be used to capture the next frame
//
// If keys is NULL or num is 0, captures keys will be disabled
typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureKeys)(RENDERDOC_InputButton *keys, int num);
typedef enum RENDERDOC_OverlayBits
{
// This single bit controls whether the overlay is enabled or disabled globally
eRENDERDOC_Overlay_Enabled = 0x1,
// Show the average framerate over several seconds as well as min/max
eRENDERDOC_Overlay_FrameRate = 0x2,
// Show the current frame number
eRENDERDOC_Overlay_FrameNumber = 0x4,
// Show a list of recent captures, and how many captures have been made
eRENDERDOC_Overlay_CaptureList = 0x8,
// Default values for the overlay mask
eRENDERDOC_Overlay_Default = (eRENDERDOC_Overlay_Enabled | eRENDERDOC_Overlay_FrameRate |
eRENDERDOC_Overlay_FrameNumber | eRENDERDOC_Overlay_CaptureList),
// Enable all bits
eRENDERDOC_Overlay_All = 0x7ffffff,
// Disable all bits
eRENDERDOC_Overlay_None = 0,
} RENDERDOC_OverlayBits;
// returns the overlay bits that have been set
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetOverlayBits)(void);
// sets the overlay bits with an and & or mask
typedef void(RENDERDOC_CC *pRENDERDOC_MaskOverlayBits)(uint32_t And, uint32_t Or);
// this function will attempt to remove RenderDoc's hooks in the application.
//
// Note: that this can only work correctly if done immediately after
// the module is loaded, before any API work happens. RenderDoc will remove its
// injected hooks and shut down. Behaviour is undefined if this is called
// after any API functions have been called, and there is still no guarantee of
// success.
typedef void(RENDERDOC_CC *pRENDERDOC_RemoveHooks)(void);
// DEPRECATED: compatibility for code compiled against pre-1.4.1 headers.
typedef pRENDERDOC_RemoveHooks pRENDERDOC_Shutdown;
// This function will unload RenderDoc's crash handler.
//
// If you use your own crash handler and don't want RenderDoc's handler to
// intercede, you can call this function to unload it and any unhandled
// exceptions will pass to the next handler.
typedef void(RENDERDOC_CC *pRENDERDOC_UnloadCrashHandler)(void);
// Sets the capture file path template
//
// pathtemplate is a UTF-8 string that gives a template for how captures will be named
// and where they will be saved.
//
// Any extension is stripped off the path, and captures are saved in the directory
// specified, and named with the filename and the frame number appended. If the
// directory does not exist it will be created, including any parent directories.
//
// If pathtemplate is NULL, the template will remain unchanged
//
// Example:
//
// SetCaptureFilePathTemplate("my_captures/example");
//
// Capture #1 -> my_captures/example_frame123.rdc
// Capture #2 -> my_captures/example_frame456.rdc
typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureFilePathTemplate)(const char *pathtemplate);
// returns the current capture path template, see SetCaptureFileTemplate above, as a UTF-8 string
typedef const char *(RENDERDOC_CC *pRENDERDOC_GetCaptureFilePathTemplate)(void);
// DEPRECATED: compatibility for code compiled against pre-1.1.2 headers.
typedef pRENDERDOC_SetCaptureFilePathTemplate pRENDERDOC_SetLogFilePathTemplate;
typedef pRENDERDOC_GetCaptureFilePathTemplate pRENDERDOC_GetLogFilePathTemplate;
// returns the number of captures that have been made
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetNumCaptures)(void);
// This function returns the details of a capture, by index. New captures are added
// to the end of the list.
//
// filename will be filled with the absolute path to the capture file, as a UTF-8 string
// pathlength will be written with the length in bytes of the filename string
// timestamp will be written with the time of the capture, in seconds since the Unix epoch
//
// Any of the parameters can be NULL and they'll be skipped.
//
// The function will return 1 if the capture index is valid, or 0 if the index is invalid
// If the index is invalid, the values will be unchanged
//
// Note: when captures are deleted in the UI they will remain in this list, so the
// capture path may not exist anymore.
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_GetCapture)(uint32_t idx, char *filename,
uint32_t *pathlength, uint64_t *timestamp);
// Sets the comments associated with a capture file. These comments are displayed in the
// UI program when opening.
//
// filePath should be a path to the capture file to add comments to. If set to NULL or ""
// the most recent capture file created made will be used instead.
// comments should be a NULL-terminated UTF-8 string to add as comments.
//
// Any existing comments will be overwritten.
typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureFileComments)(const char *filePath,
const char *comments);
// returns 1 if the RenderDoc UI is connected to this application, 0 otherwise
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_IsTargetControlConnected)(void);
// DEPRECATED: compatibility for code compiled against pre-1.1.1 headers.
// This was renamed to IsTargetControlConnected in API 1.1.1, the old typedef is kept here for
// backwards compatibility with old code, it is castable either way since it's ABI compatible
// as the same function pointer type.
typedef pRENDERDOC_IsTargetControlConnected pRENDERDOC_IsRemoteAccessConnected;
// This function will launch the Replay UI associated with the RenderDoc library injected
// into the running application.
//
// if connectTargetControl is 1, the Replay UI will be launched with a command line parameter
// to connect to this application
// cmdline is the rest of the command line, as a UTF-8 string. E.g. a captures to open
// if cmdline is NULL, the command line will be empty.
//
// returns the PID of the replay UI if successful, 0 if not successful.
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_LaunchReplayUI)(uint32_t connectTargetControl,
const char *cmdline);
// RenderDoc can return a higher version than requested if it's backwards compatible,
// this function returns the actual version returned. If a parameter is NULL, it will be
// ignored and the others will be filled out.
typedef void(RENDERDOC_CC *pRENDERDOC_GetAPIVersion)(int *major, int *minor, int *patch);
// Requests that the replay UI show itself (if hidden or not the current top window). This can be
// used in conjunction with IsTargetControlConnected and LaunchReplayUI to intelligently handle
// showing the UI after making a capture.
//
// This will return 1 if the request was successfully passed on, though it's not guaranteed that
// the UI will be on top in all cases depending on OS rules. It will return 0 if there is no current
// target control connection to make such a request, or if there was another error
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_ShowReplayUI)(void);
//////////////////////////////////////////////////////////////////////////
// Capturing functions
//
// A device pointer is a pointer to the API's root handle.
//
// This would be an ID3D11Device, HGLRC/GLXContext, ID3D12Device, etc
typedef void *RENDERDOC_DevicePointer;
// A window handle is the OS's native window handle
//
// This would be an HWND, GLXDrawable, etc
typedef void *RENDERDOC_WindowHandle;
// A helper macro for Vulkan, where the device handle cannot be used directly.
//
// Passing the VkInstance to this macro will return the RENDERDOC_DevicePointer to use.
//
// Specifically, the value needed is the dispatch table pointer, which sits as the first
// pointer-sized object in the memory pointed to by the VkInstance. Thus we cast to a void** and
// indirect once.
#define RENDERDOC_DEVICEPOINTER_FROM_VKINSTANCE(inst) (*((void **)(inst)))
// This sets the RenderDoc in-app overlay in the API/window pair as 'active' and it will
// respond to keypresses. Neither parameter can be NULL
typedef void(RENDERDOC_CC *pRENDERDOC_SetActiveWindow)(RENDERDOC_DevicePointer device,
RENDERDOC_WindowHandle wndHandle);
// capture the next frame on whichever window and API is currently considered active
typedef void(RENDERDOC_CC *pRENDERDOC_TriggerCapture)(void);
// capture the next N frames on whichever window and API is currently considered active
typedef void(RENDERDOC_CC *pRENDERDOC_TriggerMultiFrameCapture)(uint32_t numFrames);
// When choosing either a device pointer or a window handle to capture, you can pass NULL.
// Passing NULL specifies a 'wildcard' match against anything. This allows you to specify
// any API rendering to a specific window, or a specific API instance rendering to any window,
// or in the simplest case of one window and one API, you can just pass NULL for both.
//
// In either case, if there are two or more possible matching (device,window) pairs it
// is undefined which one will be captured.
//
// Note: for headless rendering you can pass NULL for the window handle and either specify
// a device pointer or leave it NULL as above.
// Immediately starts capturing API calls on the specified device pointer and window handle.
//
// If there is no matching thing to capture (e.g. no supported API has been initialised),
// this will do nothing.
//
// The results are undefined (including crashes) if two captures are started overlapping,
// even on separate devices and/oror windows.
typedef void(RENDERDOC_CC *pRENDERDOC_StartFrameCapture)(RENDERDOC_DevicePointer device,
RENDERDOC_WindowHandle wndHandle);
// Returns whether or not a frame capture is currently ongoing anywhere.
//
// This will return 1 if a capture is ongoing, and 0 if there is no capture running
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_IsFrameCapturing)(void);
// Ends capturing immediately.
//
// This will return 1 if the capture succeeded, and 0 if there was an error capturing.
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_EndFrameCapture)(RENDERDOC_DevicePointer device,
RENDERDOC_WindowHandle wndHandle);
// Ends capturing immediately and discard any data stored without saving to disk.
//
// This will return 1 if the capture was discarded, and 0 if there was an error or no capture
// was in progress
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_DiscardFrameCapture)(RENDERDOC_DevicePointer device,
RENDERDOC_WindowHandle wndHandle);
// Only valid to be called between a call to StartFrameCapture and EndFrameCapture. Gives a custom
// title to the capture produced which will be displayed in the UI.
//
// If multiple captures are ongoing, this title will be applied to the first capture to end after
// this call. The second capture to end will have no title, unless this function is called again.
//
// Calling this function has no effect if no capture is currently running, and if it is called
// multiple times only the last title will be used.
typedef void(RENDERDOC_CC *pRENDERDOC_SetCaptureTitle)(const char *title);
// Annotations API:
//
// These functions allow you to specify annotations either on a per-command level, or a per-object
// level.
//
// Basic types of annotations are supported, as well as vector versions and references to API objects.
//
// The annotations are stored as keys, with the key being a dot-separated path allowing arbitrary
// nesting and user organisation. The keys are sorted in human order so `foo.2.bar` will be displayed
// before `foo.10.bar` to allow creation of arrays if desired.
//
// Deleting an annotation can be done by assigning an empty value to it.
// the type of an annotation value, or Empty to delete an annotation
typedef enum RENDERDOC_AnnotationType
{
eRENDERDOC_Empty,
eRENDERDOC_Bool,
eRENDERDOC_Int32,
eRENDERDOC_UInt32,
eRENDERDOC_Int64,
eRENDERDOC_UInt64,
eRENDERDOC_Float,
eRENDERDOC_Double,
eRENDERDOC_String,
eRENDERDOC_APIObject,
eRENDERDOC_AnnotationMax = 0x7FFFFFFF,
} RENDERDOC_AnnotationType;
// a union with vector annotation value data
typedef union RENDERDOC_AnnotationVectorValue
{
bool boolean[4];
int32_t int32[4];
int64_t int64[4];
uint32_t uint32[4];
uint64_t uint64[4];
float float32[4];
double float64[4];
} RENDERDOC_AnnotationVectorValue;
// a union with scalar annotation value data
typedef union RENDERDOC_AnnotationValue
{
bool boolean;
int32_t int32;
int64_t int64;
uint32_t uint32;
uint64_t uint64;
float float32;
double float64;
RENDERDOC_AnnotationVectorValue vector;
const char *string;
void *apiObject;
} RENDERDOC_AnnotationValue;
// a struct for specifying a GL object, as we don't have pointers we can use so instead we specify a
// pointer to this struct giving both the type and the name
typedef struct RENDERDOC_GLResourceReference
{
// this is the same GLenum identifier as passed to glObjectLabel
uint32_t identifier;
uint32_t name;
} GLResourceReference;
// simple C++ helpers to avoid the need for a temporary objects for value passing and GL object specification
#ifdef __cplusplus
struct RDGLObjectHelper
{
RENDERDOC_GLResourceReference gl;
RDGLObjectHelper(uint32_t identifier, uint32_t name)
{
gl.identifier = identifier;
gl.name = name;
}
operator RENDERDOC_GLResourceReference *() { return &gl; }
};
struct RDAnnotationHelper
{
RENDERDOC_AnnotationValue val;
RDAnnotationHelper(bool b) { val.boolean = b; }
RDAnnotationHelper(int32_t i) { val.int32 = i; }
RDAnnotationHelper(int64_t i) { val.int64 = i; }
RDAnnotationHelper(uint32_t i) { val.uint32 = i; }
RDAnnotationHelper(uint64_t i) { val.uint64 = i; }
RDAnnotationHelper(float f) { val.float32 = f; }
RDAnnotationHelper(double d) { val.float64 = d; }
RDAnnotationHelper(const char *s) { val.string = s; }
operator RENDERDOC_AnnotationValue *() { return &val; }
};
#endif
// The device is specified in the same way as other API calls that take a RENDERDOC_DevicePointer
// to specify the device.
//
// The object or queue/commandbuffer will depend on the graphics API in question.
//
// Return value:
// 0 - The annotation was applied successfully.
// 1 - The device is unknown/invalid
// 2 - The device is valid but the annotation is not supported for API-specific reasons, such as an
// unrecognised or invalid object or queue/commandbuffer
// 3 - The call is ill-formed or invalid e.g. empty is specified with a value pointer, or non-empty
// is specified with a NULL value pointer
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_SetObjectAnnotation)(RENDERDOC_DevicePointer device,
void *object, const char *key,
RENDERDOC_AnnotationType valueType,
uint32_t valueVectorWidth,
const RENDERDOC_AnnotationValue *value);
typedef uint32_t(RENDERDOC_CC *pRENDERDOC_SetCommandAnnotation)(
RENDERDOC_DevicePointer device, void *queueOrCommandBuffer, const char *key,
RENDERDOC_AnnotationType valueType, uint32_t valueVectorWidth,
const RENDERDOC_AnnotationValue *value);
//////////////////////////////////////////////////////////////////////////////////////////////////
// RenderDoc API versions
//
// RenderDoc uses semantic versioning (http://semver.org/).
//
// MAJOR version is incremented when incompatible API changes happen.
// MINOR version is incremented when functionality is added in a backwards-compatible manner.
// PATCH version is incremented when backwards-compatible bug fixes happen.
//
// Note that this means the API returned can be higher than the one you might have requested.
// e.g. if you are running against a newer RenderDoc that supports 1.0.1, it will be returned
// instead of 1.0.0. You can check this with the GetAPIVersion entry point
typedef enum RENDERDOC_Version
{
eRENDERDOC_API_Version_1_0_0 = 10000, // RENDERDOC_API_1_0_0 = 1 00 00
eRENDERDOC_API_Version_1_0_1 = 10001, // RENDERDOC_API_1_0_1 = 1 00 01
eRENDERDOC_API_Version_1_0_2 = 10002, // RENDERDOC_API_1_0_2 = 1 00 02
eRENDERDOC_API_Version_1_1_0 = 10100, // RENDERDOC_API_1_1_0 = 1 01 00
eRENDERDOC_API_Version_1_1_1 = 10101, // RENDERDOC_API_1_1_1 = 1 01 01
eRENDERDOC_API_Version_1_1_2 = 10102, // RENDERDOC_API_1_1_2 = 1 01 02
eRENDERDOC_API_Version_1_2_0 = 10200, // RENDERDOC_API_1_2_0 = 1 02 00
eRENDERDOC_API_Version_1_3_0 = 10300, // RENDERDOC_API_1_3_0 = 1 03 00
eRENDERDOC_API_Version_1_4_0 = 10400, // RENDERDOC_API_1_4_0 = 1 04 00
eRENDERDOC_API_Version_1_4_1 = 10401, // RENDERDOC_API_1_4_1 = 1 04 01
eRENDERDOC_API_Version_1_4_2 = 10402, // RENDERDOC_API_1_4_2 = 1 04 02
eRENDERDOC_API_Version_1_5_0 = 10500, // RENDERDOC_API_1_5_0 = 1 05 00
eRENDERDOC_API_Version_1_6_0 = 10600, // RENDERDOC_API_1_6_0 = 1 06 00
eRENDERDOC_API_Version_1_7_0 = 10700, // RENDERDOC_API_1_7_0 = 1 07 00
} RENDERDOC_Version;
// API version changelog:
//
// 1.0.0 - initial release
// 1.0.1 - Bugfix: IsFrameCapturing() was returning false for captures that were triggered
// by keypress or TriggerCapture, instead of Start/EndFrameCapture.
// 1.0.2 - Refactor: Renamed eRENDERDOC_Option_DebugDeviceMode to eRENDERDOC_Option_APIValidation
// 1.1.0 - Add feature: TriggerMultiFrameCapture(). Backwards compatible with 1.0.x since the new
// function pointer is added to the end of the struct, the original layout is identical
// 1.1.1 - Refactor: Renamed remote access to target control (to better disambiguate from remote
// replay/remote server concept in replay UI)
// 1.1.2 - Refactor: Renamed "log file" in function names to just capture, to clarify that these
// are captures and not debug logging files. This is the first API version in the v1.0
// branch.
// 1.2.0 - Added feature: SetCaptureFileComments() to add comments to a capture file that will be
// displayed in the UI program on load.
// 1.3.0 - Added feature: New capture option eRENDERDOC_Option_AllowUnsupportedVendorExtensions
// which allows users to opt-in to allowing unsupported vendor extensions to function.
// Should be used at the user's own risk.
// Refactor: Renamed eRENDERDOC_Option_VerifyMapWrites to
// eRENDERDOC_Option_VerifyBufferAccess, which now also controls initialisation to
// 0xdddddddd of uninitialised buffer contents.
// 1.4.0 - Added feature: DiscardFrameCapture() to discard a frame capture in progress and stop
// capturing without saving anything to disk.
// 1.4.1 - Refactor: Renamed Shutdown to RemoveHooks to better clarify what is happening
// 1.4.2 - Refactor: Renamed 'draws' to 'actions' in callstack capture option.
// 1.5.0 - Added feature: ShowReplayUI() to request that the replay UI show itself if connected
// 1.6.0 - Added feature: SetCaptureTitle() which can be used to set a title for a
// capture made with StartFrameCapture() or EndFrameCapture()
// 1.7.0 - Added feature: SetObjectAnnotation() / SetCommandAnnotation() for adding rich
// annotations to objects and command streams
typedef struct RENDERDOC_API_1_7_0
{
pRENDERDOC_GetAPIVersion GetAPIVersion;
pRENDERDOC_SetCaptureOptionU32 SetCaptureOptionU32;
pRENDERDOC_SetCaptureOptionF32 SetCaptureOptionF32;
pRENDERDOC_GetCaptureOptionU32 GetCaptureOptionU32;
pRENDERDOC_GetCaptureOptionF32 GetCaptureOptionF32;
pRENDERDOC_SetFocusToggleKeys SetFocusToggleKeys;
pRENDERDOC_SetCaptureKeys SetCaptureKeys;
pRENDERDOC_GetOverlayBits GetOverlayBits;
pRENDERDOC_MaskOverlayBits MaskOverlayBits;
// Shutdown was renamed to RemoveHooks in 1.4.1.
// These unions allow old code to continue compiling without changes
union
{
pRENDERDOC_Shutdown Shutdown;
pRENDERDOC_RemoveHooks RemoveHooks;
};
pRENDERDOC_UnloadCrashHandler UnloadCrashHandler;
// Get/SetLogFilePathTemplate was renamed to Get/SetCaptureFilePathTemplate in 1.1.2.
// These unions allow old code to continue compiling without changes
union
{
// deprecated name
pRENDERDOC_SetLogFilePathTemplate SetLogFilePathTemplate;
// current name
pRENDERDOC_SetCaptureFilePathTemplate SetCaptureFilePathTemplate;
};
union
{
// deprecated name
pRENDERDOC_GetLogFilePathTemplate GetLogFilePathTemplate;
// current name
pRENDERDOC_GetCaptureFilePathTemplate GetCaptureFilePathTemplate;
};
pRENDERDOC_GetNumCaptures GetNumCaptures;
pRENDERDOC_GetCapture GetCapture;
pRENDERDOC_TriggerCapture TriggerCapture;
// IsRemoteAccessConnected was renamed to IsTargetControlConnected in 1.1.1.
// This union allows old code to continue compiling without changes
union
{
// deprecated name
pRENDERDOC_IsRemoteAccessConnected IsRemoteAccessConnected;
// current name
pRENDERDOC_IsTargetControlConnected IsTargetControlConnected;
};
pRENDERDOC_LaunchReplayUI LaunchReplayUI;
pRENDERDOC_SetActiveWindow SetActiveWindow;
pRENDERDOC_StartFrameCapture StartFrameCapture;
pRENDERDOC_IsFrameCapturing IsFrameCapturing;
pRENDERDOC_EndFrameCapture EndFrameCapture;
// new function in 1.1.0
pRENDERDOC_TriggerMultiFrameCapture TriggerMultiFrameCapture;
// new function in 1.2.0
pRENDERDOC_SetCaptureFileComments SetCaptureFileComments;
// new function in 1.4.0
pRENDERDOC_DiscardFrameCapture DiscardFrameCapture;
// new function in 1.5.0
pRENDERDOC_ShowReplayUI ShowReplayUI;
// new function in 1.6.0
pRENDERDOC_SetCaptureTitle SetCaptureTitle;
// new functions in 1.7.0
pRENDERDOC_SetObjectAnnotation SetObjectAnnotation;
pRENDERDOC_SetCommandAnnotation SetCommandAnnotation;
} RENDERDOC_API_1_7_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_1;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_0_2;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_1;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_1_2;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_2_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_3_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_1;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_4_2;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_5_0;
typedef RENDERDOC_API_1_7_0 RENDERDOC_API_1_6_0;
//////////////////////////////////////////////////////////////////////////////////////////////////
// RenderDoc API entry point
//
// This entry point can be obtained via GetProcAddress/dlsym if RenderDoc is available.
//
// The name is the same as the typedef - "RENDERDOC_GetAPI"
//
// This function is not thread safe, and should not be called on multiple threads at once.
// Ideally, call this once as early as possible in your application's startup, before doing
// any API work, since some configuration functionality etc has to be done also before
// initialising any APIs.
//
// Parameters:
// version is a single value from the RENDERDOC_Version above.
//
// outAPIPointers will be filled out with a pointer to the corresponding struct of function
// pointers.
//
// Returns:
// 1 - if the outAPIPointers has been filled with a pointer to the API struct requested
// 0 - if the requested version is not supported or the arguments are invalid.
//
typedef int(RENDERDOC_CC *pRENDERDOC_GetAPI)(RENDERDOC_Version version, void **outAPIPointers);
#ifdef __cplusplus
} // extern "C"
#endif

61
gpu/render/rx.h Normal file
View File

@@ -0,0 +1,61 @@
/* The renderer: the Maxwell 3D engine's work drawn with WebGPU (webgpu.h: wgpu-native on the PC, the browser's
WebGPU on the web). hos/gpu.c runs the command streams and hands the renderer the 3D methods that draw; the
renderer reads the engine's registers (suyu's Maxwell3D::Regs layout) and guest memory, and turns them into
WebGPU passes, pipelines and draws. hos/display.c gives it the window; nn_vi.c tells it what to present.
Threads: every entry point may be called from any thread; the renderer serializes them. */
#pragma once
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/* The window to present into (a Windows HWND), from the window's thread once it exists; null: no window (the
renderer draws offscreen). Returns 1 when the renderer runs, 0 when the window should show frames itself. */
int rx_start(void *hwnd);
int rx_active(void);
/* The window's client area changed size. */
void rx_resize(int width, int height);
/* One renderer state per GPU channel (its constant buffer bindings and the draw being assembled). */
void *rx_channel_new(void);
/* the channel's ZPASS_PIXEL_COUNT estimate (occlusion query reports) */
uint64_t rx_channel_zpass(void *channel);
/* The 3D methods the renderer acts on: a table of 0x1000 flags (hos/gpu.c calls rx_3d_method for those only).
0xFFFF (not a method): a macro finished. */
const uint8_t *rx_3d_methods(void);
/* A 3D method was written: regs is the channel's 3D register file after the write (regs[method] == arg). */
void rx_3d_method(void *channel, const uint32_t *regs, uint32_t method, uint32_t arg);
/* The compute engine (B1C0) was launched: regs is its register file (the launch descriptor at regs[0xAD] << 8). */
void rx_compute_launch(void *channel, const uint32_t *regs);
/* A submission ended: what was recorded goes to the GPU. */
void rx_flush(void);
/* A submission starts: memory the CPU changed since the last one is uploaded again when used. */
void rx_begin_submission(void);
/* The same, split for a GPU thread: the pages the CPU wrote are collected by the thread that submits (at its submit,
in submission order: a malloc'd array of guest page numbers, or null with *count -1 when nothing is tracked), and
applied on the GPU thread when the submission starts (the array is freed there). */
uint32_t *rx_collect_written(int64_t *count);
void rx_begin_submission_written(uint32_t *pages, int64_t count);
/* The command processor is about to run guest memory as commands (an indirect draw's arguments, say): 1 when a
shader wrote part of it on the GPU and the result is not in guest memory yet. rx_sync_gpu_writes puts it there
(it submits what was recorded and waits for the GPU). */
int rx_gpu_writes_pending(uint64_t gpu_va, uint64_t size);
/* The same for dispatch-written memory only, without the renderer's lock (a hint: split mode's parsing thread). */
int rx_gpu_written_hint(uint64_t gpu_va, uint64_t size);
void rx_sync_gpu_writes(uint64_t gpu_va, uint64_t size);
/* The command processor wrote guest memory (inline uploads, copies) while running a submission. */
void rx_memory_written(uint64_t gpu_va, uint64_t size);
/* The game queued a window buffer for display: guest address, size in pixels. Returns 1 when the renderer drew
that buffer and presented it. */
int rx_present(uint64_t guest_address, uint32_t width, uint32_t height);
/* What the window title shows: "<adapter> (<backend>)". */
const char *rx_describe(void);
/* Draws done / skipped (unsupported state) since the start. */
void rx_stats(int64_t *draws, int64_t *skipped, int64_t *pipelines);
#ifdef __cplusplus
}
#endif

201
gpu/render/rx_bc7.cpp Normal file
View File

@@ -0,0 +1,201 @@
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstring>
#include "video_core/textures/workers.h"
namespace rx {
namespace {
constexpr int kWeights[16] = {0, 4, 9, 13, 17, 21, 26, 30, 34, 38, 43, 47, 51, 55, 60, 64};
struct Fit {
int q[2][4];
int p[2];
uint8_t index[16];
int64_t error;
};
void quantize(const float e[4], int q[4], int& p) {
int64_t best = INT64_MAX;
for (int pb = 0; pb < 2; pb++) {
int64_t err = 0;
int t[4];
for (int c = 0; c < 4; c++) {
t[c] = std::clamp((int)((e[c] - pb) * 0.5f + 0.5f), 0, 127);
const float d = e[c] - (float)(t[c] * 2 + pb);
err += (int64_t)(d * d * 16.0f);
}
if (err < best) {
best = err;
p = pb;
std::memcpy(q, t, sizeof t);
}
}
}
void choose_indices(const uint8_t px[16][4], Fit& f) {
int pal[16][4];
for (int k = 0; k < 16; k++)
for (int c = 0; c < 4; c++) {
const int a = f.q[0][c] * 2 + f.p[0], b = f.q[1][c] * 2 + f.p[1];
pal[k][c] = ((64 - kWeights[k]) * a + kWeights[k] * b + 32) >> 6;
}
f.error = 0;
for (int i = 0; i < 16; i++) {
int best = 0;
int64_t best_err = INT64_MAX;
for (int k = 0; k < 16; k++) {
int64_t e = 0;
for (int c = 0; c < 4; c++) {
const int d = px[i][c] - pal[k][c];
e += d * d;
}
if (e < best_err) {
best_err = e;
best = k;
}
}
f.index[i] = (uint8_t)best;
f.error += best_err;
}
}
Fit fit_endpoints(const uint8_t px[16][4], const float e0[4], const float e1[4]) {
Fit f;
quantize(e0, f.q[0], f.p[0]);
quantize(e1, f.q[1], f.p[1]);
choose_indices(px, f);
return f;
}
void put(uint64_t (&w)[2], int& pos, uint64_t v, int n) {
if (pos < 64) {
w[0] |= v << pos;
if (pos + n > 64) w[1] |= v >> (64 - pos);
} else {
w[1] |= v << (pos - 64);
}
pos += n;
}
void encode_block(const uint8_t px[16][4], uint8_t out[16]) {
float mean[4] = {0, 0, 0, 0};
for (int i = 0; i < 16; i++)
for (int c = 0; c < 4; c++) mean[c] += px[i][c];
for (float& m : mean) m *= 1.0f / 16.0f;
float cov[4][4] = {};
for (int i = 0; i < 16; i++) {
float d[4];
for (int c = 0; c < 4; c++) d[c] = px[i][c] - mean[c];
for (int a = 0; a < 4; a++)
for (int b = 0; b < 4; b++) cov[a][b] += d[a] * d[b];
}
float axis[4] = {cov[0][0], cov[1][1], cov[2][2], cov[3][3]};
for (int it = 0; it < 8; it++) {
float n[4] = {0, 0, 0, 0};
for (int a = 0; a < 4; a++)
for (int b = 0; b < 4; b++) n[a] += cov[a][b] * axis[b];
float m = 0;
for (float v : n) m = std::max(m, std::fabs(v));
if (m < 1e-6f) break;
for (int a = 0; a < 4; a++) axis[a] = n[a] / m;
}
float len = 0;
for (float v : axis) len += v * v;
len = std::sqrt(len);
float tmin = 0, tmax = 0;
if (len > 1e-6f) {
for (float& v : axis) v /= len;
tmin = 1e30f;
tmax = -1e30f;
for (int i = 0; i < 16; i++) {
float t = 0;
for (int c = 0; c < 4; c++) t += (px[i][c] - mean[c]) * axis[c];
tmin = std::min(tmin, t);
tmax = std::max(tmax, t);
}
}
float e0[4], e1[4];
for (int c = 0; c < 4; c++) {
e0[c] = std::clamp(mean[c] + axis[c] * tmin, 0.0f, 255.0f);
e1[c] = std::clamp(mean[c] + axis[c] * tmax, 0.0f, 255.0f);
}
Fit best = fit_endpoints(px, e0, e1);
if (best.error > 64) {
float aa = 0, ab = 0, bb = 0, ax[4] = {0, 0, 0, 0}, bx[4] = {0, 0, 0, 0};
for (int i = 0; i < 16; i++) {
const float b = kWeights[best.index[i]] / 64.0f, a = 1.0f - b;
aa += a * a;
ab += a * b;
bb += b * b;
for (int c = 0; c < 4; c++) {
ax[c] += a * px[i][c];
bx[c] += b * px[i][c];
}
}
const float det = aa * bb - ab * ab;
if (std::fabs(det) > 1e-6f) {
float r0[4], r1[4];
for (int c = 0; c < 4; c++) {
r0[c] = std::clamp((bb * ax[c] - ab * bx[c]) / det, 0.0f, 255.0f);
r1[c] = std::clamp((aa * bx[c] - ab * ax[c]) / det, 0.0f, 255.0f);
}
const Fit refit = fit_endpoints(px, r0, r1);
if (refit.error < best.error) best = refit;
}
}
if (best.index[0] & 8) {
std::swap(best.q[0], best.q[1]);
std::swap(best.p[0], best.p[1]);
for (uint8_t& k : best.index) k = (uint8_t)(15 - k);
}
uint64_t w[2] = {0, 0};
int pos = 0;
put(w, pos, 1u << 6, 7);
for (int c = 0; c < 4; c++) {
put(w, pos, (uint64_t)best.q[0][c], 7);
put(w, pos, (uint64_t)best.q[1][c], 7);
}
put(w, pos, (uint64_t)best.p[0], 1);
put(w, pos, (uint64_t)best.p[1], 1);
put(w, pos, best.index[0], 3);
for (int i = 1; i < 16; i++) put(w, pos, best.index[i], 4);
std::memcpy(out, w, 16);
}
void encode_rows(const uint8_t* rgba, uint32_t width, uint32_t height, uint32_t pitch, uint32_t by0, uint32_t by1,
uint8_t* out) {
const uint32_t bw = (width + 3) / 4;
uint8_t px[16][4];
for (uint32_t by = by0; by < by1; by++)
for (uint32_t bx = 0; bx < bw; bx++) {
for (uint32_t y = 0; y < 4; y++)
for (uint32_t x = 0; x < 4; x++) {
const uint32_t sx = std::min(bx * 4 + x, width - 1), sy = std::min(by * 4 + y, height - 1);
std::memcpy(px[y * 4 + x], rgba + (size_t)sy * pitch + (size_t)sx * 4, 4);
}
encode_block(px, out + ((size_t)by * bw + bx) * 16);
}
}
} // namespace
void bc7_encode(const uint8_t* rgba, uint32_t width, uint32_t height, uint32_t pitch, uint8_t* out) {
if (!width || !height) return;
const uint32_t bh = (height + 3) / 4;
const uint32_t per_job = std::max<uint32_t>(1, 16384 / std::max<uint32_t>(1, (width + 3) / 4));
if (bh <= per_job) {
encode_rows(rgba, width, height, pitch, 0, bh, out);
return;
}
auto& workers = Tegra::Texture::GetThreadWorkers();
for (uint32_t by = 0; by < bh; by += per_job) {
const uint32_t end = std::min(bh, by + per_job);
workers.QueueWork([=] { encode_rows(rgba, width, height, pitch, by, end, out); });
}
workers.WaitForRequests();
}
} // namespace rx

113
gpu/render/rx_bcn.h Normal file
View File

@@ -0,0 +1,113 @@
// BC1 through BC5 CPU fallback for compressed textures WebGPU cannot represent directly.
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
namespace rx::bcn {
enum class Format { BC1, BC2, BC3, BC4, BC4Snorm, BC5, BC5Snorm };
inline uint16_t read16(const uint8_t* p) {
return uint16_t(p[0] | uint16_t(p[1]) << 8);
}
inline uint32_t read32(const uint8_t* p) {
return uint32_t(p[0]) | uint32_t(p[1]) << 8 | uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24;
}
inline void decode_bc1_block(const uint8_t* src, uint8_t dst[16][4], bool force_opaque = false) {
const uint16_t c0 = read16(src), c1 = read16(src + 2);
uint8_t palette[4][4]{};
auto unpack565 = [](uint16_t c, uint8_t* rgba) {
rgba[0] = uint8_t(((c >> 11) * 255 + 15) / 31);
rgba[1] = uint8_t((((c >> 5) & 63) * 255 + 31) / 63);
rgba[2] = uint8_t(((c & 31) * 255 + 15) / 31);
rgba[3] = 255;
};
unpack565(c0, palette[0]);
unpack565(c1, palette[1]);
for (unsigned c = 0; c < 3; ++c) {
if (c0 > c1 || force_opaque) {
palette[2][c] = uint8_t((2 * palette[0][c] + palette[1][c]) / 3);
palette[3][c] = uint8_t((palette[0][c] + 2 * palette[1][c]) / 3);
} else {
palette[2][c] = uint8_t((palette[0][c] + palette[1][c]) / 2);
palette[3][c] = 0;
}
}
palette[2][3] = 255;
palette[3][3] = c0 > c1 || force_opaque ? 255 : 0;
const uint32_t indices = read32(src + 4);
for (unsigned i = 0; i < 16; ++i)
std::memcpy(dst[i], palette[(indices >> (2 * i)) & 3], 4);
}
inline void decode_bc4_block(const uint8_t* src, uint8_t dst[16], bool signed_values) {
int palette[8];
palette[0] = signed_values ? std::max(-127, int(int8_t(src[0]))) : src[0];
palette[1] = signed_values ? std::max(-127, int(int8_t(src[1]))) : src[1];
if (palette[0] > palette[1]) {
for (int i = 2; i < 8; ++i)
palette[i] = ((8 - i) * palette[0] + (i - 1) * palette[1]) / 7;
} else {
for (int i = 2; i < 6; ++i)
palette[i] = ((6 - i) * palette[0] + (i - 1) * palette[1]) / 5;
palette[6] = signed_values ? -127 : 0;
palette[7] = signed_values ? 127 : 255;
}
uint64_t indices = 0;
for (unsigned i = 0; i < 6; ++i)
indices |= uint64_t(src[i + 2]) << (8 * i);
for (unsigned i = 0; i < 16; ++i)
dst[i] = uint8_t(palette[(indices >> (3 * i)) & 7]);
}
/// Decodes source block bytes into tightly packed RGBA8 texels and clips partial edge blocks.
inline void decode(const uint8_t* src, unsigned width, unsigned height, unsigned depth, Format format,
uint8_t* dst) {
const size_t blocks_x = (width + 3) / 4;
const size_t blocks_y = (height + 3) / 4;
const bool color_format = format == Format::BC1 || format == Format::BC2 || format == Format::BC3;
const bool two_channel = format == Format::BC5 || format == Format::BC5Snorm;
const bool signed_values = format == Format::BC4Snorm || format == Format::BC5Snorm;
const size_t block_bytes = format == Format::BC1 || format == Format::BC4 || format == Format::BC4Snorm ? 8 : 16;
for (unsigned z = 0; z < depth; ++z) {
for (size_t by = 0; by < blocks_y; ++by) {
for (size_t bx = 0; bx < blocks_x; ++bx) {
const uint8_t* block = src + ((size_t(z) * blocks_y + by) * blocks_x + bx) * block_bytes;
uint8_t rgba[16][4]{};
for (unsigned i = 0; i < 16; ++i) rgba[i][3] = signed_values ? 127 : 255;
if (color_format) {
const uint8_t* color = block + (format == Format::BC1 ? 0 : 8);
decode_bc1_block(color, rgba, format != Format::BC1);
}
if (format == Format::BC2) {
for (unsigned i = 0; i < 16; ++i)
rgba[i][3] = uint8_t(((block[i / 2] >> ((i & 1) * 4)) & 15) * 17);
} else if (format == Format::BC3) {
uint8_t alpha[16];
decode_bc4_block(block, alpha, false);
for (unsigned i = 0; i < 16; ++i) rgba[i][3] = alpha[i];
} else if (!color_format) {
uint8_t red[16];
decode_bc4_block(block, red, signed_values);
for (unsigned i = 0; i < 16; ++i) rgba[i][0] = red[i];
if (two_channel) {
uint8_t green[16];
decode_bc4_block(block + 8, green, signed_values);
for (unsigned i = 0; i < 16; ++i) rgba[i][1] = green[i];
}
}
for (unsigned iy = 0; iy < 4 && by * 4 + iy < height; ++iy) {
for (unsigned ix = 0; ix < 4 && bx * 4 + ix < width; ++ix) {
const size_t pixel = ((size_t(z) * height + by * 4 + iy) * width + bx * 4 + ix);
std::memcpy(dst + pixel * 4, rgba[iy * 4 + ix], 4);
}
}
}
}
}
}
} // namespace rx::bcn

764
gpu/render/rx_buffer.cpp Normal file
View File

@@ -0,0 +1,764 @@
// The buffer cache: guest memory mirrored in GPU buffers, so that vertex, index and read-only storage data stays on
// the GPU across draws and frames and only pages the game changed are uploaded again.
//
// Guest memory (4 GB at hos_guest_base) is covered by segments of 16 MB that start every 8 MB: any range of up to
// 8 MB lies inside one segment, so it binds as one buffer: an existing one that holds it, else an even one (they tile
// memory; an odd one only for a range across a 16 MB boundary; RX_SEGMENT_OVERLAP=1: always the one starting at or
// below it). A segment is created when a draw first needs it; its pages are uploaded when a draw needs them (one
// valid bit per 4 KB page; pages a dispatch wrote are copied from the neighbour that holds them).
//
// Changes: the guest arena is allocated with MEM_WRITE_WATCH (hos_main.c). At the start of each GPU submission the
// pages written since the last one are invalidated (GetWriteWatch), which catches every CPU write the submission's
// data needs. Writes the command processor makes while it runs a submission (inline uploads, copies: hos/gpu.c)
// are reported through rx_memory_written; if a page they change was read earlier in the open batch, the batch is
// submitted first, so earlier draws see the old contents.
#include <algorithm>
#include <array>
#include <cstring>
#include <memory>
#include <unordered_map>
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include "rx_internal.h"
extern "C" {
uint8_t* hos_guest_base(void);
uint64_t hos_guest_size(void);
int hos_gpu_guest_ranges(uint64_t (*out)[2], int max);
}
namespace rx {
namespace {
constexpr uint64_t kPage = 4096;
constexpr uint64_t kStep = 8ull << 20; // a segment starts every kStep bytes...
constexpr uint64_t kSegment = 16ull << 20; // ...and covers kSegment
constexpr uint64_t kPagesPerSegment = kSegment / kPage;
struct Segment {
WGPUBuffer buffer = nullptr;
uint64_t base = 0, size = 0;
std::vector<uint64_t> valid; // one bit per page
int64_t batch = -1; // the last batch that bound it
int64_t last_frame = 0;
bool page_valid(uint64_t p) const { return valid[p >> 6] >> (p & 63) & 1; }
void set(uint64_t p) { valid[p >> 6] |= 1ull << (p & 63); }
void clear(uint64_t p) { valid[p >> 6] &= ~(1ull << (p & 63)); }
};
std::vector<Segment*> segments; // by base / kStep
// ((mkw-recomp) by buffer: the table has a slot per 8 MB of the 512 GB window, 65536, which a search by buffer walked)
std::unordered_map<WGPUBuffer, Segment*> segment_of_buffer;
int64_t batch_id;
bool watch_ok = true;
#if defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
extern "C" uint8_t a64_dirty[1ull << 27]; // (rt/a64_rt.c: (mkw-recomp) A64_DIRTY_PAGES, a mark per page of 512 GB)
#endif
// (mkw-recomp) A table with an entry per guest page, as a std::vector would hold it, but in memory committed
// untouched: MKW's guest window is 512 GB (a 1 GB table of 64-bit entries), of which the GPU sees a few GB. Pages
// of the table come into being when first touched (zero filled). assign's value must be 0.
template <class T>
struct PageTable {
T* p = nullptr;
size_t n = 0;
bool empty() const { return !n; }
size_t size() const { return n; }
void assign(size_t count, T) {
p = static_cast<T*>(VirtualAlloc(nullptr, count * sizeof(T), MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
n = p ? count : 0;
}
T& operator[](size_t i) { return p[i]; }
const T& operator[](size_t i) const { return p[i]; }
T* data() { return p; }
};
// when each guest page was last written, in memory_seq() ticks (one per scan and per command-processor write)
PageTable<uint64_t> page_seq;
uint64_t mem_seq = 1;
PageTable<uint8_t> gpu_pages; // pages a dispatch wrote (newer on the GPU than in guest memory)
std::atomic<const uint8_t*> gpu_pages_data{nullptr}; // (its storage, for rx_gpu_written_hint: allocated once)
// ...and what guest memory held in each when the dispatch wrote it: when the CPU writes such a page too, only the
// bytes it changed go to the GPU (the dispatch's results in the rest of the page stay). RX_NO_PAGE_MERGE=1: the
// whole page comes from guest memory again (the old behaviour: the GPU's results in it are lost).
std::unordered_map<uint64_t, std::unique_ptr<uint8_t[]>> gpu_page_shadow;
// (the render thread and the readbacks' map callbacks, which write the GPU's results into guest memory and the
// shadows together: readback_to_guest)
std::mutex shadow_mu;
bool page_merge_on() {
static const bool off = getenv("RX_NO_PAGE_MERGE") != nullptr;
return !off;
}
void page_written(uint64_t page) {
if (page_seq.empty()) page_seq.assign(hos_guest_size() / kPage, 0);
if (page < page_seq.size()) page_seq[page] = mem_seq;
if (page < gpu_pages.size() && gpu_pages[page]) {
count("cpu_wrote_gpu_page");
std::lock_guard lk(shadow_mu);
gpu_page_shadow.erase(page);
}
if (page < gpu_pages.size()) gpu_pages[page] = 0;
textures_written(page * kPage, kPage);
}
Segment* segment(uint64_t index, bool create) {
if (segments.empty()) segments.resize(hos_guest_size() / kStep + 1, nullptr);
if (index >= segments.size()) return nullptr;
Segment* s = segments[index];
if (s || !create) return s;
s = new Segment();
s->base = index * kStep;
s->size = std::min<uint64_t>(kSegment, hos_guest_size() - s->base);
s->valid.assign((kPagesPerSegment + 63) / 64, 0);
WGPUBufferDescriptor d{};
d.label = sv("rx guest memory");
d.usage = WGPUBufferUsage_Vertex | WGPUBufferUsage_Index | WGPUBufferUsage_Storage | WGPUBufferUsage_CopyDst |
WGPUBufferUsage_CopySrc | WGPUBufferUsage_Indirect; // ((mkw-recomp) indirect arguments read in place)
d.size = s->size;
scope_push();
s->buffer = wgpuDeviceCreateBuffer(dev.device, &d);
if (const std::string e = scope_pop(); !e.empty()) {
log_limited("segment", 4, "rx: guest memory segment not created: %s", e.c_str());
delete s;
return nullptr;
}
segments[index] = s;
segment_of_buffer[s->buffer] = s;
count("segments_new");
return s;
}
/// The guest page was written: no segment holds it any more.
void invalidate_page(uint64_t page) {
const uint64_t addr = page * kPage, index = addr / kStep;
for (uint64_t i : {index, index - 1}) {
if (i >= segments.size()) continue;
if (Segment* s = segments[i]; s && addr >= s->base && addr < s->base + s->size) s->clear((addr - s->base) / kPage);
}
}
/// The CPU (or the command processor) wrote a page a dispatch also wrote: the words it changed since the dispatch
/// go to every segment holding the page, the dispatch's results stay. False when the page is not such a page.
bool merge_cpu_write(uint64_t page) {
if (!page_merge_on() || page >= gpu_pages.size() || !gpu_pages[page]) return false;
// the words the CPU changed, taken under the shadows' lock (a readback writes guest memory and the shadow at once)
static std::vector<std::pair<uint32_t, uint32_t>> runs;
static uint32_t words[kPage / 4];
runs.clear();
{
std::lock_guard lk(shadow_mu);
auto it = gpu_page_shadow.find(page);
if (it == gpu_page_shadow.end()) return false;
const uint32_t* now = reinterpret_cast<const uint32_t*>(hos_guest_base() + page * kPage);
uint32_t* old = reinterpret_cast<uint32_t*>(it->second.get());
for (uint32_t w = 0; w < kPage / 4;) {
const uint32_t v = now[w];
if (v == old[w]) {
w++;
continue;
}
uint32_t e = w;
for (uint32_t x; e < kPage / 4 && (x = now[e]) != old[e]; e++) words[e] = old[e] = x;
runs.push_back({w, e});
w = e;
}
}
const uint64_t addr = page * kPage, index = addr / kStep;
for (const auto& [w, e] : runs) {
for (uint64_t i : {index, index - 1}) {
if (i >= segments.size()) continue;
Segment* sg = segments[i];
if (!sg || addr < sg->base || addr >= sg->base + sg->size || !sg->page_valid((addr - sg->base) / kPage))
continue;
if (sg->batch == batch_id) { // (a draw of the open batch may read the old words)
count("flush_written");
flush_batch();
}
queue_upload(sg->buffer, addr - sg->base + w * 4, words + w, (uint64_t)(e - w) * 4);
}
count("merged_words", (int64_t)(e - w));
}
count("merged_pages");
return true;
}
bool holds_range(const Segment& s, uint64_t guest, uint64_t size) {
const uint64_t first = (guest - s.base) / kPage, last = (guest + size - 1 - s.base) / kPage;
for (uint64_t w = first >> 6; w <= last >> 6; w++) {
uint64_t need = ~0ull;
if (w == first >> 6) need &= ~0ull << (first & 63);
if (w == last >> 6) need &= ~0ull >> (63 - (last & 63));
if ((s.valid[w] & need) != need) return false;
}
return true;
}
bool range_gpu_written(uint64_t guest, uint64_t size) {
if (gpu_pages.empty()) return false;
for (uint64_t p = guest / kPage; p <= (guest + size - 1) / kPage && p < gpu_pages.size(); p++)
if (gpu_pages[p]) return true;
return false;
}
bool holds_gpu_pages(const Segment& s, uint64_t guest, uint64_t size) {
for (uint64_t p = guest / kPage; p <= (guest + size - 1) / kPage && p < gpu_pages.size(); p++)
if (gpu_pages[p] && !s.page_valid(p - s.base / kPage)) return false;
return true;
}
Segment* gpu_page_holder(const Segment& s, uint64_t page) {
const uint64_t a = page * kPage, index = s.base / kStep;
for (uint64_t i : {index - 1, index + 1}) {
if (i >= segments.size() || !segments[i]) continue;
Segment* n = segments[i];
if (a >= n->base && a < n->base + n->size && n->page_valid((a - n->base) / kPage)) return n;
}
return nullptr;
}
void fill_gpu_pages(Segment& s, uint64_t p0, uint64_t p1) {
if (gpu_pages.empty()) return;
const uint64_t g0 = s.base / kPage;
for (uint64_t p = p0; p < p1;) {
const uint64_t gp = g0 + p;
Segment* from = gp < gpu_pages.size() && gpu_pages[gp] ? gpu_page_holder(s, gp) : nullptr;
if (!from) {
p++;
continue;
}
uint64_t q = p + 1;
while (q < p1 && g0 + q < gpu_pages.size() && gpu_pages[g0 + q] && gpu_page_holder(s, g0 + q) == from) q++;
end_render_pass("pe_segment_fill");
const uint64_t a = gp * kPage;
ce_copy_buffer_to_buffer(encoder(), from->buffer, a - from->base, s.buffer, a - s.base, (q - p) * kPage);
from->batch = batch_id;
count("segment_gpu_fill", (int64_t)((q - p) * kPage));
p = q;
}
}
} // namespace
std::vector<WGPUBuffer> retired_segment_buffers;
void retire_unused_segments() {
static const bool on = env_flag("RX_SEGMENT_RETIRE");
if (!on) return;
int retired = 0;
for (size_t i = 0; i < segments.size(); i++) {
Segment* s = segments[i];
if (!s || s->last_frame >= frame_number - 1800 || s->batch >= batch_id) continue;
bool gpu_data = false;
const uint64_t p0 = s->base / kPage, p1 = std::min<uint64_t>((s->base + s->size) / kPage, gpu_pages.size());
for (uint64_t p = p0; p < p1 && !gpu_data; p++) gpu_data = gpu_pages[p] != 0;
if (gpu_data) continue;
retired_segment_buffers.push_back(s->buffer);
segment_of_buffer.erase(s->buffer);
delete s;
segments[i] = nullptr;
retired++;
}
if (retired) count("segments_retired", retired);
}
bool segments_retired_pending() {
return !retired_segment_buffers.empty();
}
std::vector<WGPUBuffer> take_retired_segment_buffers() {
std::vector<WGPUBuffer> out;
out.swap(retired_segment_buffers);
return out;
}
size_t buffer_segments() {
size_t n = 0;
for (const Segment* s : segments) n += s != nullptr;
return n;
}
} // namespace rx
/// (HOS_MEM_MAP) Whether a segment holds the guest page at this offset (its contents were read for an upload).
extern "C" int rx_guest_page_uploaded(uint64_t off) {
using namespace rx;
const uint64_t index = off / kStep;
for (uint64_t i : {index, index - 1}) {
if (i >= segments.size()) continue;
if (const Segment* s = segments[i]; s && off >= s->base && off < s->base + s->size &&
s->page_valid((off - s->base) / kPage))
return 1;
}
return 0;
}
namespace rx {
/// The open batch was submitted: segments bound from now on belong to the next one.
void buffer_cache_new_batch() {
batch_id++;
}
BufferRef guest_buffer(uint64_t gpu_va, uint64_t size) {
if (!size || size > kStep) return count("gb_size"), BufferRef{};
uint64_t avail = 0;
const uint8_t* host = hos_gpu_ptr(gpu_va, &avail);
if (!host || avail < size) return count(!host ? "gb_unmapped" : "gb_avail"), BufferRef{};
const uint64_t guest = (uint64_t)((uintptr_t)host - (uintptr_t)hos_guest_base());
const uint64_t index = guest / kStep;
Segment* s = nullptr;
bool held = false;
static const bool overlap = env_flag("RX_SEGMENT_OVERLAP");
if (overlap) {
s = segment(index, true);
held = s && guest + size <= s->base + s->size && holds_range(*s, guest, size);
} else {
const bool lower_fits = index > 0 && guest + size <= index * kStep + kStep;
const uint64_t none = UINT64_MAX;
const uint64_t pick[2] = {(index & 1) ? (lower_fits ? index - 1 : none) : index,
(index & 1) ? index : (lower_fits ? index - 1 : none)};
Segment* have[2] = {};
for (int k = 0; k < 2 && !s; k++) {
if (pick[k] == none) continue;
Segment* c = segment(pick[k], false);
if (!c || guest + size > c->base + c->size) continue;
have[k] = c;
if (holds_range(*c, guest, size)) s = c, held = true;
}
if (!s && (have[0] || have[1]) && range_gpu_written(guest, size))
for (int k = 0; k < 2 && !s; k++)
if (have[k] && holds_gpu_pages(*have[k], guest, size)) s = have[k];
if (!s) s = have[0] ? have[0] : have[1];
if (!s) s = segment(pick[0] != none ? pick[0] : pick[1], true);
}
if (!s || guest + size > s->base + s->size) return count(!s ? "gb_no_segment" : "gb_bounds"), BufferRef{};
// upload the pages the range needs that the segment does not hold (in runs)
const uint64_t first = (guest - s->base) / kPage, last = (guest + size - 1 - s->base) / kPage;
if (held) {
s->batch = batch_id;
s->last_frame = frame_number;
return {s->buffer, guest - s->base, size};
}
for (uint64_t p = first; p <= last;) {
if (s->page_valid(p)) {
p++;
continue;
}
uint64_t q = p;
while (q <= last && !s->page_valid(q)) s->set(q++);
const uint64_t off = p * kPage, n = std::min((q - p) * kPage, s->size - off);
queue_upload(s->buffer, off, hos_guest_base() + s->base + off, n);
count("up_cache", (int64_t)n);
fill_gpu_pages(*s, p, q);
p = q;
}
s->batch = batch_id;
s->last_frame = frame_number;
return {s->buffer, guest - s->base, size};
}
/// The guest pages the CPU wrote since the last call (GetWriteWatch, which also resets them): any thread. False when
/// write watching does not work (then nothing is tracked).
bool collect_written_pages(std::vector<uint32_t>& out) {
out.clear();
#if defined(A64_WRITE_TRACK) && !defined(__EMSCRIPTEN__)
{ // the soft write watch (rt/a64.h A64_MARK, hos_mark_written): the marks of the GPU-visible ranges, read and
// cleared (an exchange: a mark set again meanwhile is the next scan's); ~0.1 ms where GetWriteWatch took 2-50
std::vector<std::array<uint64_t, 2>> range_buf(4096);
auto* ranges = reinterpret_cast<uint64_t(*)[2]>(range_buf.data());
const int nranges = hos_gpu_guest_ranges(ranges, 4096);
for (int r = 0; r < nranges; r++) {
const uint64_t lo = ranges[r][0], hi = std::min<uint64_t>(ranges[r][1], hos_guest_size());
const uint64_t p1 = std::min<uint64_t>((hi + kPage - 1) / kPage, 1ull << 27);
for (uint64_t p = lo / kPage; p < p1;) {
if (!(p & 7) && p + 8 <= p1) { // (eight pages a load while none is marked)
uint64_t w;
std::memcpy(&w, &a64_dirty[p], 8);
if (!w) {
p += 8;
continue;
}
}
if (__atomic_load_n(&a64_dirty[p], __ATOMIC_RELAXED) && __atomic_exchange_n(&a64_dirty[p], 0, __ATOMIC_ACQUIRE))
out.push_back((uint32_t)p);
p++;
}
}
return true;
}
#endif
if (!watch_ok) return false;
// only the memory mapped into the GPU's address space (what the renderer can read); the cost of GetWriteWatch
// grows with the range, not with the pages written
std::vector<std::array<uint64_t, 2>> range_buf(4096); // (any thread may scan)
auto* ranges = reinterpret_cast<uint64_t(*)[2]>(range_buf.data());
const int nranges = hos_gpu_guest_ranges(ranges, 4096);
// (room for 1 GB of written pages a call: each extra call walks the whole range again)
thread_local std::vector<void*> addrs(1 << 18);
const int64_t t0 = hos_now_ns();
int calls = 0;
int64_t slowest = 0, slowest_range = 0;
for (int r = 0; r < nranges; r++) {
const uint64_t lo = ranges[r][0], hi = std::min<uint64_t>(ranges[r][1], hos_guest_size());
if (lo >= hi) continue;
// RX_SCAN_PROBE=1 (experiment): a range is first asked without a reset (no TLB flush) for one written page;
// only a range that has one is read and reset (atomically, as before)
static const bool probe = env_flag("RX_SCAN_PROBE");
if (probe) {
void* one[1];
ULONG_PTR n1 = 1;
DWORD gran1 = 0;
const int64_t c0 = hos_now_ns();
const UINT bad = GetWriteWatch(0, hos_guest_base() + lo, hi - lo, one, &n1, &gran1);
calls++;
if (hos_now_ns() - c0 > slowest) slowest = hos_now_ns() - c0, slowest_range = (int64_t)(hi - lo);
if (!bad && n1 == 0) continue;
}
for (;;) {
ULONG_PTR n = addrs.size();
DWORD gran = 0;
const int64_t c0 = hos_now_ns();
if (GetWriteWatch(WRITE_WATCH_FLAG_RESET, hos_guest_base() + lo, hi - lo, addrs.data(), &n, &gran)) {
hos_log("rx: GetWriteWatch failed (%lu): every buffer is uploaded at each draw", GetLastError());
watch_ok = false;
return false;
}
calls++;
if (hos_now_ns() - c0 > slowest) slowest = hos_now_ns() - c0, slowest_range = (int64_t)(hi - lo);
for (ULONG_PTR i = 0; i < n; i++)
out.push_back((uint32_t)(((uintptr_t)addrs[i] - (uintptr_t)hos_guest_base()) / kPage));
if (n < addrs.size()) break;
}
}
if (const int64_t took = hos_now_ns() - t0; took > 20000000) // (a hitch: the game waits for this at its fences)
log_limited("slow_scan", 60, "rx: write scan took %.1f ms: %zu written pages, %d calls over %d ranges, slowest "
"%.1f ms (%lld MB)", took / 1e6, out.size(), calls, nranges, slowest / 1e6,
(long long)(slowest_range >> 20));
static int64_t logged_at;
const int64_t now = hos_now_ns();
if (now - logged_at > 30000000000LL) { // (every 30 s: how much is watched)
logged_at = now;
uint64_t total = 0;
for (int r = 0; r < nranges; r++) total += ranges[r][1] - ranges[r][0];
hos_log("rx: change tracking watches %d ranges, %llu MB", nranges, (unsigned long long)(total >> 20));
}
return true;
}
/// Pages the CPU wrote (collect_written_pages) are invalidated (called with no batch open).
void buffer_cache_apply_written(const uint32_t* pages, size_t n) {
mem_seq++;
for (size_t i = 0; i < n; i++) {
if (merge_cpu_write(pages[i])) { // (the page stays valid and GPU-written)
if (page_seq.empty()) page_seq.assign(hos_guest_size() / kPage, 0);
if (pages[i] < page_seq.size()) page_seq[pages[i]] = mem_seq;
textures_written((uint64_t)pages[i] * kPage, kPage);
continue;
}
page_written(pages[i]);
if (!segments.empty()) invalidate_page(pages[i]);
}
count("dirty_pages", (int64_t)n);
}
/// Pages the CPU wrote since the last call are invalidated (called with no batch open).
void buffer_cache_scan() {
static std::vector<uint32_t> pages;
const int64_t t0 = hos_now_ns();
if (!collect_written_pages(pages)) return;
count("t_scan", hos_now_ns() - t0);
buffer_cache_apply_written(pages.data(), pages.size());
}
void buffer_cache_written(uint64_t gpu_va, uint64_t size) {
if (!size) return;
uint64_t avail = 0;
const uint8_t* host = hos_gpu_ptr(gpu_va, &avail);
if (!host) return;
const uint64_t guest = (uint64_t)((uintptr_t)host - (uintptr_t)hos_guest_base());
const uint64_t p0 = guest / kPage, p1 = (guest + std::min(size, avail) - 1) / kPage;
mem_seq++;
static std::vector<uint8_t> merged; // (reused: called for every constant buffer upload and semaphore write)
merged.assign(p1 - p0 + 1, 0);
for (uint64_t p = p0; p <= p1; p++) {
if (merge_cpu_write(p)) {
merged[p - p0] = 1;
if (page_seq.empty()) page_seq.assign(hos_guest_size() / kPage, 0);
if (p < page_seq.size()) page_seq[p] = mem_seq;
textures_written(p * kPage, kPage);
continue;
}
page_written(p);
}
if (segments.empty()) return;
for (uint64_t p = p0; p <= p1; p++) {
if (merged[p - p0]) continue;
const uint64_t addr = p * kPage, index = addr / kStep;
for (uint64_t i : {index, index - 1}) {
if (i >= segments.size()) continue;
Segment* s = segments[i];
if (!s || addr < s->base || addr >= s->base + s->size) continue;
const uint64_t sp = (addr - s->base) / kPage;
if (!s->page_valid(sp)) continue;
// a draw of the open batch may read the old contents: they go to the GPU first
if (s->batch == batch_id) {
count("flush_written");
flush_batch();
}
s->clear(sp);
}
}
}
/// A readback's results written into guest memory (a map callback: any thread), the words that changed in the batch
/// (now against was). They are the GPU's own results, not CPU writes: a page's shadow takes them too, so the next scan
/// does not upload them over what the GPU wrote since. (They were merged as CPU writes: TOTK's lighting tile flags,
/// rewritten every frame, came back a frame late over the newer ones and lit some tiles wrongly: light boxes at the
/// horizon that moved from frame to frame.)
int64_t readback_to_guest(uint8_t* guest, const uint8_t* now, const uint8_t* was, uint64_t n) {
int64_t changed = 0;
std::lock_guard lk(shadow_mu);
uint64_t shadow_page = ~0ull;
uint8_t* shadow = nullptr;
for (uint64_t i = 0; i + 4 <= n; i += 4) {
uint32_t v, before;
std::memcpy(&v, now + i, 4);
std::memcpy(&before, was + i, 4);
if (v == before) continue;
std::memcpy(guest + i, &v, 4);
changed++;
const uint64_t at = (uint64_t)(guest + i - hos_guest_base()), page = at / kPage;
if (page != shadow_page) {
shadow_page = page;
auto it = gpu_page_shadow.find(page);
shadow = it == gpu_page_shadow.end() ? nullptr : it->second.get();
}
if (shadow) std::memcpy(shadow + at % kPage, &v, 4);
}
return changed;
}
void buffer_cache_gpu_written(const BufferRef& ref) {
const auto found = segment_of_buffer.find(ref.buffer);
if (found == segment_of_buffer.end()) return;
Segment* s = found->second;
const uint64_t g0 = s->base + ref.offset, g1 = g0 + ref.size;
if (gpu_pages.empty()) {
gpu_pages.assign(hos_guest_size() / kPage, 0);
gpu_pages_data.store(gpu_pages.data(), std::memory_order_release);
}
for (uint64_t p = g0 / kPage; p <= (g1 - 1) / kPage && p < gpu_pages.size(); p++) {
if (!gpu_pages[p] && page_merge_on()) { // (what the CPU's side holds now: later CPU writes are diffed against it)
std::lock_guard lk(shadow_mu);
auto& sh = gpu_page_shadow[p];
if (!sh) sh.reset(new uint8_t[kPage]);
std::memcpy(sh.get(), hos_guest_base() + p * kPage, kPage);
}
gpu_pages[p] = 1;
}
const uint64_t index = s->base / kStep;
for (uint64_t i : {index - 1, index + 1}) {
if (i >= segments.size() || !segments[i]) continue;
Segment* n = segments[i];
const uint64_t a = std::max(g0, n->base), b = std::min(g1, n->base + n->size);
if (a >= b) continue;
// pages of the overlap the neighbour does not hold come from guest memory first (queue order: before the
// batch), then the written bytes are copied over them in the batch, after the dispatch
for (uint64_t p = (a - n->base) / kPage; p <= (b - 1 - n->base) / kPage; p++)
if (!n->page_valid(p)) {
const uint64_t off = p * kPage, len = std::min(kPage, n->size - off);
queue_upload(n->buffer, off, hos_guest_base() + n->base + off, len);
n->set(p);
}
ce_copy_buffer_to_buffer(encoder(), s->buffer, a - s->base, n->buffer, a - n->base, b - a);
n->batch = batch_id;
}
}
/// (mkw-recomp) The pages of [guest, guest + size) a dispatch wrote, copied into dst (which mirrors that range from
/// offset 0) from the segments holding them, in the batch. Bytes copied.
uint64_t copy_gpu_pages(WGPUBuffer dst, uint64_t guest, uint64_t size) {
if (gpu_pages.empty() || !size) return 0;
auto holder = [](uint64_t page) -> Segment* {
const uint64_t a = page * kPage, index = a / kStep;
for (uint64_t i : {index, index - 1})
if (i < segments.size() && segments[i] && a >= segments[i]->base &&
a < segments[i]->base + segments[i]->size && segments[i]->page_valid((a - segments[i]->base) / kPage))
return segments[i];
return nullptr;
};
uint64_t copied = 0;
const uint64_t p1 = std::min<uint64_t>((guest + size - 1) / kPage + 1, gpu_pages.size());
for (uint64_t p = guest / kPage; p < p1;) {
Segment* from = gpu_pages[p] ? holder(p) : nullptr;
if (!from) {
p++;
continue;
}
uint64_t q = p + 1;
while (q < p1 && gpu_pages[q] && (q * kPage < from->base + from->size) && from->page_valid(q - from->base / kPage))
q++;
const uint64_t lo = std::max(p * kPage, guest), hi = std::min(q * kPage, guest + size);
const uint64_t off = (lo - guest) & ~3ull, n = ((hi - guest + 3) & ~3ull) - off;
end_render_pass("pe_big_gpu_fill");
ce_copy_buffer_to_buffer(encoder(), from->buffer, guest + off - from->base, dst, off, n);
from->batch = batch_id;
copied += n;
p = q;
}
return copied;
}
uint64_t memory_seq() {
return watch_ok ? mem_seq : 0;
}
bool memory_written_since(uint64_t guest, uint64_t size, uint64_t seq) {
if (page_seq.empty() || !size) return false;
const uint64_t p1 = std::min<uint64_t>((guest + size - 1) / kPage, page_seq.size() - 1);
for (uint64_t p = guest / kPage; p <= p1; p++)
if (page_seq[p] > seq) return true;
return false;
}
bool gpu_written(uint64_t gpu_va, uint64_t size) {
if (gpu_pages.empty() || !size) return false;
uint64_t avail = 0;
const uint8_t* host = hos_gpu_ptr(gpu_va, &avail);
if (!host) return false;
const uint64_t guest = (uint64_t)((uintptr_t)host - (uintptr_t)hos_guest_base());
for (uint64_t p = guest / kPage; p <= (guest + size - 1) / kPage && p < gpu_pages.size(); p++)
if (gpu_pages[p]) return true;
return false;
}
void buffer_cache_sync_to_guest(uint64_t gpu_va, uint64_t size) {
if (gpu_pages.empty() || !size) return;
uint64_t avail = 0;
const uint8_t* host = hos_gpu_ptr(gpu_va, &avail);
if (!host) return;
const uint64_t guest = (uint64_t)((uintptr_t)host - (uintptr_t)hos_guest_base());
uint64_t p0 = guest / kPage, p1 = (guest + std::min(size, avail) - 1) / kPage;
// the segment holding the range (a dispatch's results are in every segment that holds its pages)
Segment* s = nullptr;
for (uint64_t i : {guest / kStep, guest / kStep - 1})
if (i < segments.size() && segments[i] && guest >= segments[i]->base &&
(p1 + 1) * kPage <= segments[i]->base + segments[i]->size && segments[i]->page_valid(p0 - segments[i]->base / kPage)) {
s = segments[i];
break;
}
if (!s) return;
// with the GPU-written pages around it (up to 1 MB each way): one round trip for the neighbouring arguments
const uint64_t s0 = s->base / kPage, s1 = (s->base + s->size) / kPage - 1;
for (int k = 0; k < 256 && p0 > s0 && gpu_pages[p0 - 1] && s->page_valid(p0 - 1 - s0); k++) p0--;
for (int k = 0; k < 256 && p1 < s1 && gpu_pages[p1 + 1] && s->page_valid(p1 + 1 - s0); k++) p1++;
const uint64_t off = p0 * kPage - s->base, n = (p1 - p0 + 1) * kPage;
WGPUBufferDescriptor d{};
d.label = sv("rx guest memory readback");
d.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
d.size = n;
WGPUBuffer staging = wgpuDeviceCreateBuffer(dev.device, &d);
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
wgpuCommandEncoderCopyBufferToBuffer(e, s->buffer, off, staging, 0, n);
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
wgpuQueueSubmit(dev.queue, 1, &cb);
wgpuCommandBufferRelease(cb);
wgpuCommandEncoderRelease(e);
struct Done {
bool complete = false;
WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error;
} done;
WGPUBufferMapCallbackInfo mc{};
mc.mode = WGPUCallbackMode_AllowSpontaneous;
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
auto* dn = static_cast<Done*>(u1);
dn->status = status;
dn->complete = true;
};
mc.userdata1 = &done;
wgpuBufferMapAsync(staging, WGPUMapMode_Read, 0, n, mc);
while (!done.complete) wgpuDevicePoll(dev.device, true, nullptr);
if (done.status == WGPUMapAsyncStatus_Success) {
uint8_t* dst = hos_guest_base() + p0 * kPage;
// (pages the CPU wrote since the last scan keep their write-watch mark; the others are not reported as CPU
// writes: the segment already holds this)
std::vector<void*> dirty(p1 - p0 + 1);
ULONG_PTR ndirty = dirty.size();
DWORD gran = 0;
const bool watched = !GetWriteWatch(0, dst, n, dirty.data(), &ndirty, &gran);
const auto* src = static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(staging, 0, n));
std::lock_guard lk(shadow_mu);
for (uint64_t p = p0; p <= p1; p++) {
uint8_t* g = dst + (p - p0) * kPage;
const uint8_t* q = src + (p - p0) * kPage;
auto it = gpu_page_shadow.find(p);
if (it == gpu_page_shadow.end()) {
std::memcpy(g, q, kPage);
continue;
}
// only the words the GPU changed: what the CPU wrote since (not scanned yet) stays
const uint32_t* gw = reinterpret_cast<const uint32_t*>(q);
const uint32_t* sw = reinterpret_cast<const uint32_t*>(it->second.get());
uint32_t* dw = reinterpret_cast<uint32_t*>(g);
for (uint32_t w = 0; w < kPage / 4; w++)
if (gw[w] != sw[w]) dw[w] = gw[w];
}
if (watched && ndirty == 0) ResetWriteWatch(dst, n);
wgpuBufferUnmap(staging);
for (uint64_t p = p0; p <= p1; p++) {
gpu_pages[p] = 0;
gpu_page_shadow.erase(p);
}
count("gpu_sync_bytes", (int64_t)n);
}
wgpuBufferRelease(staging);
}
namespace {
std::vector<std::pair<uint64_t, uint64_t>> dispatch_writes;
}
void note_dispatch_write(uint64_t gpu_va, uint64_t size) {
static const bool on = getenv("RX_SYNC_DISPATCH_WRITES") != nullptr;
if (!on) return;
for (auto& [v, n] : dispatch_writes)
if (v == gpu_va) { n = std::max(n, size); return; }
dispatch_writes.push_back({gpu_va, size});
}
void sync_dispatch_writes() {
if (dispatch_writes.empty()) return;
flush_batch();
wait_submitted();
for (const auto& [v, n] : dispatch_writes) buffer_cache_sync_to_guest(v, n);
count("dispatch_writes_synced", (int64_t)dispatch_writes.size());
dispatch_writes.clear();
}
int64_t batch_number() {
return batch_id;
}
bool buffer_cache_active() {
static const bool off = getenv("RX_NO_CACHE") != nullptr; // (debugging: every draw copies its data)
return watch_ok && !off;
}
} // namespace rx
/// Whether a dispatch may have written this memory, without the renderer's lock (the parsing thread asks for every
/// command list; a byte per page, read while the render thread may set it: a stale answer only delays a sync).
extern "C" int rx_gpu_written_hint(uint64_t gpu_va, uint64_t size) {
const uint8_t* pages = rx::gpu_pages_data.load(std::memory_order_acquire);
if (!pages || !size) return 0;
uint64_t avail = 0;
const uint8_t* host = hos_gpu_ptr(gpu_va, &avail);
if (!host) return 0;
const uint64_t guest = (uint64_t)((uintptr_t)host - (uintptr_t)hos_guest_base()), n = hos_guest_size() / rx::kPage;
for (uint64_t p = guest / rx::kPage; p <= (guest + size - 1) / rx::kPage && p < n; p++)
if (reinterpret_cast<const volatile uint8_t*>(pages)[p]) return 1;
return 0;
}

657
gpu/render/rx_compute.cpp Normal file
View File

@@ -0,0 +1,657 @@
// Compute: the compute engine's launches (B1C0 LAUNCH with a launch descriptor, the QMD) -> WebGPU compute passes.
// Storage buffers are bound from the buffer cache, read-write: what a dispatch writes stays on the GPU, where the
// draws and dispatches after it read it (the guest's copy of that memory is not updated).
#include <algorithm>
#include "common/div_ceil.h"
#include "rx_internal.h"
namespace rx {
uint64_t dispatch_write_epoch = 0;
namespace {
struct ComputePipeline {
WGPUComputePipeline pipeline = nullptr;
WGPUBindGroupLayout layout = nullptr;
bool failed = false;
};
std::unordered_map<uint64_t, std::unique_ptr<ComputePipeline>> pipelines;
/// The launch descriptor (64 words; suyu's KeplerCompute::LaunchParams).
ComputeLaunch read_launch(const uint32_t* regs) {
uint32_t q[64];
gpu_read((uint64_t)regs[0xAD] << 8, q, sizeof q);
ComputeLaunch l;
l.code_base = (uint64_t)regs[0x582] << 32 | regs[0x583];
l.program_start = q[8];
l.linked_tsc = q[11] >> 30 & 1;
l.grid[0] = q[12] & 0x7FFFFFFF;
l.grid[1] = q[13] & 0xFFFF;
l.grid[2] = q[13] >> 16;
l.shared_size = q[17] & 0x3FFFF;
l.block[0] = q[18] >> 16;
l.block[1] = q[19] & 0xFFFF;
l.block[2] = q[19] >> 16;
const uint32_t mask = q[20] & 0xFF;
for (int i = 0; i < 8; i++) {
const uint32_t lo = q[29 + 2 * i], hi = q[30 + 2 * i];
l.cbufs[i].address = (uint64_t)(hi & 0xFF) << 32 | lo;
l.cbufs[i].size = hi >> 15;
l.cbufs[i].enabled = mask >> i & 1;
}
l.local_size = (q[45] & 0xFFFFF) + (q[47] & 0xFFFFF);
l.texture_bound = regs[0x982];
return l;
}
uint32_t read_handle(const ComputeLaunch& l, const Shader::TextureDescriptor& d) {
const uint64_t addr = l.cbufs[d.cbuf_index % 8].address + d.cbuf_offset;
if (d.has_secondary) {
const uint64_t sec = l.cbufs[d.secondary_cbuf_index % 8].address + d.secondary_cbuf_offset;
return (gpu_get<uint32_t>(addr) << d.shift_left) | (gpu_get<uint32_t>(sec) << d.secondary_shift_left);
}
return gpu_get<uint32_t>(addr);
}
/// The web: the browser forbids writable storage bindings that overlap. Overlapping ranges of one buffer are bound
/// once, the lowest one's binding covering them all (end[j]: where each binding ends), and the others read it at
/// their offset in a variant of the program (alias[j] = that storage index + 1, alias_words[j] = the offset in
/// words). RX_FORCE_SSBO_ALIAS=1: the web's rule natively (testing). Returns whether any range was aliased.
bool alias_overlapping_storage(const std::vector<std::pair<uint32_t, BufferRef>>& storage, std::vector<uint64_t>& end,
std::array<uint8_t, 64>& alias, std::array<uint32_t, 64>& alias_words) {
end.resize(storage.size());
for (size_t j = 0; j < storage.size(); j++) end[j] = storage[j].second.offset + storage[j].second.size;
static const bool force_alias = env_flag("RX_FORCE_SSBO_ALIAS");
if (!(RX_WEB || force_alias) || storage.size() <= 1 || storage.size() > 64) return false;
std::vector<size_t> order(storage.size());
for (size_t j = 0; j < order.size(); j++) order[j] = j;
std::sort(order.begin(), order.end(), [&](size_t a, size_t b2) {
const auto& x = storage[a].second;
const auto& y = storage[b2].second;
return x.buffer != y.buffer ? (uintptr_t)x.buffer < (uintptr_t)y.buffer : x.offset < y.offset;
});
bool any = false;
size_t base = order[0];
for (size_t k = 1; k < order.size(); k++) {
const size_t j = order[k];
if (storage[j].second.buffer == storage[base].second.buffer && storage[j].second.offset < end[base]) {
alias[j] = (uint8_t)(base + 1);
alias_words[j] = (uint32_t)((storage[j].second.offset - storage[base].second.offset) / 4);
end[base] = std::max(end[base], end[j]);
any = true;
} else {
base = j;
}
}
return any;
}
void dispatch(const ComputeLaunch& l) {
ShaderModule* m = compute_module(l);
if (!m) return count("cs_skipped");
const Shader::Info& info = m->info;
if (!info.image_descriptors.empty() || !info.image_buffer_descriptors.empty() ||
!info.texture_buffer_descriptors.empty()) {
log_limited("cs_images", 8, "rx: a compute program uses images or texture buffers: not dispatched yet");
return count("cs_images");
}
if (!l.grid[0] || !l.grid[1] || !l.grid[2]) return count("cs_empty");
if (l.block[0] * l.block[1] * l.block[2] > dev.limits.maxComputeInvocationsPerWorkgroup ||
l.shared_size > dev.limits.maxComputeWorkgroupStorageSize)
return count("cs_too_big");
// room in the upload streams first: a flush must not come between the cache's bindings and the dispatch
const uint64_t estimate = 64 * 1024 + info.constant_buffer_descriptors.size() * 0x10100 +
info.texture_descriptors.size() * 256 + info.storage_buffers_descriptors.size() * (1 << 20);
if (stream_ro.used + estimate > stream_ro.capacity || stream_rw.used + (8u << 20) > stream_rw.capacity)
flush_batch();
std::vector<WGPUBindGroupLayoutEntry> layout;
std::vector<WGPUBindGroupEntry> entries;
std::vector<uint32_t> key = {(uint32_t)(uintptr_t)m, (uint32_t)((uintptr_t)m >> 32)};
std::vector<BufferRef> written;
std::vector<std::pair<uint32_t, BufferRef>> storage; // (binding, range) of each storage descriptor
auto buffer_entry = [&](uint32_t b, WGPUBufferBindingType type, WGPUBuffer buf, uint64_t off, uint64_t size) {
WGPUBindGroupLayoutEntry e{};
e.binding = b;
e.visibility = WGPUShaderStage_Compute;
e.buffer.type = type;
layout.push_back(e);
WGPUBindGroupEntry g{};
g.binding = b;
g.buffer = buf;
g.offset = off;
g.size = size;
entries.push_back(g);
};
// binding 0: the uniform block (as the WGSL backend lays it out: rescale, swizzles, the constant buffers that fit)
const bool indirect = info.uses_global_memory || info.uses_cbuf_indirect;
std::array<uint32_t, 18> base{};
std::array<bool, 18> in_ub{};
uint32_t ub_size = 4 + 2 * (uint32_t)info.texture_descriptors.size();
if (!indirect)
for (const auto& d : info.constant_buffer_descriptors) {
const uint32_t size = std::max(Common::DivCeil(info.constant_buffer_used_sizes[d.index], 16U), 1U);
if (ub_size + size > 0x1000) continue;
in_ub[d.index % 18] = true;
base[d.index % 18] = ub_size;
ub_size += size;
}
const uint64_t ub = stream_ro.alloc((uint64_t)ub_size * 16, dev.limits.minUniformBufferOffsetAlignment);
if (ub == UINT64_MAX) return count("cs_space");
uint8_t* block = stream_ro.at(ub);
std::memset(block, 0, 64);
const float scaling_z = 1.0f;
std::memcpy(block + 8, &scaling_z, 4);
for (const auto& d : info.constant_buffer_descriptors) {
if (!in_ub[d.index % 18]) continue;
const auto& cb = l.cbufs[d.index % 8];
const uint64_t size = (uint64_t)std::max(Common::DivCeil(info.constant_buffer_used_sizes[d.index], 16U), 1U) * 16;
const uint64_t n = cb.enabled ? std::min<uint64_t>(size, cb.size) : 0;
uint8_t* dst = block + (uint64_t)base[d.index % 18] * 16;
if (n) gpu_read(cb.address, dst, n);
if (n < size) std::memset(dst + n, 0, size - n);
count("up_cbuf", (int64_t)size);
}
buffer_entry(0, WGPUBufferBindingType_Uniform, stream_ro.buffer, ub, (uint64_t)ub_size * 16);
uint32_t b = 1;
for (const auto& d : info.constant_buffer_descriptors) {
if (in_ub[d.index % 18]) continue;
const auto& cb = l.cbufs[d.index % 8];
const uint64_t size = std::clamp<uint64_t>((cb.enabled ? cb.size : 0) + 15 & ~15ull, 16, 0x10000);
const uint64_t off = stream_ro.alloc(size, dev.limits.minStorageBufferOffsetAlignment);
if (off == UINT64_MAX) return count("cs_space");
const uint64_t n = cb.enabled ? std::min<uint64_t>(size, cb.size) : 0;
if (n) gpu_read(cb.address, stream_ro.at(off), n);
if (n < size) std::memset(stream_ro.at(off + n), 0, size - n);
count("up_cbuf_storage", (int64_t)size);
buffer_entry(b, WGPUBufferBindingType_ReadOnlyStorage, stream_ro.buffer, off, size);
b += d.count;
}
for (const auto& d : info.storage_buffers_descriptors) {
const auto& cb = l.cbufs[d.cbuf_index % 8];
// the program reads from the address aligned down to the storage offset alignment (the frontend's
// min_ssbo_alignment): the binding starts there
const uint64_t raw_va = gpu_get<uint64_t>(cb.address + d.cbuf_offset);
const uint64_t va = raw_va & ~(uint64_t)(dev.limits.minStorageBufferOffsetAlignment - 1);
uint64_t size = gpu_get<uint32_t>(cb.address + d.cbuf_offset + 8) + (raw_va - va);
size = std::clamp<uint64_t>((size + 3) & ~3ull, 16, 16ull << 20);
BufferRef ref = guest_buffer(va, size);
if (!ref.buffer || ref.offset % dev.limits.minStorageBufferOffsetAlignment) {
// not bindable in place: a copy (what the dispatch writes there is lost)
count("cs_ssbo_copied");
const uint64_t off = stream_rw.alloc(size, dev.limits.minStorageBufferOffsetAlignment);
if (off == UINT64_MAX) return count("cs_space");
gpu_read(va, stream_rw.at(off), size);
ref = {stream_rw.buffer, off, size};
if (d.is_written) writeback_after_batch(off, size, va);
} else if (d.is_written) {
written.push_back(ref);
note_dispatch_write(va, size);
}
// RX_CS_WATCH=<GPU VA hex> (debugging): dispatches that write storage covering it are logged
if (static const uint64_t watch = getenv("RX_CS_WATCH") ? strtoull(getenv("RX_CS_WATCH"), nullptr, 16) : 0;
(watch && d.is_written && va <= watch && watch < va + size) || (d.is_written && guest_watch_hit(va, size))) {
static int logged;
if (logged++ < 100)
hos_log("rx: dispatch %#llx grid %u,%u,%u writes %#llx+%#llx (watch %#llx)",
(unsigned long long)(l.code_base + l.program_start), l.grid[0], l.grid[1], l.grid[2],
(unsigned long long)va, (unsigned long long)size, (unsigned long long)watch);
}
storage.push_back({b, ref});
b += d.count;
}
std::array<uint8_t, 64> alias{};
std::array<uint32_t, 64> alias_words{};
std::vector<uint64_t> end;
if (alias_overlapping_storage(storage, end, alias, alias_words)) {
m = compute_module(l, &alias, &alias_words);
if (!m) return count("cs_skipped");
key[0] = (uint32_t)(uintptr_t)m;
key[1] = (uint32_t)((uintptr_t)m >> 32);
count("cs_ssbo_aliased");
}
for (size_t j = 0; j < storage.size(); j++)
if (j >= 64 || !alias[j])
buffer_entry(storage[j].first, WGPUBufferBindingType_Storage, storage[j].second.buffer,
storage[j].second.offset, end[j] - storage[j].second.offset);
uint32_t t_index = 0;
for (const auto& d : info.texture_descriptors) {
const uint32_t handle = read_handle(l, d);
const BoundTexture t = texture_for_handle(*l.tex_regs, handle, d);
// the header's component swizzle and the sampler's border, in the uniform block
const auto tic = read_tic(*l.tex_regs, Tegra::Texture::TexturePair(handle, l.linked_tsc).first);
texture_slots(block + (uint64_t)(4 + 2 * t_index++) * 16, tic, read_tsc_for_dump(*l.tex_regs, handle));
WGPUBindGroupLayoutEntry tl{};
tl.binding = b;
tl.visibility = WGPUShaderStage_Compute;
tl.texture.sampleType = t.kind == TexKind::Float ? WGPUTextureSampleType_Float
: t.kind == TexKind::UnfilterableFloat ? WGPUTextureSampleType_UnfilterableFloat
: t.kind == TexKind::Depth ? WGPUTextureSampleType_Depth
: t.kind == TexKind::Uint ? WGPUTextureSampleType_Uint
: WGPUTextureSampleType_Sint;
tl.texture.viewDimension = view_dimension(d.type);
tl.texture.multisampled = d.is_multisample;
layout.push_back(tl);
WGPUBindGroupLayoutEntry sl{};
sl.binding = b + 1;
sl.visibility = WGPUShaderStage_Compute;
sl.sampler.type = t.kind == TexKind::Depth ? WGPUSamplerBindingType_Comparison
: t.kind == TexKind::UnfilterableFloat ? WGPUSamplerBindingType_NonFiltering
: WGPUSamplerBindingType_Filtering;
layout.push_back(sl);
WGPUBindGroupEntry te{};
te.binding = b;
te.textureView = t.view;
entries.push_back(te);
WGPUBindGroupEntry se{};
se.binding = b + 1;
se.sampler = t.sampler;
entries.push_back(se);
key.push_back((uint32_t)t.kind);
b += 2;
}
{
const uint64_t lh = hash_bytes(layout.data(), layout.size() * sizeof(WGPUBindGroupLayoutEntry));
key.insert(key.end(), {(uint32_t)lh, (uint32_t)(lh >> 32)});
}
const uint64_t h = hash_bytes(key.data(), key.size() * 4);
auto it = pipelines.find(h);
if (it == pipelines.end()) {
auto p = std::make_unique<ComputePipeline>();
WGPUBindGroupLayoutDescriptor ld{};
ld.entryCount = layout.size();
ld.entries = layout.data();
scope_push();
p->layout = wgpuDeviceCreateBindGroupLayout(dev.device, &ld);
WGPUPipelineLayoutDescriptor pld{};
pld.bindGroupLayoutCount = 1;
pld.bindGroupLayouts = &p->layout;
WGPUPipelineLayout pl = wgpuDeviceCreatePipelineLayout(dev.device, &pld);
WGPUComputePipelineDescriptor cd{};
cd.layout = pl;
cd.compute.module = m->module;
cd.compute.entryPoint = sv("main");
p->pipeline = wgpuDeviceCreateComputePipeline(dev.device, &cd);
const std::string error = scope_pop();
if (RX_WEB) pipecache_record_compute(m->source(), layout);
wgpuPipelineLayoutRelease(pl);
if (!error.empty()) {
log_limited("cs_pipeline", 20, "rx: compute pipeline not created: %s", error.c_str());
p->failed = true;
}
count("pipelines");
it = pipelines.emplace(h, std::move(p)).first;
}
if (it->second->failed) return count("cs_skipped");
WGPUBindGroupDescriptor bd{};
bd.layout = it->second->layout;
bd.entryCount = entries.size();
bd.entries = entries.data();
// (the threaded web build: made where the pass is replayed, rx_record.cpp cp_set_bind_group_desc)
WGPUBindGroup group = RX_WEB_MT ? nullptr : checked_bind_group(bd, "dispatch");
if (!RX_WEB_MT && !group) return count("cs_bind_invalid");
end_render_pass("pe_compute");
WGPUComputePassEncoder pass = ce_begin_compute_pass(encoder());
cp_set_pipeline(pass, it->second->pipeline);
if (group) cp_set_bind_group(pass, 0, group, 0, nullptr);
else cp_set_bind_group_desc(pass, 0, bd.layout, entries.data(), entries.size());
const uint32_t max = dev.limits.maxComputeWorkgroupsPerDimension;
cp_dispatch(pass, std::min(l.grid[0], max), std::min(l.grid[1], max),
std::min(l.grid[2], max));
cp_end(pass);
if (group) wgpuBindGroupRelease(group);
for (const BufferRef& w : written) buffer_cache_gpu_written(w);
if (!written.empty()) dispatch_write_epoch++;
count("dispatches");
if (frame_dump_active()) {
char b[512];
int n = std::snprintf(b, sizeof b, "dispatch %#llx grid %u,%u,%u block %u,%u,%u: %zu ssbo (%zu written), %zu tex",
(unsigned long long)(l.code_base + l.program_start), l.grid[0], l.grid[1], l.grid[2],
l.block[0], l.block[1], l.block[2], info.storage_buffers_descriptors.size(),
written.size(), info.texture_descriptors.size());
for (const auto& d : info.storage_buffers_descriptors) {
const auto& cb = l.cbufs[d.cbuf_index % 8];
const uint64_t va = gpu_get<uint64_t>(cb.address + d.cbuf_offset);
const uint32_t size = gpu_get<uint32_t>(cb.address + d.cbuf_offset + 8);
if (n < 480) n += std::snprintf(b + n, sizeof b - n, " %s%#llx+%#x", d.is_written ? "w" : "r",
(unsigned long long)va, size);
}
dump_draw(b);
}
}
} // namespace
// ---- the native NVN layer's dispatches: the bound program's compute stage, its banks from NVN state (c0: the
// driver's storage buffer table at 0x310, the grid at 0x200, the uniform buffer table at 0x210; c2: handles) ----
void nvn_dispatch(NvnState& s, uint32_t x, uint32_t y, uint32_t z, uint64_t indirect_va) {
static const bool off = getenv("RX_NO_COMPUTE") != nullptr; // (debugging)
if (off || !s.render_enabled) return;
// (mkw-recomp) an indirect dispatch: the grid stays on the GPU, where the dispatch that wrote it left it (in the
// buffer cache); copied into c0 there for the program. Reading it back first stopped the GPU 6 times a frame
BufferRef args;
if (indirect_va) {
args = guest_buffer(indirect_va, 12);
if (!args.buffer || args.offset % 4) return count("cs_indirect_unmapped");
x = y = z = 1;
}
s.grid[0] = x, s.grid[1] = y, s.grid[2] = z;
if (!x || !y || !z) return count("cs_empty");
ShaderModule* m = nvn_compute_module(s);
if (!m) return count("cs_skipped");
const Shader::Info& info = m->info;
if (!info.image_buffer_descriptors.empty() || !info.texture_buffer_descriptors.empty()) {
log_limited("nvn_cs_images", 8, "rx: an NVN compute program uses image or texture buffers: not dispatched yet");
return count("cs_image_buffers");
}
// (mkw-recomp) images: the render targets of the textures their handles name (2D for now), bound as storage
// textures, with the program translated for their formats
std::array<uint16_t, 16> img_formats{};
std::vector<Image*> img_targets;
if (!info.image_descriptors.empty()) {
NvnBanks ib;
ib.c1 = nvn_c1(s, 5);
for (size_t i = 0; i < info.image_descriptors.size(); i++) {
const auto& d = info.image_descriptors[i];
uint32_t handle = 0;
ib.read(s, 5, d.cbuf_index, d.cbuf_offset, &handle, 4);
nvn_tex t{};
nvn_view v{};
bool has_view = false;
Image* im = nullptr;
const bool layered = d.type == Shader::TextureType::ColorArray2D, volume = d.type == Shader::TextureType::Color3D;
if (i < img_formats.size() && d.count == 1 && handle && nvn_texture_of((int)(handle & 0xFFFFF), &t, &v, &has_view)) {
if (d.type == Shader::TextureType::Color2D) im = nvn_target(&t, has_view ? &v : nullptr);
else if (layered || volume) im = nvn_storage_image(&t, has_view ? &v : nullptr, volume);
}
if (!im || !wgsl_storage_format(im->format)) {
log_limited("nvn_cs_image_kind", 24, "rx: NVN compute image %zu (type %d, handle %#x, NVN format %#x %ux%ux%u "
"target %d): %s (WebGPU format %#x)", i, (int)d.type, handle,
has_view && v.set_format ? v.format : t.d.format, t.d.width, t.d.height, t.d.depth,
t.d.target, im ? "its format is not a storage format" : "no 2D target for it",
im ? (unsigned)im->format : 0u);
return count("cs_images");
}
img_formats[i] = (uint16_t)im->format;
img_targets.push_back(im);
}
m = nvn_compute_module(s, nullptr, nullptr, &img_formats);
if (!m) return count("cs_skipped");
}
const uint64_t estimate = 64 * 1024 + info.constant_buffer_descriptors.size() * 0x10100 +
info.texture_descriptors.size() * 256 + info.storage_buffers_descriptors.size() * (1 << 20);
if (stream_ro.used + estimate > stream_ro.capacity || stream_rw.used + (8u << 20) > stream_rw.capacity)
flush_batch();
NvnBanks banks;
banks.c1 = nvn_c1(s, 5);
std::vector<WGPUBindGroupLayoutEntry> layout;
std::vector<WGPUBindGroupEntry> entries;
std::vector<uint32_t> key = {(uint32_t)(uintptr_t)m, (uint32_t)((uintptr_t)m >> 32)};
std::vector<BufferRef> written;
std::vector<std::pair<uint32_t, BufferRef>> storage; // (binding, range) of each storage descriptor
auto buffer_entry = [&](uint32_t b, WGPUBufferBindingType type, WGPUBuffer buf, uint64_t off, uint64_t size) {
WGPUBindGroupLayoutEntry e{};
e.binding = b;
e.visibility = WGPUShaderStage_Compute;
e.buffer.type = type;
layout.push_back(e);
WGPUBindGroupEntry g{};
g.binding = b;
g.buffer = buf;
g.offset = off;
g.size = size;
entries.push_back(g);
};
const bool indirect = info.uses_global_memory || info.uses_cbuf_indirect;
std::array<uint32_t, 18> base{};
std::array<bool, 18> in_ub{};
uint32_t ub_size = 4 + 2 * (uint32_t)info.texture_descriptors.size();
if (!indirect)
for (const auto& d : info.constant_buffer_descriptors) {
const uint32_t size = std::max(Common::DivCeil(info.constant_buffer_used_sizes[d.index], 16U), 1U);
if (ub_size + size > 0x1000) continue;
in_ub[d.index % 18] = true;
base[d.index % 18] = ub_size;
ub_size += size;
}
const uint64_t ub = stream_ro.alloc((uint64_t)ub_size * 16, dev.limits.minUniformBufferOffsetAlignment);
if (ub == UINT64_MAX) return count("cs_space");
uint8_t* block = stream_ro.at(ub);
std::memset(block, 0, 64);
const float scaling_z = 1.0f;
std::memcpy(block + 8, &scaling_z, 4);
for (const auto& d : info.constant_buffer_descriptors) {
if (!in_ub[d.index % 18]) continue;
const uint64_t size = (uint64_t)std::max(Common::DivCeil(info.constant_buffer_used_sizes[d.index], 16U), 1U) * 16;
banks.read(s, 5, d.index, 0, block + (uint64_t)base[d.index % 18] * 16, size);
count("up_cbuf", (int64_t)size);
}
buffer_entry(0, WGPUBufferBindingType_Uniform, stream_ro.buffer, ub, (uint64_t)ub_size * 16);
uint64_t c0_at = in_ub[0] ? ub + (uint64_t)base[0] * 16 : UINT64_MAX; // (where c0's grid goes, in stream_ro)
uint32_t b = 1;
for (const auto& d : info.constant_buffer_descriptors) {
if (in_ub[d.index % 18]) continue;
uint8_t probe[4];
const uint64_t bank = banks.read(s, 5, d.index, 0, probe, 4);
const uint64_t size = std::clamp<uint64_t>(bank + 15 & ~15ull, 16, 0x10000);
const uint64_t off = stream_ro.alloc(size, dev.limits.minStorageBufferOffsetAlignment);
if (off == UINT64_MAX) return count("cs_space");
banks.read(s, 5, d.index, 0, stream_ro.at(off), size);
count("up_cbuf_storage", (int64_t)size);
if (d.index == 0) c0_at = off;
buffer_entry(b, WGPUBufferBindingType_ReadOnlyStorage, stream_ro.buffer, off, size);
b += d.count;
}
for (const auto& d : info.storage_buffers_descriptors) {
uint64_t raw_va = 0;
uint32_t raw_size = 0;
banks.read(s, 5, d.cbuf_index, d.cbuf_offset, &raw_va, 8);
banks.read(s, 5, d.cbuf_index, d.cbuf_offset + 8, &raw_size, 4);
const uint64_t va = raw_va & ~(uint64_t)(dev.limits.minStorageBufferOffsetAlignment - 1);
uint64_t size = (uint64_t)raw_size + (raw_va - va);
size = std::clamp<uint64_t>((size + 3) & ~3ull, 16, 16ull << 20);
BufferRef ref = raw_va ? guest_buffer(va, size) : BufferRef{};
if (!ref.buffer || ref.offset % dev.limits.minStorageBufferOffsetAlignment) {
count("cs_ssbo_copied");
const uint64_t off = stream_rw.alloc(size, dev.limits.minStorageBufferOffsetAlignment);
if (off == UINT64_MAX) return count("cs_space");
if (raw_va) gpu_read(va, stream_rw.at(off), size);
else std::memset(stream_rw.at(off), 0, size);
ref = {stream_rw.buffer, off, size};
if (d.is_written && raw_va) writeback_after_batch(off, size, va);
} else if (d.is_written) {
written.push_back(ref);
note_dispatch_write(va, size);
}
storage.push_back({b, ref});
b += d.count;
}
std::array<uint8_t, 64> alias{};
std::array<uint32_t, 64> alias_words{};
std::vector<uint64_t> end;
if (alias_overlapping_storage(storage, end, alias, alias_words)) {
m = nvn_compute_module(s, &alias, &alias_words, &img_formats);
if (!m) return count("cs_skipped");
key[0] = (uint32_t)(uintptr_t)m;
key[1] = (uint32_t)((uintptr_t)m >> 32);
count("cs_ssbo_aliased");
}
for (size_t j = 0; j < storage.size(); j++)
if (j >= 64 || !alias[j])
buffer_entry(storage[j].first, WGPUBufferBindingType_Storage, storage[j].second.buffer,
storage[j].second.offset, end[j] - storage[j].second.offset);
uint32_t t_index = 0;
for (const auto& d : info.texture_descriptors) {
uint32_t primary = 0, secondary = 0;
banks.read(s, 5, d.cbuf_index, d.cbuf_offset, &primary, 4);
if (d.has_secondary) banks.read(s, 5, d.secondary_cbuf_index, d.secondary_cbuf_offset, &secondary, 4);
const uint32_t handle = d.has_secondary ? (primary << d.shift_left) | (secondary << d.secondary_shift_left)
: primary;
const BoundTexture t = nvn_texture(s, handle, 0, d);
std::memcpy(block + (uint64_t)(4 + 2 * t_index++) * 16, t.slots, sizeof t.slots);
WGPUBindGroupLayoutEntry tl{};
tl.binding = b;
tl.visibility = WGPUShaderStage_Compute;
tl.texture.sampleType = t.kind == TexKind::Float ? WGPUTextureSampleType_Float
: t.kind == TexKind::UnfilterableFloat ? WGPUTextureSampleType_UnfilterableFloat
: t.kind == TexKind::Depth ? WGPUTextureSampleType_Depth
: t.kind == TexKind::Uint ? WGPUTextureSampleType_Uint
: WGPUTextureSampleType_Sint;
tl.texture.viewDimension = view_dimension(d.type);
tl.texture.multisampled = d.is_multisample;
layout.push_back(tl);
WGPUBindGroupLayoutEntry sl{};
sl.binding = b + 1;
sl.visibility = WGPUShaderStage_Compute;
sl.sampler.type = t.kind == TexKind::Depth ? WGPUSamplerBindingType_Comparison
: t.kind == TexKind::UnfilterableFloat ? WGPUSamplerBindingType_NonFiltering
: WGPUSamplerBindingType_Filtering;
layout.push_back(sl);
WGPUBindGroupEntry te{};
te.binding = b;
te.textureView = t.view;
entries.push_back(te);
WGPUBindGroupEntry se{};
se.binding = b + 1;
se.sampler = t.sampler;
entries.push_back(se);
key.push_back((uint32_t)t.kind);
b += 2;
}
for (size_t i = 0; i < img_targets.size(); i++) { // (mkw-recomp) the images, after the textures (as the WGSL)
const auto& d = info.image_descriptors[i];
WGPUBindGroupLayoutEntry il{};
il.binding = b;
il.visibility = WGPUShaderStage_Compute;
il.storageTexture.access = d.is_written ? (d.is_read ? WGPUStorageTextureAccess_ReadWrite : WGPUStorageTextureAccess_WriteOnly)
: WGPUStorageTextureAccess_ReadOnly;
il.storageTexture.format = img_targets[i]->format;
const WGPUTextureViewDimension vdim = d.type == Shader::TextureType::ColorArray2D ? WGPUTextureViewDimension_2DArray
: d.type == Shader::TextureType::Color3D ? WGPUTextureViewDimension_3D
: WGPUTextureViewDimension_2D;
il.storageTexture.viewDimension = vdim;
layout.push_back(il);
WGPUBindGroupEntry g{};
g.binding = b;
g.textureView = vdim == WGPUTextureViewDimension_2D ? img_targets[i]->attachment
: img_targets[i]->view(vdim, false, 0, 0, 1, img_targets[i]->format);
entries.push_back(g);
key.push_back(0x1000u | img_targets[i]->format);
b += 1;
}
{
const uint64_t lh = hash_bytes(layout.data(), layout.size() * sizeof(WGPUBindGroupLayoutEntry));
key.insert(key.end(), {(uint32_t)lh, (uint32_t)(lh >> 32)});
}
const uint64_t h = hash_bytes(key.data(), key.size() * 4);
auto it = pipelines.find(h);
if (it == pipelines.end()) {
auto p = std::make_unique<ComputePipeline>();
WGPUBindGroupLayoutDescriptor ld{};
ld.entryCount = layout.size();
ld.entries = layout.data();
scope_push();
p->layout = wgpuDeviceCreateBindGroupLayout(dev.device, &ld);
WGPUPipelineLayoutDescriptor pld{};
pld.bindGroupLayoutCount = 1;
pld.bindGroupLayouts = &p->layout;
WGPUPipelineLayout pl = wgpuDeviceCreatePipelineLayout(dev.device, &pld);
WGPUComputePipelineDescriptor cd{};
cd.layout = pl;
cd.compute.module = m->module;
cd.compute.entryPoint = sv("main");
p->pipeline = wgpuDeviceCreateComputePipeline(dev.device, &cd);
const std::string error = scope_pop();
if (RX_WEB) pipecache_record_compute(m->source(), layout);
wgpuPipelineLayoutRelease(pl);
if (!error.empty()) {
log_limited("cs_pipeline", 20, "rx: NVN compute pipeline not created: %s", error.c_str());
p->failed = true;
}
count("pipelines");
it = pipelines.emplace(h, std::move(p)).first;
}
if (it->second->failed) return count("cs_skipped");
WGPUBindGroupDescriptor bd{};
bd.layout = it->second->layout;
bd.entryCount = entries.size();
bd.entries = entries.data();
// (the threaded web build: made where the pass is replayed, rx_record.cpp cp_set_bind_group_desc)
WGPUBindGroup group = RX_WEB_MT ? nullptr : checked_bind_group(bd, "dispatch");
if (!RX_WEB_MT && !group) return count("cs_bind_invalid");
end_render_pass("pe_compute");
if (args.buffer) {
if (c0_at != UINT64_MAX) ce_copy_buffer_to_buffer(encoder(), args.buffer, args.offset, stream_ro.buffer, c0_at, 12);
// (into a buffer of their own: the segment holding them may be bound for writing by this dispatch, and a buffer
// cannot be both in one dispatch. Slots are reused in turn: each copy runs before its dispatch, in order)
static WGPUBuffer ring;
static uint64_t next;
constexpr uint64_t kSlots = 4096;
if (!ring) {
WGPUBufferDescriptor bd{};
bd.label = sv("rx indirect arguments");
bd.size = kSlots * 16;
bd.usage = WGPUBufferUsage_Indirect | WGPUBufferUsage_CopyDst;
ring = wgpuDeviceCreateBuffer(dev.device, &bd);
}
const uint64_t slot = (next++ % kSlots) * 16;
ce_copy_buffer_to_buffer(encoder(), args.buffer, args.offset, ring, slot, 12);
args = {ring, slot, 12};
}
WGPUComputePassEncoder pass = ce_begin_compute_pass(encoder());
cp_set_pipeline(pass, it->second->pipeline);
if (group) cp_set_bind_group(pass, 0, group, 0, nullptr);
else cp_set_bind_group_desc(pass, 0, bd.layout, entries.data(), entries.size());
const uint32_t max = dev.limits.maxComputeWorkgroupsPerDimension;
if (args.buffer) cp_dispatch_indirect(pass, args.buffer, args.offset), count("cs_indirect_gpu");
else cp_dispatch(pass, std::min(x, max), std::min(y, max), std::min(z, max));
cp_end(pass);
if (group) wgpuBindGroupRelease(group);
for (const BufferRef& w : written) buffer_cache_gpu_written(w);
if (!written.empty()) dispatch_write_epoch++;
for (size_t i = 0; i < img_targets.size(); i++) // (the images it wrote: drawn on the GPU now)
if (info.image_descriptors[i].is_written) img_targets[i]->rendered = true, img_targets[i]->version++;
if (!img_targets.empty()) count("cs_with_images");
count("dispatches");
if (frame_dump_active()) { // (which program wrote which memory: what draws later read)
char line[640];
int n = std::snprintf(line, sizeof line, "nvn dispatch cs %016llx grid %u,%u,%u: %zu ssbo",
(unsigned long long)nvn_cs_key(s), x, y, z, info.storage_buffers_descriptors.size());
for (const auto& d : info.storage_buffers_descriptors) {
uint64_t va = 0;
uint32_t size = 0;
banks.read(s, 5, d.cbuf_index, d.cbuf_offset, &va, 8);
banks.read(s, 5, d.cbuf_index, d.cbuf_offset + 8, &size, 4);
if (n < 600) n += std::snprintf(line + n, sizeof line - n, " %s%#llx+%#x", d.is_written ? "w" : "r",
(unsigned long long)va, size);
}
dump_draw(line);
watch_buffer_after_draw(line);
}
}
} // namespace rx
using namespace rx;
extern "C" void rx_compute_launch(void* channel, const uint32_t* regs) {
(void)channel;
if (!rx_active()) return;
static const bool off = getenv("RX_NO_COMPUTE") != nullptr; // (debugging)
if (off) return;
std::lock_guard lk(mu);
// the texture pools sit where the 3D engine has its own (tex_sampler 0x557, tex_header 0x55D)
static Regs tex_regs;
ComputeLaunch l = read_launch(regs);
std::memcpy(&tex_regs.tex_sampler, regs + RX_REG(tex_sampler), sizeof tex_regs.tex_sampler);
std::memcpy(&tex_regs.tex_header, regs + RX_REG(tex_header), sizeof tex_regs.tex_header);
tex_regs.sampler_binding =
l.linked_tsc ? Regs::SamplerBinding::ViaHeaderBinding : Regs::SamplerBinding::Independently;
l.tex_regs = &tex_regs;
dispatch(l);
}

4708
gpu/render/rx_core.cpp Normal file

File diff suppressed because it is too large Load Diff

733
gpu/render/rx_debug.cpp Normal file
View File

@@ -0,0 +1,733 @@
// Frame dumps (debugging): RX_DUMP_FRAME=<n> (or RX_DUMP_AT=<seconds>) logs frame n's passes in order (their targets, how many draws went
// into each, dispatches, clears) and, at its present, saves every render target as a PNG in RX_DUMP_DIR (default
// "rx_frame"): <order of last use>_<guest address>_<width>x<height>_<format>.png. Float formats are clamped to
// [0, 1]; depth is shown as gray.
#include <algorithm>
#include <array>
#include <map>
#include <tuple>
#include <cstring>
#include <cmath>
#include <cstdlib>
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include "renderdoc_app.h"
#include "rx_internal.h"
namespace rx {
namespace {
int64_t dump_frame = -2;
int pass_draws;
bool pass_open;
/// The render targets in the order passes used them (their textures held: an image can be replaced mid-frame
/// when the game reuses its memory for another surface).
struct Touched {
WGPUTexture tex;
uint64_t guest;
uint32_t width, height;
WGPUTextureFormat format;
};
std::vector<Touched> touched;
float half(uint16_t h) {
const uint32_t s = h >> 15, e = h >> 10 & 31, m = h & 1023;
const float f = e == 0 ? std::ldexp((float)m, -24) : e == 31 ? 65504.0f : std::ldexp(1.0f + m / 1024.0f, (int)e - 15);
return s ? -f : f;
}
float small_float(uint32_t v, int mbits) {
const uint32_t e = v >> mbits, m = v & ((1u << mbits) - 1);
if (e == 0) return std::ldexp((float)m, -14 - mbits);
if (e == 31) return 65504.0f;
return std::ldexp(1.0f + (float)m / (1u << mbits), (int)e - 15);
}
uint8_t to8(float f) {
return (uint8_t)std::lround(std::clamp(f, 0.0f, 1.0f) * 255.0f);
}
/// The texel at p in RGBA8 (false: a format this does not show).
bool texel(WGPUTextureFormat f, const uint8_t* p, uint8_t out[4]) {
out[3] = 255;
switch (f) {
case WGPUTextureFormat_RGBA8Unorm:
case WGPUTextureFormat_RGBA8UnormSrgb:
std::memcpy(out, p, 4);
out[3] = 255;
return true;
case WGPUTextureFormat_BGRA8Unorm:
case WGPUTextureFormat_BGRA8UnormSrgb:
out[0] = p[2], out[1] = p[1], out[2] = p[0];
return true;
case WGPUTextureFormat_R8Unorm:
out[0] = out[1] = out[2] = p[0];
return true;
case WGPUTextureFormat_RG8Unorm:
out[0] = p[0], out[1] = p[1], out[2] = 0;
return true;
case WGPUTextureFormat_RGBA16Float: {
uint16_t c[4];
std::memcpy(c, p, 8);
for (int i = 0; i < 3; i++) out[i] = to8(half(c[i]));
return true;
}
case WGPUTextureFormat_RG16Float: {
uint16_t c[2];
std::memcpy(c, p, 4);
out[0] = to8(half(c[0])), out[1] = to8(half(c[1])), out[2] = 0;
return true;
}
case WGPUTextureFormat_R16Float: {
uint16_t c;
std::memcpy(&c, p, 2);
out[0] = out[1] = out[2] = to8(half(c));
return true;
}
case WGPUTextureFormat_Depth16Unorm: {
uint16_t v;
std::memcpy(&v, p, 2);
out[0] = out[1] = out[2] = (uint8_t)(v >> 8);
return true;
}
case WGPUTextureFormat_R32Float:
case WGPUTextureFormat_Depth32Float: {
float v;
std::memcpy(&v, p, 4);
out[0] = out[1] = out[2] = to8(v);
return true;
}
case WGPUTextureFormat_RG11B10Ufloat: {
uint32_t v;
std::memcpy(&v, p, 4);
out[0] = to8(small_float(v & 0x7FF, 6)), out[1] = to8(small_float(v >> 11 & 0x7FF, 6));
out[2] = to8(small_float(v >> 22, 5));
return true;
}
case WGPUTextureFormat_RGB10A2Unorm: {
uint32_t v;
std::memcpy(&v, p, 4);
out[0] = (uint8_t)((v & 1023) >> 2), out[1] = (uint8_t)((v >> 10 & 1023) >> 2);
out[2] = (uint8_t)((v >> 20 & 1023) >> 2);
return true;
}
default:
return false;
}
}
uint32_t texel_bytes(WGPUTextureFormat f) {
switch (f) {
case WGPUTextureFormat_R8Unorm: return 1;
case WGPUTextureFormat_RG8Unorm: case WGPUTextureFormat_R16Float: case WGPUTextureFormat_Depth16Unorm: return 2;
case WGPUTextureFormat_RGBA16Float: return 8;
default: return 4;
}
}
void save(const Touched* im, const char* dir, int order) {
uint8_t probe[4];
const uint8_t zero[8]{};
if (!texel(im->format, zero, probe)) return;
const uint32_t bpp = texel_bytes(im->format);
const size_t stride = ((size_t)im->width * bpp + 255) & ~size_t(255), size = stride * im->height;
WGPUBufferDescriptor bd{};
bd.size = size;
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
WGPUBuffer buffer = wgpuDeviceCreateBuffer(dev.device, &bd);
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
WGPUTexelCopyTextureInfo src{};
src.texture = im->tex;
src.aspect = is_depth_format(im->format) ? WGPUTextureAspect_DepthOnly : WGPUTextureAspect_All;
WGPUTexelCopyBufferInfo dst{};
dst.buffer = buffer;
dst.layout.bytesPerRow = (uint32_t)stride;
dst.layout.rowsPerImage = im->height;
const WGPUExtent3D extent{im->width, im->height, 1};
wgpuCommandEncoderCopyTextureToBuffer(e, &src, &dst, &extent);
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
wgpuQueueSubmit(dev.queue, 1, &cb);
wgpuCommandBufferRelease(cb);
wgpuCommandEncoderRelease(e);
struct Done {
bool complete = false;
WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error;
} done;
WGPUBufferMapCallbackInfo mc{};
mc.mode = WGPUCallbackMode_AllowSpontaneous;
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
auto* d = static_cast<Done*>(u1);
d->status = status;
d->complete = true;
};
mc.userdata1 = &done;
wgpuBufferMapAsync(buffer, WGPUMapMode_Read, 0, size, mc);
while (!done.complete) wgpuDevicePoll(dev.device, true, nullptr);
if (done.status == WGPUMapAsyncStatus_Success) {
const auto* px = static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(buffer, 0, size));
std::vector<uint8_t> rgba((size_t)im->width * im->height * 4);
for (uint32_t y = 0; y < im->height; y++)
for (uint32_t x = 0; x < im->width; x++)
texel(im->format, px + y * stride + (size_t)x * bpp, &rgba[((size_t)y * im->width + x) * 4]);
char path[512];
std::snprintf(path, sizeof path, "%s/%03d_%08llx_%ux%u_%x.png", dir, order, (unsigned long long)im->guest,
im->width, im->height, (unsigned)im->format);
write_png(path, rgba.data(), im->width, im->height, (size_t)im->width * 4);
if (im->format == WGPUTextureFormat_R32Float || im->format == WGPUTextureFormat_Depth32Float ||
im->format == WGPUTextureFormat_RGBA16Float || im->format == WGPUTextureFormat_RG11B10Ufloat ||
im->format == WGPUTextureFormat_RGBA8Unorm || im->format == WGPUTextureFormat_RGBA8UnormSrgb ||
im->format == WGPUTextureFormat_Depth16Unorm || im->format == WGPUTextureFormat_RG16Float ||
im->format == WGPUTextureFormat_R16Float) { // raw too
std::snprintf(path, sizeof path, "%s/%03d_%08llx_%ux%u_%x.raw", dir, order, (unsigned long long)im->guest,
im->width, im->height, (unsigned)im->format);
if (FILE* f = std::fopen(path, "wb")) {
for (uint32_t y = 0; y < im->height; y++) std::fwrite(px + y * stride, 1, (size_t)im->width * bpp, f);
std::fclose(f);
}
}
wgpuBufferUnmap(buffer);
}
wgpuBufferRelease(buffer);
}
void touch(const Image* im) {
if (!im) return;
for (const Touched& t : touched)
if (t.tex == im->tex) return;
wgpuTextureAddRef(im->tex);
touched.push_back({im->tex, im->guest, im->width, im->height, im->format});
}
std::vector<std::pair<Touched, std::string>> snapshots; // RX_DUMP_AFTER: copies taken after chosen draws
} // namespace
/// RX_DUMP_AFTER=<fs address>[,...] (with a frame dump): color target 0 is copied after each draw of those pixel
/// programs and saved with the frame (snap_<n>_<fs>.png); with RX_DUMP_AFTER_DEPTH=1 the depth target instead.
void dump_after_draw(uint64_t fs_address, Image* target) {
if (!frame_dump_active() || !target) return;
static std::vector<uint64_t> wanted;
static bool parsed;
if (!parsed) {
parsed = true;
if (const char* s = getenv("RX_DUMP_AFTER"))
for (const char* p = s; *p;) {
char* end;
wanted.push_back(strtoull(p, &end, 16));
p = *end == ',' ? end + 1 : end;
if (end == p && !*end) break;
}
}
// RX_DUMP_AFTER_TARGET=<guest hex> (with a frame dump): after every draw into that target, whatever its program
static const uint64_t target_guest = getenv("RX_DUMP_AFTER_TARGET") ? strtoull(getenv("RX_DUMP_AFTER_TARGET"), nullptr, 16) : 0;
if (!(target_guest && target->guest == target_guest) &&
std::find(wanted.begin(), wanted.end(), fs_address) == wanted.end())
return;
end_render_pass();
WGPUTextureDescriptor d{};
d.usage = WGPUTextureUsage_CopySrc | WGPUTextureUsage_CopyDst;
d.dimension = WGPUTextureDimension_2D;
d.size = {target->width, target->height, 1};
d.format = target->format;
d.mipLevelCount = 1;
d.sampleCount = 1;
WGPUTexture copy = wgpuDeviceCreateTexture(dev.device, &d);
WGPUTexelCopyTextureInfo src{}, dst{};
src.texture = target->tex;
dst.texture = copy;
const WGPUExtent3D extent{target->width, target->height, 1};
ce_copy_texture_to_texture(encoder(), &src, &dst, &extent);
char label[64];
std::snprintf(label, sizeof label, "snap_%03zu_%llx", snapshots.size(), (unsigned long long)fs_address);
snapshots.push_back({{copy, target->guest, target->width, target->height, target->format}, label});
}
namespace {
struct PixelRead {
WGPUTextureFormat format;
uint64_t guest;
int slot;
uint32_t x, y;
std::string what;
};
std::vector<PixelRead> pixel_reads;
WGPUBuffer pixel_buffer = nullptr;
constexpr uint32_t kPixelReads = 16384;
} // namespace
/// RX_PIXEL=<x>,<y> (with a frame dump; 1920x1080 coordinates, scaled to each target's size): after every draw each
/// colour attachment's texel there is read back and logged with the draw at the frame's end (rx-pixel lines): which
/// draw put a mark on the screen.
void dump_pixel_after(const Attachments& at, const char* what) {
// (RX_PIXEL=<x>,<y>[;<x>,<y>...]: up to 8 points)
static std::vector<std::pair<int, int>> points;
static const bool parsed = [] {
if (const char* s = getenv("RX_PIXEL"))
for (const char* p = s; *p;) {
int x, y;
if (std::sscanf(p, "%d,%d", &x, &y) != 2) break;
points.push_back({x, y});
const char* semi = std::strchr(p, ';');
if (!semi || points.size() >= 8) break;
p = semi + 1;
}
return true;
}();
(void)parsed;
if (points.empty() || !frame_dump_active()) return;
// RX_PIXEL_TARGET=<guest hex>: only after draws into that target (the passes before it are left as they are)
static const uint64_t only = getenv("RX_PIXEL_TARGET") ? strtoull(getenv("RX_PIXEL_TARGET"), nullptr, 16) : 0;
// RX_PIXEL_FROM=<text>: only from the first draw whose description contains it on (every target after it)
static const char* from = getenv("RX_PIXEL_FROM");
static int64_t from_frame = -1;
if (from) {
if (from_frame != frame_number) {
if (!what || !std::strstr(what, from)) return;
from_frame = frame_number;
}
} else if (only) {
bool hit = false;
for (const Image* c : at.color) hit |= c && c->guest == only;
if (!hit) return;
}
if (!pixel_buffer) {
WGPUBufferDescriptor bd{};
bd.size = (uint64_t)kPixelReads * 256;
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
pixel_buffer = wgpuDeviceCreateBuffer(dev.device, &bd);
}
bool ended = false;
for (int i = 0; i < 8; i++) {
const Image* t = at.color[i];
if (!t || pixel_reads.size() >= kPixelReads) continue;
uint8_t probe[4];
const uint8_t zero[8]{};
if (!texel(t->format, zero, probe)) continue;
if (!ended) end_render_pass(), ended = true;
for (const auto& [px, py] : points) {
if (pixel_reads.size() >= kPixelReads) break;
const uint32_t x = std::min<uint32_t>(t->width - 1, (uint32_t)((uint64_t)px * t->width / 1920));
const uint32_t y = std::min<uint32_t>(t->height - 1, (uint32_t)((uint64_t)py * t->height / 1080));
WGPUTexelCopyTextureInfo src{};
src.texture = t->tex;
src.origin = {x, y, 0};
WGPUTexelCopyBufferInfo dst{};
dst.buffer = pixel_buffer;
dst.layout.offset = (uint64_t)pixel_reads.size() * 256;
dst.layout.bytesPerRow = 256;
dst.layout.rowsPerImage = 1;
const WGPUExtent3D one{1, 1, 1};
ce_copy_texture_to_buffer(encoder(), &src, &dst, &one);
pixel_reads.push_back({t->format, t->guest, i, x, y, what ? what : ""});
}
}
}
namespace {
uint64_t watch_va, watch_size;
int64_t watch_frames = 600;
void watch_parse() {
static const bool parsed = [] {
if (const char* s = getenv("RX_WATCH_BUF")) {
char* e = nullptr;
watch_va = strtoull(s, &e, 16);
if (e && *e == ':') watch_size = strtoull(e + 1, &e, 16);
if (e && *e == ':') watch_frames = atoll(e + 1);
}
return true;
}();
(void)parsed;
}
// the watched range's buffer cache copy read back; the 16-byte records changed since the last read logged (rx-buf
// lines, after <when>), and the copy appended to RX_WATCH_BUF_FILE under <tag>
void watch_buffer_read(const char* when, int64_t tag) {
const BufferRef ref = guest_buffer(watch_va, watch_size);
if (!ref.buffer) return;
flush_batch(); // (the uploads guest_buffer queued)
wait_submitted();
WGPUBufferDescriptor bd{};
bd.size = (watch_size + 3) & ~3ull;
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
WGPUBuffer staging = wgpuDeviceCreateBuffer(dev.device, &bd);
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
wgpuCommandEncoderCopyBufferToBuffer(e, ref.buffer, ref.offset, staging, 0, bd.size);
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
wgpuQueueSubmit(dev.queue, 1, &cb);
wgpuCommandBufferRelease(cb);
wgpuCommandEncoderRelease(e);
struct Done {
bool complete = false;
WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error;
} done;
WGPUBufferMapCallbackInfo mc{};
mc.mode = WGPUCallbackMode_AllowSpontaneous;
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
auto* d = static_cast<Done*>(u1);
d->status = status;
d->complete = true;
};
mc.userdata1 = &done;
wgpuBufferMapAsync(staging, WGPUMapMode_Read, 0, bd.size, mc);
while (!done.complete) wgpuDevicePoll(dev.device, true, nullptr);
static std::vector<uint32_t> last;
if (done.status == WGPUMapAsyncStatus_Success) {
const auto* w = static_cast<const uint32_t*>(wgpuBufferGetConstMappedRange(staging, 0, bd.size));
const size_t n = bd.size / 4;
if (last.size() != n) last.assign(n, 0xDEADBEEFu);
size_t nonzero = 0, changed = 0;
std::string line;
for (size_t i = 0; i < n; i++) nonzero += w[i] != 0;
for (size_t r = 0; r + 4 <= n; r += 4) {
if (!std::memcmp(w + r, last.data() + r, 16)) continue;
if (changed++ < 48) {
char b[80];
std::snprintf(b, sizeof b, " %zx:%x,%x,%x,%x", r * 4, w[r], w[r + 1], w[r + 2], w[r + 3]);
line += b;
}
}
if (changed || tag < (1ll << 40)) // (after draws: only the ones that changed it)
hos_log("rx-buf: %s (tag %lld): %zu nonzero words, %zu records changed:%s%s", when, (long long)tag, nonzero,
changed, line.c_str(), changed > 48 ? " ..." : "");
// RX_WATCH_BUF_FILE=<path>: every copy appended (its tag, 8 bytes, then the range)
if (static FILE* f = getenv("RX_WATCH_BUF_FILE") ? std::fopen(getenv("RX_WATCH_BUF_FILE"), "wb") : nullptr; f) {
std::fwrite(&tag, 8, 1, f);
std::fwrite(w, 4, n, f);
std::fflush(f);
}
last.assign(w, w + n);
wgpuBufferUnmap(staging);
}
wgpuBufferRelease(staging);
}
} // namespace
/// RX_WATCH_BUF=<GPU VA hex>:<size hex>[:<frames>] (debugging): at each present the buffer cache's copy of that range
/// (what the GPU's draws read) is read back, and the 16-byte records that changed since the last present are logged
/// (rx-buf lines), for that many frames (default 600) from RX_WATCH_BUF_AFTER=<seconds> on.
void watch_buffer_frame() {
watch_parse();
if (!watch_va || !watch_size || watch_frames <= 0) return;
static const int64_t after = getenv("RX_WATCH_BUF_AFTER") ? (int64_t)(atof(getenv("RX_WATCH_BUF_AFTER")) * 1e9) : 0;
static const int64_t start = hos_now_ns();
if (hos_now_ns() - start < after) return;
char when[48];
std::snprintf(when, sizeof when, "frame %lld", (long long)frame_number);
watch_buffer_read(when, frame_number);
watch_frames--;
}
/// RX_WATCH_BUF_DRAWS=1 (with RX_WATCH_BUF and a frame dump): the range is also read back after each draw of the
/// dumped frame (ending its pass), and the draws that changed it are logged (rx-buf: after <draw>; tagged 1 << 40 +
/// the draw's number in the frame, in RX_WATCH_BUF_FILE): which draw wrote a record, and what the next one read
void watch_buffer_after_draw(const char* what) {
static const bool per_draw = env_flag("RX_WATCH_BUF_DRAWS");
if (!per_draw || !frame_dump_active()) return;
watch_parse();
if (!watch_va || !watch_size) return;
static int64_t draw_frame = -1, draw = 0;
if (draw_frame != frame_number) draw_frame = frame_number, draw = 0;
end_render_pass();
std::string when = "after ";
when += what ? what : "?";
watch_buffer_read(when.c_str(), (1ll << 40) + draw++);
}
/// RX_DUMP_BEFORE=<fs address>[,...] (with a frame dump): every colour attachment and the depth are copied just
/// before each draw of those pixel programs (what the draw reads through feedback, what it blends with), saved as
/// pre_<n>_<fs>_c<slot>.png/.raw
void dump_before_draw(uint64_t fs_address, const Attachments& at) {
if (!frame_dump_active()) return;
static std::vector<uint64_t> wanted;
static bool parsed;
if (!parsed) {
parsed = true;
if (const char* s = getenv("RX_DUMP_BEFORE"))
for (const char* p = s; *p;) {
char* end;
wanted.push_back(strtoull(p, &end, 16));
if (end == p) break;
p = *end == ',' ? end + 1 : end;
}
}
if (std::find(wanted.begin(), wanted.end(), fs_address) == wanted.end()) return;
end_render_pass();
static int n;
for (int i = 0; i < 9; i++) {
const Image* t = i < 8 ? at.color[i] : at.depth;
if (!t) continue;
WGPUTextureDescriptor d{};
d.usage = WGPUTextureUsage_CopySrc | WGPUTextureUsage_CopyDst;
d.dimension = WGPUTextureDimension_2D;
d.size = {t->width, t->height, 1};
d.format = t->format;
d.mipLevelCount = 1;
d.sampleCount = 1;
WGPUTexture copy = wgpuDeviceCreateTexture(dev.device, &d);
WGPUTexelCopyTextureInfo src{}, dst{};
src.texture = t->tex;
src.aspect = is_depth_format(t->format) ? WGPUTextureAspect_DepthOnly : WGPUTextureAspect_All;
dst.texture = copy;
dst.aspect = src.aspect;
const WGPUExtent3D extent{t->width, t->height, 1};
ce_copy_texture_to_texture(encoder(), &src, &dst, &extent);
char label[64];
std::snprintf(label, sizeof label, "pre_%03d_%llx_%s%d", n, (unsigned long long)fs_address, i < 8 ? "c" : "z",
i < 8 ? i : 0);
snapshots.push_back({{copy, t->guest, t->width, t->height, t->format}, label});
}
n++;
}
/// RX_DUMP_ON_RT=<GPU VA hex> (with RX_DUMP_FRAME set to a far frame): the dump starts at the first draw into that
/// color target, mid-frame (a target drawn once, at loading, say)
void dump_start_now() {
if (dump_frame < 0 || dump_frame > frame_number) dump_frame = frame_number;
}
bool frame_dump_enabled = false;
std::atomic<int> dump_requested{0}; // rx_request_frame_dump (the live input file's "!dump")
bool frame_dump_active_slow() {
if (dump_frame == -2) {
const char* s = getenv("RX_DUMP_FRAME");
dump_frame = s ? atoll(s) : -1;
}
if (dump_requested.exchange(0)) dump_frame = frame_number + 1;
// RX_DUMP_AT=<seconds>: the frame that starts after that much time (since the renderer started)
static const char* at = getenv("RX_DUMP_AT");
static int64_t start = hos_now_ns();
if (at && dump_frame < 0 && hos_now_ns() - start >= (int64_t)(atof(at) * 1e9)) dump_frame = frame_number + 1;
// RX_DUMP_COUNT=<n>: n frames from there (each in a folder of its own)
static const int64_t n = getenv("RX_DUMP_COUNT") ? std::max(1, atoi(getenv("RX_DUMP_COUNT"))) : 1;
return dump_frame >= 0 && frame_number >= dump_frame && frame_number < dump_frame + n;
}
/// RX_DUMP_ON_GUEST=<guest hex> (with RX_DUMP_FRAME set to a far frame): the dump starts at the first draw or pass into
/// that target (either path; RX_DUMP_ON_RT takes the emulated path's GPU VA). Native draws check before they bind.
/// RX_DUMP_ON_SIZE=<width>x<height>: the same at the first color target of that size (both: that target at that
/// size). RX_DUMP_ON_ARM=<seconds>: not
/// before that much time (past the title screen's scene); RX_DUMP_ON_DELAY=<frames>: that many frames later.
void dump_on_targets(const Attachments& at) {
static const uint64_t on_guest = getenv("RX_DUMP_ON_GUEST") ? strtoull(getenv("RX_DUMP_ON_GUEST"), nullptr, 16) : 0;
static uint32_t on_w = 0, on_h = 0;
static const bool size_read = [] {
if (const char* s = getenv("RX_DUMP_ON_SIZE")) std::sscanf(s, "%ux%u", &on_w, &on_h);
return true;
}();
(void)size_read;
if ((!on_guest && !on_w) || !frame_dump_enabled) return;
static const int64_t arm_ns = getenv("RX_DUMP_ON_ARM") ? (int64_t)(atof(getenv("RX_DUMP_ON_ARM")) * 1e9) : 0;
static const int64_t start = hos_now_ns();
if (arm_ns && hos_now_ns() - start < arm_ns) return;
static const int64_t delay = getenv("RX_DUMP_ON_DELAY") ? atoll(getenv("RX_DUMP_ON_DELAY")) : 0;
static int64_t fired_at = -1;
if (fired_at >= 0) {
if (frame_number >= fired_at + delay) dump_start_now();
return;
}
for (int i = 0; i < 8; i++)
if (at.color[i] && (!on_guest || at.color[i]->guest == on_guest) &&
(!on_w || (at.color[i]->width == on_w && at.color[i]->height == on_h))) {
fired_at = frame_number;
if (!delay) dump_start_now();
return;
}
}
void dump_pass(const Attachments& at, const char* kind) {
// RX_WATCH_RT=<guest hex>: every pass into that target is logged (any frame)
static const uint64_t watch = getenv("RX_WATCH_RT") ? strtoull(getenv("RX_WATCH_RT"), nullptr, 16) : 0;
if (watch) {
static int logged[2];
for (int i = 0; i < 8; i++)
if (at.color[i] && at.color[i]->guest == watch && logged[kind[0] == 'd']++ < 400)
hos_log("rx: watch %08llx: %s pass %ux%u as c%d (format %#x, pixel format %d) frame %lld",
(unsigned long long)watch, kind, at.width, at.height, i, (unsigned)at.color[i]->format,
(int)at.color[i]->pixel_format, (long long)frame_number);
}
dump_on_targets(at);
if (!frame_dump_active()) return;
if (pass_open) hos_log("rx-frame: %d draws", pass_draws);
pass_draws = 0;
pass_open = true;
char b[512];
int n = std::snprintf(b, sizeof b, "rx-frame: %s pass %ux%u:", kind, at.width, at.height);
for (int i = 0; i < 8; i++)
if (at.color[i]) {
touch(at.color[i]);
n += std::snprintf(b + n, sizeof b - n, " c%d %08llx@%04x fmt %x", i, (unsigned long long)at.color[i]->guest,
(unsigned)(((uintptr_t)at.color[i] >> 4) & 0xFFFF),
(unsigned)at.color[i]->format);
}
if (at.depth) {
touch(at.depth);
std::snprintf(b + n, sizeof b - n, " z %08llx fmt %x", (unsigned long long)at.depth->guest,
(unsigned)at.depth->format);
}
hos_log("%s", b);
}
void dump_sampled(const Image* im) { // RX_DUMP_SAMPLED=1: render targets read as textures are saved too
static const bool on = env_flag("RX_DUMP_SAMPLED");
if (on && frame_dump_active()) touch(im);
}
void dump_draw(const char* what) {
if (!frame_dump_active()) return;
if (what) hos_log("rx-frame: %s", what);
else pass_draws++;
}
void dump_texture(WGPUTexture tex, uint64_t va, uint32_t width, uint32_t height, WGPUTextureFormat format,
const char* dir, int order) {
const Touched t{tex, va, width, height, format};
save(&t, dir, order);
}
void dump_frame_images(const char* dir) {
if (!frame_dump_active()) return;
if (pass_open) hos_log("rx-frame: %d draws", pass_draws);
pass_open = false;
CreateDirectoryA(dir, nullptr);
char sub[512];
if (getenv("RX_DUMP_COUNT")) {
std::snprintf(sub, sizeof sub, "%s/%lld", dir, (long long)frame_number);
CreateDirectoryA(sub, nullptr);
dir = sub;
hos_log("rx-frame: end of frame %lld", (long long)frame_number);
}
int order = 0;
// RX_DUMP_ONLY=<guest hex>[,...]: only those targets are saved
static std::vector<uint64_t> only;
static bool only_parsed;
if (!only_parsed) {
only_parsed = true;
if (const char* s = getenv("RX_DUMP_ONLY"))
for (const char* p = s; *p;) {
char* end;
only.push_back(strtoull(p, &end, 16));
if (end == p) break;
p = *end == ',' ? end + 1 : end;
}
}
for (const Touched& t : touched) {
if (only.empty() || std::find(only.begin(), only.end(), t.guest) != only.end()) save(&t, dir, order);
order++;
wgpuTextureRelease(t.tex);
}
hos_log("rx-frame: %zu render targets saved in %s", touched.size(), dir);
touched.clear();
dump_tex_list(dir);
for (auto& [t, label] : snapshots) {
save(&t, dir, 900 + order++);
char from[512], to[512];
std::snprintf(from, sizeof from, "%s/%03d_%08llx_%ux%u_%x.png", dir, 900 + order - 1, (unsigned long long)t.guest,
t.width, t.height, (unsigned)t.format);
std::snprintf(to, sizeof to, "%s/%s.png", dir, label.c_str());
MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING);
std::strcpy(from + std::strlen(from) - 4, ".raw"); // (and the raw texels, when the format has them)
std::snprintf(to, sizeof to, "%s/%s.raw", dir, label.c_str());
MoveFileExA(from, to, MOVEFILE_REPLACE_EXISTING);
wgpuTextureRelease(t.tex);
}
snapshots.clear();
if (!pixel_reads.empty()) { // (RX_PIXEL: the frame's work was submitted)
struct Done {
bool complete = false;
WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error;
} done;
WGPUBufferMapCallbackInfo mc{};
mc.mode = WGPUCallbackMode_AllowSpontaneous;
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
auto* d = static_cast<Done*>(u1);
d->status = status;
d->complete = true;
};
mc.userdata1 = &done;
const uint64_t size = (uint64_t)pixel_reads.size() * 256;
wgpuBufferMapAsync(pixel_buffer, WGPUMapMode_Read, 0, size, mc);
while (!done.complete) wgpuDevicePoll(dev.device, true, nullptr);
if (done.status == WGPUMapAsyncStatus_Success) {
const auto* px = static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(pixel_buffer, 0, size));
std::map<std::tuple<uint64_t, uint32_t, uint32_t>, std::array<uint8_t, 16>> last;
for (size_t i = 0; i < pixel_reads.size(); i++) {
const PixelRead& r = pixel_reads[i];
const uint8_t* v = px + i * 256;
const uint32_t bpp = texel_bytes(r.format);
// (only what changed: a draw that did not touch the texel is left out)
auto& l = last[{r.guest, r.x, r.y}];
if (!std::memcmp(l.data(), v, bpp) && i) continue;
std::memcpy(l.data(), v, bpp);
uint8_t c[4];
texel(r.format, v, c);
char hex[40] = {};
for (uint32_t k = 0; k < bpp && k < 16; k++) std::snprintf(hex + 2 * k, 3, "%02x", v[k]);
hos_log("rx-pixel: %4zu c%d %08llx (%u,%u) = %3u %3u %3u %3u [%s] after %s", i, r.slot,
(unsigned long long)r.guest, r.x, r.y, c[0], c[1], c[2], c[3], hex, r.what.c_str());
}
wgpuBufferUnmap(pixel_buffer);
}
pixel_reads.clear();
}
}
/// RenderDoc captures of the renderer's own frames (debugging). Run with RenderDoc's Vulkan layer loaded through the
/// environment (VK_ADD_IMPLICIT_LAYER_PATH=<RenderDoc's folder> ENABLE_VULKAN_RENDERDOC_CAPTURE=1: the log and the live
/// input keep working, headless too); then the live input's "!rdoc [frames]" or RX_RDOC_AT=<frame number> captures
/// that many frames (default 1) from the next present on, every batch submitted in between, into RX_RDOC_FILE=<path
/// template> (RenderDoc's default folder otherwise).
std::atomic<int> rdoc_requested{0};
#if RX_WEB
void rdoc_present() {}
#else
RENDERDOC_API_1_1_2* rdoc_api() {
static RENDERDOC_API_1_1_2* const api = [] {
RENDERDOC_API_1_1_2* a = nullptr;
if (HMODULE m = GetModuleHandleA("renderdoc.dll"))
if (auto get = reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(m, "RENDERDOC_GetAPI")))
if (!get(eRENDERDOC_API_Version_1_1_2, reinterpret_cast<void**>(&a))) a = nullptr;
if (a && getenv("RX_RDOC_FILE")) a->SetCaptureFilePathTemplate(getenv("RX_RDOC_FILE"));
return a;
}();
return api;
}
void rdoc_present() {
static int left = 0;
static const int64_t at = getenv("RX_RDOC_AT") ? atoll(getenv("RX_RDOC_AT")) : -1;
if (left > 0) {
if (--left == 0) {
wait_submitted(); // (all of the captured frames' batches reached the queue)
rdoc_api()->EndFrameCapture(nullptr, nullptr);
hos_log("rx: RenderDoc capture ended at frame %lld", (long long)frame_number);
}
return;
}
int n = rdoc_requested.exchange(0);
if (frame_number == at) n = 1;
if (n <= 0) return;
if (!rdoc_api()) {
log_limited("rdoc", 4, "rx: no RenderDoc capture: its Vulkan layer is not loaded (ENABLE_VULKAN_RENDERDOC_CAPTURE)");
return;
}
wait_submitted(); // (nothing recorded before the capture lands in it)
rdoc_api()->StartFrameCapture(nullptr, nullptr);
left = n;
hos_log("rx: RenderDoc capture of %d frame(s) from frame %lld", n, (long long)frame_number);
}
#endif
} // namespace rx
/// A frame dump of the next frame, as RX_DUMP_AT would make (the live input file's "!dump" line: hos/nn_hid.c).
extern "C" void rx_request_frame_dump(void) {
rx::frame_dump_enabled = true;
rx::dump_requested = 1;
}
/// RenderDoc captures the next `frames` frames (the live input file's "!rdoc [frames]" line: rdoc_present).
extern "C" void rx_request_rdoc_capture(int frames) {
rx::rdoc_requested = frames > 0 ? frames : 1;
}

4212
gpu/render/rx_draw.cpp Normal file

File diff suppressed because it is too large Load Diff

577
gpu/render/rx_framegen.cpp Normal file
View File

@@ -0,0 +1,577 @@
// Frame interpolation (F7, RX_FRAMEGEN=1): frames made between the game's, from its own two last frames, shown in
// between by a display thread, so a game at 60 fps is shown at 120 (RX_FRAMEGEN_MULT=3: 180) on a fast display.
// TOTK hands out no motion vectors, so the motion is estimated from the images (optical flow):
// 1. each frame is copied (it may be drawn over next frame) and reduced to quarter-size luma;
// 2. block matching: for every 16x16 pixels, the offset (within +-32 pixels) where the last frame's 8x8 quarter-size
// patch matches the new frame's best (a small penalty for long vectors keeps still areas still), then a 3x3
// median of those vectors;
// 3. each in-between frame at time t: every pixel takes the last frame back along t of its motion and the new frame
// forward along 1 - t, blended by t.
// The display thread shows the in-between frames, then the new one, evenly over the game's frame time (its recent
// mean): what is shown runs about one frame behind the game. The HUD over moving scenery and things moving faster
// than the search reaches smear a little; loading screens (frames over 100 ms apart) are not interpolated.
// The threaded web build too: only its WebGPU thread may present to the canvas, and it may not sleep (the batches go
// through it), so there the made frames are shown by that thread's loop as they come due (framegen_web_tick). The
// single-threaded web build: not done (its one render thread is the replay).
#include "rx_internal.h"
#include <algorithm>
#include <chrono>
#include <condition_variable>
#include <cstdlib>
#include <cstring>
#include <deque>
#include <mutex>
#include <string>
#include <thread>
extern "C" void hos_host_thread_register_self(const char* name);
namespace rx {
namespace {
const char* kCopyWgsl = R"(
@group(0) @binding(0) var src: texture_2d<f32>;
@group(0) @binding(1) var dst: texture_storage_2d<rgba8unorm, write>;
@compute @workgroup_size(8, 8) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
let d = textureDimensions(dst);
if (g.x >= d.x || g.y >= d.y) { return; }
textureStore(dst, g.xy, vec4<f32>(textureLoad(src, g.xy, 0).rgb, 1.0));
}
)";
const char* kLumaWgsl = R"(
@group(0) @binding(0) var src: texture_2d<f32>;
@group(0) @binding(1) var dst: texture_storage_2d<rgba16float, write>;
@compute @workgroup_size(8, 8) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
let d = textureDimensions(dst);
if (g.x >= d.x || g.y >= d.y) { return; }
let s = textureDimensions(src) - vec2<u32>(1u);
var sum = 0.0;
for (var y = 0u; y < 4u; y++) {
for (var x = 0u; x < 4u; x++) {
sum += dot(textureLoad(src, min(g.xy * 4u + vec2<u32>(x, y), s), 0).rgb, vec3<f32>(0.299, 0.587, 0.114));
}
}
textureStore(dst, g.xy, vec4<f32>(sum / 16.0, 0.0, 0.0, 1.0));
}
)";
// one vector per 4x4 quarter-size texels (16x16 pixels); the search covers +-8 quarter texels (+-32 pixels)
const char* kFlowWgsl = R"(
@group(0) @binding(0) var la: texture_2d<f32>;
@group(0) @binding(1) var lb: texture_2d<f32>;
@group(0) @binding(2) var flow: texture_storage_2d<rgba16float, write>;
@compute @workgroup_size(8, 8) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
let fd = textureDimensions(flow);
if (g.x >= fd.x || g.y >= fd.y) { return; }
let lm = vec2<i32>(textureDimensions(la)) - vec2<i32>(1);
let c = vec2<i32>(g.xy) * 4 + vec2<i32>(2);
var pa: array<f32, 64>;
for (var i = 0; i < 64; i++) {
pa[i] = textureLoad(la, clamp(c + vec2<i32>(i % 8 - 4, i / 8 - 4), vec2<i32>(0), lm), 0).r;
}
var best = 1e9;
var bv = vec2<i32>(0);
for (var dy = -RADIUS; dy <= RADIUS; dy++) {
for (var dx = -RADIUS; dx <= RADIUS; dx++) {
var sad = 0.0;
for (var i = 0; i < 64; i++) {
let q = clamp(c + vec2<i32>(i % 8 - 4 + dx, i / 8 - 4 + dy), vec2<i32>(0), lm);
sad += abs(pa[i] - textureLoad(lb, q, 0).r);
}
let cost = sad + 0.04 * f32(abs(dx) + abs(dy));
if (cost < best) { best = cost; bv = vec2<i32>(dx, dy); }
}
}
textureStore(flow, g.xy, vec4<f32>(vec2<f32>(bv) * 4.0, best / 64.0, 1.0));
}
)";
const char* kMedianWgsl = R"(
@group(0) @binding(0) var src: texture_2d<f32>;
@group(0) @binding(1) var dst: texture_storage_2d<rgba16float, write>;
fn med9(v: array<f32, 9>) -> f32 {
var a = v;
for (var i = 0; i < 5; i++) { // (the 5 smallest in order: the 5th is the median)
for (var j = i + 1; j < 9; j++) {
if (a[j] < a[i]) { let t = a[i]; a[i] = a[j]; a[j] = t; }
}
}
return a[4];
}
@compute @workgroup_size(8, 8) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
let d = textureDimensions(dst);
if (g.x >= d.x || g.y >= d.y) { return; }
let m = vec2<i32>(d) - vec2<i32>(1);
var xs: array<f32, 9>;
var ys: array<f32, 9>;
var e = 0.0;
for (var i = 0; i < 9; i++) {
let f = textureLoad(src, clamp(vec2<i32>(g.xy) + vec2<i32>(i % 3 - 1, i / 3 - 1), vec2<i32>(0), m), 0);
xs[i] = f.x;
ys[i] = f.y;
e = max(e, f.z);
}
textureStore(dst, g.xy, vec4<f32>(med9(xs), med9(ys), e, 1.0));
}
)";
const char* kSynthWgsl = R"(
@group(0) @binding(0) var fa: texture_2d<f32>;
@group(0) @binding(1) var fb: texture_2d<f32>;
@group(0) @binding(2) var flow: texture_2d<f32>;
@group(0) @binding(3) var samp: sampler;
@group(0) @binding(4) var dst: texture_storage_2d<rgba8unorm, write>;
@group(0) @binding(5) var<uniform> u: vec4<f32>; // t
fn fl(q: vec2<i32>) -> vec2<f32> {
return textureLoad(flow, clamp(q, vec2<i32>(0), vec2<i32>(textureDimensions(flow)) - vec2<i32>(1)), 0).xy;
}
@compute @workgroup_size(8, 8) fn main(@builtin(global_invocation_id) g: vec3<u32>) {
let d = textureDimensions(dst);
if (g.x >= d.x || g.y >= d.y) { return; }
let t = u.x;
let p = vec2<f32>(g.xy) + 0.5;
// the motion here: the vectors (one per 16x16 pixels, at their centres) bilinearly
let fp = p / 16.0 - 0.5;
let i = vec2<i32>(floor(fp));
let w = fp - floor(fp);
let v = mix(mix(fl(i), fl(i + vec2<i32>(1, 0)), w.x), mix(fl(i + vec2<i32>(0, 1)), fl(i + vec2<i32>(1, 1)), w.x), w.y);
let size = vec2<f32>(d);
let a = textureSampleLevel(fa, samp, (p - t * v) / size, 0.0).rgb;
let b = textureSampleLevel(fb, samp, (p + (1.0 - t) * v) / size, 0.0).rgb;
textureStore(dst, g.xy, vec4<f32>(mix(a, b, t), 1.0));
}
)";
constexpr int kMaxMult = 4;
struct Tex {
WGPUTexture tex = nullptr;
WGPUTextureView view = nullptr;
};
Tex make_tex(uint32_t w, uint32_t h, WGPUTextureFormat f, WGPUTextureUsage usage, const char* label) {
WGPUTextureDescriptor d{};
d.label = sv(label);
d.usage = usage;
d.dimension = WGPUTextureDimension_2D;
d.size = {w, h, 1};
d.format = f;
d.mipLevelCount = 1;
d.sampleCount = 1;
Tex t;
t.tex = wgpuDeviceCreateTexture(dev.device, &d);
t.view = wgpuTextureCreateView(t.tex, nullptr);
return t;
}
void drop_tex(Tex& t) {
if (t.view) wgpuTextureViewRelease(t.view);
if (t.tex) wgpuTextureRelease(t.tex);
t = {};
}
/// What the display thread shows for one game frame: the made frames, then the game's (its own copy each)
struct OutSet {
Tex frame[kMaxMult];
int made = 0; // (how many of frame[] are made frames; frame[made] is the game's)
bool busy = false;
};
struct Job {
OutSet* set;
uint32_t sw, sh;
int32_t ww, wh;
int64_t at;
};
struct State {
bool init = false;
uint32_t w = 0, h = 0;
WGPUComputePipeline copy = nullptr, luma = nullptr, flow = nullptr, median = nullptr, synth = nullptr;
WGPUSampler sampler = nullptr;
WGPUBuffer t_buf[kMaxMult] = {};
Tex frames[2], lumas[2], flows[2];
int cur = 0;
bool have_prev = false;
int64_t last_at = 0;
OutSet sets[2];
int next_set = 0;
// the display thread's
std::mutex mu;
std::condition_variable cv;
std::deque<Job> jobs;
bool thread_started = false;
} fg;
WGPUComputePipeline make_compute(const char* wgsl, const char* label) {
WGPUShaderSourceWGSL src{};
src.chain.sType = WGPUSType_ShaderSourceWGSL;
src.code = sv(wgsl);
WGPUShaderModuleDescriptor md{};
md.nextInChain = &src.chain;
md.label = sv(label);
WGPUShaderModule mod = wgpuDeviceCreateShaderModule(dev.device, &md);
WGPUComputePipelineDescriptor cd{};
cd.label = sv(label);
cd.compute.module = mod;
cd.compute.entryPoint = sv("main");
WGPUComputePipeline p = wgpuDeviceCreateComputePipeline(dev.device, &cd); // (layout: from the shader)
wgpuShaderModuleRelease(mod);
return p;
}
void init_pipelines() {
if (fg.init) return;
fg.init = true;
fg.copy = make_compute(kCopyWgsl, "rx framegen copy");
fg.luma = make_compute(kLumaWgsl, "rx framegen luma");
{ // RX_FRAMEGEN_RADIUS=<quarter-size texels> searched each way (default 8: +-32 pixels)
const char* e = getenv("RX_FRAMEGEN_RADIUS");
const int r = e ? std::clamp(atoi(e), 1, 16) : 8;
std::string src = std::string("const RADIUS: i32 = ") + std::to_string(r) + ";\n" + kFlowWgsl;
fg.flow = make_compute(src.c_str(), "rx framegen flow");
}
fg.median = make_compute(kMedianWgsl, "rx framegen median");
fg.synth = make_compute(kSynthWgsl, "rx framegen synth");
WGPUSamplerDescriptor sd{};
sd.addressModeU = sd.addressModeV = sd.addressModeW = WGPUAddressMode_ClampToEdge;
sd.magFilter = sd.minFilter = WGPUFilterMode_Linear;
sd.mipmapFilter = WGPUMipmapFilterMode_Nearest;
sd.lodMaxClamp = 1.0f;
sd.maxAnisotropy = 1;
fg.sampler = wgpuDeviceCreateSampler(dev.device, &sd);
for (WGPUBuffer& b : fg.t_buf) {
WGPUBufferDescriptor d{};
d.size = 16;
d.usage = WGPUBufferUsage_Uniform | WGPUBufferUsage_CopyDst;
b = wgpuDeviceCreateBuffer(dev.device, &d);
}
}
void ensure_size(uint32_t w, uint32_t h) {
if (fg.w == w && fg.h == h) return;
{ // (the display thread is done with the old sets first)
std::unique_lock lk(fg.mu);
fg.cv.wait(lk, [] { return fg.jobs.empty() && !fg.sets[0].busy && !fg.sets[1].busy; });
}
for (int i = 0; i < 2; i++) {
drop_tex(fg.frames[i]);
drop_tex(fg.lumas[i]);
drop_tex(fg.flows[i]);
for (Tex& t : fg.sets[i].frame) drop_tex(t);
}
fg.w = w, fg.h = h;
const uint32_t qw = (w + 3) / 4, qh = (h + 3) / 4, cw = (qw + 3) / 4, ch = (qh + 3) / 4;
const WGPUTextureUsage rw = WGPUTextureUsage_StorageBinding | WGPUTextureUsage_TextureBinding;
for (int i = 0; i < 2; i++) {
fg.frames[i] = make_tex(w, h, WGPUTextureFormat_RGBA8Unorm, rw | WGPUTextureUsage_CopySrc, "rx framegen frame");
fg.lumas[i] = make_tex(qw, qh, WGPUTextureFormat_RGBA16Float, rw, "rx framegen luma");
fg.flows[i] = make_tex(cw, ch, WGPUTextureFormat_RGBA16Float, rw, "rx framegen flow");
for (Tex& t : fg.sets[i].frame)
t = make_tex(w, h, WGPUTextureFormat_RGBA8Unorm, rw | WGPUTextureUsage_CopyDst | WGPUTextureUsage_CopySrc,
"rx framegen shown");
}
fg.have_prev = false;
}
/// A compute pass: `pipe` over x * y groups of 8x8 with these bindings (textures, then a sampler / buffer)
void dispatch(WGPUCommandEncoder e, WGPUComputePipeline pipe, std::initializer_list<WGPUBindGroupEntry> entries,
uint32_t w, uint32_t h) {
std::vector<WGPUBindGroupEntry> be(entries);
for (size_t i = 0; i < be.size(); i++) be[i].binding = (uint32_t)i;
WGPUBindGroupLayout layout = wgpuComputePipelineGetBindGroupLayout(pipe, 0);
WGPUBindGroupDescriptor bd{};
bd.layout = layout;
bd.entryCount = be.size();
bd.entries = be.data();
WGPUBindGroup bg = wgpuDeviceCreateBindGroup(dev.device, &bd);
WGPUComputePassEncoder p = wgpuCommandEncoderBeginComputePass(e, nullptr);
wgpuComputePassEncoderSetPipeline(p, pipe);
wgpuComputePassEncoderSetBindGroup(p, 0, bg, 0, nullptr);
wgpuComputePassEncoderDispatchWorkgroups(p, (w + 7) / 8, (h + 7) / 8, 1);
wgpuComputePassEncoderEnd(p);
wgpuComputePassEncoderRelease(p);
wgpuBindGroupRelease(bg);
wgpuBindGroupLayoutRelease(layout);
}
WGPUBindGroupEntry tex_entry(WGPUTextureView v) {
WGPUBindGroupEntry e{};
e.textureView = v;
return e;
}
/// RX_FRAMEGEN_DUMP=<seconds> (checking): that long after interpolation started, the last frame, the first made one and the new
/// one as fg_prev.png, fg_mid.png, fg_cur.png (the GPU waited for)
void dump_png(WGPUTexture tex, uint32_t w, uint32_t h, const char* path) {
const uint32_t row = (w * 4 + 255) & ~255u;
WGPUBufferDescriptor bd{};
bd.size = (uint64_t)row * h;
bd.usage = WGPUBufferUsage_CopyDst | WGPUBufferUsage_MapRead;
WGPUBuffer buf = wgpuDeviceCreateBuffer(dev.device, &bd);
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
WGPUTexelCopyTextureInfo src{};
src.texture = tex;
WGPUTexelCopyBufferInfo dst{};
dst.buffer = buf;
dst.layout.bytesPerRow = row;
dst.layout.rowsPerImage = h;
const WGPUExtent3D extent{w, h, 1};
wgpuCommandEncoderCopyTextureToBuffer(e, &src, &dst, &extent);
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
wgpuQueueSubmit(dev.queue, 1, &cb);
wgpuCommandBufferRelease(cb);
wgpuCommandEncoderRelease(e);
struct Done { volatile bool done = false; WGPUMapAsyncStatus status = WGPUMapAsyncStatus_Error; } done;
WGPUBufferMapCallbackInfo mc{};
mc.mode = WGPUCallbackMode_AllowSpontaneous;
mc.callback = [](WGPUMapAsyncStatus st, WGPUStringView, void* u, void*) {
static_cast<Done*>(u)->status = st;
static_cast<Done*>(u)->done = true;
};
mc.userdata1 = &done;
wgpuBufferMapAsync(buf, WGPUMapMode_Read, 0, bd.size, mc);
while (!done.done) wgpuDevicePoll(dev.device, true, nullptr);
if (done.status == WGPUMapAsyncStatus_Success) {
write_png(path, static_cast<const uint8_t*>(wgpuBufferGetConstMappedRange(buf, 0, bd.size)), w, h, row);
wgpuBufferUnmap(buf);
hos_log("rx: frame interpolation: wrote %s", path);
}
wgpuBufferRelease(buf);
}
int multiplier() {
static const int m = [] {
const char* e = getenv("RX_FRAMEGEN_MULT");
return e ? std::clamp(atoi(e), 2, kMaxMult) : 2;
}();
return m;
}
/// The display thread: each set's made frames, then the game's, spread over the game's recent frame time
void display_main() {
hos_host_thread_register_self("rx display");
int64_t interval = 16666667, last_at = 0;
struct { int on_time, late, skipped; int64_t since; } stat{}; // (logged every 5 s)
for (;;) {
Job j;
{
std::unique_lock lk(fg.mu);
fg.cv.wait(lk, [] { return !fg.jobs.empty(); });
j = fg.jobs.front();
fg.jobs.pop_front();
}
if (last_at) {
const int64_t d = j.at - last_at;
if (d > 0 && d < 100000000) interval = (interval * 7 + std::clamp<int64_t>(d, 4000000, 50000000)) / 8;
}
last_at = j.at;
if (!stat.since) stat.since = j.at;
if (j.at - stat.since > 5000000000LL) {
log_limited("framegen_pace", 40, "rx: frame interpolation: %d frames shown on time, %d late (over 2 ms), %d "
"left out (the game's next frame came first)", stat.on_time, stat.late, stat.skipped);
stat = {0, 0, 0, j.at};
}
const int shown = j.set->made + 1;
const int64_t step = interval / shown;
for (int k = 0; k <= j.set->made; k++) {
wgpuTextureViewAddRef(j.set->frame[k].view);
surface_present(j.set->frame[k].view, j.sw, j.sh, j.ww, j.wh, false);
if (k == j.set->made) break;
// the next one at its time; if the game's next frame comes first, the rest are left out (shown at once,
// two presents back to back: the display shows one of them)
const int64_t due = j.at + step * (k + 1);
std::unique_lock lk(fg.mu);
if (fg.cv.wait_until(lk, std::chrono::steady_clock::now() +
std::chrono::nanoseconds(std::max<int64_t>(0, due - hos_now_ns())),
[] { return !fg.jobs.empty(); })) {
stat.skipped += j.set->made - k;
break;
}
(hos_now_ns() - due > 2000000 ? stat.late : stat.on_time)++;
}
std::lock_guard lk(fg.mu);
j.set->busy = false;
fg.cv.notify_all();
}
}
#if RX_WEB
// ---- the web (the WebGPU thread): the set being shown, frame by frame as they come due ----
struct WebShow {
bool on = false;
Job j{};
int k = 0;
int64_t step = 0;
} ws;
int64_t web_interval = 16666667, web_last_at = 0;
struct { int on_time, late, skipped; int64_t since; } wstat; // (logged every 5 s)
void web_present_k(int64_t due) {
if (due) (hos_now_ns() - due > 2000000 ? wstat.late : wstat.on_time)++;
const Tex& t = ws.j.set->frame[ws.k];
wgpuTextureViewAddRef(t.view);
surface_present(t.view, ws.j.sw, ws.j.sh, ws.j.ww, ws.j.wh, false);
}
void web_finish() {
ws.on = false;
ws.j.set->busy = false;
framegen_web_due = 0;
}
/// the game's next frame is here: the last set's frames not shown yet are left out (shown now, they would go out
/// back to back with the next set's first, and the browser shows one of the two)
void web_flush() {
if (!ws.on) return;
wstat.skipped += ws.j.set->made - ws.k;
web_finish();
}
void web_start(const Job& j) {
if (web_last_at) {
const int64_t d = j.at - web_last_at;
if (d > 0 && d < 100000000) web_interval = (web_interval * 7 + std::clamp<int64_t>(d, 4000000, 50000000)) / 8;
}
web_last_at = j.at;
if (!wstat.since) wstat.since = j.at;
if (j.at - wstat.since > 5000000000LL) {
log_limited("framegen_pace", 40, "rx: frame interpolation: %d frames shown on time, %d late (over 2 ms), %d "
"left out (the WebGPU thread busy when they were due)", wstat.on_time, wstat.late, wstat.skipped);
wstat = {0, 0, 0, j.at};
}
ws.on = true;
ws.j = j;
ws.k = 0;
ws.step = web_interval / (j.set->made + 1);
web_present_k(0);
if (!j.set->made) web_finish();
else framegen_web_due = j.at + ws.step;
}
#endif
} // namespace
#if RX_WEB
int64_t framegen_web_due;
int64_t framegen_web_tick() {
static bool in_tick; // (a present yields to the event loop)
if (!ws.on || in_tick) return -1;
const int64_t due = ws.j.at + ws.step * (ws.k + 1), now = hos_now_ns();
if (now < due) return due - now;
in_tick = true;
ws.k++;
web_present_k(due);
in_tick = false;
if (ws.k >= ws.j.set->made) {
web_finish();
return -1;
}
framegen_web_due = due + ws.step;
return std::max<int64_t>(0, framegen_web_due - hos_now_ns());
}
#endif
bool framegen_frame(WGPUTextureView view, uint32_t sw, uint32_t sh, int32_t ww, int32_t wh) {
#if RX_WEB && !RX_WEB_MT
(void)view, (void)sw, (void)sh, (void)ww, (void)wh;
log_limited("framegen_web", 1, "rx: frame interpolation: the threaded web build only (index_mt.html)");
return false;
#else
if (!sw || !sh) return false;
#if RX_WEB
web_flush(); // (the set still being shown: done now; this thread is the one that would free it)
#endif
init_pipelines();
ensure_size(sw, sh);
if (!fg.thread_started) {
fg.thread_started = true;
#if !RX_WEB
std::thread(display_main).detach();
#endif
hos_log("rx: frame interpolation on: %dx the game's frames (made from its last two, shown evenly)", multiplier());
}
const int64_t now = hos_now_ns();
const bool gap = fg.last_at && now - fg.last_at > 100000000; // (a loading hitch: nothing to blend)
fg.last_at = now;
// a set the display thread is not showing
OutSet* set;
static int64_t waited, frames, since;
const int64_t w0 = hos_now_ns();
{
std::unique_lock lk(fg.mu);
fg.cv.wait(lk, [] { return !fg.sets[fg.next_set].busy; });
set = &fg.sets[fg.next_set];
set->busy = true;
fg.next_set ^= 1;
}
waited += hos_now_ns() - w0;
frames++;
if (!since) since = w0;
#if !RX_WEB // (the web's sets are free by now: web_flush)
if (w0 - since > 5000000000LL) { // (the submit thread's wait for a set the display thread is done with)
log_limited("framegen_wait", 40, "rx: frame interpolation: %.2f ms a frame waiting for the display thread",
waited / 1e6 / frames);
waited = frames = 0;
since = w0;
}
#endif
const int cur = fg.cur, prev = cur ^ 1;
const int m = multiplier();
const bool blend = fg.have_prev && !gap;
for (int k = 1; k < m && blend; k++) {
const float t[4] = {(float)k / (float)m, 0, 0, 0};
wgpuQueueWriteBuffer(dev.queue, fg.t_buf[k], 0, t, sizeof t);
}
WGPUCommandEncoder e = wgpuDeviceCreateCommandEncoder(dev.device, nullptr);
const uint32_t qw = (sw + 3) / 4, qh = (sh + 3) / 4, cw = (qw + 3) / 4, ch = (qh + 3) / 4;
dispatch(e, fg.copy, {tex_entry(view), tex_entry(fg.frames[cur].view)}, sw, sh);
dispatch(e, fg.luma, {tex_entry(fg.frames[cur].view), tex_entry(fg.lumas[cur].view)}, qw, qh);
set->made = 0;
if (blend) {
dispatch(e, fg.flow, {tex_entry(fg.lumas[prev].view), tex_entry(fg.lumas[cur].view), tex_entry(fg.flows[0].view)},
cw, ch);
dispatch(e, fg.median, {tex_entry(fg.flows[0].view), tex_entry(fg.flows[1].view)}, cw, ch);
for (int k = 1; k < m; k++) {
WGPUBindGroupEntry s{}, b{};
s.sampler = fg.sampler;
b.buffer = fg.t_buf[k];
b.size = 16;
dispatch(e, fg.synth,
{tex_entry(fg.frames[prev].view), tex_entry(fg.frames[cur].view), tex_entry(fg.flows[1].view), s,
tex_entry(set->frame[k - 1].view), b},
sw, sh);
}
set->made = m - 1;
}
WGPUTexelCopyTextureInfo src{}, dst{};
src.texture = fg.frames[cur].tex;
dst.texture = set->frame[set->made].tex;
const WGPUExtent3D extent{sw, sh, 1};
wgpuCommandEncoderCopyTextureToTexture(e, &src, &dst, &extent);
WGPUCommandBuffer cb = wgpuCommandEncoderFinish(e, nullptr);
wgpuQueueSubmit(dev.queue, 1, &cb);
wgpuCommandBufferRelease(cb);
wgpuCommandEncoderRelease(e);
wgpuTextureViewRelease(view);
if (static const double dump_at = getenv("RX_FRAMEGEN_DUMP") ? atof(getenv("RX_FRAMEGEN_DUMP")) : 0; dump_at && blend) {
static const int64_t t0 = now;
static bool dumped;
if (!dumped && now - t0 > (int64_t)(dump_at * 1e9)) {
dumped = true;
dump_png(fg.frames[prev].tex, sw, sh, "fg_prev.png");
dump_png(set->frame[0].tex, sw, sh, "fg_mid.png");
dump_png(fg.frames[cur].tex, sw, sh, "fg_cur.png");
}
}
fg.have_prev = true;
fg.cur = prev;
#if RX_WEB
web_start({set, sw, sh, ww, wh, now});
#else
{
std::lock_guard lk(fg.mu);
fg.jobs.push_back({set, sw, sh, ww, wh, now});
fg.cv.notify_all();
}
#endif
return true;
#endif
}
} // namespace rx

833
gpu/render/rx_internal.h Normal file
View File

@@ -0,0 +1,833 @@
// The renderer's shared state and helpers (gpu/render/rx_*.cpp). See rx.h.
#pragma once
#include <array>
#include <bitset>
#include <cstdint>
#include <cstring>
#include <memory>
#include <atomic>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
#include <webgpu/webgpu.h>
#ifdef __EMSCRIPTEN__
// The browser build (the browser's WebGPU through emdawnwebgpu): wgpu-native's extras stand in here. Every WebGPU
// call is made on the render thread (objects belong to the worker that made them); waits on the GPU yield to that
// worker's event loop (JSPI), where the browser runs WebGPU's callbacks.
#define RX_WEB 1
#define WGPUNativeFeature_TextureFormat16bitNorm 0x7FFF0001
bool wgpuDevicePoll(WGPUDevice device, bool wait, const void* submission); // rx_core.cpp
#else
#define RX_WEB 0
#include <webgpu/wgpu.h>
#endif
#ifndef RX_WEB_MT
#define RX_WEB_MT 0 // (the threaded web build: rx_web_mt.h)
#endif
#include "shader_recompiler/shader_info.h"
#include "video_core/maxwell_3d_regs.h"
#include "video_core/surface.h"
#include "video_core/textures/texture.h"
#include "rx.h"
#include "rx_nvn.h"
extern "C" {
void hos_log(const char* fmt, ...);
void hos_process_memory(uint64_t* private_bytes, uint64_t* working_set);
void hos_memory_map_log(void);
uint64_t hos_gpu_translate(uint64_t va);
uint8_t* hos_gpu_ptr(uint64_t va, uint64_t* avail);
int64_t hos_now_ns(void);
uint64_t hos_ticks(void); // (hos_thread.c: the guest tick clock, 31.25 MHz from the start)
void hos_host_thread_register_self(const char* name);
void hos_thread_boost(int level);
}
namespace rx {
using Regs = Tegra::Engines::Maxwell3D::Regs;
using PixelFormat = VideoCore::Surface::PixelFormat;
#define RX_REG(field) (offsetof(::rx::Regs, field) / sizeof(uint32_t))
inline WGPUStringView sv(const char* s) {
return WGPUStringView{s, WGPU_STRLEN};
}
inline std::string str(WGPUStringView v) {
return v.data ? std::string(v.data, v.length == WGPU_STRLEN ? std::strlen(v.data) : v.length) : std::string{};
}
// ---- guest memory, through the GPU's address space (hos/nvdrv.c's mappings) ----
/// Reads n bytes at a GPU VA; unmapped bytes read as 0. False when any byte was unmapped.
bool gpu_read(uint64_t va, void* dst, uint64_t n);
/// A cheap nanosecond clock for the renderer's own time statistics (the CPU's invariant TSC, scaled once against
/// hos_now_ns): ten reads a draw through QueryPerformanceCounter were ~4% of the GPU thread.
int64_t clock_ns();
extern "C" volatile long maps_generation; // hos/nvdrv.c: bumped by every GPU mapping change
bool render_enabled(const Regs& r);
/// A host pointer to n contiguous bytes at a GPU VA, or null when they are not all mapped in one piece.
const uint8_t* gpu_span(uint64_t va, uint64_t n);
template <class T>
T gpu_get(uint64_t va) {
T v{};
gpu_read(va, &v, sizeof v);
return v;
}
// ---- images: render targets and textures, by the guest address of their memory ----
inline uint64_t next_image_id = 1;
struct Image {
uint64_t id = next_image_id++; // unique (a cache that remembers an image compares this, not the pointer)
uint64_t gpu_va = 0, guest = 0;
uint32_t width = 0, height = 0, layers = 1, levels = 1;
PixelFormat pixel_format{};
WGPUTextureFormat format = WGPUTextureFormat_Undefined;
WGPUTextureDimension dimension = WGPUTextureDimension_2D;
WGPUTexture tex = nullptr;
WGPUTextureView attachment = nullptr; // level 0, layer 0: what render passes draw into
bool rendered = false; // the GPU wrote it: its contents live only in the texture
bool storage = false; // decoded by a compute shader (ASTC): views ask for sampling only
// a render target: its memory layout (from the target registers), for writing it back to guest memory
uint32_t bl_height = 0, bl_depth = 0, reg_width = 0;
bool pitch_linear = false;
uint64_t storage_limit = 0; // a native NVN target: the bytes its texture's storage has there (0: unknown)
bool writeback_queued = false; // a one-off draw rendered it: copied to guest memory at the submission's end
uint32_t slice = 0, volume_depth = 0; // a 3D texture's slice drawn (nvn_slice_key): which, of how many
bool block_view = false; // drawn through an uncompressed view of a compressed texture: a texel a block
uint64_t content_hash = 0; // a texture decoded from memory: samples of the memory it came from
int64_t checked_frame = -1;
uint64_t decoded_seq = 0; // memory_seq() when it was decoded (0: no tracking: content_hash)
int64_t decoded_frame = 0; // (mkw-recomp) the frame it was decoded in
uint64_t guest_bytes = 0; // a texture decoded from memory: how much it covers
bool stale = false; // ...and its memory was written since
int64_t last_use = 0;
std::vector<std::pair<uint64_t, WGPUTextureView>> views;
uint64_t version = 0; // a render target: bumped by each pass that may write it
std::vector<uint64_t> sub_versions; // a texture assembled from render targets: the version copied into
// each (layer, level), layer-major
std::vector<uint64_t> sub_addresses; // ...where each starts in guest memory
std::vector<Image*> sub_images; // ...image_at(each address), as of sub_images_generation
uint64_t sub_images_generation = 0;
uint64_t sub_end = 0; // ...the end of the last subresource's start (the range image changes
// must touch for sub_images to change)
std::vector<uint32_t> sub_present; // ...the subresources with an image (the ones a pass may need to copy)
uint64_t assembled_passes = ~0ull; // ...pass_count() when last brought up to date
std::unique_ptr<Image> sampled_copy; // reusable pre-draw snapshot for attachment feedback
uint64_t snapshot_epoch = 0; // the texture barrier epoch it was taken in (snapshot_for_feedback)
/// A view of the image as a shader binds it (cached).
WGPUTextureView view(WGPUTextureViewDimension dim, bool depth_aspect, uint32_t base_layer = 0,
uint32_t base_level = 0, uint32_t level_count = 0,
WGPUTextureFormat view_format = WGPUTextureFormat_Undefined);
~Image();
};
/// Views of destroyed images are kept until no cached bind group can name them (a released handle's address could
/// come back as a new view's).
void retire_view(WGPUTextureView view);
bool retired_views_pending();
void release_retired_views();
const std::vector<WGPUTextureView>& retired_view_list();
/// A bind group the recorded batches may still name: released once they are replayed (at once when not deferred).
void release_bind_group_later(WGPUBindGroup g);
void release_pipeline_later(WGPURenderPipeline p);
void trim_pipelines();
/// The image a render target register set describes (created, or replaced when its format/size changed).
/// key: where the image is filed instead of the translated address (a 3D texture's slice: nvn_slice_key)
Image* find_render_target(uint64_t gpu_va, uint32_t width, uint32_t height, PixelFormat format, uint64_t key = 0);
/// The key a slice of a 3D render target's images are filed under: above any guest address (guest memory is below
/// 4 GB), so no texture lookup meets it, and never written back. The 3D texture is assembled from them when sampled.
inline uint64_t nvn_slice_key(uint64_t guest, uint32_t slice) { return guest | (uint64_t)(slice + 1) << 40; }
/// Every render target, by guest address (debugging).
const std::unordered_map<uint64_t, std::unique_ptr<Image>>& all_images();
/// A PNG of RGBA8 rows.
void write_png(const char* path, const uint8_t* rgba, uint32_t w, uint32_t h, size_t stride);
/// The image whose memory starts at this guest address, if any.
Image* image_at(uint64_t guest);
/// Changes whenever image_at could answer differently for some address (a cache of its answers is valid while this
/// stays the same).
uint64_t images_generation();
/// Whether image_at could answer differently for an address in [lo, hi) than at generation since.
bool images_changed_in(uint64_t since, uint64_t lo, uint64_t hi);
/// Ends the current pass and snapshots an attachment before a draw samples and writes it.
Image* snapshot_for_feedback(Image& image);
/// A texture barrier (texture data cache invalidation, tiled cache barrier) in the command stream.
void texture_barrier(uint32_t method, uint32_t arg);
WGPUTextureFormat wgpu_format(PixelFormat format);
/// The same format with the other sRGB encoding (RGBA8 <-> RGBA8 sRGB, ...), or Undefined. Textures are created
/// with it as a view format: what the game writes or samples as sRGB and what it reads raw are one memory.
WGPUTextureFormat srgb_pair(WGPUTextureFormat f);
bool is_depth_format(WGPUTextureFormat f);
bool has_stencil(WGPUTextureFormat f);
// ---- per-channel state ----
struct ConstBuffer {
uint64_t address = 0;
uint32_t size = 0;
bool enabled = false;
};
struct Channel {
std::array<std::array<ConstBuffer, 18>, 5> cbufs{};
// the draw being assembled (suyu's DrawManager state)
enum class DrawMode { General, Instance, InlineIndex } draw_mode = DrawMode::General;
bool draw_indexed = false;
uint32_t instance_count = 0;
std::vector<uint8_t> inline_index;
uint32_t topology = 0;
// the render enable condition (draws and clears run while it holds), evaluated when its mode or override is
// written (suyu's ProcessQueryCondition): the game can change the values it compares before the draws it covers
bool render_on = true;
// ZPASS_PIXEL_COUNT: an estimate of the samples that passed (draws while counting is enabled add their size),
// reset by CLEAR_REPORT_VALUE. TOTK culls objects by occlusion queries on boxes: a report of 0 hides them.
uint64_t zpass = 0;
};
/// One draw, as the 3D engine describes it at the moment it runs.
struct DrawParams {
bool indexed = false;
uint32_t topology = 0;
uint32_t first = 0, count = 0; // vertices, or indices of the index buffer
uint32_t instance_count = 1, base_instance = 0;
int32_t base_vertex = 0;
const std::vector<uint8_t>* inline_index = nullptr; // u32 indices given in the command stream
uint64_t index_va = 0; // otherwise the index buffer
uint32_t index_size = 4; // bytes per index
};
void draw(Channel& ch, const Regs& regs, const DrawParams& p);
void clear(Channel& ch, const Regs& regs);
// ---- recording: one command encoder per batch, render passes inside it ----
struct Attachments {
std::array<Image*, 8> color{};
Image* depth = nullptr;
uint32_t width = 0, height = 0;
bool operator==(const Attachments& o) const {
return color == o.color && depth == o.depth;
}
};
/// The render targets a draw writes (one size: WebGPU requires it).
Attachments draw_attachments(const Regs& r);
/// RX_DUMP_FRAME=<n> (debugging): frame n's passes are logged and its render targets saved as PNGs at its present.
extern bool frame_dump_enabled;
bool frame_dump_active_slow();
inline bool frame_dump_active() { return frame_dump_enabled && frame_dump_active_slow(); }
void dump_pass(const Attachments& at, const char* kind);
void dump_on_targets(const Attachments& at);
void watch_buffer_frame();
void watch_buffer_after_draw(const char* what);
void dump_draw(const char* what);
/// RX_DUMP_SAMPLED=1 (with a frame dump): render targets sampled by draws are saved with the frame's targets.
void dump_sampled(const Image* im);
/// A dispatch wrote this range in the buffer cache (RX_SYNC_DISPATCH_WRITES=1: copied to guest memory at the present).
void note_dispatch_write(uint64_t gpu_va, uint64_t size);
void sync_dispatch_writes();
/// The native NVN layer's fences: the last batch flushed; whether one ran on the GPU with its readbacks in guest
/// memory; waiting for that (rx_core.cpp)
uint64_t last_batch_serial();
bool batch_done(uint64_t serial);
void wait_batch(uint64_t serial);
void dump_frame_images(const char* dir);
void note_texels_written(uint64_t guest); // (rx_texture.cpp: RX_DUMP_TEX=texels)
/// RX_DUMP_TEX=<GPU VA hex>[,...] (with a frame dump): the decoded textures at those addresses, saved like targets.
void dump_tex_list(const char* dir);
void dump_texture(WGPUTexture tex, uint64_t va, uint32_t width, uint32_t height, WGPUTextureFormat format,
const char* dir, int order);
void dump_after_draw(uint64_t fs_address, Image* target);
void dump_before_draw(uint64_t fs_address, const Attachments& at);
void dump_pixel_after(const Attachments& at, const char* what);
/// The pass draws go into: (re)started when the attachments differ from the current pass's.
WGPURenderPassEncoder render_pass(const Attachments& at);
/// Ends the current render pass (a clear or a copy comes next).
void end_render_pass(const char* why = nullptr); // (why: a stats counter when a pass was open)
/// Before a draw: a pass that has had RX_PASS_SPLIT draws (default 256; 0: never) is ended, so the draw begins it again
/// with its attachments loaded. wgpu encodes a pass in one encoder, and the submit thread's encoders (one each of up to
/// RX_FINISH_THREADS, cut where passes end) were 5.1, 4.3 and 0.4 ms a frame at the castle: one pass was most of it.
void split_long_pass();
/// Render passes begun so far (a render target can have changed only if this did).
uint64_t pass_count();
WGPUCommandEncoder encoder();
/// Occlusion counts (NVN SAMPLES_PASSED) from occlusion queries (rx_core.cpp; RX_OCCLUSION_QUERIES=0: the estimate).
/// A reset starts counting: each draw after it brackets its rp_draw with occlusion_draw_begin (true: counted) and
/// occlusion_draw_end; a report is false when its draws cannot be summed (the estimate is written then), else the
/// counter word at addr gets the draws' samples plus extra once the GPU is done.
bool occlusion_queries();
void occlusion_reset();
bool occlusion_draw_begin();
void occlusion_draw_end();
uint64_t occlusion_queried(); // draws counted so far
bool occlusion_report(uint64_t addr, uint64_t extra);
int rx_target_writebacks_pending();
/// The window's present (rx_present_fx.cpp): where the frame goes in the window (its aspect kept), and the passes
/// that put it there (FSR 1 when that is larger than the frame)
struct PresentRect {
uint32_t x, y, w, h;
};
PresentRect present_rect(uint32_t sw, uint32_t sh, uint32_t tw, uint32_t th);
bool present_framegen(); // (the player's frame interpolation setting, F7)
void present_window_size(int32_t w, int32_t h); // (rx_resize: ultrawide's aspect)
void surface_present(WGPUTextureView view, uint32_t sw, uint32_t sh, int32_t ww, int32_t wh, bool debug_present);
/// Frame interpolation (rx_framegen.cpp): the frame (its view released) and the frames made between it and the last
/// handed to the display thread; false: not done (the caller presents it as usual)
bool framegen_frame(WGPUTextureView view, uint32_t sw, uint32_t sh, int32_t ww, int32_t wh);
/// (the threaded web build's WebGPU thread: its loop, between jobs and between a batch's commands) the next made frame
/// shown if due; ns until the one after (-1: none waiting). framegen_web_due: when it is due (0: none).
int64_t framegen_web_tick();
extern int64_t framegen_web_due;
void present_fx_draw(WGPUCommandEncoder e, WGPUTextureView src, uint32_t sw, uint32_t sh, WGPUTextureView target,
WGPUTextureFormat target_format, uint32_t tw, uint32_t th, const WGPUColor& clear); // (the threaded web build: render target writebacks not in guest memory yet)
/// The attachments of the current pass (valid while render_pass's result is).
const Attachments& current_attachments();
/// Per-batch upload memory: GPU buffers written once before the batch's commands are submitted.
struct Stream {
WGPUBuffer buffer = nullptr;
uint8_t* cpu = nullptr; // this batch's contents (each batch gets its own: the submit thread writes them)
uint64_t used = 0, capacity = 0;
WGPUBufferUsage usage{};
const char* name = "";
/// Space for n bytes; the offset, or UINT64_MAX when the batch must be flushed first.
uint64_t alloc(uint64_t n, uint64_t align);
uint8_t* at(uint64_t offset) {
return cpu + offset;
}
};
/// A buffer write that must reach the GPU after the batches already flushed and before the open one (the buffer
/// cache's page uploads): it goes to the queue with the open batch, on the submit thread.
void queue_upload(WGPUBuffer buffer, uint64_t offset, const void* data, uint64_t size);
/// Waits until every flushed batch was submitted (before work that reads results back or bypasses the batches).
void wait_submitted();
/// RenderDoc frame captures (rx_debug.cpp: "!rdoc", RX_RDOC_AT): at each present, after its batch was flushed
void rdoc_present();
/// A writable storage binding the open batch copied into stream_rw at rw_offset from guest memory at gpu_va: after the
/// batch runs, the words the GPU changed there are written back into guest memory (the game reads what its shaders
/// write: visibility flags from material shaders, ...).
void writeback_after_batch(uint64_t rw_offset, uint64_t size, uint64_t gpu_va);
/// A pixel program of the open pass writes this stream_rw copy of guest memory: at the pass's end it is copied into
/// the buffer cache (later passes read the results there).
void storage_written_in_pass(uint64_t rw_offset, uint64_t size, uint64_t gpu_va);
// ---- the buffer cache (rx_buffer.cpp): guest memory mirrored in GPU buffers ----
struct BufferRef {
WGPUBuffer buffer = nullptr;
uint64_t offset = 0, size = 0;
};
/// A guest range as a binding of a cached buffer, its contents current; empty when it cannot be (too big,
/// unmapped, spans mappings).
BufferRef guest_buffer(uint64_t gpu_va, uint64_t size);
bool buffer_cache_active();
/// Change tracking of guest memory (the buffer cache's): a tick (0 when there is no tracking), and whether any page
/// of a guest range was written after a tick.
uint64_t memory_seq();
bool memory_written_since(uint64_t guest, uint64_t size, uint64_t seq);
/// Guest memory [guest, guest + size) was written: textures decoded from it are stale (rx_texture.cpp).
void textures_written(uint64_t guest, uint64_t size);
void buffer_cache_new_batch();
/// The open batch (changes at each flush).
int64_t batch_number();
/// Whether a dispatch wrote some of this range (its contents are then only on the GPU).
bool gpu_written(uint64_t gpu_va, uint64_t size);
uint64_t copy_gpu_pages(WGPUBuffer dst, uint64_t guest, uint64_t size);
/// A readback's changed words (now against was) into guest memory, as the GPU's results rather than CPU writes (any
/// thread).
/// Whether a pixel program of the open render pass writes storage overlapping this range.
bool storage_written_this_pass(uint64_t gpu_va, uint64_t size);
bool pass_wrote_storage(); // (a pixel program of the open pass wrote storage: its results are pending)
int64_t readback_to_guest(uint8_t* guest, const uint8_t* now, const uint8_t* was, uint64_t n);
/// Guest memory gets what dispatches wrote into the range (and the GPU-written pages next to it) in the buffer cache:
/// a GPU round trip, after the batches that wrote it were submitted.
void buffer_cache_sync_to_guest(uint64_t gpu_va, uint64_t size);
void buffer_cache_scan();
bool collect_written_pages(std::vector<uint32_t>& out);
void buffer_cache_apply_written(const uint32_t* pages, size_t n);
void buffer_cache_written(uint64_t gpu_va, uint64_t size);
/// The GPU wrote a cached range (a dispatch's storage buffer): the other segment holding it gets the new contents.
void buffer_cache_gpu_written(const BufferRef& ref);
/// Bumped by every dispatch that writes storage: the draws' shared copies of writable storage (one per range and
/// batch) are made again after it, from what the dispatch wrote.
extern uint64_t dispatch_write_epoch;
/// The draw being made goes to targets no draw of the last frames used (a map made once, at loading, say): its
/// programs are translated and its pipeline compiled before it is drawn, not skipped meanwhile (RX_SKIP_ONE_OFF=1:
/// skipped as any other)
extern bool draw_must_run;
/// RX_RT_HISTORY=1 (debugging): the color target addresses in [lo, hi) any draw or clear named, with their last frame
std::string rt_history_find(uint64_t lo, uint64_t hi);
void dump_start_now();
/// A one-off draw rendered into it: its contents go to guest memory at the end of the submission (before its fence),
/// as the console's memory would hold them (TOTK's terrain copies the maps it renders with the CPU)
void queue_target_writeback(Image* im);
void writeback_targets();
/// RX_GUEST_WATCH=<lo>-<hi> guest hex (debugging): whether GPU memory [va, va + size) touches that guest range
bool guest_watch_hit(uint64_t va, uint64_t size);
extern Stream stream_ro; // uniforms, vertices, indices, read-only storage
extern Stream stream_rw; // writable storage
/// Allocates, flushing the batch first when the stream is full. UINT64_MAX when n can never fit.
uint64_t stream_alloc(Stream& s, uint64_t n, uint64_t align);
void flush_batch();
// ---- the device ----
struct Device {
WGPUInstance instance = nullptr;
WGPUAdapter adapter = nullptr;
WGPUDevice device = nullptr;
WGPUQueue queue = nullptr;
WGPULimits limits{};
bool tier1 = false; // (mkw-recomp) texture-formats-tier1: more storage texture formats
bool multi_draw_count = false; // (mkw-recomp) wgpu's multi-draw-indirect-count (NVN's MultiDraw*IndirectCount)
bool adapter_formats = false; // (mkw-recomp) wgpu's adapter-specific format features: the same, natively
bool bc = false, float32_filterable = false, float32_blendable = false, depth32_stencil8 = false, norm16 = false, depth_clip_control = false,
early_depth_test = false, timestamps = false;
std::string description;
};
extern Device dev;
/// The renderer's lock: recursive, over a Windows SRW lock (std::recursive_mutex was ~3.5% of the GPU thread: it is
/// taken for every renderer method, several a draw, nearly always by the GPU thread itself).
class RecursiveLock {
alignas(8) unsigned char srw_[64] = {}; // an SRWLOCK (SRWLOCK_INIT is all zero; the web's is a pthread rwlock)
std::atomic<unsigned long> owner_{0};
unsigned depth_ = 0;
public:
void lock();
void unlock();
};
extern RecursiveLock mu;
extern int64_t frame_number;
// statistics
extern int64_t st_draws, st_skipped, st_pipelines, st_clears;
/// Validation error scopes: the error the calls in between made (empty: none).
void scope_push(bool out_of_memory = true); // (mkw-recomp: false: validation errors only, one scope)
std::string scope_pop();
WGPUBindGroup checked_bind_group(const WGPUBindGroupDescriptor& d, const char* what); // (null: invalid)
/// An environment switch, read once (the hot paths test them per draw).
bool env_flag(const char* name);
/// HOS_AB_SWITCH=<name> with HOS_AB=<ms> (hos_prof.c: an A/B inside one run, for a busy machine): true while the
/// switch of that name is in its flipped phase. Callers keep `ab_named(name)` in a static and test `ab && ab_flipped()`.
bool ab_named(const char* name);
bool ab_flipped();
/// A translated program's module. RX_SHADER_CHECKS=<bits> (native, experiment): the runtime checks naga adds to it
/// (WGPUShaderRuntimeChecks: 1 bounds checks, 2 loop bounding); unset: wgpu's default, all of them.
WGPUShaderModule program_module(const WGPUShaderModuleDescriptor* d);
void background_thread();
/// Statistics of the current period, by key (skipped draws by reason, ...): logged with RX_STATS.
void count(const char* key, int64_t n = 1);
/// Logs a message at most `limit` times for its key.
void log_limited(const char* key, int limit, const char* fmt, ...);
// ---- shaders and pipelines (rx_draw.cpp, rx_shader.cpp) ----
struct ShaderModule {
WGPUShaderModule module = nullptr;
Shader::Info info{};
std::string wgsl; // (until interned: then text, and this is emptied)
std::string* text = nullptr; // the interned WGSL (graphics stages: one copy per distinct text; a pair
// each held its own, ~7000 pairs x 2 x ~15 KB of host memory)
const std::string& source() const { return text ? *text : wgsl; }
bool failed = false;
const char* what = ""; // (graphics stages: created by the pipeline that first needs them, ensure_module)
uint64_t key = 0;
uint32_t text_id = 0; // the same for the same WGSL (module sharing; ids are per session)
/// info.loads.Generic(i) for the 32 attributes as one mask, made once (each draw testing 128 bits was ~1.6% of the
/// replay thread; info is set when the module is made and not changed after)
uint32_t generic_loads() const {
if (!generic_loads_ok) {
uint32_t m = 0;
for (uint32_t i = 0; i < 32; i++) m |= (info.loads.Generic(i) ? 1u : 0u) << i;
generic_loads_mask = m, generic_loads_ok = true;
}
return generic_loads_mask;
}
mutable uint32_t generic_loads_mask = 0;
mutable bool generic_loads_ok = false;
uint64_t text_hash = 0; // the interned text's hash: pipelines are keyed by these (the same in every session)
uint64_t cache_hash = 0;
std::shared_ptr<std::mutex> creating = std::make_shared<std::mutex>(); // (one creation; others wait)
};
/// The module from the WGSL (once; any thread: the pipeline compilers create the graphics stages' modules, as
/// creating them waits for the device while pipelines compile). False when it does not compile.
bool ensure_module(ShaderModule& m);
struct StageKey {
uint64_t address = 0; // GPU VA of the program header
uint64_t code_hash = 0;
bool operator==(const StageKey& o) const {
return address == o.address && code_hash == o.code_hash;
}
};
/// The vertex + fragment modules of a draw's programs (translated together: their interfaces must agree).
struct ProgramPair {
ShaderModule vs, fs;
bool has_fs = false;
bool failed = false;
/// A vertex program that stores to memory, as a compute program (WebGPU's vertex stage cannot store): run before
/// the draw (nvn_vs_compute, rx_draw.cpp). Empty when the vertex program stores nothing.
std::string vs_compute_wgsl;
WGPUShaderModule vs_compute_module = nullptr;
bool vs_compute_failed = false;
};
ProgramPair* programs_for_draw(Channel& ch, const Regs& regs);
/// Vertex attributes the vertex stage reads from memory itself (layouts WebGPU's vertex fetch cannot describe):
/// up to 3 pulled streams, as the WGSL backend's Options::vertex_pull wants them.
struct PullPlan {
struct Attr {
bool enabled = false;
uint8_t stream = 0; // the pulled stream (0-2)
bool instanced = false;
uint32_t attribute = 0, stride = 0;
} attr[32];
uint32_t streams[3]{}; // the vertex stream (0-15) of each pulled stream
uint32_t count = 0;
};
PullPlan pull_plan(const Regs& r, const std::bitset<32>& loads);
/// How the vertex stage unpacks an attribute fetched as one u32 (Shader::Backend::WGSL::VertexUnpack; 0: it is
/// fetched in its own format).
uint8_t vertex_unpack(const Regs::VertexAttribute& a);
/// A compute dispatch as the launch descriptor (QMD) and the compute engine's registers describe it.
struct ComputeLaunch {
uint64_t code_base = 0; // code_loc
uint32_t program_start = 0;
uint32_t grid[3]{}, block[3]{};
uint32_t shared_size = 0, local_size = 0, texture_bound = 0;
bool linked_tsc = false;
struct Cbuf {
uint64_t address = 0;
uint32_t size = 0;
bool enabled = false;
} cbufs[8];
const Regs* tex_regs = nullptr; // the texture/sampler pools (the compute engine's, in the 3D layout)
};
/// The compute module of a launch (cached by program); null when it does not translate.
/// ssbo_alias (web): storage descriptors bound to one same range use one binding (see WGSL::ssbo_alias_next)
ShaderModule* compute_module(const ComputeLaunch& launch, const std::array<uint8_t, 64>* ssbo_alias = nullptr,
const std::array<uint32_t, 64>* ssbo_alias_words = nullptr);
// ---- textures and samplers (rx_texture.cpp) ----
/// How a texture binding is declared in a pipeline layout (the format decides; part of the pipeline key).
enum class TexKind : uint8_t { Float, UnfilterableFloat, Depth, Uint, Sint };
struct BoundTexture {
WGPUTextureView view = nullptr;
WGPUSampler sampler = nullptr;
TexKind kind = TexKind::Float;
Image* image = nullptr; // a render target sampled (must not be an attachment of the same pass)
char source = 'd'; // (debugging) d placeholder, t texture from memory, r render target
uint64_t guest = 0;
uint32_t width = 0, height = 0;
int format = 0;
uint8_t slots[32]{}; // the header's swizzle and the sampler's border/compare/bias, as the uniform block takes them
};
BoundTexture texture_for_handle(const Regs& regs, uint32_t raw_handle, const Shader::TextureDescriptor& desc);
void bc7_encode(const uint8_t* rgba, uint32_t width, uint32_t height, uint32_t pitch, uint8_t* out);
/// The placeholder a binding gets when its texture cannot be used.
BoundTexture dummy_texture(const Shader::TextureDescriptor& desc);
Tegra::Texture::TICEntry read_tic(const Regs& regs, uint32_t tic_index);
Tegra::Texture::TSCEntry read_tsc_for_dump(const Regs& regs, uint32_t raw_handle);
/// A texture's two uniform block slots (the WGSL backend's): the TIC's component swizzle with the sampler's border
/// axes and depth compare function, then the border color.
void texture_slots(uint8_t* dst, const Tegra::Texture::TICEntry& tic, const Tegra::Texture::TSCEntry& tsc);
WGPUTextureViewDimension view_dimension(Shader::TextureType type);
uint64_t hash_bytes(const void* p, size_t n, uint64_t seed = 0xcbf29ce484222325ull);
/// The browser's pipeline cache (rx_pipecache.cpp): pipelines recorded as made, made again in the background when the
/// page opens next (start: at device creation; step: each present).
std::string web_shader_source(const std::string& wgsl); // (the WGSL as the device gets it)
void pipecache_start();
bool sampler_nonfiltering(WGPUSampler s); // (debugging)
void pipecache_step();
void pipecache_record_render(const WGPURenderPipelineDescriptor& d, const ShaderModule& vs, const ShaderModule* fs,
const std::vector<WGPUBindGroupLayoutEntry>* layouts, int nlayouts);
uint64_t pipecache_text_hash(const std::string& wgsl);
bool pipecache_has_module(uint64_t h);
void release_module_text(ShaderModule& m);
void pipecache_record_compute(const std::string& wgsl, const std::vector<WGPUBindGroupLayoutEntry>& layout);
/// A render pipeline as the native pipeline cache keeps it (rx_pipecache_native.cpp): its key in the pipeline map,
/// the WGSL texts of its stages (by interned-text hash) and everything its creation reads.
struct PipelineDesc {
uint64_t key = 0, vs_text = 0, fs_text = 0;
std::vector<uint32_t> layout[2]; // packed bind group layouts (pack_layout)
int nlayouts = 1;
std::vector<WGPUVertexBufferLayout> buffers; // (attributes: below, one list per buffer)
std::vector<std::vector<WGPUVertexAttribute>> attributes;
std::vector<WGPUColorTargetState> targets; // (blend: has_blend)
std::vector<uint8_t> has_blend;
std::array<WGPUBlendState, 8> blends{};
WGPUPrimitiveState primitive{};
bool has_depth = false;
WGPUDepthStencilState depth{};
};
void native_cache_start(); // (render thread, at device creation: reads the cache in the background)
void native_cache_step(); // (render thread, each frame: saves new records, makes cached pipelines a few at a time)
void native_cache_text(uint64_t h, const std::string& text); // a new interned WGSL text (any thread)
void native_cache_pipeline(const PipelineDesc& d); // a new pipeline (render thread)
void native_cache_used(uint64_t key); // a draw used this pipeline (render thread; once a session per pipeline)
bool native_cache_on();
void native_cache_compute_text(uint64_t h, const std::string& text); // a compute program's WGSL that compiled
WGPUShaderModule native_cache_compute_module(uint64_t h); // its module, made by the cache's thread (or null)
/// The module of a cached WGSL text (interned as a translation would: draws of the same text share the module)
ShaderModule* prewarm_stage_module(uint64_t h, const std::string& text, const char* what);
/// Makes a cached pipeline, unless the map has its key (render thread); false when there was nothing to make
bool prewarm_pipeline(const PipelineDesc& d, ShaderModule* vs, ShaderModule* fs);
int prewarm_in_flight(); // cached pipelines still compiling
// ---- textures made by the GPU from guest memory as it is (rx_texture_gpu.cpp) ----
struct BlockLinearLevel {
uint32_t offset; // the level's first byte in the guest texture
uint32_t tiles_w, tiles_h; // its size in blocks
uint32_t block_size, x_shift, block_height; // its block-linear layout (as the CPU unswizzle's)
};
/// Whether ASTC textures are decoded by the GPU (native; RX_ASTC_CPU=1: by the CPU)
bool astc_gpu_on();
/// Decodes every level of a 2D (array) ASTC texture in guest layout into tex (RGBA8Unorm, storage binding), submitted
/// at once (any thread). False: nothing was done (the CPU decodes it).
bool astc_gpu_decode(WGPUTexture tex, const uint8_t* src, uint64_t total, const BlockLinearLevel* levels, int nlevels,
uint32_t layers, uint32_t layer_stride, uint32_t block_w, uint32_t block_h);
/// Whether other block-linear textures are unswizzled by the GPU (native; RX_UNSWIZZLE_CPU=1: by the CPU)
bool unswizzle_gpu_on();
/// Unswizzles every level of a 2D (array) texture of 4, 8 or 16 byte blocks (bpp_log2 2-4) into tex, as it is
/// (submitted at once, any thread). False: nothing was done.
bool unswizzle_gpu(WGPUTexture tex, const uint8_t* src, uint64_t total, const BlockLinearLevel* levels, int nlevels,
uint32_t layers, uint32_t layer_stride, uint32_t bpp_log2, uint32_t block_w, uint32_t block_h);
/// The bytes of one level and layer, row_bytes x rows (debugging: waits for the GPU; width/height: the copy's extent)
std::vector<uint8_t> read_level(WGPUTexture tex, uint32_t level, uint32_t layer, uint32_t width, uint32_t height,
uint32_t row_bytes, uint32_t rows);
// ---- the native NVN layer (HOS_NVN_NATIVE=1): rx_nvn.cpp replays the game's NVN command records with these, each
// built from NVN state directly (no Switch GPU registers or command buffers in between). See rx_nvn.h. ----
PixelFormat nvn_pixel_format(int nvn_format);
/// The queue's state as the recordings set it (one queue; it carries across submits, as the console's does).
struct NvnState {
nvn_prog prog{}; // the bound program, copied at the bind (the game can set new shaders on the object later)
bool has_prog = false;
// per NVN stage (0 vertex, 1 fragment, 2 geometry, 3 tess control, 4 tess evaluation, 5 compute)
nvn_p_bind_buf ubo[6][16]{}, ssbo[6][16]{};
uint64_t tex[6][128]{}, samp[6][128]{}, img[6][32]{};
nvn_blend_st blend[8]{}; // by target
nvn_color_st color{};
nvn_chmask_st chmask{0xffffffffu};
nvn_ms_st ms{};
nvn_poly_st poly{};
nvn_ds_st ds{};
uint32_t vattr[16]{}; // nvn_va_st
int nvattr = 0;
nvn_vs_st vstream[16]{};
NVNbufferRange vbuf[16]{};
nvn_p_set_rt rt{};
Attachments at; // the render targets' images (made from rt when at_dirty)
bool at_dirty = true;
uint64_t at_images = ~0ull;
int64_t at_frame = -1;
float viewport[16][4]{}; // x, y, width, height
float depth_near = 0.0f, depth_far = 1.0f;
int32_t scissor[4]{0, 0, 0, 0}; // (width 0: none set)
uint32_t stencil_value_mask[2]{0xff, 0xff}, stencil_mask[2]{0xff, 0xff}, stencil_ref[2]{0, 0}; // front, back
float poly_offset[3]{}; // factor, units, clamp
float blend_color[4]{};
bool restart = false;
uint32_t restart_index = 0;
bool depth_clamp = false, raster_discard = false;
bool render_enabled = true;
int64_t copy_stride = 0; // SetCopyRowStride: the buffer side's bytes per row (0: packed)
uint32_t grid[3]{}; // the dispatch being made (compute programs read it from c0)
// bumped when what picks a program variant changes (program, vertex attributes / streams, texture, sampler and
// image handles; a rebind of the same values does not count): nvn_programs reuses an answer while it stays.
uint64_t prog_epoch = 1;
// the vertex attributes and streams by content (set when they change): the program variants and pull plans
// memoized for each, as draws of different materials take turns (an epoch bumped at every program bind was not)
uint64_t vtx_hash = 0;
// the occlusion counter (SAMPLES_PASSED) the queue reports: it grows with each draw that rasterizes (no real
// count is read back), so the difference between two reports says whether anything was drawn between them
uint64_t samples_passed = 0;
uint64_t samples_unqueried = 0; // (the estimate for draws since the reset that got no occlusion query)
};
inline uint64_t nvn_vertex_hash(const NvnState& s) { // (NvnState::vtx_hash)
return hash_bytes(s.vattr, sizeof s.vattr, hash_bytes(s.vstream, sizeof s.vstream, (uint64_t)s.nvattr));
}
/// The images a render target set draws into (one size: WebGPU requires it)
Attachments nvn_attachments(NvnState& s);
/// A render target: the texture's level / layer the view selects, as an image at its storage address
Image* nvn_target(const nvn_tex* t, const nvn_view* v);
Image* nvn_storage_image(const nvn_tex* t, const nvn_view* v, bool volume); // (mkw-recomp: layered / 3D images)
struct NvnDraw {
bool indexed = false;
int prim = 0, index_type = 0; // NVNdrawPrimitive, NVNindexType
uint32_t first = 0, count = 0, base_instance = 0, instances = 1;
int32_t base_vertex = 0;
uint64_t index_gpu = 0;
// (mkw-recomp) a GPU-driven draw: its arguments (and count) in these buffers, read by the GPU; the fields above
// other than the index buffer and its type are not used
WGPUBuffer indirect = nullptr, count_buffer = nullptr;
uint64_t indirect_offset = 0, count_offset = 0;
uint32_t indirect_max = 0;
};
extern bool nvn_draw_indirect_refused; // (mkw-recomp) set by nvn_draw when a GPU-driven draw cannot be made so
void nvn_draw(NvnState& s, const NvnDraw& d); // rx_draw.cpp
// (mkw-recomp) indirect_va: the grid read on the GPU from that memory (x, y, z unused)
void nvn_dispatch(NvnState& s, uint32_t x, uint32_t y, uint32_t z, uint64_t indirect_va = 0); // rx_compute.cpp
void nvn_clear_color(Image* im, const float rgba[4], uint32_t mask, const int32_t* scissor); // rx_core.cpp
void nvn_clear_depth_stencil(Image* im, bool depth, float d, bool stencil, uint32_t sv, uint32_t smask,
const int32_t* scissor);
/// An NVN vertex attribute (nvn_va_st) in the encoding the vertex fetch and the WGSL backend take
Regs::VertexAttribute nvn_attribute(uint32_t va); // rx_draw.cpp
/// The attributes the vertex program reads from memory itself (see PullPlan), for the bound NVN vertex state
PullPlan nvn_pull_plan(const NvnState& s, const std::bitset<32>& loads);
/// The program pair of the bound program (vertex + fragment, translated together) for these target output types
/// (Shader::AttributeType of each color target); null: not drawable
ProgramPair* nvn_programs(const NvnState& s, const uint8_t out_types[8]); // rx_shader.cpp
/// The c1 constants (the shader package's own constant bank) of the bound program's vertex (0) / fragment (1) stage
const std::vector<uint8_t>* nvn_c1(const NvnState& s, int stage);
/// The bound program's compute code key (its RX_DUMP_WGSL name nvn_cs_<key>), 0 without one.
uint64_t nvn_cs_key(const NvnState& s);
ShaderModule* nvn_compute_module(const NvnState& s, const std::array<uint8_t, 64>* ssbo_alias = nullptr,
const std::array<uint32_t, 64>* ssbo_alias_words = nullptr,
const std::array<uint16_t, 16>* image_formats = nullptr);
/// (mkw-recomp) WGSL's name of a storage texture format ("rgba16float"...); null when it cannot be a storage texture
const char* wgsl_storage_format(WGPUTextureFormat f);
/// A texture binding from NVN handles (texture handle; separate sampler handle, 0: the texture handle's sampler)
BoundTexture nvn_texture(const NvnState& s, uint64_t tex_handle, uint64_t samp_handle,
const Shader::TextureDescriptor& desc); // rx_texture.cpp
void nvn_register_texture(int id, const nvn_tex& t, const nvn_view* v);
void nvn_register_sampler(int id, const nvn_samp_par& p);
void nvn_texture_released(uint64_t serial);
/// The registered texture of an id, its storage resolved to the texture living there now (false: none)
bool nvn_texture_of(int id, nvn_tex* t, nvn_view* v, bool* has_view);
// per handle, bumped when what the handle resolves to may change (rx_nvn.cpp; read without a lock)
uint32_t nvn_texture_version(int id);
uint32_t nvn_sampler_version(int id);
// the handle's texture was finalized: it names whatever texture lives at that storage now (may change any frame)
bool nvn_texture_follows_storage(int id);
bool nvn_sampler_of(int id, nvn_samp_par* p); // rx_nvn.cpp: the registered sampler (false: none)
/// The driver's constant bank c0 (storage buffer table) / c2 (texture and sampler handle tables) of a stage as the
/// NVN driver lays them out (0xA00 bytes): what translated programs read for their bindings
void nvn_driver_bank(const NvnState& s, int bank, int stage, uint8_t out[0xA00]);
/// What a shader can read of a uniform buffer bound with `size` bytes: the console's driver binds whole 256-byte
/// blocks (RX_NVN_UBO_EXACT=1: only the bytes bound)
inline uint64_t nvn_ubo_size(uint64_t size) {
static const bool exact = getenv("RX_NVN_UBO_EXACT") != nullptr;
return exact ? size : (size + 255) & ~255ull;
}
/// One stage's constant banks as an NVN draw or dispatch sees them: c0 / c2 the driver's, c1 the package's, c3+ uniform buffers
uint32_t nvn_driver_bank_word(const NvnState& s, int bank, int stage, uint32_t offset);
/// Native programs: uniform buffers they read past this many bytes are bound in place as read-only storage, not
/// copied into each draw's uniform block (RX_NVN_UB_LIMIT=<bytes>, default 512; 0: as before). The translation
/// (RuntimeInfo::cbuf_ub_limit) and nvn_stage_resources lay the block out with it.
uint32_t nvn_ub_limit();
struct NvnBanks {
uint8_t bank0[0xA00], bank2[0xA00];
bool have0 = false, have2 = false;
const std::vector<uint8_t>* c1 = nullptr;
/// up to n bytes of constant bank `index` from `offset` (the rest zeros); the bank's size
uint64_t read(const NvnState& s, int stage, uint32_t index, uint64_t offset, void* dst, uint64_t n) {
if ((index == 0 || index == 2) && (n == 4 || n == 8) && !(offset & 3) && offset + n <= 0xA00) {
// (a handle or a buffer address: the words themselves, not the whole bank built)
const uint32_t w[2] = {nvn_driver_bank_word(s, (int)index, stage, (uint32_t)offset),
n == 8 ? nvn_driver_bank_word(s, (int)index, stage, (uint32_t)offset + 4) : 0u};
std::memcpy(dst, w, n);
return 0xA00;
}
std::memset(dst, 0, n);
if (index == 0 || index == 2) {
uint8_t* b = index == 0 ? bank0 : bank2;
bool& have = index == 0 ? have0 : have2;
if (!have) nvn_driver_bank(s, (int)index, stage, b), have = true;
if (offset < 0xA00) std::memcpy(dst, b + offset, std::min<uint64_t>(n, 0xA00 - offset));
return 0xA00;
}
if (index == 1) {
if (c1 && offset < c1->size()) std::memcpy(dst, c1->data() + offset, std::min<uint64_t>(n, c1->size() - offset));
return c1 ? c1->size() : 0;
}
if (index >= 3 && index < 3 + 16) {
const nvn_p_bind_buf& u = s.ubo[stage][index - 3];
const uint64_t size = nvn_ubo_size(u.size);
if (u.addr && offset < size) gpu_read(u.addr + offset, dst, std::min<uint64_t>(n, size - offset));
return u.addr ? size : 0;
}
return 0;
}
};
uint32_t nvn_driver_bank_word(const NvnState& s, int bank, int stage, uint32_t offset);
/// What a frame spent on the usual causes of slow frames (logged for frames over 18 ms with HOS_FRAME_HIST).
struct FrameSpike {
int64_t decode_ns, wait_ns, trim_ns, upload_bytes, bind_groups, draws, translations;
int64_t translate_ns, pipeline_ns; // one-off draws: programs translated and pipelines made in place
};
extern FrameSpike spike;
/// Set on the renderer's helper threads (translation, texture decoding): count() and log_limited() are the render
/// thread's alone and do nothing there.
extern thread_local bool rx_side_thread;
/// Textures decoded on the worker since the last call go into the decoded map (render thread, each frame).
void install_decoded_textures();
void trim_decoded_textures();
uint64_t image_bytes(const Image& im);
void decoded_texture_stats(size_t* n, uint64_t* bytes);
size_t buffer_segments();
void retire_unused_segments();
bool segments_retired_pending();
std::vector<WGPUBuffer> take_retired_segment_buffers();
void release_buffer_later(WGPUBuffer b);
size_t pipeline_count();
void wgsl_text_stats(size_t* n, uint64_t* bytes);
// ---- deferred command encoding (rx_record.cpp) ----
struct Recording;
bool encoding_deferred();
WGPUCommandEncoder fake_encoder();
Recording* recording_take();
void recording_replay(Recording* r, WGPUCommandEncoder e);
/// A replay in parts (words [from, to): each part whole passes), for encoders finished on several threads. With a
/// GPU timer (RX_GPU_TIMING), each pass writes timestamps into it.
struct GpuTimer;
void recording_replay(Recording* r, WGPUCommandEncoder e, size_t from, size_t to, GpuTimer* t = nullptr);
/// RX_GPU_TIMING=1 (profiling): each pass's GPU time, summed by what it draws into and logged every 5 s. begin_pass
/// tags the next pass (its first target's address; its size and formats); a timer is taken per encoder, resolved into
/// it before its finish and read after its submit.
bool gpu_timing();
void ce_pass_tag(uint64_t va, uint64_t desc);
GpuTimer* gpu_timer_take();
void gpu_timer_resolve(GpuTimer* t, WGPUCommandEncoder e);
void gpu_timer_read(GpuTimer* t);
/// Up to `parts` ranges of about equal size, split where passes end (one range when it is small).
std::vector<std::pair<size_t, size_t>> recording_parts(const Recording* r, uint32_t parts, size_t min_words);
void recording_free(Recording* r);
WGPURenderPassEncoder ce_begin_render_pass(WGPUCommandEncoder e, const WGPURenderPassDescriptor* pd);
void ce_copy_buffer_to_buffer(WGPUCommandEncoder e, WGPUBuffer src, uint64_t src_offset, WGPUBuffer dst,
uint64_t dst_offset, uint64_t size);
void ce_copy_texture_to_texture(WGPUCommandEncoder e, const WGPUTexelCopyTextureInfo* src,
const WGPUTexelCopyTextureInfo* dst, const WGPUExtent3D* extent);
void ce_copy_texture_to_buffer(WGPUCommandEncoder e, const WGPUTexelCopyTextureInfo* src,
const WGPUTexelCopyBufferInfo* dst, const WGPUExtent3D* extent);
WGPUComputePassEncoder ce_begin_compute_pass(WGPUCommandEncoder e);
void rp_end(WGPURenderPassEncoder p);
void rp_set_viewport(WGPURenderPassEncoder p, float x, float y, float w, float h, float zmin, float zmax);
void rp_set_pipeline(WGPURenderPassEncoder p, WGPURenderPipeline pipeline);
void rp_set_bind_group(WGPURenderPassEncoder p, uint32_t index, WGPUBindGroup g, size_t n, const uint32_t* offsets);
void rp_set_vertex_buffer(WGPURenderPassEncoder p, uint32_t slot, WGPUBuffer b, uint64_t offset, uint64_t size);
void rp_set_index_buffer(WGPURenderPassEncoder p, WGPUBuffer b, WGPUIndexFormat format, uint64_t offset, uint64_t size);
// (mkw-recomp) up to max draws whose arguments are at offset (packed); count: how many, read by the GPU (or null)
void rp_multi_draw_indirect(WGPURenderPassEncoder p, WGPUBuffer args, uint64_t offset, WGPUBuffer count,
uint64_t count_offset, uint32_t max, bool indexed);
void rp_set_scissor(WGPURenderPassEncoder p, uint32_t x, uint32_t y, uint32_t w, uint32_t h);
void rp_set_blend_constant(WGPURenderPassEncoder p, const WGPUColor* c);
void rp_set_stencil_reference(WGPURenderPassEncoder p, uint32_t ref);
void rp_begin_occlusion_query(WGPURenderPassEncoder p, uint32_t index);
void rp_end_occlusion_query(WGPURenderPassEncoder p);
void ce_resolve_query_set(WGPUCommandEncoder e, WGPUQuerySet set, uint32_t first, uint32_t n, WGPUBuffer dst,
uint64_t offset);
void rp_draw(WGPURenderPassEncoder p, uint32_t vertices, uint32_t instances, uint32_t first, uint32_t first_instance);
void rp_draw_indexed(WGPURenderPassEncoder p, uint32_t indices, uint32_t instances, uint32_t first_index,
int32_t base_vertex, uint32_t first_instance);
void cp_set_pipeline(WGPUComputePassEncoder p, WGPUComputePipeline pipeline);
void cp_set_bind_group(WGPUComputePassEncoder p, uint32_t index, WGPUBindGroup g, size_t n, const uint32_t* offsets);
void cp_set_bind_group_desc(WGPUComputePassEncoder p, uint32_t index, WGPUBindGroupLayout layout,
const WGPUBindGroupEntry* entries, size_t n);
void cp_dispatch(WGPUComputePassEncoder p, uint32_t x, uint32_t y, uint32_t z);
void cp_dispatch_indirect(WGPUComputePassEncoder p, WGPUBuffer args, uint64_t offset); // (mkw-recomp)
void cp_end(WGPUComputePassEncoder p);
} // namespace rx
#include "rx_web_mt.h" // (last: its macros wrap the WebGPU calls of the files that include this)

22
gpu/render/rx_none.c Normal file
View File

@@ -0,0 +1,22 @@
/* The renderer's API when the build has no WebGPU (build.py native without --wgpu): nothing is drawn and the
window shows the window buffers' memory (hos/display.c). */
#include "rx.h"
int rx_start(void *hwnd) { (void)hwnd; return 0; }
int rx_active(void) { return 0; }
void rx_resize(int width, int height) { (void)width; (void)height; }
void *rx_channel_new(void) { return 0; }
const uint8_t *rx_3d_methods(void) { return 0; }
void rx_3d_method(void *channel, const uint32_t *regs, uint32_t method, uint32_t arg) {
(void)channel; (void)regs; (void)method; (void)arg;
}
void rx_flush(void) {}
void rx_compute_launch(void *channel, const uint32_t *regs) { (void)channel; (void)regs; }
void rx_begin_submission(void) {}
void rx_memory_written(uint64_t gpu_va, uint64_t size) { (void)gpu_va; (void)size; }
int rx_present(uint64_t guest_address, uint32_t width, uint32_t height) {
(void)guest_address; (void)width; (void)height;
return 0;
}
const char *rx_describe(void) { return "off"; }
void rx_stats(int64_t *draws, int64_t *skipped, int64_t *pipelines) { *draws = *skipped = *pipelines = 0; }

1603
gpu/render/rx_nvn.cpp Normal file

File diff suppressed because it is too large Load Diff

267
gpu/render/rx_nvn.h Normal file
View File

@@ -0,0 +1,267 @@
/* The native NVN layer: TOTK's graphics API implemented directly (HOS_NVN_NATIVE=1), with no Switch GPU in between.
hos/nvn_native.c is the API: objects live in place in the memory the game gives NVN objects, state-only and GPU work
is recorded as commands in host memory, and a submit hands the recordings to gpu/render/rx_nvn.cpp, which replays
them with WebGPU. This header is shared by both (C and C++).
Design and TOTK specifics follow the MIT-licensed NVN layer of d2r-mobile / MacNativeGames (jhaizhou), which runs
TOTK v1.0.0 on Metal: object layouts, command records, deferred texture binds, recording copies, the driver constant
buffers' layout.
Addresses: a GPU address is a guest address for ordinary memory pools (identity map). A virtual pool (NVN sparse
memory) gets GPU addresses above 4 GB, which no guest pointer can reach, and a page table (64 KiB pages) to the
physical pools' guest addresses; nvn_gpu_* resolve both. Host pointers in the objects below are only read by us. */
#pragma once
#include <stddef.h>
#include <stdint.h>
#include "nvn_api.h"
#ifdef __cplusplus
extern "C" {
#endif
#define NVN_MAGIC(a, b, c, d) ((uint32_t)(a) | (uint32_t)(b) << 8 | (uint32_t)(c) << 16 | (uint32_t)(d) << 24)
/* ---- texture storage layout (Tegra X1 block linear; hos/nvn_layout.c, from deko3d) ---- */
typedef struct {
int target, format, flags, samples; /* NVNtextureTarget / NVNformat / NVNtextureFlags / samples (0 or 1: one) */
uint32_t width, height, depth, levels, stride;
signed int pkg_block_h : 8; /* packaged layout byte 0 (log2 block height) + 1; 0: the automatic choice */
unsigned int swizzle_r : 3, swizzle_g : 3, swizzle_b : 3, swizzle_a : 3, swizzle_pad : 12;
} nvn_texdesc;
typedef struct {
uint64_t size, layer_size; /* storage bytes, bytes per layer */
uint32_t align, layers, bpb; /* alignment, layers, bytes per block */
unsigned tileH, tileD; /* block height / depth (log2 GOBs) */
} nvn_texlayout;
void nvn_tex_layout(const nvn_texdesc *d, nvn_texlayout *o);
uint64_t nvn_tex_level_offset(const nvn_texdesc *d, unsigned level);
int nvn_fmt_bpb(int format, int *bw, int *bh); /* bytes per block (0: unknown format), block width / height */
void nvn_tex_level_tiles(const nvn_texdesc *d, unsigned level, unsigned *tileH, unsigned *tileD);
/* LINEAR_RENDER_TARGET (0x40) is pitch storage even without LINEAR (0x10) */
static inline int nvn_tex_is_linear(const nvn_texdesc *d) { return (d->flags & 0x50) != 0; }
uint32_t nvn_tex_linear_stride(const nvn_texdesc *d);
/* ---- objects (in place in the game's NVN object memory; each fits the reserved size of its type) ---- */
#define NVN_FITS(T, H) _Static_assert(sizeof(T) <= sizeof(H), #T " is larger than " #H)
typedef struct { uint32_t flags; } nvn_dev_builder;
typedef struct { uint32_t magic, flags, origin, depth_mode; } nvn_dev;
typedef struct { void *dev; uint32_t flags; uint64_t cmd_sz, comp_sz, ctl_sz, flush; uint64_t mem, mem_sz; } nvn_queue_builder;
typedef struct { uint32_t magic; uint64_t submits; } nvn_queue;
/* a memory pool: mem = guest address of its storage (0: a virtual pool), gpu = its GPU address. (hookshot) The object
is guest memory, so it holds no host pointers: a virtual pool's page table is found by its GPU address. */
typedef struct { void *dev; uint64_t mem; uint64_t size; uint32_t flags; } nvn_pool_builder; /* mem: guest address */
typedef struct {
uint32_t magic, flags;
uint64_t mem; /* guest address of the storage (0 for a virtual pool) */
uint64_t size, gpu; /* GPU address: the guest address, or a virtual pool's range above 4 GB */
uint64_t reserved;
} nvn_pool;
typedef struct { const nvn_pool *pool; int64_t off; int32_t n; } nvn_dpool; /* texture / sampler descriptor pool */
typedef struct { void *dev; nvn_pool *pool; int64_t off; uint64_t size; } nvn_buffer_builder;
typedef struct { uint32_t magic; nvn_pool *pool; int64_t off; uint64_t size; } nvn_buffer;
typedef struct {
void *dev; nvn_texdesc d; nvn_pool *pool; int64_t off;
uint8_t has_layout, layout[8];
uint64_t pkg_data; /* SetPackagedTextureData (guest address): copied into the storage at Initialize */
} nvn_tex_builder;
typedef struct {
uint32_t magic; nvn_texdesc d; uint8_t has_layout, layout[8];
nvn_texlayout L;
uint64_t gpu; /* GPU address of the storage (pool gpu + offset) */
const nvn_pool *pool; int64_t off;
uint64_t serial; /* one per initialization: the game copies texture objects by value */
} nvn_tex;
/* TOTK reserves 16 bytes for a view on its stack (main+0x823398): packed */
typedef struct {
uint8_t set_levels : 1, set_layers : 1, set_format : 1, set_target : 1, set_swizzle : 1, pad0 : 3;
uint8_t base_level, nlevels, target; uint16_t min_layer, nlayers; uint16_t format; uint8_t swz[4]; uint8_t pad1[2];
} nvn_view;
_Static_assert(sizeof(nvn_view) == 16, "NVNtextureView is 16 bytes in TOTK's NVN");
typedef struct {
uint8_t minf, magf, ws, wt, wr, cmp_mode, cmp_func, reduction;
float lod_min, lod_max, lod_bias, aniso, border[4], lod_snap;
} nvn_samp_par;
typedef struct { void *dev; nvn_samp_par p; } nvn_samp_builder;
typedef struct { uint32_t magic; nvn_samp_par p; } nvn_samp;
/* a program: up to 6 stages, each the shader's code address (GPU) and its control section (guest address) */
typedef struct { uint64_t data, control; } nvn_stage_data;
typedef struct { uint32_t magic, nstage; nvn_stage_data sd[6]; uint64_t serial; } nvn_prog;
typedef struct { uint32_t magic, fenced; uint64_t fence; } nvn_sync;
typedef struct { const nvn_pool *pool; int64_t off; } nvn_event_builder;
typedef struct { uint32_t magic; uint64_t gpu; } nvn_event; /* gpu: the 32-bit value's address */
typedef struct { void *dev; uint64_t native; int32_t ntex, interval, active; uint64_t tex[4]; } nvn_win_builder; /* tex: guest addresses */
typedef struct {
uint32_t magic; int32_t ntex, interval, next, active, crop[4];
uint64_t tex[4]; /* (hookshot) guest addresses, checked at each use */
uint64_t presents;
} nvn_win;
/* a command buffer: records go to a growable host buffer (never into the game's command memory) */
/* (hookshot) The object is guest memory: the recording buffer's pointers (seg, cur, end) are kept in a host table
(host/nvn_native.c cmd_rec), and the object holds only its slot there, which names the object it belongs to. */
typedef struct {
uint32_t magic, recording;
uint32_t slot, slot_pad; /* the host table's slot (0: none yet) */
uint64_t guest_flags; /* +0x10: NVN 55 inline code writes a dirty flag here */
uint64_t reserved_18;
/* NVN 55 headers inline fast paths that read the write pointer at +0x20 and its end at +0x28 (main+0x70644c) and
emit raw GPU methods when 0x30 bytes fit: ~0 / 0 never fits, so every call takes the API (*Deferred) path */
uint64_t inline_cur, inline_end;
uint64_t cb, cb_data; /* the memory callback (guest), never called: records live in host memory */
uint64_t reserved_40;
uint64_t ctl, ctl_size; /* control memory the game gave (guest): a handle lives as long as it */
uint64_t game_size; /* command memory the game gave (accounting only) */
uint64_t cmd_used, ctl_used, rec_start;
uint64_t hw_used, last_rec, hw_start; /* what the real driver would have used (the game sizes heaps from it) */
} nvn_cmd;
_Static_assert(offsetof(nvn_cmd, guest_flags) == 0x10, "NVN inline dirty flag");
_Static_assert(offsetof(nvn_cmd, inline_cur) == 0x20 && offsetof(nvn_cmd, inline_end) == 0x28, "NVN inline fast path");
/* state objects (the game copies them by value): packed in place */
typedef struct { uint32_t blend_en : 8, logic_op : 4, alpha_func : 4; } nvn_color_st;
typedef struct { uint32_t mask; } nvn_chmask_st; /* 4 bits a target: r 1 g 2 b 4 a 8 (set: written) */
typedef struct { uint64_t target : 3, src_rgb : 7, dst_rgb : 7, src_a : 7, dst_a : 7, eq_rgb : 3, eq_a : 3; } nvn_blend_st;
typedef struct { uint8_t enable, samples, a2c, a2c_dither, raster_samples, cov_mod, cov2color_en, cov2color_out, sl_en, sl_grid; } nvn_ms_st;
typedef struct { uint32_t cull : 2, front : 1, mode : 2, offset_en : 3; } nvn_poly_st;
typedef struct {
uint64_t dt : 1, dw : 1, df : 4, st : 1, sf_f : 4, sf_b : 4,
fail_f : 4, zfail_f : 4, zpass_f : 4, fail_b : 4, zfail_b : 4, zpass_b : 4;
} nvn_ds_st;
typedef struct { uint32_t fmt : 8, offset : 19, stream : 5; } nvn_va_st;
typedef struct { uint32_t stride, divisor; } nvn_vs_st;
/* ---- command records: {op, size (header included, 8-aligned)} + payload ---- */
typedef struct { uint32_t op, size; } nvn_op_hdr;
enum {
NOP_END, NOP_JUMP, NOP_BEGIN, NOP_CALL,
NOP_SIGNAL_EVENT, NOP_REPORT_COUNTER, NOP_COPY_B2B, NOP_UPDATE_UB, /* CPU-visible side effects */
NOP_SET_RT, NOP_CLEAR_COLOR, NOP_CLEAR_DS, NOP_VIEWPORTS, NOP_SCISSOR, NOP_DEPTH_RANGE,
NOP_BIND_PROGRAM, NOP_BIND_STATE, NOP_BIND_VATTR, NOP_BIND_VSTREAM, NOP_BIND_VBUF,
NOP_BIND_UBO, NOP_BIND_SSBO, NOP_BIND_TEX, NOP_BIND_SAMP,
NOP_STENCIL, NOP_POLY_OFFSET, NOP_DRAW, NOP_DRAW_IDX, NOP_DISPATCH, NOP_BARRIER,
NOP_SET_TEX_POOL, NOP_SET_SAMP_POOL, NOP_COPY_B2T, NOP_COPY_STRIDE, NOP_BLEND_COLOR, NOP_COPY_T2B, NOP_BIND_IMG,
NOP_COPY_T2T, NOP_RESET_COUNTER, NOP_CLEAR_TEX, NOP_DEPTH_CLAMP, NOP_RENDER_ENABLE, NOP_RASTER_DISCARD,
NOP_RENDER_COND, NOP_PRIMITIVE_RESTART, NOP_BIND_BUFS,
NOP_DRAW_INDIRECT, NOP_DISPATCH_INDIRECT, /* (mkw-recomp: NVN2's GPU-driven draws) */
NOP_DRAW_MESH, /* (mkw-recomp: NVN2 mesh shading; the WebGPU replay leaves it out) */
NOP_WAIT_EVENT, /* (mkw-recomp: a wait for an event another queue's work signals) */
NOP_PUSH_CONST, /* (hookshot: nvnCommandBufferPushComputeConstants) */
NOP_CLEAR_BUF, /* (hookshot: nvnCommandBufferClearBuffer) */
NOP_COUNT_
};
typedef struct { int32_t n; const nvn_tex *color[8]; nvn_view cview[8]; uint8_t has_cview[8]; const nvn_tex *depth; nvn_view dview; uint8_t has_dview; } nvn_p_set_rt;
typedef struct { int32_t index, mask; float rgba[4]; } nvn_p_clear_color;
typedef struct { float depth; int32_t depth_mask, stencil, stencil_mask; } nvn_p_clear_ds;
typedef struct { int32_t first, n; float v[16][4]; } nvn_p_viewports;
typedef struct { int32_t x, y, w, h; } nvn_p_scissor;
typedef struct { int32_t value; int32_t index; } nvn_p_flag;
typedef struct { int32_t mode; int32_t snapped; uint64_t addr, x, y; } nvn_p_render_cond; /* render when the counters at addr, addr + 16 are equal (0) / differ (1) */
typedef struct { float n, f; } nvn_p_depth_range;
typedef struct { const nvn_prog *prog; int32_t stages; } nvn_p_bind_program;
typedef struct { int32_t kind; uint8_t bytes[0x18]; } nvn_p_bind_state; /* kind: 0 blend 1 color 2 channel mask 3 multisample 4 polygon 5 depth-stencil */
typedef struct { int32_t first, n; uint32_t st[16]; } nvn_p_bind_vattr;
typedef struct { int32_t first, n; nvn_vs_st st[16]; } nvn_p_bind_vstream;
typedef struct { int32_t first, n; NVNbufferRange r[16]; } nvn_p_bind_vbuf;
typedef struct { int32_t stage, index; uint64_t addr, size; } nvn_p_bind_buf; /* uniform / storage buffer */
/* src: deferred binds keep the game's handle array (guest address) and read it at submit; 0: immediate */
typedef struct { int32_t stage, first, n, pad_; uint64_t src; uint64_t h[32]; } nvn_p_bind_handles;
/* deferred uniform / storage buffer ranges: read from the game's array (src, guest address) at submit */
typedef struct { int32_t ssbo, stage, first, n; uint64_t src; NVNbufferRange r[16]; } nvn_p_bind_bufs;
typedef struct { int32_t face, value_mask, mask, ref, which; } nvn_p_stencil; /* which: 0 value mask 1 write mask 2 reference */
typedef struct { float factor, units, clamp; } nvn_p_poly_offset;
typedef struct { int32_t prim, first, count, base_instance, instances; } nvn_p_draw;
typedef struct { int32_t prim, itype, count, base_vertex, base_instance, instances; uint64_t ibuf; } nvn_p_draw_idx;
typedef struct { int32_t x, y, z, pad; } nvn_p_dispatch;
typedef struct { const nvn_dpool *pool; } nvn_p_set_pool;
typedef struct { uint64_t buf; const nvn_tex *tex; nvn_view view; uint8_t has_view; NVNcopyRegion r; int32_t flags; } nvn_p_copy_bt; /* buffer <-> texture */
typedef struct { const nvn_tex *src, *dst; nvn_view sv, dv; uint8_t has_sv, has_dv; NVNcopyRegion sr, dr; int32_t flags; } nvn_p_copy_tt;
typedef struct { const nvn_tex *tex; nvn_view view; uint8_t has_view, is_uint; NVNcopyRegion r; int32_t mask; union { float f[4]; uint32_t u[4]; } c; } nvn_p_clear_tex;
typedef struct { int64_t stride; } nvn_p_copy_stride;
typedef struct { float rgba[4]; } nvn_p_blend_color;
typedef struct { uint64_t gpu; uint32_t mode, value; } nvn_p_signal; /* event: its 32-bit value's GPU address */
typedef struct { uint64_t gpu; uint32_t mode, value; } nvn_p_wait_event; /* (the same, and the wait's condition) */
typedef struct { uint64_t type, addr; } nvn_p_counter;
#define NVN_COUNTER_WRITTEN (1ull << 32) /* (type: the report's counter word is in memory already: a clear flag, host/nvn_native.c) */
typedef struct { uint64_t src, dst, size; } nvn_p_copy_bb;
typedef struct { uint64_t dst, size; /* data follows */ } nvn_p_update_ub;
/* (hookshot) compute push constants: the bytes at offset in the compute driver bank's push area */
typedef struct { uint32_t offset, size; /* data follows */ } nvn_p_push_const;
/* (hookshot) a buffer range filled with a 32-bit value */
typedef struct { uint64_t dst, size; uint32_t value, pad; } nvn_p_clear_buf;
typedef struct { int32_t enable, index; } nvn_p_restart;
/* (mkw-recomp) indirect draws: up to max draws of `stride`-byte arguments at `indirect` (indexed: count, instances,
first index, base vertex, base instance; arrays: count, instances, first, base instance), as many as the 32-bit
value at count_addr says when there is one; read at replay, after the GPU's writes there reached guest memory */
typedef struct { int32_t indexed, prim, itype, max, stride, pad; uint64_t ibuf, indirect, count_addr; } nvn_p_draw_indirect;
typedef struct { uint64_t addr; } nvn_p_dispatch_indirect; /* three 32-bit group counts */
/* DrawMeshTasks (x, y, z: the call's arguments as they are), DrawMeshTasksIndirect (indirect, max 1),
MultiDrawMeshTasksIndirectCount (indirect, count_addr, max, stride) */
typedef struct { int32_t x, y, z, max, stride, kind; uint64_t indirect, count_addr; } nvn_p_draw_mesh;
/* ---- the API layer's services the replay uses (hos/nvn_native.c) ---- */
/* host pointer of a GPU address (virtual pools resolved through their page table); null when not mapped */
uint8_t *nvn_gpu_host(uint64_t gpu);
/* how many bytes from gpu on are contiguous in host memory (at most n) */
uint64_t nvn_gpu_contiguous(uint64_t gpu, uint64_t n);
/* the guest address of a GPU address (0: not mapped) */
uint64_t nvn_gpu_guest(uint64_t gpu);
/* copies n bytes at a GPU address (page by page for virtual pools); 0 when part of it is not mapped */
int nvn_gpu_read(uint64_t gpu, void *dst, uint64_t n);
int nvn_gpu_write(uint64_t gpu, const void *src, uint64_t n);
/* the texture currently living at a storage address (the game re-initialises transient targets every frame at the
same memory without registering them again): 1 and *out when there is one */
int nvn_tex_live_at(uint64_t gpu, nvn_tex *out);
/* NVN_DEPTH_MODE_NEAR_IS_MINUS_W (GL clip z) was selected */
int nvn_depth_minus_w(void);
/* ---- the replay (gpu/render/rx_nvn.cpp) ---- */
/* the recordings of a submit, in order (handles: host pointers of NOP_BEGIN records), replayed now */
void rx_nvn_submit(const uint64_t *handles, int n);
/* (the same with the queue it came from: the game's queues have their own state and order each other with events) */
void rx_nvn_submit_q(const uint64_t *handles, int n, uint64_t queue);
/* fence value of everything submitted so far; waits until the GPU reached a fence (1: reached, 0: timed out) */
uint64_t rx_nvn_fence(void);
uint64_t rx_nvn_fence_q(uint64_t queue); /* (hookshot) a fence of one queue */
/* submits and presents queued so far / replayed so far (recordings released after a submit stay until it replayed) */
uint64_t rx_nvn_pushed(void);
uint64_t rx_nvn_replayed(void);
int rx_nvn_wait(uint64_t fence, uint64_t timeout_ns);
/* (the threaded web build) waits while a render target writeback not in guest memory yet will write [p, p + n): the
game's memcpy reads the targets it copies with the CPU through this, so fences need not wait for them */
void rx_wait_writebacks_reading(const void *p, uint64_t n);
void rx_nvn_finish(void);
/* a window texture to the screen (crop: x, y, w, h; w = 0 whole) */
void rx_nvn_present(const nvn_tex *t, const int crop[4]);
/* texture / sampler descriptor registrations (copies are taken) */
void rx_nvn_register_texture(int id, const nvn_tex *t, const nvn_view *v);
void rx_nvn_register_sampler(int id, const nvn_samp_par *p);
void rx_nvn_set_texture_pool_size(int n);
/* objects that went away: what the replay made for them can go */
void rx_nvn_texture_released(const nvn_tex *t);
void rx_nvn_texture_created(const nvn_tex *t);
void rx_nvn_program_released(const nvn_prog *p);
/* CPU reads / writes of a texture (ReadTexels / WriteTexels) */
void rx_nvn_read_texels(const nvn_tex *t, unsigned level, unsigned layer0, const NVNcopyRegion *r, uint8_t *dst,
size_t row, size_t img);
void rx_nvn_write_texels(const nvn_tex *t, unsigned level, unsigned layer0, const NVNcopyRegion *r, const uint8_t *src,
size_t row, size_t img);
/* the CPU is about to read memory the GPU may have written: GPU results there go to guest memory now */
void rx_nvn_invalidate(uint64_t gpu, uint64_t size);
void rx_nvn_pool_created(uint64_t gpu, uint64_t size); /* (the Vulkan back end imports it early; WebGPU: nothing) */
/* virtual pages [gpu, gpu + size) mapped now are about to be unmapped or remapped: what the GPU drew into textures
there goes into their memory first (the Vulkan back end; WebGPU: nothing) */
void rx_nvn_vmap_changing(uint64_t gpu, uint64_t size);
/* the CPU set an event's value (nvnEventSignal): in order with the submits, for the GPU's waits on it */
void rx_nvn_event_signal(uint64_t gpu, uint32_t value);
#ifdef __cplusplus
}
#endif

486
gpu/render/rx_pipecache.cpp Normal file
View File

@@ -0,0 +1,486 @@
// The browser's pipeline cache: every render and compute pipeline the game makes is recorded (its WGSL and its whole
// descriptor) to /rxcache/pipelines.bin, which the page keeps in IndexedDB; the next time the page opens they are all
// made again in the background, a few at a time, and kept. The browser's WebGPU (Dawn) gives back the object it
// already has for an identical descriptor, so when the game first needs one of them it is ready at once instead of a
// Tint -> HLSL -> D3D compile in the middle of play (first jumps, new areas, camera cuts all stalled on these), and
// Chrome's own disk cache of compiled shaders makes the background compiles cheap after the first time.
// /rxcache/seed.bin (the page fetches pipelines.bin from the server when there is one) is made in the same way.
// The records hold the WebGPU structs as bytes: a cache from another build of the page may make pipelines the game
// never asks for, which costs only their compile.
#include <algorithm>
#include <array>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <mutex>
#include <cstring>
#include <deque>
#include <unordered_map>
#include <unordered_set>
#include "rx_internal.h"
namespace rx {
namespace {
#if RX_WEB
constexpr uint32_t kMagic = 0x43505852; // "RXPC"
constexpr uint32_t kVersion = 1;
const char* const kCachePath = "/rxcache/pipelines.bin";
const char* const kSeedPath = "/rxcache/seed.bin";
uint32_t layout_signature() {
return (uint32_t)(sizeof(WGPUBindGroupLayoutEntry) * 1 + sizeof(WGPUVertexBufferLayout) * 3 +
sizeof(WGPUVertexAttribute) * 7 + sizeof(WGPUColorTargetState) * 11 +
sizeof(WGPUBlendState) * 13 + sizeof(WGPUPrimitiveState) * 17 +
sizeof(WGPUDepthStencilState) * 19 + sizeof(WGPUMultisampleState) * 23);
}
struct Writer {
std::string& out;
template <typename T> void put(const T& v) { out.append(reinterpret_cast<const char*>(&v), sizeof v); }
void bytes(const void* p, size_t n) { out.append(static_cast<const char*>(p), n); }
};
struct Reader {
const uint8_t* p;
const uint8_t* end;
bool ok = true;
template <typename T> T get() {
T v{};
if (end - p < (ptrdiff_t)sizeof v) { ok = false; return v; }
std::memcpy(&v, p, sizeof v);
p += sizeof v;
return v;
}
const uint8_t* take(size_t n) {
if ((size_t)(end - p) < n) { ok = false; return nullptr; }
const uint8_t* r = p;
p += n;
return r;
}
};
std::mutex cache_mu; // (the recorders: the render thread and the compute path)
std::string pending; // records made since the last save (appended to the file: the whole file is ~60-95 MB)
size_t saved_size = 0;
int64_t last_save_ns = 0;
std::unordered_set<uint64_t> known_modules, known_records;
// A loaded record, where it is in its file: read again when the prewarm makes it. (The files stay in the page's file
// system, JavaScript memory: held in the host heap, the cache and the seed were ~120 MB of its ~570 for minutes of
// prewarm, and the open world ran it out of memory.)
struct Item {
uint8_t file; // 0 the page's cache, 1 the seed
uint32_t off, len;
char type;
};
FILE* files[2];
std::unordered_map<uint64_t, Item> module_texts; // (loaded, for the prewarm)
/// A record's bytes (into buf, reused)
bool read_item(const Item& item, std::string& buf) {
FILE* f = files[item.file & 1];
buf.resize(item.len);
return f && !std::fseek(f, (long)item.off, SEEK_SET) && std::fread(buf.data(), 1, item.len, f) == item.len;
}
// the prewarm (render thread: everything here, recording included, runs on it)
std::deque<Item> prewarm; // render ('R') and compute ('C') records to make
std::unordered_map<uint64_t, WGPUShaderModule> warm_modules;
std::vector<WGPURenderPipeline> warm_render;
std::vector<WGPUComputePipeline> warm_compute;
std::atomic<int> in_flight{0};
int64_t warmed = 0, prewarm_total = 0, prewarm_t0 = 0;
uint64_t text_hash(const std::string& s) { return hash_bytes(s.data(), s.size(), 0x9e3779b97f4a7c15ull); }
void add_record(char type, const std::string& payload) { // (mu held)
const uint64_t h = hash_bytes(payload.data(), payload.size(), (uint64_t)type);
if (!known_records.insert(h).second) return;
Writer w{pending};
w.put(type);
w.put((uint32_t)payload.size());
w.bytes(payload.data(), payload.size());
}
uint64_t add_module(const std::string& text) { // (mu held)
const uint64_t h = text_hash(text);
if (known_modules.insert(h).second) {
std::string payload;
Writer w{payload};
w.put(h);
w.bytes(text.data(), text.size());
add_record('M', payload);
}
return h;
}
uint64_t add_module_hashed(uint64_t h, const std::string& wgsl) { // (mu held)
if (!h) return add_module(web_shader_source(wgsl));
if (known_modules.count(h) || wgsl.empty()) return h;
return add_module(web_shader_source(wgsl));
}
void put_layouts(Writer& w, const std::vector<WGPUBindGroupLayoutEntry>* layouts, int n) {
w.put((uint32_t)n);
for (int i = 0; i < n; i++) {
w.put((uint32_t)layouts[i].size());
w.bytes(layouts[i].data(), layouts[i].size() * sizeof(WGPUBindGroupLayoutEntry));
}
}
/// Reads a cache file's records (the page's own cache, or a seed): modules go to module_texts and the rest onto the
/// prewarm, as places in the file, which stays open until the prewarm is done.
void load(const char* path, bool keep) {
FILE* f = std::fopen(path, "rb");
if (!f) return;
uint32_t head[3] = {};
if (std::fread(head, 1, sizeof head, f) != sizeof head || head[0] != kMagic || head[1] != kVersion ||
head[2] != layout_signature()) {
std::fclose(f);
hos_log("rx: pipeline cache %s is from another build: not used", path);
return;
}
std::fseek(f, 0, SEEK_END);
const long size = std::ftell(f) - (long)sizeof head;
std::fseek(f, sizeof head, SEEK_SET);
if (keep && size > (96l << 20)) { // (grown past use: started again)
std::fclose(f);
std::remove(path);
hos_log("rx: pipeline cache %s: %ld MB, started again", path, size >> 20);
return;
}
const uint8_t which = keep ? 0 : 1;
files[which] = f;
std::string payload;
long pos = (long)sizeof head;
int modules = 0, records = 0;
for (;;) {
char type;
uint32_t n;
if (std::fread(&type, 1, 1, f) != 1 || std::fread(&n, 1, 4, f) != 4) break;
pos += 5;
payload.resize(n);
if (std::fread(payload.data(), 1, n, f) != n) break;
const uint32_t off = (uint32_t)pos;
pos += n;
const uint64_t h = hash_bytes(payload.data(), n, (uint64_t)type);
if (!keep && known_records.count(h)) continue;
if (keep) known_records.insert(h);
if (type == 'M' && n >= 8) {
uint64_t mh;
std::memcpy(&mh, payload.data(), 8);
if (keep) known_modules.insert(mh);
module_texts.emplace(mh, Item{which, off + 8, n - 8, 'M'});
modules++;
} else if (type == 'R' || type == 'C') {
prewarm.push_back(Item{which, off, n, type});
records++;
}
}
hos_log("rx: pipeline cache %s: %d modules, %d pipelines to make in the background", path, modules, records);
}
WGPUShaderModule warm_module(uint64_t h) {
auto it = warm_modules.find(h);
if (it != warm_modules.end()) return it->second;
auto t = module_texts.find(h);
static std::string text;
if (t == module_texts.end() || !read_item(t->second, text)) return warm_modules[h] = nullptr;
WGPUShaderSourceWGSL src{};
src.chain.sType = WGPUSType_ShaderSourceWGSL;
src.code = WGPUStringView{text.data(), text.size()};
WGPUShaderModuleDescriptor d{};
d.nextInChain = &src.chain;
return warm_modules[h] = program_module(&d);
}
WGPUPipelineLayout warm_layout(Reader& r, WGPUShaderStage visibility_unused) {
(void)visibility_unused;
const uint32_t n = r.get<uint32_t>();
WGPUBindGroupLayout bgl[4]{};
if (n > 4) { r.ok = false; return nullptr; }
for (uint32_t i = 0; i < n; i++) {
const uint32_t count = r.get<uint32_t>();
const uint8_t* p = r.take((size_t)count * sizeof(WGPUBindGroupLayoutEntry));
if (!r.ok) return nullptr;
std::vector<WGPUBindGroupLayoutEntry> e(count);
std::memcpy(e.data(), p, (size_t)count * sizeof(WGPUBindGroupLayoutEntry));
WGPUBindGroupLayoutDescriptor ld{};
ld.entryCount = count;
ld.entries = e.data();
bgl[i] = wgpuDeviceCreateBindGroupLayout(dev.device, &ld);
}
WGPUPipelineLayoutDescriptor pld{};
pld.bindGroupLayoutCount = n;
pld.bindGroupLayouts = bgl;
WGPUPipelineLayout pl = wgpuDeviceCreatePipelineLayout(dev.device, &pld);
for (uint32_t i = 0; i < n; i++) wgpuBindGroupLayoutRelease(bgl[i]);
return pl;
}
void warm_one(const Item& item) {
static std::string bytes;
if (!read_item(item, bytes)) return;
const uint8_t* p = reinterpret_cast<const uint8_t*>(bytes.data());
Reader r{p, p + item.len};
const char type = item.type;
if (type == 'C') {
const uint64_t mh = r.get<uint64_t>();
WGPUPipelineLayout pl = warm_layout(r, WGPUShaderStage_Compute);
WGPUShaderModule m = warm_module(mh);
if (!r.ok || !m || !pl) {
if (pl) wgpuPipelineLayoutRelease(pl);
return;
}
WGPUComputePipelineDescriptor cd{};
cd.layout = pl;
cd.compute.module = m;
cd.compute.entryPoint = WGPUStringView{"main", 4};
WGPUCreateComputePipelineAsyncCallbackInfo cb{};
cb.mode = WGPUCallbackMode_AllowProcessEvents;
cb.callback = [](WGPUCreatePipelineAsyncStatus status, WGPUComputePipeline p, WGPUStringView, void*, void*) {
in_flight--;
if (status == WGPUCreatePipelineAsyncStatus_Success && p) warm_compute.push_back(p);
};
in_flight++;
wgpuDeviceCreateComputePipelineAsync(dev.device, &cd, cb);
wgpuPipelineLayoutRelease(pl);
return;
}
if (type != 'R') return;
const uint64_t vh = r.get<uint64_t>(), fh = r.get<uint64_t>();
WGPUPipelineLayout pl = warm_layout(r, WGPUShaderStage_Vertex);
const uint32_t nbuf = r.get<uint32_t>();
if (!r.ok || nbuf > 32) {
if (pl) wgpuPipelineLayoutRelease(pl);
return;
}
std::vector<WGPUVertexBufferLayout> buffers(nbuf);
std::vector<std::vector<WGPUVertexAttribute>> attributes(nbuf);
for (uint32_t i = 0; i < nbuf; i++) {
buffers[i] = r.get<WGPUVertexBufferLayout>();
const uint32_t na = r.get<uint32_t>();
if (!r.ok || na > 64) { r.ok = false; break; }
attributes[i].resize(na);
for (uint32_t a = 0; a < na; a++) attributes[i][a] = r.get<WGPUVertexAttribute>();
buffers[i].attributes = attributes[i].data();
buffers[i].attributeCount = na;
}
const uint32_t nt = r.get<uint32_t>();
if (!r.ok || nt > 8) {
if (pl) wgpuPipelineLayoutRelease(pl);
return;
}
std::vector<WGPUColorTargetState> targets(nt);
std::array<WGPUBlendState, 8> blends{};
for (uint32_t i = 0; i < nt; i++) {
targets[i] = r.get<WGPUColorTargetState>();
const uint32_t has_blend = r.get<uint32_t>();
blends[i] = r.get<WGPUBlendState>();
targets[i].blend = has_blend ? &blends[i] : nullptr;
targets[i].nextInChain = nullptr;
}
WGPUPrimitiveState prim = r.get<WGPUPrimitiveState>();
const uint32_t has_depth = r.get<uint32_t>();
WGPUDepthStencilState depth = r.get<WGPUDepthStencilState>();
WGPUMultisampleState ms = r.get<WGPUMultisampleState>();
WGPUShaderModule vs = warm_module(vh), fs = fh ? warm_module(fh) : nullptr;
if (!r.ok || !pl || !vs || (fh && !fs)) {
if (pl) wgpuPipelineLayoutRelease(pl);
return;
}
prim.nextInChain = nullptr;
depth.nextInChain = nullptr;
ms.nextInChain = nullptr;
WGPUFragmentState fst{};
if (fs) {
fst.module = fs;
fst.entryPoint = WGPUStringView{"main", 4};
fst.targetCount = nt;
fst.targets = targets.data();
}
WGPURenderPipelineDescriptor d{};
d.layout = pl;
d.vertex.module = vs;
d.vertex.entryPoint = WGPUStringView{"main", 4};
d.vertex.bufferCount = nbuf;
d.vertex.buffers = buffers.data();
d.primitive = prim;
d.depthStencil = has_depth ? &depth : nullptr;
d.multisample = ms;
d.fragment = fs ? &fst : nullptr;
WGPUCreateRenderPipelineAsyncCallbackInfo cb{};
cb.mode = WGPUCallbackMode_AllowProcessEvents;
cb.callback = [](WGPUCreatePipelineAsyncStatus status, WGPURenderPipeline p, WGPUStringView, void*, void*) {
in_flight--;
if (status == WGPUCreatePipelineAsyncStatus_Success && p) warm_render.push_back(p);
};
in_flight++;
wgpuDeviceCreateRenderPipelineAsync(dev.device, &d, cb);
wgpuPipelineLayoutRelease(pl);
}
void save() { // (render thread: the records made since the last save, appended)
std::lock_guard lk(cache_mu);
if (pending.empty()) return;
FILE* f = std::fopen(kCachePath, "rb");
const bool fresh = !f;
if (f) std::fclose(f);
f = std::fopen(kCachePath, "ab");
if (!f) return;
if (fresh) {
const uint32_t head[3] = {kMagic, kVersion, layout_signature()};
std::fwrite(head, 1, sizeof head, f);
}
std::fwrite(pending.data(), 1, pending.size(), f);
std::fclose(f);
std::string().swap(pending);
}
#endif
} // namespace
extern "C" void rx_pipecache_memory(int64_t* recorded_bytes, int64_t* seed_bytes, int64_t* prewarm_left) {
#if RX_WEB
*recorded_bytes = (int64_t)pending.capacity();
*seed_bytes = 0;
*prewarm_left = (int64_t)prewarm.size();
#else
*recorded_bytes = *seed_bytes = *prewarm_left = 0;
#endif
}
std::string web_shader_source(const std::string& wgsl) {
// the browser (Tint) enforces derivative uniformity, which wgpu (naga) does not: the translated programs sample
// in control flow it cannot prove uniform
return RX_WEB ? "diagnostic(off, derivative_uniformity);\n" + wgsl : wgsl;
}
void pipecache_start() {
#if RX_WEB
static const bool off = env_flag("RX_NO_PIPELINE_CACHE");
if (off) return;
std::lock_guard lk(cache_mu);
load(kCachePath, true);
load(kSeedPath, false);
prewarm_total = (int64_t)prewarm.size();
prewarm_t0 = hos_now_ns();
last_save_ns = hos_now_ns();
#endif
}
uint64_t pipecache_text_hash(const std::string& wgsl) {
#if RX_WEB
return text_hash(web_shader_source(wgsl));
#else
(void)wgsl;
return 0;
#endif
}
bool pipecache_has_module(uint64_t h) {
#if RX_WEB
std::lock_guard lk(cache_mu);
return known_modules.count(h) != 0;
#else
(void)h;
return false;
#endif
}
void pipecache_record_render(const WGPURenderPipelineDescriptor& d, const ShaderModule& vs, const ShaderModule* fs,
const std::vector<WGPUBindGroupLayoutEntry>* layouts, int nlayouts) {
#if RX_WEB
std::lock_guard lk(cache_mu);
std::string payload;
Writer w{payload};
w.put(add_module_hashed(vs.cache_hash, vs.source()));
w.put(fs ? add_module_hashed(fs->cache_hash, fs->source()) : (uint64_t)0);
put_layouts(w, layouts, nlayouts);
w.put((uint32_t)d.vertex.bufferCount);
for (size_t i = 0; i < d.vertex.bufferCount; i++) {
WGPUVertexBufferLayout b = d.vertex.buffers[i];
b.attributes = nullptr;
b.nextInChain = nullptr;
w.put(b);
w.put((uint32_t)d.vertex.buffers[i].attributeCount);
for (size_t a = 0; a < d.vertex.buffers[i].attributeCount; a++) {
WGPUVertexAttribute at = d.vertex.buffers[i].attributes[a];
at.nextInChain = nullptr;
w.put(at);
}
}
const uint32_t nt = d.fragment ? (uint32_t)d.fragment->targetCount : 0;
w.put(nt);
for (uint32_t i = 0; i < nt; i++) {
WGPUColorTargetState t = d.fragment->targets[i];
const WGPUBlendState blend = t.blend ? *t.blend : WGPUBlendState{};
const uint32_t has_blend = t.blend != nullptr;
t.blend = nullptr;
t.nextInChain = nullptr;
w.put(t);
w.put(has_blend);
w.put(blend);
}
WGPUPrimitiveState prim = d.primitive;
prim.nextInChain = nullptr;
w.put(prim);
w.put((uint32_t)(d.depthStencil != nullptr));
WGPUDepthStencilState depth = d.depthStencil ? *d.depthStencil : WGPUDepthStencilState{};
depth.nextInChain = nullptr;
w.put(depth);
WGPUMultisampleState ms = d.multisample;
ms.nextInChain = nullptr;
w.put(ms);
add_record('R', payload);
#else
(void)d; (void)vs; (void)fs; (void)layouts; (void)nlayouts;
#endif
}
void pipecache_record_compute(const std::string& wgsl, const std::vector<WGPUBindGroupLayoutEntry>& layout) {
#if RX_WEB
std::lock_guard lk(cache_mu);
std::string payload;
Writer w{payload};
w.put(add_module(web_shader_source(wgsl)));
put_layouts(w, &layout, 1);
add_record('C', payload);
#else
(void)wgsl; (void)layout;
#endif
}
void pipecache_step() {
#if RX_WEB
// as many in the browser's compilers as it keeps up with (their callbacks: at present), up to ~6 ms a frame here:
// done fast, while the title screen and the first loads run (spread thin, 2026-10-15, it was still going when the
// opening area loaded and took that to 11 fps; fast, it was done in ~30 s and the area came in at 55-60 fps)
int made = 0;
const int64_t t0 = hos_now_ns();
while (!prewarm.empty() && in_flight.load() < 16 && made < 24 && hos_now_ns() - t0 < 6000000) {
warm_one(prewarm.front());
prewarm.pop_front();
made++;
warmed++;
}
if (prewarm_total && prewarm.empty() && in_flight.load() == 0) {
hos_log("rx: pipeline cache: %lld pipelines made in the background in %.1f s", (long long)warmed,
(hos_now_ns() - prewarm_t0) / 1e9);
prewarm_total = 0;
module_texts.clear();
for (FILE*& f : files)
if (f) std::fclose(f), f = nullptr;
}
const int64_t now = hos_now_ns();
if (now - last_save_ns > 10000000000LL) { // (every 10 s, when something new was made)
last_save_ns = now;
save();
}
#endif
}
} // namespace rx
#ifdef __EMSCRIPTEN__
// (debugging: HOS_BIG_ALLOC_MB and RX_LEAK_SAMPLE, hos/web/stubs.c: they hook malloc itself, which this goes to)
#endif

View File

@@ -0,0 +1,491 @@
// The native pipeline cache: every WGSL text the translator makes and every render pipeline made from them are
// appended to rxcache/pipelines.bin next to the exe; at the next start a thread reads the file and the render thread
// hands its pipelines to the pipeline compilers a few at a time (while the title screen shows), under the keys the
// draws will look them up by. Shader modules and pipelines (WGSL -> naga -> SPIR-V -> the driver, most of the
// pipeline compilers' time: 2.6 s of modules for 460 pipelines at one loading) are then ready before the game first
// draws with them; only the translation of the game's programs still happens on first use (on its workers).
// Compute programs ('C' records: their WGSL) get their shader modules made by the reading thread: the render thread
// made them in place at their first dispatch (WGSL parsing: ~270 ms of the boot's first frame).
// The file holds the game's shaders (made from its data): it stays on this machine, never in the repository.
// RX_NO_PIPELINE_CACHE=1: neither read nor written. RX_PIPELINE_CACHE_MAX=<n>: how many are made at a start (6000).
// Only pipelines some draw used in the last 4 sessions are made ('U' records: the keys each session used): each made
// pipeline holds ~390 KB of host memory (wgpu keeps its shader modules' IR, the driver its code), and the file holds
// every pipeline of every place ever played: all 3783 of them were 1.2 GB of the working set where a session at the
// title save used 690. RX_PIPELINE_CACHE_SESSIONS=<n> (4): how many sessions back; RX_PIPELINE_CACHE_ALL=1: all.
#if !RX_WEB
#include <algorithm>
#include <atomic>
#include <cstdio>
#include <cstring>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>
#include "rx_internal.h"
namespace rx {
namespace {
constexpr uint32_t kMagic = 0x4E505852; // "RXPN"
constexpr uint32_t kVersion = 1;
uint32_t signature() { // (another wgpu header layout or record format: the file is started again)
return (uint32_t)(sizeof(WGPUVertexAttribute) * 3 + sizeof(WGPUColorTargetState) * 5 + sizeof(WGPUBlendState) * 7 +
sizeof(WGPUPrimitiveState) * 11 + sizeof(WGPUDepthStencilState) * 13 + kVersion * 17);
}
struct Writer {
std::string& out;
template <typename T> void put(const T& v) { out.append(reinterpret_cast<const char*>(&v), sizeof v); }
void bytes(const void* p, size_t n) { out.append(static_cast<const char*>(p), n); }
};
struct Reader {
const uint8_t* p;
const uint8_t* end;
bool ok = true;
template <typename T> T get() {
T v{};
if (end - p < (ptrdiff_t)sizeof v) { ok = false; return v; }
std::memcpy(&v, p, sizeof v);
p += sizeof v;
return v;
}
};
std::string path_of_cache() {
char exe[MAX_PATH];
DWORD n = GetModuleFileNameA(nullptr, exe, sizeof exe);
while (n && exe[n - 1] != '\\' && exe[n - 1] != '/') n--;
exe[n] = 0;
std::string dir = std::string(exe) + "rxcache";
CreateDirectoryA(dir.c_str(), nullptr);
return dir + "\\pipelines.bin";
}
bool nc_on = true;
std::mutex nc_mu; // (texts are recorded from the translation workers, pipelines from the render thread)
std::string nc_pending;
std::unordered_set<uint64_t> nc_known_texts, nc_known_pipelines;
std::string nc_path;
int64_t nc_last_save = 0;
// what the loader read (the render thread takes it once `loaded` is set)
std::atomic<bool> nc_loaded{false};
std::unordered_map<uint64_t, std::string> nc_texts;
std::vector<PipelineDesc> nc_records; // in file order (the newest last)
size_t nc_next_record = 0;
std::unordered_map<uint64_t, std::string> nc_compute_texts; // (the reading thread)
std::unordered_map<uint64_t, WGPUShaderModule> nc_compute_modules; // made from them (nc_mu)
std::unordered_set<uint64_t> nc_known_compute;
uint32_t nc_session = 0; // this session's number (one more than the file's last; set by the reader)
std::vector<uint64_t> nc_used_pending; // pipeline keys this session used, not yet written (nc_mu)
int64_t nc_made = 0, nc_t_start = 0;
bool nc_reported = false;
void put_desc(Writer& w, const PipelineDesc& d) {
w.put(d.key);
w.put(d.vs_text);
w.put(d.fs_text);
w.put((uint32_t)d.nlayouts);
for (int i = 0; i < d.nlayouts && i < 2; i++) {
w.put((uint32_t)d.layout[i].size());
w.bytes(d.layout[i].data(), d.layout[i].size() * 4);
}
w.put((uint32_t)d.buffers.size());
for (size_t i = 0; i < d.buffers.size(); i++) {
w.put((uint32_t)d.buffers[i].stepMode);
w.put(d.buffers[i].arrayStride);
const auto& attrs = d.attributes[i];
w.put((uint32_t)attrs.size());
for (const WGPUVertexAttribute& a : attrs) {
w.put((uint32_t)a.format);
w.put(a.offset);
w.put(a.shaderLocation);
}
}
w.put((uint32_t)d.targets.size());
for (size_t i = 0; i < d.targets.size(); i++) {
w.put((uint32_t)d.targets[i].format);
w.put((uint64_t)d.targets[i].writeMask);
w.put((uint8_t)(i < d.has_blend.size() ? d.has_blend[i] : 0));
const WGPUBlendState& b = d.blends[i < 8 ? i : 7];
for (const WGPUBlendComponent& c : {b.color, b.alpha}) {
w.put((uint32_t)c.operation);
w.put((uint32_t)c.srcFactor);
w.put((uint32_t)c.dstFactor);
}
}
const WGPUPrimitiveState& p = d.primitive;
w.put((uint32_t)p.topology);
w.put((uint32_t)p.stripIndexFormat);
w.put((uint32_t)p.frontFace);
w.put((uint32_t)p.cullMode);
w.put((uint32_t)p.unclippedDepth);
w.put((uint8_t)d.has_depth);
if (d.has_depth) {
const WGPUDepthStencilState& s = d.depth;
w.put((uint32_t)s.format);
w.put((uint32_t)s.depthWriteEnabled);
w.put((uint32_t)s.depthCompare);
for (const WGPUStencilFaceState& f : {s.stencilFront, s.stencilBack}) {
w.put((uint32_t)f.compare);
w.put((uint32_t)f.failOp);
w.put((uint32_t)f.depthFailOp);
w.put((uint32_t)f.passOp);
}
w.put(s.stencilReadMask);
w.put(s.stencilWriteMask);
w.put(s.depthBias);
w.put(s.depthBiasSlopeScale);
w.put(s.depthBiasClamp);
}
}
bool get_desc(Reader& r, PipelineDesc& d) {
d.key = r.get<uint64_t>();
d.vs_text = r.get<uint64_t>();
d.fs_text = r.get<uint64_t>();
d.nlayouts = (int)r.get<uint32_t>();
if (d.nlayouts < 1 || d.nlayouts > 2) return false;
for (int i = 0; i < d.nlayouts; i++) {
const uint32_t n = r.get<uint32_t>();
if (n > 256) return false;
d.layout[i].resize(n);
for (uint32_t k = 0; k < n; k++) d.layout[i][k] = r.get<uint32_t>();
}
const uint32_t nb = r.get<uint32_t>();
if (nb > 32) return false;
d.buffers.resize(nb);
d.attributes.resize(nb);
for (uint32_t i = 0; i < nb; i++) {
WGPUVertexBufferLayout& b = d.buffers[i];
b = WGPUVertexBufferLayout{};
b.stepMode = (WGPUVertexStepMode)r.get<uint32_t>();
b.arrayStride = r.get<uint64_t>();
const uint32_t na = r.get<uint32_t>();
if (na > 32) return false;
d.attributes[i].resize(na);
for (WGPUVertexAttribute& a : d.attributes[i]) {
a = WGPUVertexAttribute{};
a.format = (WGPUVertexFormat)r.get<uint32_t>();
a.offset = r.get<uint64_t>();
a.shaderLocation = r.get<uint32_t>();
}
}
const uint32_t nt = r.get<uint32_t>();
if (nt > 8) return false;
d.targets.resize(nt);
d.has_blend.resize(nt);
for (uint32_t i = 0; i < nt; i++) {
WGPUColorTargetState& t = d.targets[i];
t = WGPUColorTargetState{};
t.format = (WGPUTextureFormat)r.get<uint32_t>();
t.writeMask = (WGPUColorWriteMask)r.get<uint64_t>();
d.has_blend[i] = r.get<uint8_t>();
WGPUBlendState& b = d.blends[i];
for (WGPUBlendComponent* c : {&b.color, &b.alpha}) {
c->operation = (WGPUBlendOperation)r.get<uint32_t>();
c->srcFactor = (WGPUBlendFactor)r.get<uint32_t>();
c->dstFactor = (WGPUBlendFactor)r.get<uint32_t>();
}
}
WGPUPrimitiveState& p = d.primitive;
p = WGPUPrimitiveState{};
p.topology = (WGPUPrimitiveTopology)r.get<uint32_t>();
p.stripIndexFormat = (WGPUIndexFormat)r.get<uint32_t>();
p.frontFace = (WGPUFrontFace)r.get<uint32_t>();
p.cullMode = (WGPUCullMode)r.get<uint32_t>();
p.unclippedDepth = (WGPUBool)r.get<uint32_t>();
d.has_depth = r.get<uint8_t>() != 0;
if (d.has_depth) {
WGPUDepthStencilState& s = d.depth;
s = WGPUDepthStencilState{};
s.format = (WGPUTextureFormat)r.get<uint32_t>();
s.depthWriteEnabled = (WGPUOptionalBool)r.get<uint32_t>();
s.depthCompare = (WGPUCompareFunction)r.get<uint32_t>();
for (WGPUStencilFaceState* f : {&s.stencilFront, &s.stencilBack}) {
f->compare = (WGPUCompareFunction)r.get<uint32_t>();
f->failOp = (WGPUStencilOperation)r.get<uint32_t>();
f->depthFailOp = (WGPUStencilOperation)r.get<uint32_t>();
f->passOp = (WGPUStencilOperation)r.get<uint32_t>();
}
s.stencilReadMask = r.get<uint32_t>();
s.stencilWriteMask = r.get<uint32_t>();
s.depthBias = r.get<int32_t>();
s.depthBiasSlopeScale = r.get<float>();
s.depthBiasClamp = r.get<float>();
}
return r.ok;
}
void add_record(char type, const std::string& payload) { // (nc_mu held)
Writer w{nc_pending};
w.put(type);
w.put((uint32_t)payload.size());
w.bytes(payload.data(), payload.size());
}
/// Reads the file (a thread of its own): texts by hash and the pipeline records, for the render thread.
void load() {
FILE* f = std::fopen(nc_path.c_str(), "rb");
if (!f) {
nc_loaded = true;
return;
}
uint32_t head[3] = {};
std::string payload;
size_t n_texts = 0;
int64_t bytes = 0;
if (std::fread(head, 1, sizeof head, f) == sizeof head && head[0] == kMagic && head[1] == kVersion &&
head[2] == signature()) {
std::unordered_map<uint64_t, size_t> where; // pipeline key -> its record (a later one replaces it)
std::vector<PipelineDesc> list;
std::unordered_map<uint64_t, uint32_t> last_session; // pipeline key -> the last session that used it
uint32_t sessions = 0;
for (;;) {
char type;
uint32_t len;
if (std::fread(&type, 1, 1, f) != 1 || std::fread(&len, 1, 4, f) != 4 || len > (64u << 20)) break;
payload.resize(len);
if (std::fread(payload.data(), 1, len, f) != len) break;
bytes += 5 + len;
Reader r{reinterpret_cast<const uint8_t*>(payload.data()), reinterpret_cast<const uint8_t*>(payload.data()) + len};
if (type == 'T' && len >= 8) {
const uint64_t h = r.get<uint64_t>();
std::lock_guard lk(nc_mu);
if (nc_known_texts.insert(h).second) {
nc_texts.emplace(h, payload.substr(8));
n_texts++;
}
} else if (type == 'U' && len >= 4) {
const uint32_t session = r.get<uint32_t>();
sessions = std::max(sessions, session);
while (r.ok && r.p < r.end) {
const uint64_t key = r.get<uint64_t>();
if (!r.ok) break;
uint32_t& last = last_session[key];
last = std::max(last, session);
}
} else if (type == 'C' && len >= 8) {
const uint64_t h = r.get<uint64_t>();
std::lock_guard lk(nc_mu);
if (nc_known_compute.insert(h).second) nc_compute_texts.emplace(h, payload.substr(8));
} else if (type == 'P') {
PipelineDesc d;
if (!get_desc(r, d)) continue;
{
std::lock_guard lk(nc_mu);
nc_known_pipelines.insert(d.key);
}
auto it = where.find(d.key);
if (it != where.end()) list[it->second] = std::move(d);
else {
where.emplace(d.key, list.size());
list.push_back(std::move(d));
}
}
}
// (pipelines no recent session used are not made: their host memory is the cost, see the top)
static const int window = getenv("RX_PIPELINE_CACHE_SESSIONS") ? atoi(getenv("RX_PIPELINE_CACHE_SESSIONS")) : 4;
const bool all = getenv("RX_PIPELINE_CACHE_ALL") != nullptr;
size_t stale = 0;
for (PipelineDesc& d : list) {
auto it = last_session.find(d.key);
const uint32_t last = it == last_session.end() ? 0 : it->second; // (0: recorded before 'U' records)
if (all || (int64_t)last >= (int64_t)sessions + 1 - window) nc_records.push_back(std::move(d));
else stale++;
}
nc_session = sessions + 1;
if (stale)
hos_log("rx: pipeline cache: %zu pipelines no draw used in the last %d sessions are not made", stale, window);
} else {
std::fclose(f);
f = nullptr;
std::remove(nc_path.c_str());
hos_log("rx: pipeline cache %s is from another build or damaged: started again", nc_path.c_str());
}
if (f) std::fclose(f);
if (bytes > (400ll << 20)) { // (grown past use: the next session starts it again)
std::remove(nc_path.c_str());
std::lock_guard lk(nc_mu);
nc_known_texts.clear();
nc_known_pipelines.clear();
hos_log("rx: pipeline cache %s: %lld MB, started again", nc_path.c_str(), (long long)(bytes >> 20));
}
hos_log("rx: pipeline cache: %zu shader texts, %zu pipelines to make in the background, %zu compute programs "
"(%lld MB)", n_texts, nc_records.size(), nc_compute_texts.size(), (long long)(bytes >> 20));
nc_loaded = true;
// the compute programs' modules, here (WGSL parsing and validation; they compiled when they were recorded)
const int64_t t0 = hos_now_ns();
size_t made = 0;
for (auto& [h, text] : nc_compute_texts) {
WGPUShaderSourceWGSL src{};
src.chain.sType = WGPUSType_ShaderSourceWGSL;
src.code = WGPUStringView{text.data(), text.size()};
WGPUShaderModuleDescriptor d{};
d.nextInChain = &src.chain;
if (WGPUShaderModule m = program_module(&d)) {
std::lock_guard lk(nc_mu);
nc_compute_modules.emplace(h, m);
made++;
}
}
nc_compute_texts.clear();
if (made) hos_log("rx: pipeline cache: %zu compute modules made in %.0f ms", made, (hos_now_ns() - t0) / 1e6);
}
void save() {
std::string out;
{
std::lock_guard lk(nc_mu);
if (nc_loaded && nc_session && !nc_used_pending.empty()) { // (the session's number is known once read)
std::string payload;
Writer w{payload};
w.put(nc_session);
w.bytes(nc_used_pending.data(), nc_used_pending.size() * sizeof(uint64_t));
add_record('U', payload);
nc_used_pending.clear();
}
if (nc_pending.empty()) return;
out.swap(nc_pending);
}
FILE* f = std::fopen(nc_path.c_str(), "ab");
if (!f) return;
std::fseek(f, 0, SEEK_END); // (in append mode the position reads 0 until the first write)
if (std::ftell(f) == 0) {
const uint32_t head[3] = {kMagic, kVersion, signature()};
std::fwrite(head, 1, sizeof head, f);
}
std::fwrite(out.data(), 1, out.size(), f);
std::fclose(f);
}
} // namespace
bool native_cache_on() { return nc_on; }
void native_cache_start() {
nc_on = getenv("RX_NO_PIPELINE_CACHE") == nullptr;
if (!nc_on) return;
nc_path = path_of_cache();
nc_t_start = hos_now_ns();
std::thread([] {
background_thread();
load();
}).detach();
}
void native_cache_text(uint64_t h, const std::string& text) {
if (!nc_on) return;
std::lock_guard lk(nc_mu);
if (!nc_known_texts.insert(h).second) return;
std::string payload;
Writer w{payload};
w.put(h);
w.bytes(text.data(), text.size());
add_record('T', payload);
}
void native_cache_used(uint64_t key) {
if (!nc_on) return;
std::lock_guard lk(nc_mu);
nc_used_pending.push_back(key);
}
void native_cache_compute_text(uint64_t h, const std::string& text) {
if (!nc_on) return;
std::lock_guard lk(nc_mu);
if (!nc_known_compute.insert(h).second) return;
std::string payload;
Writer w{payload};
w.put(h);
w.bytes(text.data(), text.size());
add_record('C', payload);
}
WGPUShaderModule native_cache_compute_module(uint64_t h) {
if (!nc_on) return nullptr;
std::lock_guard lk(nc_mu);
auto it = nc_compute_modules.find(h);
if (it == nc_compute_modules.end()) return nullptr;
WGPUShaderModule m = it->second; // (the caller's now)
nc_compute_modules.erase(it);
return m;
}
void native_cache_pipeline(const PipelineDesc& d) {
if (!nc_on) return;
std::lock_guard lk(nc_mu);
if (!nc_known_pipelines.insert(d.key).second) return;
std::string payload;
Writer w{payload};
put_desc(w, d);
add_record('P', payload);
}
void native_cache_step() {
if (!nc_on) return;
const int64_t now = hos_now_ns();
if (now - nc_last_save > 5000000000LL) { // (new records every 5 s: an abrupt exit loses at most those)
nc_last_save = now;
save();
}
if (!nc_loaded.load(std::memory_order_acquire) || nc_next_record >= nc_records.size()) {
if (nc_loaded && !nc_reported && nc_next_record && nc_next_record >= nc_records.size() && prewarm_in_flight() == 0) {
nc_reported = true;
hos_log("rx: pipeline cache: %lld pipelines made in the background in %.1f s", (long long)nc_made,
(hos_now_ns() - nc_t_start) / 1e9);
nc_records.clear();
nc_records.shrink_to_fit();
std::lock_guard lk(nc_mu);
nc_texts.clear();
}
return;
}
static const size_t cap = getenv("RX_PIPELINE_CACHE_MAX") ? (size_t)atoi(getenv("RX_PIPELINE_CACHE_MAX")) : 6000;
if (nc_next_record == 0 && nc_records.size() > cap) nc_next_record = nc_records.size() - cap; // (the newest)
// a few at a time, while the compilers keep up: the game's own new pipelines go first
for (int budget = 4; budget > 0 && nc_next_record < nc_records.size() && prewarm_in_flight() < 4; budget--) {
const PipelineDesc& d = nc_records[nc_next_record++];
std::string vs, fs;
{
std::lock_guard lk(nc_mu);
auto v = nc_texts.find(d.vs_text);
if (v == nc_texts.end()) continue;
vs = v->second;
if (d.fs_text) {
auto f = nc_texts.find(d.fs_text);
if (f == nc_texts.end()) continue;
fs = f->second;
}
}
ShaderModule* vm = prewarm_stage_module(d.vs_text, vs, "vertex");
ShaderModule* fm = d.fs_text ? prewarm_stage_module(d.fs_text, fs, "pixel") : nullptr;
if (!vm || (d.fs_text && !fm)) continue;
if (prewarm_pipeline(d, vm, fm)) nc_made++;
}
}
} // namespace rx
#else
#include "rx_internal.h"
namespace rx {
bool native_cache_on() { return false; }
void native_cache_start() {}
void native_cache_step() {}
void native_cache_text(uint64_t, const std::string&) {}
void native_cache_pipeline(const PipelineDesc&) {}
void native_cache_compute_text(uint64_t, const std::string&) {}
void native_cache_used(uint64_t) {}
WGPUShaderModule native_cache_compute_module(uint64_t) { return nullptr; }
} // namespace rx
#endif

View File

@@ -0,0 +1,440 @@
// What the window's present does with the game's frame: keeps its aspect (black bars where the window is wider or
// taller; RX_ASPECT=stretch fills it) and, when that is larger than the frame, upscales it with AMD FidelityFX Super
// Resolution 1 (EASU, the edge-adaptive upscale, then RCAS, the sharpening) instead of a bilinear stretch.
// RX_FSR=0: bilinear RX_FSR_SHARPNESS=<stops> (0 the sharpest, 2 soft; default 0.25)
// The player changes them while playing (rx_present_hotkey: F8 FSR on/off, F9 sharpness, F10 fit/stretch; F7 frame
// interpolation), kept in <user>/present.txt.
// Both passes run on native (wgpu) and the web (WebGPU) alike: fragment shaders on a full-window triangle.
//
// EASU and RCAS below are a WGSL port of ffx_fsr1.h (FidelityFX Super Resolution 1.0):
// Copyright (c) 2021 Advanced Micro Devices, Inc. All rights reserved.
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
// documentation files (the "Software"), to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
// to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above
// copyright notice and this permission notice shall be included in all copies or substantial portions of the
// Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
// LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#include "rx_internal.h"
#include <algorithm>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <mutex>
#include <string>
extern "C" const char* hos_user_dir;
extern "C" float hos_view_aspect; // (hos/totk_ultrawide.c: the game's cameras' aspect; 0 their 16:9)
extern "C" void hos_ultrawide_apply(void);
namespace rx {
namespace {
// ---- the settings: the environment's, then the player's (present.txt), changed by hotkeys ----
const float kSharpness[] = {0.0f, 0.25f, 0.5f, 1.0f, 2.0f};
struct Settings {
bool fsr = true, stretch = false, framegen = false, ultrawide = false;
float sharpness = 0.25f;
};
std::mutex settings_mu;
Settings settings;
bool settings_read;
std::string settings_path() {
return std::string(hos_user_dir ? hos_user_dir : "user") + "/present.txt";
}
Settings current_settings() {
std::lock_guard lk(settings_mu);
if (!settings_read) {
settings_read = true;
if (const char* v = getenv("RX_FSR")) settings.fsr = *v != '0';
if (const char* v = getenv("RX_FSR_SHARPNESS")) settings.sharpness = std::clamp((float)atof(v), 0.0f, 2.0f);
if (const char* v = getenv("RX_ASPECT")) settings.stretch = !strcmp(v, "stretch");
if (const char* v = getenv("RX_FRAMEGEN")) settings.framegen = *v != '0';
if (const char* v = getenv("RX_ULTRAWIDE")) settings.ultrawide = *v != '0';
if (FILE* f = fopen(settings_path().c_str(), "r")) { // (the player's choices win over the defaults)
char key[32];
float val;
while (fscanf(f, "%31s %f", key, &val) == 2) {
if (!strcmp(key, "fsr")) settings.fsr = val != 0;
else if (!strcmp(key, "sharpness")) settings.sharpness = std::clamp(val, 0.0f, 2.0f);
else if (!strcmp(key, "stretch")) settings.stretch = val != 0;
else if (!strcmp(key, "framegen")) settings.framegen = val != 0;
else if (!strcmp(key, "ultrawide")) settings.ultrawide = val != 0;
}
fclose(f);
}
}
return settings;
}
void save_settings(const Settings& s) {
if (FILE* f = fopen(settings_path().c_str(), "w")) {
fprintf(f, "fsr %d\nsharpness %.2f\nstretch %d\nframegen %d\nultrawide %d\n", s.fsr, s.sharpness, s.stretch,
s.framegen, s.ultrawide);
fclose(f);
}
}
const char* kPresentWgsl = R"(
struct P { con0: vec4<f32>, src: vec4<f32>, rcas: vec4<f32>, rect: vec4<f32> };
@group(0) @binding(0) var tex: texture_2d<f32>;
@group(0) @binding(1) var<uniform> p: P;
@vertex fn vs(@builtin(vertex_index) i: u32) -> @builtin(position) vec4<f32> {
let uv = vec2<f32>(f32((i << 1u) & 2u), f32(i & 2u));
return vec4<f32>(uv.x * 2.0 - 1.0, 1.0 - uv.y * 2.0, 0.0, 1.0);
}
// the source texel q, clamped to the image (p.src.xy: its size)
fn ld(q: vec2<i32>) -> vec3<f32> {
return textureLoad(tex, clamp(q, vec2<i32>(0), vec2<i32>(p.src.xy) - vec2<i32>(1)), 0).rgb;
}
// luma times 2 (FSR's cheap approximation)
fn luma(c: vec3<f32>) -> f32 { return c.g + 0.5 * (c.r + c.b); }
// ---- EASU: edge-adaptive spatial upsampling (12 taps; output pixel -> a direction and an anisotropic lobe) ----
fn easu_set(dir: ptr<function, vec2<f32>>, len: ptr<function, f32>, w: f32, lA: f32, lB: f32, lC: f32, lD: f32, lE: f32) {
let dirX = lD - lB;
let dX = max(abs(lD - lC), abs(lC - lB));
var lenX = select(0.0, clamp(abs(dirX) / dX, 0.0, 1.0), dX > 0.0);
lenX = lenX * lenX;
let dirY = lE - lA;
let dY = max(abs(lE - lC), abs(lC - lA));
var lenY = select(0.0, clamp(abs(dirY) / dY, 0.0, 1.0), dY > 0.0);
lenY = lenY * lenY;
*dir = *dir + vec2<f32>(dirX, dirY) * w;
*len = *len + (lenX + lenY) * w;
}
fn easu_tap(aC: ptr<function, vec3<f32>>, aW: ptr<function, f32>, off: vec2<f32>, dir: vec2<f32>, len: vec2<f32>,
lob: f32, clp: f32, c: vec3<f32>) {
let v = vec2<f32>(dot(off, dir), dot(off, vec2<f32>(-dir.y, dir.x))) * len;
let d2 = min(dot(v, v), clp);
var wB = 0.4 * d2 - 1.0;
var wA = lob * d2 - 1.0;
wB = wB * wB;
wA = wA * wA;
wB = 1.5625 * wB - 0.5625;
let w = wB * wA;
*aC = *aC + c * w;
*aW = *aW + w;
}
@fragment fn easu(@builtin(position) pos: vec4<f32>) -> @location(0) vec4<f32> {
// the output pixel's position in the source (texel centers at integers)
var pp = floor(pos.xy) * p.con0.xy + p.con0.zw;
let fp = floor(pp);
pp = pp - fp;
let f0 = vec2<i32>(fp);
// b c
// e f g h
// i j k l
// n o
let bC = ld(f0 + vec2<i32>(0, -1)); let cC = ld(f0 + vec2<i32>(1, -1));
let eC = ld(f0 + vec2<i32>(-1, 0)); let fC = ld(f0); let gC = ld(f0 + vec2<i32>(1, 0)); let hC = ld(f0 + vec2<i32>(2, 0));
let iC = ld(f0 + vec2<i32>(-1, 1)); let jC = ld(f0 + vec2<i32>(0, 1)); let kC = ld(f0 + vec2<i32>(1, 1)); let lC = ld(f0 + vec2<i32>(2, 1));
let nC = ld(f0 + vec2<i32>(0, 2)); let oC = ld(f0 + vec2<i32>(1, 2));
let bL = luma(bC); let cL = luma(cC); let eL = luma(eC); let fL = luma(fC); let gL = luma(gC); let hL = luma(hC);
let iL = luma(iC); let jL = luma(jC); let kL = luma(kC); let lL = luma(lC); let nL = luma(nC); let oL = luma(oC);
// the direction and edge length, bilinearly weighted over the 4 nearest texels' neighbourhoods
var dir = vec2<f32>(0.0);
var len = 0.0;
easu_set(&dir, &len, (1.0 - pp.x) * (1.0 - pp.y), bL, eL, fL, gL, jL);
easu_set(&dir, &len, pp.x * (1.0 - pp.y), cL, fL, gL, hL, kL);
easu_set(&dir, &len, (1.0 - pp.x) * pp.y, fL, iL, jL, kL, nL);
easu_set(&dir, &len, pp.x * pp.y, gL, jL, kL, lL, oL);
let dir2 = dir * dir;
var dirR = dir2.x + dir2.y;
let zro = dirR < (1.0 / 32768.0);
dirR = select(inverseSqrt(dirR), 1.0, zro);
dir = select(dir, vec2<f32>(1.0, dir.y), zro) * dirR;
len = len * 0.5;
len = len * len;
let stretch = dot(dir, dir) / max(abs(dir.x), abs(dir.y));
let len2 = vec2<f32>(1.0 + (stretch - 1.0) * len, 1.0 - 0.5 * len);
let lob = 0.5 - 0.29 * len;
let clp = 1.0 / lob;
let min4 = min(min(fC, gC), min(jC, kC));
let max4 = max(max(fC, gC), max(jC, kC));
var aC = vec3<f32>(0.0);
var aW = 0.0;
easu_tap(&aC, &aW, vec2<f32>(0.0, -1.0) - pp, dir, len2, lob, clp, bC);
easu_tap(&aC, &aW, vec2<f32>(1.0, -1.0) - pp, dir, len2, lob, clp, cC);
easu_tap(&aC, &aW, vec2<f32>(-1.0, 1.0) - pp, dir, len2, lob, clp, iC);
easu_tap(&aC, &aW, vec2<f32>(0.0, 1.0) - pp, dir, len2, lob, clp, jC);
easu_tap(&aC, &aW, vec2<f32>(0.0, 0.0) - pp, dir, len2, lob, clp, fC);
easu_tap(&aC, &aW, vec2<f32>(-1.0, 0.0) - pp, dir, len2, lob, clp, eC);
easu_tap(&aC, &aW, vec2<f32>(1.0, 1.0) - pp, dir, len2, lob, clp, kC);
easu_tap(&aC, &aW, vec2<f32>(2.0, 1.0) - pp, dir, len2, lob, clp, lC);
easu_tap(&aC, &aW, vec2<f32>(2.0, 0.0) - pp, dir, len2, lob, clp, hC);
easu_tap(&aC, &aW, vec2<f32>(1.0, 0.0) - pp, dir, len2, lob, clp, gC);
easu_tap(&aC, &aW, vec2<f32>(1.0, 2.0) - pp, dir, len2, lob, clp, oC);
easu_tap(&aC, &aW, vec2<f32>(0.0, 2.0) - pp, dir, len2, lob, clp, nC);
// normalized, and clamped to the 4 nearest texels (no ringing)
let c = select(fC, aC / aW, abs(aW) > 1e-6);
return vec4<f32>(min(max4, max(min4, c)), 1.0);
}
// ---- RCAS: robust contrast-adaptive sharpening (5 taps; the lobe limited so it never clips) ----
@fragment fn rcas(@builtin(position) pos: vec4<f32>) -> @location(0) vec4<f32> {
let ip = vec2<i32>(floor(pos.xy - p.rect.xy));
let b = ld(ip + vec2<i32>(0, -1));
let d = ld(ip + vec2<i32>(-1, 0));
let e = ld(ip);
let f = ld(ip + vec2<i32>(1, 0));
let h = ld(ip + vec2<i32>(0, 1));
let mn4 = min(min(b, d), min(f, h));
let mx4 = max(max(b, d), max(f, h));
let hitMin = mn4 / max(4.0 * mx4, vec3<f32>(1e-5));
let hitMax = (1.0 - mx4) / min(4.0 * mn4 - 4.0, vec3<f32>(-1e-5));
let lobeRGB = max(-hitMin, hitMax);
let lobe = max(-0.1875, min(max(lobeRGB.r, max(lobeRGB.g, lobeRGB.b)), 0.0)) * p.rcas.x;
return vec4<f32>((lobe * (b + d + h + f) + e) / (4.0 * lobe + 1.0), 1.0);
}
// ---- bilinear (RX_FSR=0, or no upscale): the source stretched over the rectangle ----
@fragment fn bilinear(@builtin(position) pos: vec4<f32>) -> @location(0) vec4<f32> {
let s = (pos.xy - p.rect.xy) * p.src.xy / p.rect.zw - 0.5;
let i = vec2<i32>(floor(s));
let t = s - floor(s);
let c = mix(mix(ld(i), ld(i + vec2<i32>(1, 0)), t.x), mix(ld(i + vec2<i32>(0, 1)), ld(i + vec2<i32>(1, 1)), t.x), t.y);
return vec4<f32>(c, 1.0);
}
)";
struct Params {
float con0[4], src[4], rcas[4], rect[4];
};
struct State {
WGPUTextureFormat target_format = WGPUTextureFormat_Undefined;
WGPUBindGroupLayout layout = nullptr;
WGPURenderPipeline easu = nullptr, rcas = nullptr, bilinear = nullptr;
WGPUBuffer params[2] = {}; // (the EASU pass's and the window pass's)
WGPUTexture mid = nullptr; // EASU's output, the rectangle's size
WGPUTextureView mid_view = nullptr;
uint32_t mid_w = 0, mid_h = 0;
} st;
WGPURenderPipeline make_pipeline(WGPUShaderModule mod, WGPUPipelineLayout pl, const char* fs, WGPUTextureFormat format) {
WGPUColorTargetState target{};
target.format = format;
target.writeMask = WGPUColorWriteMask_All;
WGPUFragmentState f{};
f.module = mod;
f.entryPoint = sv(fs);
f.targetCount = 1;
f.targets = &target;
WGPURenderPipelineDescriptor rd{};
rd.layout = pl;
rd.vertex.module = mod;
rd.vertex.entryPoint = sv("vs");
rd.primitive.topology = WGPUPrimitiveTopology_TriangleList;
rd.primitive.cullMode = WGPUCullMode_None;
rd.primitive.frontFace = WGPUFrontFace_CCW;
rd.multisample.count = 1;
rd.multisample.mask = 0xFFFFFFFF;
rd.fragment = &f;
return wgpuDeviceCreateRenderPipeline(dev.device, &rd);
}
void init(WGPUTextureFormat target_format) {
if (st.layout && st.target_format == target_format) return;
st.target_format = target_format;
if (!st.layout) {
WGPUBindGroupLayoutEntry e[2]{};
e[0].binding = 0;
e[0].visibility = WGPUShaderStage_Fragment;
e[0].texture.sampleType = WGPUTextureSampleType_UnfilterableFloat;
e[0].texture.viewDimension = WGPUTextureViewDimension_2D;
e[1].binding = 1;
e[1].visibility = WGPUShaderStage_Fragment;
e[1].buffer.type = WGPUBufferBindingType_Uniform;
e[1].buffer.minBindingSize = sizeof(Params);
WGPUBindGroupLayoutDescriptor bd{};
bd.entryCount = 2;
bd.entries = e;
st.layout = wgpuDeviceCreateBindGroupLayout(dev.device, &bd);
for (WGPUBuffer& b : st.params) {
WGPUBufferDescriptor d{};
d.size = sizeof(Params);
d.usage = WGPUBufferUsage_Uniform | WGPUBufferUsage_CopyDst;
b = wgpuDeviceCreateBuffer(dev.device, &d);
}
}
WGPUShaderSourceWGSL src{};
src.chain.sType = WGPUSType_ShaderSourceWGSL;
src.code = sv(kPresentWgsl);
WGPUShaderModuleDescriptor md{};
md.nextInChain = &src.chain;
WGPUShaderModule mod = wgpuDeviceCreateShaderModule(dev.device, &md);
WGPUPipelineLayoutDescriptor pd{};
pd.bindGroupLayoutCount = 1;
pd.bindGroupLayouts = &st.layout;
WGPUPipelineLayout pl = wgpuDeviceCreatePipelineLayout(dev.device, &pd);
for (WGPURenderPipeline* p : {&st.easu, &st.rcas, &st.bilinear})
if (*p) wgpuRenderPipelineRelease(*p);
st.easu = make_pipeline(mod, pl, "easu", WGPUTextureFormat_RGBA8Unorm);
st.rcas = make_pipeline(mod, pl, "rcas", target_format);
st.bilinear = make_pipeline(mod, pl, "bilinear", target_format);
wgpuPipelineLayoutRelease(pl);
wgpuShaderModuleRelease(mod);
}
void ensure_mid(uint32_t w, uint32_t h) {
if (st.mid && st.mid_w == w && st.mid_h == h) return;
if (st.mid_view) wgpuTextureViewRelease(st.mid_view);
if (st.mid) wgpuTextureRelease(st.mid);
WGPUTextureDescriptor d{};
d.label = sv("rx FSR upscale");
d.usage = WGPUTextureUsage_RenderAttachment | WGPUTextureUsage_TextureBinding;
d.dimension = WGPUTextureDimension_2D;
d.size = {w, h, 1};
d.format = WGPUTextureFormat_RGBA8Unorm;
d.mipLevelCount = 1;
d.sampleCount = 1;
st.mid = wgpuDeviceCreateTexture(dev.device, &d);
st.mid_view = wgpuTextureCreateView(st.mid, nullptr);
st.mid_w = w;
st.mid_h = h;
}
/// A pass drawing one full-window triangle with `pipe` over `rect` of `target` (cleared to `clear` first)
void draw_pass(WGPUCommandEncoder e, WGPUTextureView target, const WGPUColor& clear, WGPURenderPipeline pipe,
WGPUTextureView src, WGPUBuffer params, const float rect[4]) {
WGPURenderPassColorAttachment ca{};
ca.view = target;
ca.depthSlice = WGPU_DEPTH_SLICE_UNDEFINED;
ca.loadOp = WGPULoadOp_Clear;
ca.storeOp = WGPUStoreOp_Store;
ca.clearValue = clear;
WGPURenderPassDescriptor pd{};
pd.colorAttachmentCount = 1;
pd.colorAttachments = &ca;
WGPURenderPassEncoder p = wgpuCommandEncoderBeginRenderPass(e, &pd);
WGPUBindGroupEntry be[2]{};
be[0].binding = 0;
be[0].textureView = src;
be[1].binding = 1;
be[1].buffer = params;
be[1].size = sizeof(Params);
WGPUBindGroupDescriptor bd{};
bd.layout = st.layout;
bd.entryCount = 2;
bd.entries = be;
WGPUBindGroup bg = wgpuDeviceCreateBindGroup(dev.device, &bd);
wgpuRenderPassEncoderSetViewport(p, rect[0], rect[1], rect[2], rect[3], 0.0f, 1.0f);
wgpuRenderPassEncoderSetScissorRect(p, (uint32_t)rect[0], (uint32_t)rect[1], (uint32_t)rect[2], (uint32_t)rect[3]);
wgpuRenderPassEncoderSetPipeline(p, pipe);
wgpuRenderPassEncoderSetBindGroup(p, 0, bg, 0, nullptr);
wgpuRenderPassEncoderDraw(p, 3, 1, 0, 0);
wgpuRenderPassEncoderEnd(p);
wgpuRenderPassEncoderRelease(p);
wgpuBindGroupRelease(bg);
}
} // namespace
PresentRect present_rect(uint32_t sw, uint32_t sh, uint32_t tw, uint32_t th) {
const Settings set = current_settings();
// (ultrawide: the frame holds the window's field of view, so it is stretched over the window)
const bool stretch = set.stretch || (set.ultrawide && hos_view_aspect > 0);
PresentRect r{0, 0, tw, th};
if (stretch || !sw || !sh || !tw || !th) return r;
// the largest rectangle of the frame's aspect that fits, centred (pixels: at most one off the exact ratio)
if ((uint64_t)tw * sh > (uint64_t)th * sw) {
r.w = std::max<uint32_t>(1, (uint32_t)((uint64_t)th * sw / sh));
r.x = (tw - r.w) / 2;
} else {
r.h = std::max<uint32_t>(1, (uint32_t)((uint64_t)tw * sh / sw));
r.y = (th - r.h) / 2;
}
return r;
}
void present_fx_draw(WGPUCommandEncoder e, WGPUTextureView src, uint32_t sw, uint32_t sh, WGPUTextureView target,
WGPUTextureFormat target_format, uint32_t tw, uint32_t th, const WGPUColor& clear) {
init(target_format);
const Settings set = current_settings();
const float sharpness = set.sharpness;
static std::string logged;
const PresentRect r = present_rect(sw, sh, tw, th);
const float rect[4] = {(float)r.x, (float)r.y, (float)r.w, (float)r.h};
const bool upscale = set.fsr && (r.w > sw || r.h > sh);
char what[96];
std::snprintf(what, sizeof what, "%ux%u %ux%u %ux%u %d", sw, sh, r.w, r.h, tw, th, upscale);
if (logged != what) { // (each new size or mode)
logged = what;
hos_log("rx: present: %ux%u frames into %ux%u at %u,%u of a %ux%u window, %s", sw, sh, r.w, r.h, r.x, r.y, tw, th,
upscale ? "FSR 1 (EASU + RCAS)" : "bilinear");
}
Params pm{};
pm.src[0] = (float)sw;
pm.src[1] = (float)sh;
pm.rect[0] = rect[0];
pm.rect[1] = rect[1];
pm.rect[2] = rect[2];
pm.rect[3] = rect[3];
if (!upscale) {
wgpuQueueWriteBuffer(dev.queue, st.params[1], 0, &pm, sizeof pm);
draw_pass(e, target, clear, st.bilinear, src, st.params[1], rect);
return;
}
ensure_mid(r.w, r.h);
Params easu = pm;
easu.con0[0] = (float)sw / (float)r.w;
easu.con0[1] = (float)sh / (float)r.h;
easu.con0[2] = 0.5f * easu.con0[0] - 0.5f;
easu.con0[3] = 0.5f * easu.con0[1] - 0.5f;
const float whole[4] = {0, 0, (float)r.w, (float)r.h};
std::memcpy(easu.rect, whole, sizeof whole);
wgpuQueueWriteBuffer(dev.queue, st.params[0], 0, &easu, sizeof easu);
draw_pass(e, st.mid_view, WGPUColor{0, 0, 0, 1}, st.easu, src, st.params[0], whole);
Params sharpen = pm;
sharpen.src[0] = (float)r.w; // (RCAS reads EASU's output)
sharpen.src[1] = (float)r.h;
sharpen.rcas[0] = std::exp2(-sharpness);
wgpuQueueWriteBuffer(dev.queue, st.params[1], 0, &sharpen, sizeof sharpen);
draw_pass(e, target, clear, st.rcas, st.mid_view, st.params[1], rect);
}
bool present_framegen() { return current_settings().framegen; }
/// The window's size (rx_resize): with ultrawide on, a window wider than 16:9, or narrower (16:10, 4:3: SkyboundTotk
/// issue 10), gives the game's cameras its aspect
void present_window_size(int32_t w, int32_t h) {
const float a = w > 0 && h > 0 ? (float)w / (float)h : 0.0f;
const bool other = a > 16.0f / 9.0f + 0.01f || (a > 0 && a < 16.0f / 9.0f - 0.01f);
hos_view_aspect = current_settings().ultrawide && other ? std::clamp(a, 1.25f, 4.0f) : 0.0f;
hos_ultrawide_apply();
}
} // namespace rx
/* A hotkey (7..10: F7 frame interpolation, F8 FSR, F9 sharpness, F10 fit/stretch; 0: none): the settings after it,
as a line for the window's title */
extern "C" const char* rx_present_hotkey(int key) {
using namespace rx;
Settings s = current_settings();
if (key) {
if (key == 7) s.framegen = !s.framegen;
if (key == 8) s.fsr = !s.fsr;
if (key == 9) {
int i = 0;
while (i < 5 && kSharpness[i] < s.sharpness - 0.01f) i++;
s.sharpness = kSharpness[(i + 1) % 5];
}
if (key == 10) s.stretch = !s.stretch;
if (key == 6) s.ultrawide = !s.ultrawide;
{
std::lock_guard lk(settings_mu);
settings = s;
}
save_settings(s);
}
static thread_local char line[160];
std::snprintf(line, sizeof line,
"FSR %s (F8), sharpness %.2f (F9), %s (F10), frame interpolation %s (F7), ultrawide %s (F6)",
s.fsr ? "on" : "off", s.sharpness, s.stretch ? "stretched" : "aspect kept", s.framegen ? "on" : "off",
s.ultrawide ? (hos_view_aspect > 0 ? "on" : "on (the window is 16:9)") : "off");
return line;
}

828
gpu/render/rx_record.cpp Normal file
View File

@@ -0,0 +1,828 @@
// Deferred command encoding: the render thread records a batch's encoder commands (render and compute passes,
// copies) into a plain list, and the submit thread replays them into a WebGPU command encoder just before it finishes
// it. wgpu's encoding calls (SetBindGroup above all) were ~10% of the render thread, the frame rate's limit; the submit
// thread had room. Objects the render thread may release before the replay are held (AddRef) until it is done: texture
// views and textures (passes, copies) and compute bind groups (released after their dispatch). Pipelines and buffers
// live as long as the renderer, and the cached render bind groups are only released after a flush and a wait
// (trim_bind_groups): an AddRef each was as costly as the call it deferred. RX_DEFER_ENCODING=0: the calls go to the
// encoder at once, as before.
#include "rx_internal.h"
#include <algorithm>
#include <map>
namespace rx {
namespace {
enum Op : uint64_t {
kBeginRenderPass = 1, kEndRenderPass, kSetViewport, kSetPipeline, kSetBindGroup, kSetVertexBuffer, kSetIndexBuffer,
kSetScissor, kSetBlendConstant, kSetStencilReference, kDraw, kDrawIndexed, kCopyBufferToBuffer,
kCopyTextureToTexture, kBeginComputePass, kComputeSetPipeline, kComputeSetBindGroup, kDispatch, kEndComputePass,
kCopyTextureToBuffer, kComputeSetBindGroupDesc, kBeginOcclusionQuery, kEndOcclusionQuery, kResolveQuerySet,
kDispatchIndirect, kMultiDrawIndirect,
};
enum RefKind : uint8_t { kView, kTexture, kBuffer, kRenderPipeline, kComputePipeline, kBindGroup };
WGPURenderPassEncoder const kFakePass = reinterpret_cast<WGPURenderPassEncoder>(uintptr_t{0x10});
WGPUComputePassEncoder const kFakeCompute = reinterpret_cast<WGPUComputePassEncoder>(uintptr_t{0x20});
WGPUCommandEncoder const kFakeEncoder = reinterpret_cast<WGPUCommandEncoder>(uintptr_t{0x30});
} // namespace
struct Recording {
std::vector<uint64_t> words;
std::vector<size_t> pass_ends; // where each pass ended (in words): the places a replay can be split
std::vector<std::pair<RefKind, void*>> refs; // released after the replay
};
namespace {
Recording* open_rec; // the open batch's commands (render thread)
size_t last_words, last_refs, last_passes; // (the last batch's sizes: a new one starts at them instead of growing)
Recording& rec() {
if (!open_rec) {
open_rec = new Recording();
open_rec->words.reserve(last_words);
open_rec->refs.reserve(last_refs);
open_rec->pass_ends.reserve(last_passes);
}
return *open_rec;
}
void put(uint64_t w) { rec().words.push_back(w); }
// RX_REC_STATS=1: the commands recorded, by kind, logged every 600 frames (per frame): what each draw costs the
// encoding threads in wgpu calls
uint64_t op_counts[32];
const char* const op_names[32] = {"", "begin pass", "end pass", "viewport", "pipeline", "bind group", "vertex buffer",
"index buffer", "scissor", "blend constant", "stencil ref", "draw", "draw indexed",
"copy b2b", "copy t2t", "begin compute", "compute pipeline", "compute bind group",
"dispatch", "end compute", "copy t2b", "compute bind group desc",
"begin occlusion", "end occlusion", "resolve queries"};
void put_op(Op o) {
static const bool stats = env_flag("RX_REC_STATS");
if (stats) {
op_counts[o & 31]++;
static int64_t since = frame_number;
if (frame_number - since >= 600) {
std::string line;
for (int i = 1; i < 32; i++)
if (op_counts[i]) {
char b[64];
std::snprintf(b, sizeof b, " %s %.1f;", op_names[i], (double)op_counts[i] / (double)(frame_number - since));
line += b;
}
hos_log("rx: recorded per frame:%s", line.c_str());
std::memset(op_counts, 0, sizeof op_counts);
since = frame_number;
}
}
put(o);
}
void put_ptr(const void* p) { put(reinterpret_cast<uintptr_t>(p)); }
uint64_t bits(float f) { return std::bit_cast<uint32_t>(f); }
uint64_t bits(double d) { return std::bit_cast<uint64_t>(d); }
void hold(RefKind k, void* h) {
if (!h) return;
switch (k) {
case kView: wgpuTextureViewAddRef(static_cast<WGPUTextureView>(h)); break;
case kTexture: wgpuTextureAddRef(static_cast<WGPUTexture>(h)); break;
case kBuffer: wgpuBufferAddRef(static_cast<WGPUBuffer>(h)); break;
case kRenderPipeline: wgpuRenderPipelineAddRef(static_cast<WGPURenderPipeline>(h)); break;
case kComputePipeline: wgpuComputePipelineAddRef(static_cast<WGPUComputePipeline>(h)); break;
case kBindGroup: wgpuBindGroupAddRef(static_cast<WGPUBindGroup>(h)); break;
}
rec().refs.push_back({k, h});
}
void drop(RefKind k, void* h) {
switch (k) {
case kView: wgpuTextureViewRelease(static_cast<WGPUTextureView>(h)); break;
case kTexture: wgpuTextureRelease(static_cast<WGPUTexture>(h)); break;
case kBuffer: wgpuBufferRelease(static_cast<WGPUBuffer>(h)); break;
case kRenderPipeline: wgpuRenderPipelineRelease(static_cast<WGPURenderPipeline>(h)); break;
case kComputePipeline: wgpuComputePipelineRelease(static_cast<WGPUComputePipeline>(h)); break;
case kBindGroup: wgpuBindGroupRelease(static_cast<WGPUBindGroup>(h)); break;
}
}
template <class T> T as(uint64_t w) { return reinterpret_cast<T>(static_cast<uintptr_t>(w)); }
float as_float(uint64_t w) { return std::bit_cast<float>(static_cast<uint32_t>(w)); }
double as_double(uint64_t w) { return std::bit_cast<double>(w); }
void put_copy_info(const WGPUTexelCopyTextureInfo& t) {
hold(kTexture, t.texture);
put_ptr(t.texture);
put(t.mipLevel);
put(uint64_t{t.origin.x} | uint64_t{t.origin.y} << 32);
put(uint64_t{t.origin.z} | uint64_t{static_cast<uint32_t>(t.aspect)} << 32);
}
WGPUTexelCopyTextureInfo get_copy_info(const uint64_t*& w) {
WGPUTexelCopyTextureInfo t{};
t.texture = as<WGPUTexture>(*w++);
t.mipLevel = static_cast<uint32_t>(*w++);
t.origin.x = static_cast<uint32_t>(*w);
t.origin.y = static_cast<uint32_t>(*w++ >> 32);
t.origin.z = static_cast<uint32_t>(*w);
t.aspect = static_cast<WGPUTextureAspect>(*w++ >> 32);
return t;
}
} // namespace
bool encoding_deferred() {
static const bool on = [] {
const char* e = getenv("RX_DEFER_ENCODING");
return e ? std::atoi(e) != 0 : !RX_WEB || RX_WEB_MT; // (the threaded web build records, as natively)
}();
return on;
}
WGPUCommandEncoder fake_encoder() { return kFakeEncoder; }
Recording* recording_take() {
Recording* r = open_rec;
open_rec = nullptr;
if (r) {
last_words = r->words.size();
last_refs = r->refs.size();
last_passes = r->pass_ends.size();
}
return r;
}
std::vector<std::pair<size_t, size_t>> recording_parts(const Recording* r, uint32_t parts, size_t min_words) {
std::vector<std::pair<size_t, size_t>> out;
const size_t total = r->words.size();
if (parts > 1 && total >= 2 * min_words) {
// each cut at the pass end nearest its share (k * total / parts): cutting at the first one past the target left
// the last part the remainder (5.1, 4.3 and 0.4 ms a frame at the castle)
size_t from = 0, k = 1;
const std::vector<size_t>& e = r->pass_ends;
for (size_t i = 0; i < e.size() && k < parts; i++) {
const size_t share = total * k / parts;
if (e[i] < share && i + 1 < e.size() && e[i + 1] <= share) continue; // (a later one is nearer)
size_t cut = e[i];
if (cut < share && i + 1 < e.size() && e[i + 1] - share < share - cut) cut = e[++i];
if (cut - from < min_words || total - cut < min_words) continue;
out.push_back({from, cut});
from = cut;
k++;
}
out.push_back({from, total});
} else {
out.push_back({0, total});
}
return out;
}
// ---- RX_GPU_TIMING=1 (profiling): timestamps around each pass, summed by the pass's tag ----
struct GpuTimer {
static constexpr uint32_t kQueries = 1024; // two a pass
WGPUQuerySet set = nullptr;
WGPUBuffer resolve = nullptr, read = nullptr;
struct Pass {
uint64_t va, desc; // desc 0: a compute pass
uint32_t draws;
};
std::vector<Pass> passes;
/// The next pass's timestamp writes, or null when the timer is full.
const WGPUPassTimestampWrites* begin(uint64_t va, uint64_t desc) {
if (passes.size() * 2 + 2 > kQueries) return nullptr;
tw.querySet = set;
tw.beginningOfPassWriteIndex = static_cast<uint32_t>(passes.size() * 2);
tw.endOfPassWriteIndex = tw.beginningOfPassWriteIndex + 1;
passes.push_back({va, desc, 0});
return &tw;
}
WGPUPassTimestampWrites tw{};
};
namespace {
uint64_t tag_va, tag_desc; // the next render pass's (render thread)
std::mutex timer_mu;
std::vector<GpuTimer*> timer_pool;
struct TimeStat {
uint64_t ns = 0, passes = 0, draws = 0;
};
std::map<std::pair<uint64_t, uint64_t>, TimeStat> time_stats;
uint64_t time_total_ns = 0;
int64_t time_since = 0, time_frame0 = 0;
void gpu_timer_done(GpuTimer* t, WGPUMapAsyncStatus status) {
const size_t n = t->passes.size() * 2;
if (status == WGPUMapAsyncStatus_Success) {
#if RX_WEB
static const double period = 1.0; // (the browser's timestamps are nanoseconds already: no period call there)
#else
static const double period = wgpuQueueGetTimestampPeriod(dev.queue);
#endif
const auto* ts = static_cast<const uint64_t*>(wgpuBufferGetConstMappedRange(t->read, 0, n * 8));
std::lock_guard lk(timer_mu);
for (size_t i = 0; ts && i < t->passes.size(); i++) {
const uint64_t a = ts[2 * i], b = ts[2 * i + 1];
const uint64_t ns = b > a ? static_cast<uint64_t>((b - a) * period) : 0;
if (ns > 1000000000) continue; // (a pass the queue split oddly: not a real time)
TimeStat& s = time_stats[{t->passes[i].va, t->passes[i].desc}];
s.ns += ns;
s.passes++;
s.draws += t->passes[i].draws;
time_total_ns += ns;
}
wgpuBufferUnmap(t->read);
const int64_t now = hos_now_ns();
if (!time_since) {
time_since = now;
time_frame0 = frame_number;
} else if (now - time_since > 5000000000) {
const double frames = std::max<double>(1.0, static_cast<double>(frame_number - time_frame0));
std::vector<std::pair<std::pair<uint64_t, uint64_t>, TimeStat>> v(time_stats.begin(), time_stats.end());
std::sort(v.begin(), v.end(), [](const auto& x, const auto& y) { return x.second.ns > y.second.ns; });
hos_log("rx: gpu time: %.2f ms a frame in passes over %.0f frames (%zu kinds)", time_total_ns / 1e6 / frames,
frames, v.size());
for (size_t i = 0; i < v.size() && i < 30; i++) {
const auto& [key, s] = v[i];
const uint64_t d = key.second;
if (!d)
hos_log("rx: gpu time %5.2f ms x%-5.1f compute %6.1f dispatches", s.ns / 1e6 / frames,
s.passes / frames, static_cast<double>(s.draws) / s.passes);
else
hos_log("rx: gpu time %5.2f ms x%-5.1f %4llux%-4llu c%llu f%-3llu d%-3llu %6.1f draws va %llx",
s.ns / 1e6 / frames, s.passes / frames, (unsigned long long)(d >> 48 & 0x7FFF),
(unsigned long long)(d >> 32 & 0xFFFF), (unsigned long long)(d >> 24 & 0xF),
(unsigned long long)(d >> 8 & 0xFFFF), (unsigned long long)(d & 0xFF),
static_cast<double>(s.draws) / s.passes, (unsigned long long)key.first);
}
time_stats.clear();
time_total_ns = 0;
time_since = now;
time_frame0 = frame_number;
}
}
t->passes.clear();
std::lock_guard lk(timer_mu);
timer_pool.push_back(t);
}
} // namespace
bool gpu_timing() {
static const bool on = env_flag("RX_GPU_TIMING") && dev.timestamps && encoding_deferred();
return on;
}
void ce_pass_tag(uint64_t va, uint64_t desc) {
tag_va = va;
tag_desc = desc;
}
GpuTimer* gpu_timer_take() {
{
std::lock_guard lk(timer_mu);
if (!timer_pool.empty()) {
GpuTimer* t = timer_pool.back();
timer_pool.pop_back();
return t;
}
}
auto* t = new GpuTimer();
WGPUQuerySetDescriptor qd{};
qd.type = WGPUQueryType_Timestamp;
qd.count = GpuTimer::kQueries;
t->set = wgpuDeviceCreateQuerySet(dev.device, &qd);
WGPUBufferDescriptor bd{};
bd.size = GpuTimer::kQueries * 8;
bd.usage = WGPUBufferUsage_QueryResolve | WGPUBufferUsage_CopySrc;
t->resolve = wgpuDeviceCreateBuffer(dev.device, &bd);
bd.usage = WGPUBufferUsage_MapRead | WGPUBufferUsage_CopyDst;
t->read = wgpuDeviceCreateBuffer(dev.device, &bd);
return t;
}
void gpu_timer_resolve(GpuTimer* t, WGPUCommandEncoder e) {
if (!t || t->passes.empty()) return;
const uint32_t n = static_cast<uint32_t>(t->passes.size() * 2);
wgpuCommandEncoderResolveQuerySet(e, t->set, 0, n, t->resolve, 0);
wgpuCommandEncoderCopyBufferToBuffer(e, t->resolve, 0, t->read, 0, uint64_t{n} * 8);
}
void gpu_timer_read(GpuTimer* t) {
if (!t) return;
if (t->passes.empty()) {
std::lock_guard lk(timer_mu);
timer_pool.push_back(t);
return;
}
WGPUBufferMapCallbackInfo mc{};
mc.mode = WGPUCallbackMode_AllowSpontaneous;
mc.callback = [](WGPUMapAsyncStatus status, WGPUStringView, void* u1, void*) {
gpu_timer_done(static_cast<GpuTimer*>(u1), status);
};
mc.userdata1 = t;
wgpuBufferMapAsync(t->read, WGPUMapMode_Read, 0, t->passes.size() * 16, mc);
}
void recording_replay(Recording* r, WGPUCommandEncoder e) {
recording_replay(r, e, 0, r->words.size());
}
void recording_replay(Recording* r, WGPUCommandEncoder e, size_t from, size_t to, GpuTimer* t) {
WGPURenderPassEncoder pass = nullptr;
WGPUComputePassEncoder compute = nullptr;
GpuTimer::Pass* timed = nullptr; // the open pass's entry in t (a pointer into a vector reserved below)
if (t) t->passes.reserve(GpuTimer::kQueries / 2);
const bool tagged = gpu_timing();
const uint64_t* w = r->words.data() + from;
const uint64_t* end = r->words.data() + to;
#if RX_WEB_MT
uint32_t ops = 0;
#endif
while (w < end) {
#if RX_WEB_MT
serve_gpu_calls_now_if_any(); // (creations other threads wait for: not after the whole batch)
// (frame interpolation: a made frame shown when it is due, not after the batch: rx_framegen.cpp)
if (framegen_web_due && !(++ops & 31) && hos_now_ns() >= framegen_web_due) framegen_web_tick();
#endif
switch (static_cast<Op>(*w++)) {
case kBeginRenderPass: {
uint64_t va = 0, desc = 0;
if (tagged) {
va = *w++;
desc = *w++;
}
const uint32_t ncolor = static_cast<uint32_t>(*w++);
WGPURenderPassColorAttachment colors[8]{};
for (uint32_t i = 0; i < ncolor; i++) {
WGPURenderPassColorAttachment& c = colors[i];
c.view = as<WGPUTextureView>(*w++);
c.depthSlice = static_cast<uint32_t>(*w);
c.loadOp = static_cast<WGPULoadOp>(*w >> 32 & 0xFFFF);
c.storeOp = static_cast<WGPUStoreOp>(*w++ >> 48);
c.clearValue = {as_double(w[0]), as_double(w[1]), as_double(w[2]), as_double(w[3])};
w += 4;
}
WGPURenderPassDepthStencilAttachment depth{};
const bool has_depth = *w++ != 0;
if (has_depth) {
depth.view = as<WGPUTextureView>(*w++);
depth.depthLoadOp = static_cast<WGPULoadOp>(*w & 0xFF);
depth.depthStoreOp = static_cast<WGPUStoreOp>(*w >> 8 & 0xFF);
depth.stencilLoadOp = static_cast<WGPULoadOp>(*w >> 16 & 0xFF);
depth.stencilStoreOp = static_cast<WGPUStoreOp>(*w >> 24 & 0xFF);
depth.depthReadOnly = static_cast<WGPUBool>(*w >> 32 & 1);
depth.stencilReadOnly = static_cast<WGPUBool>(*w++ >> 33 & 1);
depth.depthClearValue = as_float(*w);
depth.stencilClearValue = static_cast<uint32_t>(*w++ >> 32);
}
WGPURenderPassDescriptor pd{};
pd.colorAttachmentCount = ncolor;
pd.colorAttachments = colors;
pd.depthStencilAttachment = has_depth ? &depth : nullptr;
pd.occlusionQuerySet = as<WGPUQuerySet>(*w++);
if (t) {
pd.timestampWrites = t->begin(va, desc | 1ull << 63);
timed = pd.timestampWrites ? &t->passes.back() : nullptr;
}
pass = wgpuCommandEncoderBeginRenderPass(e, &pd);
break;
}
case kEndRenderPass:
wgpuRenderPassEncoderEnd(pass);
wgpuRenderPassEncoderRelease(pass);
pass = nullptr;
timed = nullptr;
break;
case kSetViewport:
wgpuRenderPassEncoderSetViewport(pass, as_float(w[0]), as_float(w[1]), as_float(w[2]), as_float(w[3]),
as_float(w[4]), as_float(w[5]));
w += 6;
break;
case kSetPipeline: wgpuRenderPassEncoderSetPipeline(pass, as<WGPURenderPipeline>(*w++)); break;
case kSetBindGroup: {
const uint32_t index = static_cast<uint32_t>(w[0]), n = static_cast<uint32_t>(w[0] >> 32);
WGPUBindGroup g = as<WGPUBindGroup>(w[1]);
w += 2;
uint32_t offsets[64];
for (uint32_t i = 0; i < n; i++) offsets[i] = static_cast<uint32_t>(*w++);
wgpuRenderPassEncoderSetBindGroup(pass, index, g, n, offsets);
break;
}
case kSetVertexBuffer:
wgpuRenderPassEncoderSetVertexBuffer(pass, static_cast<uint32_t>(w[0]), as<WGPUBuffer>(w[1]), w[2], w[3]);
w += 4;
break;
case kSetIndexBuffer:
wgpuRenderPassEncoderSetIndexBuffer(pass, as<WGPUBuffer>(w[0]), static_cast<WGPUIndexFormat>(w[1]), w[2],
w[3]);
w += 4;
break;
case kSetScissor:
wgpuRenderPassEncoderSetScissorRect(pass, static_cast<uint32_t>(w[0]), static_cast<uint32_t>(w[0] >> 32),
static_cast<uint32_t>(w[1]), static_cast<uint32_t>(w[1] >> 32));
w += 2;
break;
case kSetBlendConstant: {
const WGPUColor c{as_double(w[0]), as_double(w[1]), as_double(w[2]), as_double(w[3])};
w += 4;
wgpuRenderPassEncoderSetBlendConstant(pass, &c);
break;
}
case kSetStencilReference: wgpuRenderPassEncoderSetStencilReference(pass, static_cast<uint32_t>(*w++)); break;
case kBeginOcclusionQuery: wgpuRenderPassEncoderBeginOcclusionQuery(pass, static_cast<uint32_t>(*w++)); break;
case kEndOcclusionQuery: wgpuRenderPassEncoderEndOcclusionQuery(pass); break;
case kResolveQuerySet:
wgpuCommandEncoderResolveQuerySet(e, as<WGPUQuerySet>(w[0]), static_cast<uint32_t>(w[1]),
static_cast<uint32_t>(w[1] >> 32), as<WGPUBuffer>(w[2]), w[3]);
w += 4;
break;
case kDraw:
if (timed) timed->draws++;
wgpuRenderPassEncoderDraw(pass, static_cast<uint32_t>(w[0]), static_cast<uint32_t>(w[0] >> 32),
static_cast<uint32_t>(w[1]), static_cast<uint32_t>(w[1] >> 32));
w += 2;
break;
case kDrawIndexed:
if (timed) timed->draws++;
wgpuRenderPassEncoderDrawIndexed(pass, static_cast<uint32_t>(w[0]), static_cast<uint32_t>(w[0] >> 32),
static_cast<uint32_t>(w[1]), static_cast<int32_t>(w[1] >> 32),
static_cast<uint32_t>(w[2]));
w += 3;
break;
case kMultiDrawIndirect: {
if (timed) timed->draws++;
const uint32_t max = static_cast<uint32_t>(w[4]);
const bool indexed = (w[4] >> 32) != 0;
WGPUBuffer count = as<WGPUBuffer>(w[2]);
if (count && indexed) wgpuRenderPassEncoderMultiDrawIndexedIndirectCount(pass, as<WGPUBuffer>(w[0]), w[1], count, w[3], max);
else if (count) wgpuRenderPassEncoderMultiDrawIndirectCount(pass, as<WGPUBuffer>(w[0]), w[1], count, w[3], max);
else if (indexed) wgpuRenderPassEncoderMultiDrawIndexedIndirect(pass, as<WGPUBuffer>(w[0]), w[1], max);
else wgpuRenderPassEncoderMultiDrawIndirect(pass, as<WGPUBuffer>(w[0]), w[1], max);
w += 5;
break;
}
case kCopyBufferToBuffer:
wgpuCommandEncoderCopyBufferToBuffer(e, as<WGPUBuffer>(w[0]), w[1], as<WGPUBuffer>(w[2]), w[3], w[4]);
w += 5;
break;
case kCopyTextureToTexture: {
const WGPUTexelCopyTextureInfo src = get_copy_info(w), dst = get_copy_info(w);
const WGPUExtent3D extent{static_cast<uint32_t>(w[0]), static_cast<uint32_t>(w[0] >> 32),
static_cast<uint32_t>(w[1])};
w += 2;
wgpuCommandEncoderCopyTextureToTexture(e, &src, &dst, &extent);
break;
}
case kCopyTextureToBuffer: {
const WGPUTexelCopyTextureInfo src = get_copy_info(w);
WGPUTexelCopyBufferInfo dst{};
dst.buffer = as<WGPUBuffer>(w[0]);
dst.layout.offset = w[1];
dst.layout.bytesPerRow = static_cast<uint32_t>(w[2]);
dst.layout.rowsPerImage = static_cast<uint32_t>(w[2] >> 32);
const WGPUExtent3D extent{static_cast<uint32_t>(w[3]), static_cast<uint32_t>(w[3] >> 32),
static_cast<uint32_t>(w[4])};
w += 5;
wgpuCommandEncoderCopyTextureToBuffer(e, &src, &dst, &extent);
break;
}
case kBeginComputePass: {
WGPUComputePassDescriptor cd{};
if (t) {
cd.timestampWrites = t->begin(0, 0);
timed = cd.timestampWrites ? &t->passes.back() : nullptr;
}
compute = wgpuCommandEncoderBeginComputePass(e, &cd);
break;
}
case kComputeSetPipeline: wgpuComputePassEncoderSetPipeline(compute, as<WGPUComputePipeline>(*w++)); break;
case kComputeSetBindGroupDesc: { // (made here, from what cp_set_bind_group_desc recorded)
const uint32_t index = static_cast<uint32_t>(w[0]), n = static_cast<uint32_t>(w[0] >> 32);
WGPUBindGroupDescriptor bd{};
bd.layout = as<WGPUBindGroupLayout>(w[1]);
w += 2;
WGPUBindGroupEntry entries[64]{};
for (uint32_t i = 0; i < n && i < 64; i++, w += 6) {
entries[i].binding = static_cast<uint32_t>(w[0]);
entries[i].buffer = as<WGPUBuffer>(w[1]);
entries[i].offset = w[2];
entries[i].size = w[3];
entries[i].sampler = as<WGPUSampler>(w[4]);
entries[i].textureView = as<WGPUTextureView>(w[5]);
}
bd.entryCount = std::min<uint32_t>(n, 64);
bd.entries = entries;
WGPUBindGroup g = wgpuDeviceCreateBindGroup(dev.device, &bd);
wgpuComputePassEncoderSetBindGroup(compute, index, g, 0, nullptr);
wgpuBindGroupRelease(g);
break;
}
case kComputeSetBindGroup: {
const uint32_t index = static_cast<uint32_t>(w[0]), n = static_cast<uint32_t>(w[0] >> 32);
WGPUBindGroup g = as<WGPUBindGroup>(w[1]);
w += 2;
uint32_t offsets[64];
for (uint32_t i = 0; i < n; i++) offsets[i] = static_cast<uint32_t>(*w++);
wgpuComputePassEncoderSetBindGroup(compute, index, g, n, offsets);
break;
}
case kDispatch:
if (timed) timed->draws++;
wgpuComputePassEncoderDispatchWorkgroups(compute, static_cast<uint32_t>(w[0]), static_cast<uint32_t>(w[1]),
static_cast<uint32_t>(w[2]));
w += 3;
break;
case kDispatchIndirect:
if (timed) timed->draws++;
wgpuComputePassEncoderDispatchWorkgroupsIndirect(compute, as<WGPUBuffer>(w[0]), w[1]);
w += 2;
break;
case kEndComputePass:
wgpuComputePassEncoderEnd(compute);
wgpuComputePassEncoderRelease(compute);
compute = nullptr;
timed = nullptr;
break;
default:
hos_log("rx: recording: unknown op %llu", static_cast<unsigned long long>(w[-1]));
return;
}
}
}
void recording_free(Recording* r) {
if (!r) return;
for (auto& [k, h] : r->refs) drop(k, h);
delete r;
}
// ---- the calls: recorded when deferred, made at once otherwise ----
WGPURenderPassEncoder ce_begin_render_pass(WGPUCommandEncoder e, const WGPURenderPassDescriptor* pd) {
if (!encoding_deferred()) return wgpuCommandEncoderBeginRenderPass(e, pd);
put_op(kBeginRenderPass);
if (gpu_timing()) {
put(tag_va);
put(tag_desc);
tag_va = tag_desc = 0;
}
put(pd->colorAttachmentCount);
for (uint32_t i = 0; i < pd->colorAttachmentCount; i++) {
const WGPURenderPassColorAttachment& c = pd->colorAttachments[i];
hold(kView, c.view);
put_ptr(c.view);
put(uint64_t{c.depthSlice} | uint64_t{static_cast<uint16_t>(c.loadOp)} << 32 |
uint64_t{static_cast<uint16_t>(c.storeOp)} << 48);
put(bits(c.clearValue.r));
put(bits(c.clearValue.g));
put(bits(c.clearValue.b));
put(bits(c.clearValue.a));
}
const WGPURenderPassDepthStencilAttachment* d = pd->depthStencilAttachment;
put(d ? 1 : 0);
if (d) {
hold(kView, d->view);
put_ptr(d->view);
put(uint64_t{static_cast<uint8_t>(d->depthLoadOp)} | uint64_t{static_cast<uint8_t>(d->depthStoreOp)} << 8 |
uint64_t{static_cast<uint8_t>(d->stencilLoadOp)} << 16 |
uint64_t{static_cast<uint8_t>(d->stencilStoreOp)} << 24 | uint64_t{d->depthReadOnly ? 1u : 0u} << 32 |
uint64_t{d->stencilReadOnly ? 1u : 0u} << 33);
put(bits(d->depthClearValue) | uint64_t{d->stencilClearValue} << 32);
}
put_ptr(pd->occlusionQuerySet); // (query sets live as long as the renderer)
return kFakePass;
}
void rp_end(WGPURenderPassEncoder p) {
if (!encoding_deferred()) {
wgpuRenderPassEncoderEnd(p);
wgpuRenderPassEncoderRelease(p);
return;
}
put_op(kEndRenderPass);
rec().pass_ends.push_back(rec().words.size());
}
void rp_set_viewport(WGPURenderPassEncoder p, float x, float y, float w, float h, float zmin, float zmax) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetViewport(p, x, y, w, h, zmin, zmax);
put_op(kSetViewport);
for (float f : {x, y, w, h, zmin, zmax}) put(bits(f));
}
void rp_set_pipeline(WGPURenderPassEncoder p, WGPURenderPipeline pipeline) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetPipeline(p, pipeline);
put_op(kSetPipeline);
put_ptr(pipeline);
}
void rp_set_bind_group(WGPURenderPassEncoder p, uint32_t index, WGPUBindGroup g, size_t n, const uint32_t* offsets) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetBindGroup(p, index, g, n, offsets);
put_op(kSetBindGroup);
put(uint64_t{index} | uint64_t{static_cast<uint32_t>(n)} << 32);
put_ptr(g);
for (size_t i = 0; i < n; i++) put(offsets[i]);
}
void rp_set_vertex_buffer(WGPURenderPassEncoder p, uint32_t slot, WGPUBuffer b, uint64_t offset, uint64_t size) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetVertexBuffer(p, slot, b, offset, size);
put_op(kSetVertexBuffer);
put(slot);
put_ptr(b);
put(offset);
put(size);
}
void rp_set_index_buffer(WGPURenderPassEncoder p, WGPUBuffer b, WGPUIndexFormat format, uint64_t offset,
uint64_t size) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetIndexBuffer(p, b, format, offset, size);
put_op(kSetIndexBuffer);
put_ptr(b);
put(static_cast<uint64_t>(format));
put(offset);
put(size);
}
void rp_set_scissor(WGPURenderPassEncoder p, uint32_t x, uint32_t y, uint32_t w, uint32_t h) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetScissorRect(p, x, y, w, h);
put_op(kSetScissor);
put(uint64_t{x} | uint64_t{y} << 32);
put(uint64_t{w} | uint64_t{h} << 32);
}
void rp_set_blend_constant(WGPURenderPassEncoder p, const WGPUColor* c) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetBlendConstant(p, c);
put_op(kSetBlendConstant);
put(bits(c->r));
put(bits(c->g));
put(bits(c->b));
put(bits(c->a));
}
void rp_set_stencil_reference(WGPURenderPassEncoder p, uint32_t ref) {
if (!encoding_deferred()) return wgpuRenderPassEncoderSetStencilReference(p, ref);
put_op(kSetStencilReference);
put(ref);
}
void rp_begin_occlusion_query(WGPURenderPassEncoder p, uint32_t index) {
if (!encoding_deferred()) return wgpuRenderPassEncoderBeginOcclusionQuery(p, index);
put_op(kBeginOcclusionQuery);
put(index);
}
void rp_end_occlusion_query(WGPURenderPassEncoder p) {
if (!encoding_deferred()) return wgpuRenderPassEncoderEndOcclusionQuery(p);
put_op(kEndOcclusionQuery);
}
void ce_resolve_query_set(WGPUCommandEncoder e, WGPUQuerySet set, uint32_t first, uint32_t n, WGPUBuffer dst,
uint64_t offset) {
if (!encoding_deferred()) return wgpuCommandEncoderResolveQuerySet(e, set, first, n, dst, offset);
put_op(kResolveQuerySet);
put_ptr(set);
put(uint64_t{first} | uint64_t{n} << 32);
put_ptr(dst);
put(offset);
}
void rp_draw(WGPURenderPassEncoder p, uint32_t vertices, uint32_t instances, uint32_t first, uint32_t first_instance) {
if (!encoding_deferred()) return wgpuRenderPassEncoderDraw(p, vertices, instances, first, first_instance);
put_op(kDraw);
put(uint64_t{vertices} | uint64_t{instances} << 32);
put(uint64_t{first} | uint64_t{first_instance} << 32);
}
void rp_draw_indexed(WGPURenderPassEncoder p, uint32_t indices, uint32_t instances, uint32_t first_index,
int32_t base_vertex, uint32_t first_instance) {
if (!encoding_deferred())
return wgpuRenderPassEncoderDrawIndexed(p, indices, instances, first_index, base_vertex, first_instance);
put_op(kDrawIndexed);
put(uint64_t{indices} | uint64_t{instances} << 32);
put(uint64_t{first_index} | uint64_t{static_cast<uint32_t>(base_vertex)} << 32);
put(first_instance);
}
void rp_multi_draw_indirect(WGPURenderPassEncoder p, WGPUBuffer args, uint64_t offset, WGPUBuffer count,
uint64_t count_offset, uint32_t max, bool indexed) {
if (!encoding_deferred()) {
if (count && indexed) return wgpuRenderPassEncoderMultiDrawIndexedIndirectCount(p, args, offset, count, count_offset, max);
if (count) return wgpuRenderPassEncoderMultiDrawIndirectCount(p, args, offset, count, count_offset, max);
if (indexed) return wgpuRenderPassEncoderMultiDrawIndexedIndirect(p, args, offset, max);
return wgpuRenderPassEncoderMultiDrawIndirect(p, args, offset, max);
}
hold(kBuffer, args);
hold(kBuffer, count);
put_op(kMultiDrawIndirect);
put_ptr(args);
put(offset);
put_ptr(count);
put(count_offset);
put(uint64_t{max} | uint64_t{indexed ? 1u : 0u} << 32);
}
void ce_copy_buffer_to_buffer(WGPUCommandEncoder e, WGPUBuffer src, uint64_t src_offset, WGPUBuffer dst,
uint64_t dst_offset, uint64_t size) {
if (!encoding_deferred()) return wgpuCommandEncoderCopyBufferToBuffer(e, src, src_offset, dst, dst_offset, size);
put_op(kCopyBufferToBuffer);
put_ptr(src);
put(src_offset);
put_ptr(dst);
put(dst_offset);
put(size);
}
void ce_copy_texture_to_texture(WGPUCommandEncoder e, const WGPUTexelCopyTextureInfo* src,
const WGPUTexelCopyTextureInfo* dst, const WGPUExtent3D* extent) {
if (!encoding_deferred()) return wgpuCommandEncoderCopyTextureToTexture(e, src, dst, extent);
put_op(kCopyTextureToTexture);
put_copy_info(*src);
put_copy_info(*dst);
put(uint64_t{extent->width} | uint64_t{extent->height} << 32);
put(extent->depthOrArrayLayers);
}
void ce_copy_texture_to_buffer(WGPUCommandEncoder e, const WGPUTexelCopyTextureInfo* src,
const WGPUTexelCopyBufferInfo* dst, const WGPUExtent3D* extent) {
if (!encoding_deferred()) return wgpuCommandEncoderCopyTextureToBuffer(e, src, dst, extent);
put_op(kCopyTextureToBuffer);
put_copy_info(*src);
hold(kBuffer, dst->buffer);
put_ptr(dst->buffer);
put(dst->layout.offset);
put(uint64_t{dst->layout.bytesPerRow} | uint64_t{dst->layout.rowsPerImage} << 32);
put(uint64_t{extent->width} | uint64_t{extent->height} << 32);
put(extent->depthOrArrayLayers);
}
WGPUComputePassEncoder ce_begin_compute_pass(WGPUCommandEncoder e) {
if (!encoding_deferred()) return wgpuCommandEncoderBeginComputePass(e, nullptr);
put_op(kBeginComputePass);
return kFakeCompute;
}
void cp_set_pipeline(WGPUComputePassEncoder p, WGPUComputePipeline pipeline) {
if (!encoding_deferred()) return wgpuComputePassEncoderSetPipeline(p, pipeline);
put_op(kComputeSetPipeline);
put_ptr(pipeline);
}
void cp_set_bind_group(WGPUComputePassEncoder p, uint32_t index, WGPUBindGroup g, size_t n, const uint32_t* offsets) {
if (!encoding_deferred()) return wgpuComputePassEncoderSetBindGroup(p, index, g, n, offsets);
hold(kBindGroup, g);
put_op(kComputeSetBindGroup);
put(uint64_t{index} | uint64_t{static_cast<uint32_t>(n)} << 32);
put_ptr(g);
for (size_t i = 0; i < n; i++) put(offsets[i]);
}
/// A bind group made from its entries where the pass is replayed (the threaded web build: a group that lives for one
/// dispatch is not made on the renderer's thread, which would wait for the WebGPU thread each time). The entries'
/// buffers and views are held until then; the layout and samplers live in their caches.
void cp_set_bind_group_desc(WGPUComputePassEncoder p, uint32_t index, WGPUBindGroupLayout layout,
const WGPUBindGroupEntry* entries, size_t n) {
if (!encoding_deferred()) {
WGPUBindGroupDescriptor bd{};
bd.layout = layout;
bd.entryCount = n;
bd.entries = entries;
WGPUBindGroup g = wgpuDeviceCreateBindGroup(dev.device, &bd);
wgpuComputePassEncoderSetBindGroup(p, index, g, 0, nullptr);
wgpuBindGroupRelease(g);
return;
}
put_op(kComputeSetBindGroupDesc);
put(uint64_t{index} | uint64_t{static_cast<uint32_t>(n)} << 32);
put_ptr(layout);
for (size_t i = 0; i < n; i++) {
const WGPUBindGroupEntry& e = entries[i];
hold(kBuffer, e.buffer);
hold(kView, e.textureView);
put(e.binding);
put_ptr(e.buffer);
put(e.offset);
put(e.size);
put_ptr(e.sampler);
put_ptr(e.textureView);
}
}
void cp_dispatch(WGPUComputePassEncoder p, uint32_t x, uint32_t y, uint32_t z) {
if (!encoding_deferred()) return wgpuComputePassEncoderDispatchWorkgroups(p, x, y, z);
put_op(kDispatch);
put(x);
put(y);
put(z);
}
void cp_dispatch_indirect(WGPUComputePassEncoder p, WGPUBuffer args, uint64_t offset) {
if (!encoding_deferred()) return wgpuComputePassEncoderDispatchWorkgroupsIndirect(p, args, offset);
hold(kBuffer, args);
put_op(kDispatchIndirect);
put_ptr(args);
put(offset);
}
void cp_end(WGPUComputePassEncoder p) {
if (!encoding_deferred()) {
wgpuComputePassEncoderEnd(p);
wgpuComputePassEncoderRelease(p);
return;
}
put_op(kEndComputePass);
rec().pass_ends.push_back(rec().words.size());
}
} // namespace rx

1606
gpu/render/rx_shader.cpp Normal file

File diff suppressed because it is too large Load Diff

2406
gpu/render/rx_texture.cpp Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

330
gpu/render/rx_web_mt.cpp Normal file
View File

@@ -0,0 +1,330 @@
// The threaded web build's WebGPU thread (RX_WEB_MT; see rx_web_mt.h). It owns the device and the page's canvas. It
// runs, in the order they came, the calls other threads hand it (run_on_gpu_thread: they wait; post_to_gpu_thread:
// they do not) and the batches the renderer's thread flushed (rx_core.cpp push_batch posts gpu_thread_batch: replay,
// uploads, submit, present, readbacks, as the native submit thread), and with nothing to do waits in its event loop,
// where the browser runs WebGPU's callbacks.
#include "rx_internal.h"
#if RX_WEB_MT
#include <algorithm>
#include <atomic>
#include <climits>
#include <string>
#include <unordered_map>
#include <vector>
#include <deque>
#include <mutex>
#include <emscripten/threading.h>
#include <pthread.h>
extern "C" {
void rx_web_wait_word(uint32_t addr, uint32_t value, double ms); // rx_core.cpp
void hos_host_thread_register_self(const char* name);
}
namespace rx {
namespace {
thread_local bool is_gpu_thread = false;
struct Job {
void (*fn)(void*);
void* arg;
uint32_t done;
bool owned; // (posted: nobody waits, deleted once run)
};
std::mutex jobs_mu;
std::deque<Job*> jobs;
std::deque<Job*> jobs_now; // (run_on_gpu_thread_now: ahead of jobs)
uint32_t gpu_word; // bumped when a job or a batch comes: the idle WebGPU thread wakes on it
std::mutex start_mu;
bool started_thread = false;
void finish(Job* j) {
if (j->owned) {
delete j;
} else {
__atomic_store_n(&j->done, 1, __ATOMIC_RELEASE);
emscripten_futex_wake(&j->done, INT_MAX);
}
}
bool serve_jobs() {
bool any = false;
for (;;) {
serve_gpu_calls_now_if_any();
Job* j;
{
std::lock_guard lk(jobs_mu);
if (jobs.empty()) return any;
j = jobs.front();
jobs.pop_front();
}
j->fn(j->arg);
finish(j);
any = true;
if (framegen_web_due && hos_now_ns() >= framegen_web_due) framegen_web_tick(); // (jobs keep coming at times)
}
}
void* gpu_main(void*) {
is_gpu_thread = true;
hos_host_thread_register_self("WebGPU");
for (;;) {
const uint32_t seen = __atomic_load_n(&gpu_word, __ATOMIC_SEQ_CST);
const bool did = serve_jobs();
// (the callbacks of pipelines made asynchronously and of readbacks: other threads' calls of this are no-ops)
if (dev.instance) (wgpuInstanceProcessEvents)(dev.instance);
// (frame interpolation: the made frames shown as they come due, rx_framegen.cpp)
const int64_t next = framegen_web_tick();
if (!did) rx_web_wait_word((uint32_t)(uintptr_t)&gpu_word, seen, next >= 0 ? std::min(2.0, next / 1e6) : 2.0);
}
return nullptr;
}
} // namespace
bool on_gpu_thread() { return is_gpu_thread; }
std::atomic<int> gpu_calls_now{0};
void serve_gpu_calls_now() {
for (;;) {
Job* j;
{
std::lock_guard lk(jobs_mu);
if (jobs_now.empty()) return;
j = jobs_now.front();
jobs_now.pop_front();
}
gpu_calls_now--;
j->fn(j->arg);
finish(j);
}
}
void wake_gpu_thread() {
__atomic_add_fetch(&gpu_word, 1, __ATOMIC_SEQ_CST);
emscripten_futex_wake(&gpu_word, INT_MAX);
}
/// The WebGPU thread, started once: it takes the canvas the page shows (#screen).
void start_gpu_thread() {
std::lock_guard lk(start_mu);
if (started_thread) return;
started_thread = true;
pthread_attr_t a;
pthread_attr_init(&a);
pthread_attr_setstacksize(&a, 2 << 20);
emscripten_pthread_attr_settransferredcanvases(&a, "#screen");
pthread_t t;
pthread_create(&t, &a, gpu_main, nullptr);
pthread_attr_destroy(&a);
pthread_detach(t);
}
/* RX_MT_LOG=1: every 5 s, the calls other threads handed the WebGPU thread, by function: how many and how long their
callers waited (a call waits for the batch the thread is running) */
namespace {
struct CallStat {
const char* what = nullptr;
int64_t n = 0, wait_ns = 0;
};
std::mutex stat_mu;
CallStat stats[128];
int64_t stats_since;
void note_call(const char* what, int64_t wait_ns) {
std::lock_guard lk(stat_mu);
int i = 0;
while (i < 128 && stats[i].what && stats[i].what != what) i++;
if (i < 128) {
stats[i].what = what;
stats[i].n++;
stats[i].wait_ns += wait_ns;
}
const int64_t now = hos_now_ns();
if (!stats_since) stats_since = now;
if (now - stats_since < 5000000000LL) return;
std::sort(std::begin(stats), std::end(stats), [](const CallStat& a, const CallStat& b) { return a.wait_ns > b.wait_ns; });
std::string line;
int64_t total = 0, waited = 0;
for (const CallStat& s : stats) {
if (!s.what) break;
total += s.n;
waited += s.wait_ns;
if (line.size() < 900) line += " " + std::string(s.what) + " " + std::to_string(s.n) + " (" + std::to_string(s.wait_ns / 1000000) + " ms)";
}
hos_log("rx: WebGPU thread calls from others in %.1f s: %lld, waited %.0f ms:%s", (now - stats_since) / 1e9,
(long long)total, waited / 1e6, line.c_str());
for (CallStat& s : stats) s = {};
stats_since = now;
}
} // namespace
void mt_note_wait(const char* what, int64_t ns) {
static const bool log = env_flag("RX_MT_LOG");
if (log) note_call(what, ns);
}
void run_on_gpu_thread(void (*fn)(void*), void* arg, const char* what) {
start_gpu_thread();
static const bool log = env_flag("RX_MT_LOG");
const int64_t t0 = log ? hos_now_ns() : 0;
Job j{fn, arg, 0, false};
{
std::lock_guard lk(jobs_mu);
jobs.push_back(&j);
}
wake_gpu_thread();
while (!__atomic_load_n(&j.done, __ATOMIC_ACQUIRE)) emscripten_futex_wait(&j.done, 0, 1000.0);
if (log) note_call(what, hos_now_ns() - t0);
}
void run_on_gpu_thread_now(void (*fn)(void*), void* arg, const char* what) {
start_gpu_thread();
static const bool log = env_flag("RX_MT_LOG");
const int64_t t0 = log ? hos_now_ns() : 0;
Job j{fn, arg, 0, false};
{
std::lock_guard lk(jobs_mu);
jobs_now.push_back(&j);
}
gpu_calls_now++;
wake_gpu_thread();
while (!__atomic_load_n(&j.done, __ATOMIC_ACQUIRE)) emscripten_futex_wait(&j.done, 0, 1000.0);
if (log) note_call(what, hos_now_ns() - t0);
}
void post_to_gpu_thread(void (*fn)(void*), void* arg) {
start_gpu_thread();
{
std::lock_guard lk(jobs_mu);
jobs.push_back(new Job{fn, arg, 0, true});
}
wake_gpu_thread();
}
namespace {
struct BufferWrite {
WGPUQueue q;
WGPUBuffer b;
uint64_t offset;
std::vector<uint8_t> data;
};
struct TextureWrite {
WGPUQueue q;
WGPUTexelCopyTextureInfo dst;
WGPUTexelCopyBufferLayout layout;
WGPUExtent3D extent;
std::vector<uint8_t> data;
};
// (the copies waiting in the queue: past 64 MB a write waits for them, as a load uploads faster than the WebGPU
// thread may take it and the web's host heap is ~570 MB)
std::atomic<int64_t> queued_bytes{0};
constexpr int64_t kQueuedBytesMax = 64ll << 20;
void before_write(size_t size) {
queued_bytes += (int64_t)size;
if (queued_bytes.load() > kQueuedBytesMax)
on_gpu("(queued writes)", [] {}); // (everything queued before it has run when it returns)
}
std::mutex sizes_mu;
std::unordered_map<WGPUBuffer, uint64_t> sizes; // (a reused handle is overwritten at its creation)
} // namespace
void mt_queue_write_buffer(WGPUQueue q, WGPUBuffer b, uint64_t offset, const void* data, size_t size) {
if (on_gpu_thread()) return (wgpuQueueWriteBuffer)(q, b, offset, data, size);
before_write(size);
auto* w = new BufferWrite{q, b, offset, std::vector<uint8_t>(static_cast<const uint8_t*>(data), static_cast<const uint8_t*>(data) + size)};
post_to_gpu_thread([](void* p) {
auto* w = static_cast<BufferWrite*>(p);
(wgpuQueueWriteBuffer)(w->q, w->b, w->offset, w->data.data(), w->data.size());
queued_bytes -= (int64_t)w->data.size();
delete w;
}, w);
}
void mt_queue_write_texture(WGPUQueue q, const WGPUTexelCopyTextureInfo* dst, const void* data, size_t size,
const WGPUTexelCopyBufferLayout* layout, const WGPUExtent3D* extent) {
if (on_gpu_thread()) return (wgpuQueueWriteTexture)(q, dst, data, size, layout, extent);
before_write(size);
auto* w = new TextureWrite{q, *dst, *layout, *extent, std::vector<uint8_t>(static_cast<const uint8_t*>(data), static_cast<const uint8_t*>(data) + size)};
post_to_gpu_thread([](void* p) {
auto* w = static_cast<TextureWrite*>(p);
(wgpuQueueWriteTexture)(w->q, &w->dst, w->data.data(), w->data.size(), &w->layout, &w->extent);
queued_bytes -= (int64_t)w->data.size();
delete w;
}, w);
}
WGPUBuffer mt_create_buffer(WGPUDevice d, const WGPUBufferDescriptor* desc) {
WGPUBuffer b = on_gpu<true>("wgpuDeviceCreateBuffer", [&] { return (wgpuDeviceCreateBuffer)(d, desc); });
if (b) {
std::lock_guard lk(sizes_mu);
sizes[b] = desc->size;
}
return b;
}
WGPUTexture mt_create_texture(WGPUDevice d, const WGPUTextureDescriptor* desc) {
// (named by its label in RX_MT_LOG's counts: a literal's address, so one entry each)
const char* what = desc->label.data && desc->label.length ? desc->label.data : "wgpuDeviceCreateTexture (no label)";
return on_gpu<true>(what, [&] { return (wgpuDeviceCreateTexture)(d, desc); });
}
namespace {
/// What the renderer holds for a bind group (as its WGPUBindGroup): the group, made by the WebGPU thread in the order
/// of the queue (so before the batches flushed after it, which use it), and the references the renderer counts
struct BindGroupProxy {
WGPUBindGroup group = nullptr;
std::atomic<int> refs{1};
};
struct BindGroupMake {
BindGroupProxy* proxy;
WGPUDevice device;
WGPUBindGroupLayout layout;
std::vector<WGPUBindGroupEntry> entries;
};
void free_bind_group(BindGroupProxy* g) {
if (g->group) (wgpuBindGroupRelease)(g->group);
delete g;
}
} // namespace
WGPUBindGroup mt_create_bind_group(WGPUDevice d, const WGPUBindGroupDescriptor* desc) {
auto* g = new BindGroupProxy;
if (on_gpu_thread()) {
g->group = (wgpuDeviceCreateBindGroup)(d, desc);
} else {
// (the entries' objects outlive this: their releases are queued after it)
auto* m = new BindGroupMake{g, d, desc->layout, std::vector<WGPUBindGroupEntry>(desc->entries, desc->entries + desc->entryCount)};
post_to_gpu_thread([](void* p) {
auto* m = static_cast<BindGroupMake*>(p);
WGPUBindGroupDescriptor bd{};
bd.layout = m->layout;
bd.entryCount = m->entries.size();
bd.entries = m->entries.data();
m->proxy->group = (wgpuDeviceCreateBindGroup)(m->device, &bd);
delete m;
}, m);
}
return reinterpret_cast<WGPUBindGroup>(g);
}
void mt_bind_group_add_ref(WGPUBindGroup g) {
if (g) reinterpret_cast<BindGroupProxy*>(g)->refs++;
}
void mt_bind_group_release(WGPUBindGroup g) {
auto* p = reinterpret_cast<BindGroupProxy*>(g);
if (!p || --p->refs > 0) return;
if (on_gpu_thread()) return free_bind_group(p);
post_to_gpu_thread([](void* q) { free_bind_group(static_cast<BindGroupProxy*>(q)); }, p); // (after its uses)
}
WGPUBindGroup mt_bind_group(WGPUBindGroup g) {
return g ? reinterpret_cast<BindGroupProxy*>(g)->group : nullptr;
}
uint64_t mt_buffer_size(WGPUBuffer b) {
{
std::lock_guard lk(sizes_mu);
if (auto it = sizes.find(b); it != sizes.end()) return it->second;
}
return on_gpu<true>("wgpuBufferGetSize", [&] { return (wgpuBufferGetSize)(b); });
}
} // namespace rx
#endif

153
gpu/render/rx_web_mt.h Normal file
View File

@@ -0,0 +1,153 @@
// The threaded web build (RX_WEB_MT, build.py RX_WEB_MT=1): in the browser a WebGPU object belongs to the worker
// that made it, so one thread, the WebGPU thread (rx_web_mt.cpp), makes every WebGPU call. The renderer's own thread
// prepares draws and records them (deferred encoding, as natively), and the WebGPU thread replays, submits and presents
// them, as the native submit thread does. Every other WebGPU call the renderer makes goes through these macros: on the
// WebGPU thread it is made at once, on any other it runs there while the caller waits. Made by
// scripts/web_mt_macros.py from the calls in gpu/render: a call not listed here would run on the wrong thread.
#pragma once
#if RX_WEB_MT
#include <atomic>
#include <type_traits>
namespace rx {
bool on_gpu_thread();
void run_on_gpu_thread(void (*fn)(void*), void* arg, const char* what); // (and waits for it; RX_MT_LOG: counted)
// Creations (Now): ahead of the calls and batches queued before them, and run between a batch's commands too, as they
// depend on none of those and the caller would otherwise wait for the whole batch the WebGPU thread is running
void run_on_gpu_thread_now(void (*fn)(void*), void* arg, const char* what);
extern std::atomic<int> gpu_calls_now; // (how many wait)
void serve_gpu_calls_now();
/// (the WebGPU thread, between the commands of a batch)
inline void serve_gpu_calls_now_if_any() {
if (gpu_calls_now.load(std::memory_order_relaxed)) serve_gpu_calls_now();
}
template <bool Now = false, class F>
auto on_gpu(const char* what, F&& f) -> decltype(f()) {
if (on_gpu_thread()) return f();
using R = decltype(f());
using Fn = std::remove_reference_t<F>;
auto* const run = Now ? run_on_gpu_thread_now : run_on_gpu_thread;
if constexpr (std::is_void_v<R>) {
run([](void* p) { (*static_cast<Fn*>(p))(); }, &f, what);
} else {
struct Call {
Fn* f;
R r;
} c{&f, R{}};
run([](void* p) { auto* c = static_cast<Call*>(p); c->r = (*c->f)(); }, &c, what);
return c.r;
}
}
// Calls that need no answer go without waiting, in order with the others (they run before the batches flushed after
// them): reference counting, and queue writes (their data copied)
void post_to_gpu_thread(void (*fn)(void*), void* arg);
void mt_note_wait(const char* what, int64_t ns); // (RX_MT_LOG: other waits counted with the calls)
template <auto Fn, class H>
void gpu_async_handle(H h) {
if (on_gpu_thread()) return Fn(h);
post_to_gpu_thread([](void* p) { Fn(static_cast<H>(p)); }, static_cast<void*>(h));
}
void mt_queue_write_buffer(WGPUQueue q, WGPUBuffer b, uint64_t offset, const void* data, size_t size);
void mt_queue_write_texture(WGPUQueue q, const WGPUTexelCopyTextureInfo* dst, const void* data, size_t size,
const WGPUTexelCopyBufferLayout* layout, const WGPUExtent3D* extent);
// Buffer sizes from their creation (the renderer asks ~1000 times a frame)
WGPUBuffer mt_create_buffer(WGPUDevice d, const WGPUBufferDescriptor* desc);
uint64_t mt_buffer_size(WGPUBuffer b);
WGPUTexture mt_create_texture(WGPUDevice d, const WGPUTextureDescriptor* desc);
// Bind groups are made without waiting: the handle the renderer gets is a stand-in (rx_web_mt.cpp BindGroupProxy)
// counting its own references, which the WebGPU thread fills before anything queued after it runs, and the calls that
// take one get the group itself (mt_bind_group) there
WGPUBindGroup mt_create_bind_group(WGPUDevice d, const WGPUBindGroupDescriptor* desc);
void mt_bind_group_add_ref(WGPUBindGroup g);
void mt_bind_group_release(WGPUBindGroup g);
WGPUBindGroup mt_bind_group(WGPUBindGroup g); // (the WebGPU thread)
} // namespace rx
#define wgpuAdapterGetInfo(...) ::rx::on_gpu("wgpuAdapterGetInfo", [&] { return wgpuAdapterGetInfo(__VA_ARGS__); })
#define wgpuAdapterGetLimits(...) ::rx::on_gpu("wgpuAdapterGetLimits", [&] { return wgpuAdapterGetLimits(__VA_ARGS__); })
#define wgpuAdapterHasFeature(...) ::rx::on_gpu("wgpuAdapterHasFeature", [&] { return wgpuAdapterHasFeature(__VA_ARGS__); })
#define wgpuAdapterInfoFreeMembers(...) ::rx::on_gpu("wgpuAdapterInfoFreeMembers", [&] { return wgpuAdapterInfoFreeMembers(__VA_ARGS__); })
#define wgpuAdapterRequestDevice(...) ::rx::on_gpu("wgpuAdapterRequestDevice", [&] { return wgpuAdapterRequestDevice(__VA_ARGS__); })
#define wgpuBindGroupAddRef(...) ::rx::mt_bind_group_add_ref(__VA_ARGS__)
#define wgpuBindGroupLayoutRelease(...) ::rx::gpu_async_handle<&wgpuBindGroupLayoutRelease>(__VA_ARGS__)
#define wgpuBindGroupRelease(...) ::rx::mt_bind_group_release(__VA_ARGS__)
#define wgpuBufferAddRef(...) ::rx::gpu_async_handle<&wgpuBufferAddRef>(__VA_ARGS__)
#define wgpuBufferGetConstMappedRange(...) ::rx::on_gpu("wgpuBufferGetConstMappedRange", [&] { return wgpuBufferGetConstMappedRange(__VA_ARGS__); })
#define wgpuBufferGetSize(...) ::rx::mt_buffer_size(__VA_ARGS__)
#define wgpuBufferMapAsync(...) ::rx::on_gpu("wgpuBufferMapAsync", [&] { return wgpuBufferMapAsync(__VA_ARGS__); })
#define wgpuBufferRelease(...) ::rx::gpu_async_handle<&wgpuBufferRelease>(__VA_ARGS__)
#define wgpuBufferUnmap(...) ::rx::on_gpu("wgpuBufferUnmap", [&] { return wgpuBufferUnmap(__VA_ARGS__); })
#define wgpuCommandBufferRelease(...) ::rx::gpu_async_handle<&wgpuCommandBufferRelease>(__VA_ARGS__)
#define wgpuCommandEncoderBeginComputePass(...) ::rx::on_gpu("wgpuCommandEncoderBeginComputePass", [&] { return wgpuCommandEncoderBeginComputePass(__VA_ARGS__); })
#define wgpuCommandEncoderBeginRenderPass(...) ::rx::on_gpu("wgpuCommandEncoderBeginRenderPass", [&] { return wgpuCommandEncoderBeginRenderPass(__VA_ARGS__); })
#define wgpuCommandEncoderCopyBufferToBuffer(...) ::rx::on_gpu("wgpuCommandEncoderCopyBufferToBuffer", [&] { return wgpuCommandEncoderCopyBufferToBuffer(__VA_ARGS__); })
#define wgpuCommandEncoderCopyBufferToTexture(...) ::rx::on_gpu("wgpuCommandEncoderCopyBufferToTexture", [&] { return wgpuCommandEncoderCopyBufferToTexture(__VA_ARGS__); })
#define wgpuCommandEncoderCopyTextureToBuffer(...) ::rx::on_gpu("wgpuCommandEncoderCopyTextureToBuffer", [&] { return wgpuCommandEncoderCopyTextureToBuffer(__VA_ARGS__); })
#define wgpuCommandEncoderCopyTextureToTexture(...) ::rx::on_gpu("wgpuCommandEncoderCopyTextureToTexture", [&] { return wgpuCommandEncoderCopyTextureToTexture(__VA_ARGS__); })
#define wgpuCommandEncoderFinish(...) ::rx::on_gpu("wgpuCommandEncoderFinish", [&] { return wgpuCommandEncoderFinish(__VA_ARGS__); })
#define wgpuCommandEncoderRelease(...) ::rx::gpu_async_handle<&wgpuCommandEncoderRelease>(__VA_ARGS__)
#define wgpuCommandEncoderResolveQuerySet(...) ::rx::on_gpu("wgpuCommandEncoderResolveQuerySet", [&] { return wgpuCommandEncoderResolveQuerySet(__VA_ARGS__); })
#define wgpuComputePassEncoderDispatchWorkgroups(...) ::rx::on_gpu("wgpuComputePassEncoderDispatchWorkgroups", [&] { return wgpuComputePassEncoderDispatchWorkgroups(__VA_ARGS__); })
#define wgpuComputePassEncoderEnd(...) ::rx::on_gpu("wgpuComputePassEncoderEnd", [&] { return wgpuComputePassEncoderEnd(__VA_ARGS__); })
#define wgpuComputePassEncoderRelease(...) ::rx::gpu_async_handle<&wgpuComputePassEncoderRelease>(__VA_ARGS__)
#define wgpuComputePassEncoderSetBindGroup(p, i, g, n, o) ::rx::on_gpu("wgpuComputePassEncoderSetBindGroup", [&] { return wgpuComputePassEncoderSetBindGroup(p, i, ::rx::mt_bind_group(g), n, o); })
#define wgpuComputePassEncoderSetPipeline(...) ::rx::on_gpu("wgpuComputePassEncoderSetPipeline", [&] { return wgpuComputePassEncoderSetPipeline(__VA_ARGS__); })
#define wgpuComputePipelineAddRef(...) ::rx::gpu_async_handle<&wgpuComputePipelineAddRef>(__VA_ARGS__)
#define wgpuComputePipelineGetBindGroupLayout(...) ::rx::on_gpu("wgpuComputePipelineGetBindGroupLayout", [&] { return wgpuComputePipelineGetBindGroupLayout(__VA_ARGS__); })
#define wgpuComputePipelineRelease(...) ::rx::gpu_async_handle<&wgpuComputePipelineRelease>(__VA_ARGS__)
#define wgpuCreateInstance(...) ::rx::on_gpu("wgpuCreateInstance", [&] { return wgpuCreateInstance(__VA_ARGS__); })
#define wgpuDeviceCreateBindGroup(...) ::rx::mt_create_bind_group(__VA_ARGS__)
#define wgpuDeviceCreateBindGroupLayout(...) ::rx::on_gpu<true>("wgpuDeviceCreateBindGroupLayout", [&] { return wgpuDeviceCreateBindGroupLayout(__VA_ARGS__); })
#define wgpuDeviceCreateBuffer(...) ::rx::mt_create_buffer(__VA_ARGS__)
#define wgpuDeviceCreateCommandEncoder(...) ::rx::on_gpu("wgpuDeviceCreateCommandEncoder", [&] { return wgpuDeviceCreateCommandEncoder(__VA_ARGS__); })
#define wgpuDeviceCreateComputePipeline(...) ::rx::on_gpu<true>("wgpuDeviceCreateComputePipeline", [&] { return wgpuDeviceCreateComputePipeline(__VA_ARGS__); })
#define wgpuDeviceCreateComputePipelineAsync(...) ::rx::on_gpu<true>("wgpuDeviceCreateComputePipelineAsync", [&] { return wgpuDeviceCreateComputePipelineAsync(__VA_ARGS__); })
#define wgpuDeviceCreatePipelineLayout(...) ::rx::on_gpu<true>("wgpuDeviceCreatePipelineLayout", [&] { return wgpuDeviceCreatePipelineLayout(__VA_ARGS__); })
#define wgpuDeviceCreateQuerySet(...) ::rx::on_gpu<true>("wgpuDeviceCreateQuerySet", [&] { return wgpuDeviceCreateQuerySet(__VA_ARGS__); })
#define wgpuDeviceCreateRenderPipeline(...) ::rx::on_gpu<true>("wgpuDeviceCreateRenderPipeline", [&] { return wgpuDeviceCreateRenderPipeline(__VA_ARGS__); })
#define wgpuDeviceCreateRenderPipelineAsync(...) ::rx::on_gpu<true>("wgpuDeviceCreateRenderPipelineAsync", [&] { return wgpuDeviceCreateRenderPipelineAsync(__VA_ARGS__); })
#define wgpuDeviceCreateSampler(...) ::rx::on_gpu<true>("wgpuDeviceCreateSampler", [&] { return wgpuDeviceCreateSampler(__VA_ARGS__); })
#define wgpuDeviceCreateShaderModule(...) ::rx::on_gpu<true>("wgpuDeviceCreateShaderModule", [&] { return wgpuDeviceCreateShaderModule(__VA_ARGS__); })
#define wgpuDeviceCreateShaderModuleTrusted(...) ::rx::on_gpu<true>("wgpuDeviceCreateShaderModuleTrusted", [&] { return wgpuDeviceCreateShaderModuleTrusted(__VA_ARGS__); })
#define wgpuDeviceCreateTexture(...) ::rx::mt_create_texture(__VA_ARGS__)
#define wgpuDeviceGetLimits(...) ::rx::on_gpu("wgpuDeviceGetLimits", [&] { return wgpuDeviceGetLimits(__VA_ARGS__); })
#define wgpuDeviceGetQueue(...) ::rx::on_gpu("wgpuDeviceGetQueue", [&] { return wgpuDeviceGetQueue(__VA_ARGS__); })
#define wgpuDevicePopErrorScope(...) ::rx::on_gpu("wgpuDevicePopErrorScope", [&] { return wgpuDevicePopErrorScope(__VA_ARGS__); })
#define wgpuDevicePushErrorScope(...) ::rx::on_gpu("wgpuDevicePushErrorScope", [&] { return wgpuDevicePushErrorScope(__VA_ARGS__); })
#define wgpuInstanceCreateSurface(...) ::rx::on_gpu("wgpuInstanceCreateSurface", [&] { return wgpuInstanceCreateSurface(__VA_ARGS__); })
#define wgpuInstanceProcessEvents(...) (::rx::on_gpu_thread() ? wgpuInstanceProcessEvents(__VA_ARGS__) : (void)0)
#define wgpuInstanceRequestAdapter(...) ::rx::on_gpu("wgpuInstanceRequestAdapter", [&] { return wgpuInstanceRequestAdapter(__VA_ARGS__); })
#define wgpuPipelineLayoutRelease(...) ::rx::gpu_async_handle<&wgpuPipelineLayoutRelease>(__VA_ARGS__)
#define wgpuQueueGetTimestampPeriod(...) ::rx::on_gpu("wgpuQueueGetTimestampPeriod", [&] { return wgpuQueueGetTimestampPeriod(__VA_ARGS__); })
#define wgpuQueueOnSubmittedWorkDone(...) ::rx::on_gpu("wgpuQueueOnSubmittedWorkDone", [&] { return wgpuQueueOnSubmittedWorkDone(__VA_ARGS__); })
#define wgpuQueueSubmit(...) ::rx::on_gpu("wgpuQueueSubmit", [&] { return wgpuQueueSubmit(__VA_ARGS__); })
#define wgpuQueueSubmitForIndex(...) ::rx::on_gpu("wgpuQueueSubmitForIndex", [&] { return wgpuQueueSubmitForIndex(__VA_ARGS__); })
#define wgpuQueueWriteBuffer(...) ::rx::mt_queue_write_buffer(__VA_ARGS__)
#define wgpuQueueWriteTexture(...) ::rx::mt_queue_write_texture(__VA_ARGS__)
#define wgpuRenderPassEncoderDraw(...) ::rx::on_gpu("wgpuRenderPassEncoderDraw", [&] { return wgpuRenderPassEncoderDraw(__VA_ARGS__); })
#define wgpuRenderPassEncoderDrawIndexed(...) ::rx::on_gpu("wgpuRenderPassEncoderDrawIndexed", [&] { return wgpuRenderPassEncoderDrawIndexed(__VA_ARGS__); })
#define wgpuRenderPassEncoderEnd(...) ::rx::on_gpu("wgpuRenderPassEncoderEnd", [&] { return wgpuRenderPassEncoderEnd(__VA_ARGS__); })
#define wgpuRenderPassEncoderRelease(...) ::rx::gpu_async_handle<&wgpuRenderPassEncoderRelease>(__VA_ARGS__)
#define wgpuRenderPassEncoderSetBindGroup(p, i, g, n, o) ::rx::on_gpu("wgpuRenderPassEncoderSetBindGroup", [&] { return wgpuRenderPassEncoderSetBindGroup(p, i, ::rx::mt_bind_group(g), n, o); })
#define wgpuRenderPassEncoderSetBlendConstant(...) ::rx::on_gpu("wgpuRenderPassEncoderSetBlendConstant", [&] { return wgpuRenderPassEncoderSetBlendConstant(__VA_ARGS__); })
#define wgpuRenderPassEncoderSetIndexBuffer(...) ::rx::on_gpu("wgpuRenderPassEncoderSetIndexBuffer", [&] { return wgpuRenderPassEncoderSetIndexBuffer(__VA_ARGS__); })
#define wgpuRenderPassEncoderSetPipeline(...) ::rx::on_gpu("wgpuRenderPassEncoderSetPipeline", [&] { return wgpuRenderPassEncoderSetPipeline(__VA_ARGS__); })
#define wgpuRenderPassEncoderSetScissorRect(...) ::rx::on_gpu("wgpuRenderPassEncoderSetScissorRect", [&] { return wgpuRenderPassEncoderSetScissorRect(__VA_ARGS__); })
#define wgpuRenderPassEncoderSetStencilReference(...) ::rx::on_gpu("wgpuRenderPassEncoderSetStencilReference", [&] { return wgpuRenderPassEncoderSetStencilReference(__VA_ARGS__); })
#define wgpuRenderPassEncoderSetVertexBuffer(...) ::rx::on_gpu("wgpuRenderPassEncoderSetVertexBuffer", [&] { return wgpuRenderPassEncoderSetVertexBuffer(__VA_ARGS__); })
#define wgpuRenderPassEncoderSetViewport(...) ::rx::on_gpu("wgpuRenderPassEncoderSetViewport", [&] { return wgpuRenderPassEncoderSetViewport(__VA_ARGS__); })
#define wgpuRenderPipelineAddRef(...) ::rx::gpu_async_handle<&wgpuRenderPipelineAddRef>(__VA_ARGS__)
#define wgpuRenderPipelineRelease(...) ::rx::gpu_async_handle<&wgpuRenderPipelineRelease>(__VA_ARGS__)
#define wgpuShaderModuleRelease(...) ::rx::gpu_async_handle<&wgpuShaderModuleRelease>(__VA_ARGS__)
#define wgpuSurfaceCapabilitiesFreeMembers(...) ::rx::on_gpu("wgpuSurfaceCapabilitiesFreeMembers", [&] { return wgpuSurfaceCapabilitiesFreeMembers(__VA_ARGS__); })
#define wgpuSurfaceConfigure(...) ::rx::on_gpu("wgpuSurfaceConfigure", [&] { return wgpuSurfaceConfigure(__VA_ARGS__); })
#define wgpuSurfaceGetCapabilities(...) ::rx::on_gpu("wgpuSurfaceGetCapabilities", [&] { return wgpuSurfaceGetCapabilities(__VA_ARGS__); })
#define wgpuSurfaceGetCurrentTexture(...) ::rx::on_gpu("wgpuSurfaceGetCurrentTexture", [&] { return wgpuSurfaceGetCurrentTexture(__VA_ARGS__); })
#define wgpuSurfacePresent(...) ::rx::on_gpu("wgpuSurfacePresent", [&] { return wgpuSurfacePresent(__VA_ARGS__); })
#define wgpuTextureAddRef(...) ::rx::gpu_async_handle<&wgpuTextureAddRef>(__VA_ARGS__)
#define wgpuTextureCreateView(...) ::rx::on_gpu<true>("wgpuTextureCreateView", [&] { return wgpuTextureCreateView(__VA_ARGS__); })
#define wgpuTextureRelease(...) ::rx::gpu_async_handle<&wgpuTextureRelease>(__VA_ARGS__)
#define wgpuTextureViewAddRef(...) ::rx::gpu_async_handle<&wgpuTextureViewAddRef>(__VA_ARGS__)
#define wgpuTextureViewRelease(...) ::rx::gpu_async_handle<&wgpuTextureViewRelease>(__VA_ARGS__)
#endif

View File

@@ -0,0 +1,24 @@
/* (mkw-recomp) Included first in every source of the WebGPU renderer (nx2/build.py, -include): the functions the
game executable calls are renamed rx_* -> wgr_*, and host/rx_backend.c defines the rx_* names, sending each call
to this renderer or to the Vulkan one (mkw_vk.dll, RX_BACKEND=vulkan). */
#pragma once
#define rx_start wgr_start
#define rx_resize wgr_resize
#define rx_describe wgr_describe
#define rx_stats wgr_stats
#define rx_nvn_submit wgr_nvn_submit
#define rx_nvn_fence wgr_nvn_fence
#define rx_nvn_pushed wgr_nvn_pushed
#define rx_nvn_replayed wgr_nvn_replayed
#define rx_nvn_wait wgr_nvn_wait
#define rx_wait_writebacks_reading wgr_wait_writebacks_reading
#define rx_nvn_finish wgr_nvn_finish
#define rx_nvn_present wgr_nvn_present
#define rx_nvn_register_texture wgr_nvn_register_texture
#define rx_nvn_register_sampler wgr_nvn_register_sampler
#define rx_nvn_set_texture_pool_size wgr_nvn_set_texture_pool_size
#define rx_nvn_texture_released wgr_nvn_texture_released
#define rx_nvn_program_released wgr_nvn_program_released
#define rx_nvn_read_texels wgr_nvn_read_texels
#define rx_nvn_write_texels wgr_nvn_write_texels
#define rx_nvn_invalidate wgr_nvn_invalidate