Triforce v0.2.0

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ruckus7382sfgsgdf
2026-11-10 19:32:48 -06:00
commit 34b3683441
1730 changed files with 554853 additions and 0 deletions

View File

@@ -0,0 +1,20 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include "common/common_types.h"
namespace Shader::Backend {
struct Bindings {
u32 unified{};
u32 uniform_buffer{};
u32 storage_buffer{};
u32 texture{};
u32 image{};
u32 texture_scaling_index{};
u32 image_scaling_index{};
};
} // namespace Shader::Backend

View File

@@ -0,0 +1,764 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <algorithm>
#include <span>
#include <tuple>
#include <type_traits>
#include <utility>
#include <vector>
#include "common/settings.h"
#include "shader_recompiler/backend/spirv/emit_spirv.h"
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
#include "shader_recompiler/frontend/ir/basic_block.h"
#include "shader_recompiler/frontend/ir/program.h"
#include "shader_recompiler/backend/spirv/mma_layout.h" // (hookshot)
namespace Shader::Backend::SPIRV {
namespace {
template <class Func>
struct FuncTraits {};
template <class ReturnType_, class... Args>
struct FuncTraits<ReturnType_ (*)(Args...)> {
using ReturnType = ReturnType_;
static constexpr size_t NUM_ARGS = sizeof...(Args);
template <size_t I>
using ArgType = std::tuple_element_t<I, std::tuple<Args...>>;
};
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4702) // Ignore unreachable code warning
#endif
template <auto func, typename... Args>
void SetDefinition(EmitContext& ctx, IR::Inst* inst, Args... args) {
inst->SetDefinition<Id>(func(ctx, std::forward<Args>(args)...));
}
#ifdef _MSC_VER
#pragma warning(pop)
#endif
template <typename ArgType>
ArgType Arg(EmitContext& ctx, const IR::Value& arg) {
if constexpr (std::is_same_v<ArgType, Id>) {
return ctx.Def(arg);
} else if constexpr (std::is_same_v<ArgType, const IR::Value&>) {
return arg;
} else if constexpr (std::is_same_v<ArgType, u32>) {
return arg.U32();
} else if constexpr (std::is_same_v<ArgType, IR::Attribute>) {
return arg.Attribute();
} else if constexpr (std::is_same_v<ArgType, IR::Patch>) {
return arg.Patch();
} else if constexpr (std::is_same_v<ArgType, IR::Reg>) {
return arg.Reg();
}
}
template <auto func, bool is_first_arg_inst, size_t... I>
void Invoke(EmitContext& ctx, IR::Inst* inst, std::index_sequence<I...>) {
using Traits = FuncTraits<decltype(func)>;
if constexpr (std::is_same_v<typename Traits::ReturnType, Id>) {
if constexpr (is_first_arg_inst) {
SetDefinition<func>(
ctx, inst, inst,
Arg<typename Traits::template ArgType<I + 2>>(ctx, inst->Arg(I))...);
} else {
SetDefinition<func>(
ctx, inst, Arg<typename Traits::template ArgType<I + 1>>(ctx, inst->Arg(I))...);
}
} else {
if constexpr (is_first_arg_inst) {
func(ctx, inst, Arg<typename Traits::template ArgType<I + 2>>(ctx, inst->Arg(I))...);
} else {
func(ctx, Arg<typename Traits::template ArgType<I + 1>>(ctx, inst->Arg(I))...);
}
}
}
template <auto func>
void Invoke(EmitContext& ctx, IR::Inst* inst) {
using Traits = FuncTraits<decltype(func)>;
static_assert(Traits::NUM_ARGS >= 1, "Insufficient arguments");
if constexpr (Traits::NUM_ARGS == 1) {
Invoke<func, false>(ctx, inst, std::make_index_sequence<0>{});
} else {
using FirstArgType = typename Traits::template ArgType<1>;
static constexpr bool is_first_arg_inst = std::is_same_v<FirstArgType, IR::Inst*>;
using Indices = std::make_index_sequence<Traits::NUM_ARGS - (is_first_arg_inst ? 2 : 1)>;
Invoke<func, is_first_arg_inst>(ctx, inst, Indices{});
}
}
void EmitInstOf(EmitContext& ctx, IR::Inst* inst) {
switch (inst->GetOpcode()) {
#define OPCODE(name, result_type, ...) \
case IR::Opcode::name: \
return Invoke<&Emit##name>(ctx, inst);
#include "shader_recompiler/frontend/ir/opcodes.inc"
#undef OPCODE
}
throw LogicError("Invalid opcode {}", inst->GetOpcode());
}
void EmitInst(EmitContext& ctx, IR::Inst* inst) {
// (hookshot) the emitters that are left unimplemented say only "SPIR-V Instruction": name the IR op
try {
EmitInstOf(ctx, inst);
} catch (const NotImplementedException& e) {
throw NotImplementedException("{} ({})", e.what(), IR::NameOf(inst->GetOpcode()));
}
}
Id TypeId(const EmitContext& ctx, IR::Type type) {
switch (type) {
case IR::Type::U1:
return ctx.U1;
case IR::Type::U32:
return ctx.U32[1];
default:
throw NotImplementedException("Phi node type {}", type);
}
}
void Traverse(EmitContext& ctx, IR::Program& program) {
IR::Block* current_block{};
// (EvilOOT, RX_VK_BLOCK_PROBE=1 with RX_VK_MISS_LOG=1 and RX_VK_MISS_LOG_PROG=<key>: each block's entries counted, by
// invocation, in the miss log's words 128 + the block's index in program.blocks: IR::DumpProgram's "Block $n")
static const bool probe_env{std::getenv("RX_VK_BLOCK_PROBE") != nullptr};
const bool probe{probe_env && Sirit::ValidId(ctx.probe_log)};
const auto block_index{[&](const IR::Block* b) -> u32 {
const auto it{std::find(program.blocks.begin(), program.blocks.end(), b)};
return static_cast<u32>(it - program.blocks.begin());
}};
for (const IR::AbstractSyntaxNode& node : program.syntax_list) {
switch (node.type) {
case IR::AbstractSyntaxNode::Type::Block: {
const Id label{node.data.block->Definition<Id>()};
if (current_block) {
ctx.OpBranch(label);
}
current_block = node.data.block;
ctx.AddLabel(label);
bool probed{!probe};
for (IR::Inst& inst : node.data.block->Instructions()) {
if (!probed && inst.GetOpcode() != IR::Opcode::Phi) { // (after the phis: they must lead the block)
probed = true;
const u32 at{128U + std::min<u32>(block_index(node.data.block), 4000U)};
const Id counter{ctx.OpAccessChain(ctx.storage_types.U32.element, ctx.probe_log, ctx.u32_zero_value,
ctx.Const(at))};
ctx.OpAtomicIAdd(ctx.U32[1], counter, ctx.Const(static_cast<u32>(spv::Scope::Device)), ctx.u32_zero_value,
ctx.Const(1U));
}
EmitInst(ctx, &inst);
}
break;
}
case IR::AbstractSyntaxNode::Type::If: {
const Id if_label{node.data.if_node.body->Definition<Id>()};
const Id endif_label{node.data.if_node.merge->Definition<Id>()};
ctx.OpSelectionMerge(endif_label, spv::SelectionControlMask::MaskNone);
ctx.OpBranchConditional(ctx.Def(node.data.if_node.cond), if_label, endif_label);
break;
}
case IR::AbstractSyntaxNode::Type::Loop: {
const Id body_label{node.data.loop.body->Definition<Id>()};
const Id continue_label{node.data.loop.continue_block->Definition<Id>()};
const Id endloop_label{node.data.loop.merge->Definition<Id>()};
ctx.OpLoopMerge(endloop_label, continue_label, spv::LoopControlMask::MaskNone);
ctx.OpBranch(body_label);
break;
}
case IR::AbstractSyntaxNode::Type::Break: {
const Id break_label{node.data.break_node.merge->Definition<Id>()};
const Id skip_label{node.data.break_node.skip->Definition<Id>()};
ctx.OpBranchConditional(ctx.Def(node.data.break_node.cond), break_label, skip_label);
break;
}
case IR::AbstractSyntaxNode::Type::EndIf:
if (current_block) {
ctx.OpBranch(node.data.end_if.merge->Definition<Id>());
}
break;
case IR::AbstractSyntaxNode::Type::Repeat: {
Id cond{ctx.Def(node.data.repeat.cond)};
if (!Settings::values.disable_shader_loop_safety_checks) {
const Id pointer_type{ctx.TypePointer(spv::StorageClass::Private, ctx.U32[1])};
const Id safety_counter{ctx.AddGlobalVariable(
pointer_type, spv::StorageClass::Private, ctx.Const(0x2000u))};
if (ctx.profile.supported_spirv >= 0x00010400) {
ctx.interfaces.push_back(safety_counter);
}
const Id old_counter{ctx.OpLoad(ctx.U32[1], safety_counter)};
const Id new_counter{ctx.OpISub(ctx.U32[1], old_counter, ctx.Const(1u))};
ctx.OpStore(safety_counter, new_counter);
const Id safety_cond{
ctx.OpSGreaterThanEqual(ctx.U1, new_counter, ctx.u32_zero_value)};
cond = ctx.OpLogicalAnd(ctx.U1, cond, safety_cond);
}
const Id loop_header_label{node.data.repeat.loop_header->Definition<Id>()};
const Id merge_label{node.data.repeat.merge->Definition<Id>()};
ctx.OpBranchConditional(cond, loop_header_label, merge_label);
break;
}
case IR::AbstractSyntaxNode::Type::Return:
ctx.OpReturn();
break;
case IR::AbstractSyntaxNode::Type::Unreachable:
if (ctx.stage == Stage::Mesh && !ctx.mesh_fallback) {
// (hookshot) an indirect branch to none of its table's targets: the mesh shader leaves with no outputs.
// RADV makes OpUnreachable a return that leaves its output counts undefined, and GS_ALLOC_REQ then asked
// for a garbage count and hung the GPU (issue 1000). Marker for OpSetMeshOutputsEXT 0 0
// (PatchMeshInstructions); the epilogue writes the outputs, so none were written yet
ctx.OpStore(ctx.u32_zero_value, ctx.u32_zero_value);
ctx.mesh_zero_outputs_marker = true;
ctx.OpReturn();
} else {
ctx.OpUnreachable();
}
break;
}
if (node.type != IR::AbstractSyntaxNode::Type::Block) {
current_block = nullptr;
}
}
}
Id DefineMain(EmitContext& ctx, IR::Program& program) {
const Id void_function{ctx.TypeFunction(ctx.void_id)};
const Id main{ctx.OpFunction(ctx.void_id, spv::FunctionControlMask::MaskNone, void_function)};
for (IR::Block* const block : program.blocks) {
block->SetDefinition(ctx.OpLabel());
}
Traverse(ctx, program);
ctx.OpFunctionEnd();
return main;
}
spv::ExecutionMode ExecutionMode(TessPrimitive primitive) {
switch (primitive) {
case TessPrimitive::Isolines:
return spv::ExecutionMode::Isolines;
case TessPrimitive::Triangles:
return spv::ExecutionMode::Triangles;
case TessPrimitive::Quads:
return spv::ExecutionMode::Quads;
}
throw InvalidArgument("Tessellation primitive {}", primitive);
}
spv::ExecutionMode ExecutionMode(TessSpacing spacing) {
switch (spacing) {
case TessSpacing::Equal:
return spv::ExecutionMode::SpacingEqual;
case TessSpacing::FractionalOdd:
return spv::ExecutionMode::SpacingFractionalOdd;
case TessSpacing::FractionalEven:
return spv::ExecutionMode::SpacingFractionalEven;
}
throw InvalidArgument("Tessellation spacing {}", spacing);
}
void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) {
const std::span interfaces(ctx.interfaces.data(), ctx.interfaces.size());
if (ctx.mesh_fallback && (program.stage == Stage::Task || program.stage == Stage::Mesh)) {
// (hookshot, issue 640: on a GPU without mesh shaders, a task or mesh program is a compute shader)
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U, 1U);
ctx.AddEntryPoint(spv::ExecutionModel::GLCompute, main, "main", interfaces);
return;
}
spv::ExecutionModel execution_model{};
switch (program.stage) {
case Stage::Compute: {
const std::array<u32, 3> workgroup_size{program.workgroup_size};
execution_model = spv::ExecutionModel::GLCompute;
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, workgroup_size[0],
workgroup_size[1], workgroup_size[2]);
break;
}
case Stage::VertexB:
execution_model = spv::ExecutionModel::Vertex;
break;
case Stage::Task:
execution_model = spv::ExecutionModel::TaskEXT;
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U,
1U);
break;
case Stage::Mesh:
execution_model = spv::ExecutionModel::MeshEXT;
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U,
1U);
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.mesh.max_vertices);
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputPrimitivesEXT,
program.mesh.max_primitives);
ctx.AddExecutionMode(main, program.mesh.Corners() == 1 ? spv::ExecutionMode::OutputPoints
: program.mesh.Corners() == 2 ? spv::ExecutionMode::OutputLinesEXT
: spv::ExecutionMode::OutputTrianglesEXT);
break;
case Stage::TessellationControl:
execution_model = spv::ExecutionModel::TessellationControl;
ctx.AddCapability(spv::Capability::Tessellation);
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.invocations);
break;
case Stage::TessellationEval:
execution_model = spv::ExecutionModel::TessellationEvaluation;
ctx.AddCapability(spv::Capability::Tessellation);
ctx.AddExecutionMode(main, ExecutionMode(ctx.runtime_info.tess_primitive));
ctx.AddExecutionMode(main, ExecutionMode(ctx.runtime_info.tess_spacing));
ctx.AddExecutionMode(main, ctx.runtime_info.tess_clockwise
? spv::ExecutionMode::VertexOrderCw
: spv::ExecutionMode::VertexOrderCcw);
break;
case Stage::Geometry:
execution_model = spv::ExecutionModel::Geometry;
ctx.AddCapability(spv::Capability::Geometry);
if (ctx.profile.support_geometry_streams) {
ctx.AddCapability(spv::Capability::GeometryStreams);
}
switch (ctx.runtime_info.input_topology) {
case InputTopology::Points:
ctx.AddExecutionMode(main, spv::ExecutionMode::InputPoints);
break;
case InputTopology::Lines:
ctx.AddExecutionMode(main, spv::ExecutionMode::InputLines);
break;
case InputTopology::LinesAdjacency:
ctx.AddExecutionMode(main, spv::ExecutionMode::InputLinesAdjacency);
break;
case InputTopology::Triangles:
ctx.AddExecutionMode(main, spv::ExecutionMode::Triangles);
break;
case InputTopology::TrianglesAdjacency:
ctx.AddExecutionMode(main, spv::ExecutionMode::InputTrianglesAdjacency);
break;
}
switch (program.output_topology) {
case OutputTopology::PointList:
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputPoints);
break;
case OutputTopology::LineStrip:
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputLineStrip);
break;
case OutputTopology::TriangleStrip:
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputTriangleStrip);
break;
}
if (program.info.stores[IR::Attribute::PointSize]) {
ctx.AddCapability(spv::Capability::GeometryPointSize);
}
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.output_vertices);
ctx.AddExecutionMode(main, spv::ExecutionMode::Invocations, program.invocations);
if (program.is_geometry_passthrough) {
if (ctx.profile.support_geometry_shader_passthrough) {
ctx.AddExtension("SPV_NV_geometry_shader_passthrough");
ctx.AddCapability(spv::Capability::GeometryShaderPassthroughNV);
} else {
LOG_WARNING(Shader_SPIRV, "Geometry shader passthrough used with no support");
}
}
break;
case Stage::Fragment:
execution_model = spv::ExecutionModel::Fragment;
if (ctx.profile.lower_left_origin_mode) {
ctx.AddExecutionMode(main, spv::ExecutionMode::OriginLowerLeft);
} else {
ctx.AddExecutionMode(main, spv::ExecutionMode::OriginUpperLeft);
}
if (program.info.stores_frag_depth) {
ctx.AddExecutionMode(main, spv::ExecutionMode::DepthReplacing);
}
if (ctx.runtime_info.force_early_z) {
ctx.AddExecutionMode(main, spv::ExecutionMode::EarlyFragmentTests);
}
if (ctx.profile.support_shader_quad_control && program.info.uses_quad_shuffles) {
ctx.AddExtension("SPV_KHR_quad_control");
ctx.AddCapability(spv::Capability::QuadControlKHR);
ctx.AddExecutionMode(main, spv::ExecutionMode::RequireFullQuadsKHR);
}
break;
default:
throw NotImplementedException("Stage {}", program.stage);
}
ctx.AddEntryPoint(execution_model, main, "main", interfaces);
}
void SetupDenormControl(const Profile& profile, const IR::Program& program, EmitContext& ctx,
Id main_func) {
const Info& info{program.info};
if (info.uses_fp32_denorms_flush && info.uses_fp32_denorms_preserve) {
LOG_DEBUG(Shader_SPIRV, "Fp32 denorm flush and preserve on the same shader");
} else if (info.uses_fp32_denorms_flush) {
if (profile.support_fp32_denorm_flush && !profile.has_broken_fp32_denorm_flush) {
ctx.AddCapability(spv::Capability::DenormFlushToZero);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormFlushToZero, 32U);
} else {
// Drivers will most likely flush denorms by default, no need to warn
}
} else if (info.uses_fp32_denorms_preserve) {
if (profile.support_fp32_denorm_preserve) {
ctx.AddCapability(spv::Capability::DenormPreserve);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormPreserve, 32U);
} else {
LOG_DEBUG(Shader_SPIRV, "Fp32 denorm preserve used in shader without host support");
}
}
if (!profile.support_separate_denorm_behavior || profile.has_broken_fp16_float_controls) {
// No separate denorm behavior
return;
}
if (info.uses_fp16_denorms_flush && info.uses_fp16_denorms_preserve) {
LOG_DEBUG(Shader_SPIRV, "Fp16 denorm flush and preserve on the same shader");
} else if (info.uses_fp16_denorms_flush) {
if (profile.support_fp16_denorm_flush) {
ctx.AddCapability(spv::Capability::DenormFlushToZero);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormFlushToZero, 16U);
} else {
// Same as fp32, no need to warn as most drivers will flush by default
}
} else if (info.uses_fp16_denorms_preserve) {
if (profile.support_fp16_denorm_preserve) {
ctx.AddCapability(spv::Capability::DenormPreserve);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormPreserve, 16U);
} else {
LOG_DEBUG(Shader_SPIRV, "Fp16 denorm preserve used in shader without host support");
}
}
}
void SetupSignedNanCapabilities(const Profile& profile, const IR::Program& program,
EmitContext& ctx, Id main_func) {
if (profile.has_broken_fp16_float_controls && program.info.uses_fp16) {
return;
}
if (program.info.uses_fp16 && profile.support_fp16_signed_zero_nan_preserve) {
ctx.AddCapability(spv::Capability::SignedZeroInfNanPreserve);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::SignedZeroInfNanPreserve, 16U);
}
if (profile.support_fp32_signed_zero_nan_preserve) {
ctx.AddCapability(spv::Capability::SignedZeroInfNanPreserve);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::SignedZeroInfNanPreserve, 32U);
}
if (program.info.uses_fp64 && profile.support_fp64_signed_zero_nan_preserve) {
ctx.AddCapability(spv::Capability::SignedZeroInfNanPreserve);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::SignedZeroInfNanPreserve, 64U);
}
}
void SetupTransformFeedbackCapabilities(EmitContext& ctx, Id main_func) {
if (ctx.runtime_info.xfb_count == 0) {
return;
}
ctx.AddCapability(spv::Capability::TransformFeedback);
ctx.AddExecutionMode(main_func, spv::ExecutionMode::Xfb);
}
void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ctx) {
if (info.uses_sampled_1d) {
ctx.AddCapability(spv::Capability::Sampled1D);
}
if (info.uses_image_1d) {
ctx.AddCapability(spv::Capability::Image1D);
}
if (info.uses_sparse_residency) {
ctx.AddCapability(spv::Capability::SparseResidency);
}
if (info.uses_demote_to_helper_invocation && profile.support_demote_to_helper_invocation) {
if (profile.supported_spirv < 0x00010600) {
ctx.AddExtension("SPV_EXT_demote_to_helper_invocation");
}
ctx.AddCapability(spv::Capability::DemoteToHelperInvocation);
}
if (info.stores[IR::Attribute::ViewportIndex] && profile.support_multi_viewport) {
ctx.AddCapability(spv::Capability::MultiViewport);
}
if (info.stores[IR::Attribute::ViewportMask] && profile.support_viewport_mask) {
ctx.AddExtension("SPV_NV_viewport_array2");
ctx.AddCapability(spv::Capability::ShaderViewportMaskNV);
}
if (info.stores[IR::Attribute::Layer] || info.stores[IR::Attribute::ViewportIndex]) {
if (profile.support_viewport_index_layer_non_geometry && ctx.stage != Stage::Geometry) {
ctx.AddExtension("SPV_EXT_shader_viewport_index_layer");
ctx.AddCapability(spv::Capability::ShaderViewportIndexLayerEXT);
}
}
if ((!profile.support_vertex_instance_id &&
(info.loads[IR::Attribute::InstanceId] || info.loads[IR::Attribute::VertexId])) ||
info.loads[IR::Attribute::BaseInstance] || info.loads[IR::Attribute::BaseVertex] ||
info.loads[IR::Attribute::DrawID] || (ctx.wants_mesh_first && !ctx.mesh_fallback)) {
ctx.AddExtension("SPV_KHR_shader_draw_parameters");
ctx.AddCapability(spv::Capability::DrawParameters);
}
if ((info.uses_subgroup_vote || info.uses_subgroup_invocation_id ||
info.uses_subgroup_shuffles || info.uses_subgroup_mask) &&
profile.support_vote && profile.SupportsSubgroupStage(ctx.stage)) {
ctx.AddCapability(spv::Capability::GroupNonUniformBallot);
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
if (info.uses_subgroup_shuffles && profile.support_shuffle_relative) {
ctx.AddCapability(spv::Capability::GroupNonUniformShuffleRelative);
}
if (!profile.warp_size_potentially_larger_than_guest) {
// vote ops are only used when not taking the long path
ctx.AddCapability(spv::Capability::GroupNonUniformVote);
}
}
if (info.uses_cooperative_matrix && !mma_software) {
ctx.AddExtension("SPV_KHR_cooperative_matrix");
ctx.AddCapability(spv::Capability::CooperativeMatrixKHR);
ctx.AddExtension("SPV_KHR_vulkan_memory_model");
ctx.AddCapability(spv::Capability::VulkanMemoryModel);
ctx.AddCapability(spv::Capability::VulkanMemoryModelDeviceScope);
ctx.SetMemoryModel(spv::MemoryModel::VulkanKHR);
}
if (info.uses_cooperative_matrix && mma_software) { // (hookshot: DefineSoftwareMatrixFunction's)
ctx.AddCapability(spv::Capability::GroupNonUniform);
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
}
if (info.uses_quad_shuffles) {
if (profile.support_quad_shuffles) {
ctx.AddCapability(spv::Capability::GroupNonUniformQuad);
}
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
}
if (info.uses_int64_bit_atomics && profile.support_int64_atomics) {
ctx.AddCapability(spv::Capability::Int64Atomics);
}
if (info.uses_typeless_image_reads && profile.support_typeless_image_loads) {
ctx.AddCapability(spv::Capability::StorageImageReadWithoutFormat);
}
if (info.uses_typeless_image_writes) {
ctx.AddCapability(spv::Capability::StorageImageWriteWithoutFormat);
}
if (info.uses_image_buffers) {
ctx.AddCapability(spv::Capability::ImageBuffer);
}
if (info.uses_sample_id) {
ctx.AddCapability(spv::Capability::SampleRateShading);
}
if (info.uses_derivatives) {
ctx.AddCapability(spv::Capability::DerivativeControl);
}
// TODO: Track this usage
ctx.AddCapability(spv::Capability::ImageGatherExtended);
ctx.AddCapability(spv::Capability::ImageQuery);
ctx.AddCapability(spv::Capability::SampledBuffer);
if (!ctx.non_uniform_ids.empty()) {
if (ctx.profile.supported_spirv < 0x00010500)
ctx.AddExtension("SPV_EXT_descriptor_indexing");
ctx.AddCapability(spv::Capability::ShaderNonUniform);
if (ctx.uses_nonuniform_sampled_image) {
ctx.AddCapability(spv::Capability::SampledImageArrayNonUniformIndexing);
}
if (ctx.uses_nonuniform_storage_image) {
ctx.AddCapability(spv::Capability::StorageImageArrayNonUniformIndexing);
}
if (ctx.uses_nonuniform_uniform_texel_buffer) {
ctx.AddCapability(spv::Capability::UniformTexelBufferArrayNonUniformIndexing);
}
if (ctx.uses_nonuniform_storage_texel_buffer) {
ctx.AddCapability(spv::Capability::StorageTexelBufferArrayNonUniformIndexing);
}
}
}
void PatchPhiNodes(IR::Program& program, EmitContext& ctx) {
// Flatten all leading PHIs from each block into a vector
std::vector<IR::Inst*> phi_instructions;
for (IR::Block* block : program.blocks) {
for (auto it = block->begin(); it != block->end(); ++it) {
if (it->GetOpcode() != IR::Opcode::Phi)
break;
phi_instructions.push_back(&*it);
}
}
if (phi_instructions.empty()) {
return; // nothing to patch
}
// Start "before" first PHI; advance on phi_arg == 0
size_t phi_index = static_cast<size_t>(-1);
ctx.PatchDeferredPhi([&](size_t phi_arg, Id parent) -> std::pair<Id, Id> {
if (phi_arg == 0) {
++phi_index;
}
IR::Inst* phi = phi_instructions[phi_index];
return { ctx.Def(phi->Arg(phi_arg)), parent };
});
}
void RewriteOpcodes(std::vector<u32>& code, std::span<const std::pair<u32, spv::Op>> rewrites) {
if (rewrites.empty()) {
return;
}
size_t offset = 5;
while (offset + 2 < code.size()) {
const u32 word_count{code[offset] >> 16};
const auto opcode{static_cast<spv::Op>(code[offset] & 0xFFFFu)};
if (word_count == 0) {
return;
}
if (opcode == spv::Op::OpGroupNonUniformShuffleXor ||
opcode == spv::Op::OpGroupNonUniformQuadBroadcast) {
const auto it{std::ranges::find(rewrites, code[offset + 2],
&std::pair<u32, spv::Op>::first)};
if (it != rewrites.end()) {
code[offset] = (code[offset] & 0xFFFF0000u) | static_cast<u32>(it->second);
}
}
offset += word_count;
}
}
// Replaces the OpStore markers of the mesh instructions Sirit cannot emit: OpStore(vertices,
// primitives) becomes OpSetMeshOutputsEXT, and OpStore(x, y), OpStore(z, payload), OpReturn
// becomes the OpEmitMeshTasksEXT terminator.
void PatchMeshInstructions(std::vector<u32>& code, const EmitContext& ctx) {
const bool set_outputs{Sirit::ValidId(ctx.mesh_outputs_marker[0])};
const bool emit_tasks{Sirit::ValidId(ctx.mesh_tasks_marker[0])};
const bool zero_outputs{ctx.mesh_zero_outputs_marker};
if (!set_outputs && !emit_tasks && !zero_outputs) {
return;
}
const auto store_at = [&](size_t offset, Id pointer, Id object) {
return offset + 2 < code.size() &&
code[offset] == ((3U << 16) | static_cast<u32>(spv::Op::OpStore)) &&
code[offset + 1] == pointer.value && code[offset + 2] == object.value;
};
const auto& tasks{ctx.mesh_tasks_marker};
size_t offset = 5;
bool patched{};
while (offset < code.size()) {
const u32 word_count{code[offset] >> 16};
if (word_count == 0) {
break;
}
if (set_outputs &&
store_at(offset, ctx.mesh_outputs_marker[0], ctx.mesh_outputs_marker[1])) {
code[offset] = (3U << 16) | static_cast<u32>(spv::Op::OpSetMeshOutputsEXT);
patched = true;
} else if (zero_outputs && store_at(offset, ctx.u32_zero_value, ctx.u32_zero_value)) {
code[offset] = (3U << 16) | static_cast<u32>(spv::Op::OpSetMeshOutputsEXT);
patched = true;
} else if (emit_tasks && store_at(offset, tasks[0], tasks[1]) &&
store_at(offset + 3, tasks[2], tasks[3]) && offset + 6 < code.size() &&
code[offset + 6] == ((1U << 16) | static_cast<u32>(spv::Op::OpReturn))) {
const std::array<u32, 5> emit{(5U << 16) | static_cast<u32>(spv::Op::OpEmitMeshTasksEXT),
tasks[0].value, tasks[1].value, tasks[2].value,
tasks[3].value};
code.erase(code.begin() + offset, code.begin() + offset + 7);
code.insert(code.begin() + offset, emit.begin(), emit.end());
patched = true;
offset += emit.size();
continue;
}
offset += word_count;
}
if (!patched) {
throw LogicError("Mesh instruction markers not found");
}
}
} // Anonymous namespace
std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings) {
EmitContext ctx{profile, runtime_info, program, bindings};
if (program.info.uses_cooperative_matrix && mma_software) DefineSoftwareMatrixFunction(ctx);
Id main{};
if (runtime_info.mesh_fallback_vertex) { // (hookshot: the fallback's generated vertex stage, which reads the records)
const Id void_function{ctx.TypeFunction(ctx.void_id)};
main = ctx.OpFunction(ctx.void_id, spv::FunctionControlMask::MaskNone, void_function);
ctx.AddLabel();
EmitMeshFallbackVertexBody(ctx);
ctx.OpReturn();
ctx.OpFunctionEnd();
} else {
main = DefineMain(ctx, program);
}
DefineEntryPoint(program, ctx, main);
if (profile.support_float_controls) {
ctx.AddExtension("SPV_KHR_float_controls");
SetupDenormControl(profile, program, ctx, main);
SetupSignedNanCapabilities(profile, program, ctx, main);
}
SetupCapabilities(profile, program.info, ctx);
SetupTransformFeedbackCapabilities(ctx, main);
PatchPhiNodes(program, ctx);
std::vector<u32> code{ctx.Assemble()};
RewriteOpcodes(code, ctx.opcode_rewrites);
PatchMeshInstructions(code, ctx);
return code;
}
Id EmitPhi(EmitContext& ctx, IR::Inst* inst) {
const size_t num_args{inst->NumArgs()};
boost::container::small_vector<Id, 32> blocks;
blocks.reserve(num_args);
for (size_t index = 0; index < num_args; ++index) {
blocks.push_back(inst->PhiBlock(index)->Definition<Id>());
}
// The type of a phi instruction is stored in its flags
const Id result_type{TypeId(ctx, inst->Flags<IR::Type>())};
return ctx.DeferredOpPhi(result_type, std::span(blocks.data(), blocks.size()));
}
void EmitVoid(EmitContext&) {}
Id EmitIdentity(EmitContext& ctx, const IR::Value& value) {
const Id id{ctx.Def(value)};
if (!Sirit::ValidId(id)) {
throw NotImplementedException("Forward identity declaration");
}
return id;
}
Id EmitConditionRef(EmitContext& ctx, const IR::Value& value) {
const Id id{ctx.Def(value)};
if (!Sirit::ValidId(id)) {
throw NotImplementedException("Forward identity declaration");
}
return id;
}
void EmitReference(EmitContext&) {}
void EmitPhiMove(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetZeroFromOp(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetSignFromOp(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetCarryFromOp(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetOverflowFromOp(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetSparseFromOp(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetInBoundsFromOp(EmitContext&) {
throw LogicError("Unreachable instruction");
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,72 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <vector>
#include "common/common_types.h"
#include "shader_recompiler/backend/bindings.h"
#include "shader_recompiler/frontend/ir/program.h"
#include "shader_recompiler/profile.h"
#include "shader_recompiler/runtime_info.h"
namespace Shader::Backend::SPIRV {
constexpr u32 NUM_TEXTURE_SCALING_WORDS = 4;
constexpr u32 NUM_IMAGE_SCALING_WORDS = 2;
constexpr u32 NUM_TEXTURE_AND_IMAGE_SCALING_WORDS =
NUM_TEXTURE_SCALING_WORDS + NUM_IMAGE_SCALING_WORDS;
struct RescalingLayout {
alignas(16) std::array<u32, NUM_TEXTURE_SCALING_WORDS> rescaling_textures;
alignas(16) std::array<u32, NUM_IMAGE_SCALING_WORDS> rescaling_images;
u32 down_factor;
};
struct RenderAreaLayout {
std::array<f32, 4> render_area;
};
constexpr u32 RESCALING_LAYOUT_WORDS_OFFSET = offsetof(RescalingLayout, rescaling_textures);
constexpr u32 RESCALING_LAYOUT_DOWN_FACTOR_OFFSET = offsetof(RescalingLayout, down_factor);
constexpr u32 RENDERAREA_LAYOUT_OFFSET = offsetof(RenderAreaLayout, render_area);
// (hookshot) guest memory's page table, a u64 device address pushed to every compute program that reads global memory, after the blocks
// above. It was a constant in the SPIR-V, and the address changes from run to run: the drivers' pipeline caches never hit
constexpr u32 GUEST_PAGES_PUSH_OFFSET = sizeof(RescalingLayout) + sizeof(RenderAreaLayout);
// (hookshot) a task program's first workgroup: NVN draws tasks [first, first + count), Vulkan from 0. The address of the draw's first
// (a u32, 16 bytes per draw of an indirect draw, read at + DrawIndex * 16) is pushed here
constexpr u32 MESH_FIRST_PUSH_OFFSET = GUEST_PAGES_PUSH_OFFSET + 8;
// (hookshot) the mesh shader fallback's buffers, after it: the task payloads and the vertex records (u64 device addresses).
// Paths without the fallback never push them.
constexpr u32 MESH_FALLBACK_PUSH_OFFSET = MESH_FIRST_PUSH_OFFSET + 8;
constexpr u32 MESH_FALLBACK_PAYLOAD_PUSH_OFFSET = MESH_FALLBACK_PUSH_OFFSET;
constexpr u32 MESH_FALLBACK_RECORDS_PUSH_OFFSET = MESH_FALLBACK_PUSH_OFFSET + 8;
constexpr u32 MESH_FALLBACK_INDIRECT_INPUT_PUSH_OFFSET = MESH_FALLBACK_PUSH_OFFSET + 16;
constexpr u32 MESH_FALLBACK_PUSH_END = MESH_FALLBACK_PUSH_OFFSET + 24;
// (hookshot) the fallback's mesh workgroups a draw can run: each a slot, a word of the payload buffer saying whether its task
// started it; then each task workgroup's payload. It was 1024, a record block each, and Ocarina's floor grass (433 tasks of
// 32 slots) lost 13 of 14 meshlets. At 16384, Hyrule Field's far trees (3cb76827/4f0c0b40: 721 tasks of 32) lost the tasks
// past 512, a different set each frame as the culling's list order changes: they flickered. 65535: Vulkan's least
// maxComputeWorkGroupCount, which the mesh dispatch can reach; the payload buffer is ~6.5 MB a recording thread at 800 words.
constexpr u32 MESH_FALLBACK_CAPACITY = 65535;
// (hookshot) the records buffer: a VkDrawIndirectCommand the vertex stage's draw reads (its vertex count the records written),
// the allocator's word, then the records. A mesh workgroup takes records for its primitives' corners only, so a draw covers
// what was drawn, not every slot's maximum
constexpr u32 MESH_FALLBACK_DRAW_WORD = 0; // (vertexCount; instanceCount at word 1, set to 1 by the host)
constexpr u32 MESH_FALLBACK_ALLOC_WORD = 4; // (records taken, past the end too: those that do not fit are not drawn)
// (a VkDispatchIndirectCommand for the mesh workgroups a task program started (y and z 1, set by the host): each started task
// thread takes an index of it and writes its slot there in the payload buffer's map, so the mesh dispatch runs only those. With
// a count from the GPU (DrawMeshTasksIndirect, the floor grass) the host knew none, and dispatched no mesh workgroup)
constexpr u32 MESH_FALLBACK_DISPATCH_WORD = 5;
constexpr u32 MESH_FALLBACK_HEADER_WORDS = 8;
constexpr u32 MESH_FALLBACK_RECORD_WORDS = 16u << 20; // (64 MB: ~47k grass primitives of 31 words are 17 MB)
[[nodiscard]] std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings);
[[nodiscard]] inline std::vector<u32> EmitSPIRV(const Profile& profile, IR::Program& program) {
Bindings binding;
return EmitSPIRV(profile, {}, program, binding);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,625 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <bit>
#include "shader_recompiler/backend/spirv/emit_spirv.h"
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) {
const Id shift_id{ctx.Const(2U)};
Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
if (index_offset > 0) {
index = ctx.OpIAdd(ctx.U32[1], index, ctx.Const(index_offset));
}
return ctx.uses_explicit_workgroup_layout
? ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index)
: ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index);
}
Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size) {
if (offset.IsImmediate()) {
const u32 imm_offset{static_cast<u32>(offset.U32() / element_size)};
return ctx.Const(imm_offset);
}
const u32 shift{static_cast<u32>(std::countr_zero(element_size))};
const Id index{ctx.Def(offset)};
if (shift == 0) {
return index;
}
const Id shift_id{ctx.Const(shift)};
return ctx.OpShiftRightLogical(ctx.U32[1], index, shift_id);
}
Id StoragePointer(EmitContext& ctx, const StorageTypeDefinition& type_def,
Id StorageDefinitions::*member_ptr, const IR::Value& binding,
const IR::Value& offset, size_t element_size) {
if (!binding.IsImmediate()) {
throw NotImplementedException("Dynamic storage buffer indexing");
}
const Id ssbo{ctx.ssbos[binding.U32()].*member_ptr};
const Id index{StorageIndex(ctx, offset, element_size)};
return ctx.OpAccessChain(type_def.element, ssbo, ctx.u32_zero_value, index);
}
std::pair<Id, Id> AtomicArgs(EmitContext& ctx) {
const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Device))};
const Id semantics{ctx.u32_zero_value};
return {scope, semantics};
}
Id SharedAtomicU32(EmitContext& ctx, Id offset, Id value,
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
const Id pointer{SharedPointer(ctx, offset)};
const auto [scope, semantics]{AtomicArgs(ctx)};
return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
}
Id StorageAtomicU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding,
offset, sizeof(u32))};
const auto [scope, semantics]{AtomicArgs(ctx)};
return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
}
Id StorageAtomicU64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id),
Id (Sirit::Module::*non_atomic_func)(Id, Id, Id)) {
if (!ctx.profile.support_descriptor_aliasing) {
LOG_WARNING(Shader_SPIRV, "Descriptor aliasing not supported, this cannot be atomic.");
return ctx.ConstantNull(ctx.U64);
}
if (ctx.profile.support_int64_atomics) {
const Id pointer{StoragePointer(ctx, ctx.storage_types.U64, &StorageDefinitions::U64,
binding, offset, sizeof(u64))};
const auto [scope, semantics]{AtomicArgs(ctx)};
return (ctx.*atomic_func)(ctx.U64, pointer, scope, semantics, value);
}
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
binding, offset, sizeof(u32[2]))};
const Id original_value{ctx.OpBitcast(ctx.U64, ctx.OpLoad(ctx.U32[2], pointer))};
const Id result{(ctx.*non_atomic_func)(ctx.U64, value, original_value)};
ctx.OpStore(pointer, ctx.OpBitcast(ctx.U32[2], result));
return original_value;
}
Id StorageAtomicU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
Id (Sirit::Module::*non_atomic_func)(Id, Id, Id)) {
if (!ctx.profile.support_descriptor_aliasing) {
LOG_WARNING(Shader_SPIRV, "Descriptor aliasing not supported, this cannot be atomic.");
return ctx.ConstantNull(ctx.U32[2]);
}
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
binding, offset, sizeof(u32[2]))};
const Id original_value{ctx.OpLoad(ctx.U32[2], pointer)};
const Id result{(ctx.*non_atomic_func)(ctx.U32[2], value, original_value)};
ctx.OpStore(pointer, result);
return original_value;
}
} // Anonymous namespace
Id EmitSharedAtomicIAdd32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicIAdd);
}
Id EmitSharedAtomicSMin32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicSMin);
}
Id EmitSharedAtomicUMin32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicUMin);
}
Id EmitSharedAtomicSMax32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicSMax);
}
Id EmitSharedAtomicUMax32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicUMax);
}
Id EmitSharedAtomicInc32(EmitContext& ctx, Id offset, Id value) {
const Id shift_id{ctx.Const(2U)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
return ctx.OpFunctionCall(ctx.U32[1], ctx.increment_cas_shared, index, value);
}
Id EmitSharedAtomicDec32(EmitContext& ctx, Id offset, Id value) {
const Id shift_id{ctx.Const(2U)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
return ctx.OpFunctionCall(ctx.U32[1], ctx.decrement_cas_shared, index, value);
}
Id EmitSharedAtomicAnd32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicAnd);
}
Id EmitSharedAtomicOr32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicOr);
}
Id EmitSharedAtomicXor32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicXor);
}
Id EmitSharedAtomicExchange32(EmitContext& ctx, Id offset, Id value) {
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicExchange);
}
Id EmitSharedAtomicExchange64(EmitContext& ctx, Id offset, Id value) {
if (ctx.profile.support_shared_int64_atomics && ctx.uses_explicit_workgroup_layout) {
const Id shift_id{ctx.Const(3U)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
const Id pointer{
ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
const auto [scope, semantics]{AtomicArgs(ctx)};
return ctx.OpAtomicExchange(ctx.U64, pointer, scope, semantics, value);
}
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
const Id pointer_1{SharedPointer(ctx, offset, 0)};
const Id pointer_2{SharedPointer(ctx, offset, 1)};
const Id value_1{ctx.OpLoad(ctx.U32[1], pointer_1)};
const Id value_2{ctx.OpLoad(ctx.U32[1], pointer_2)};
const Id new_vector{ctx.OpBitcast(ctx.U32[2], value)};
ctx.OpStore(pointer_1, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 0U));
ctx.OpStore(pointer_2, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 1U));
return ctx.OpBitcast(ctx.U64, ctx.OpCompositeConstruct(ctx.U32[2], value_1, value_2));
}
Id EmitSharedAtomicExchange32x2(EmitContext& ctx, Id offset, Id value) {
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
const Id pointer_1{SharedPointer(ctx, offset, 0)};
const Id pointer_2{SharedPointer(ctx, offset, 1)};
const Id value_1{ctx.OpLoad(ctx.U32[1], pointer_1)};
const Id value_2{ctx.OpLoad(ctx.U32[1], pointer_2)};
const Id new_vector{ctx.OpBitcast(ctx.U32[2], value)};
ctx.OpStore(pointer_1, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 0U));
ctx.OpStore(pointer_2, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 1U));
return ctx.OpCompositeConstruct(ctx.U32[2], value_1, value_2);
}
Id EmitStorageAtomicIAdd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicIAdd);
}
Id EmitStorageAtomicSMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMin);
}
Id EmitStorageAtomicUMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMin);
}
Id EmitStorageAtomicSMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMax);
}
Id EmitStorageAtomicUMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMax);
}
Id EmitStorageAtomicInc32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
return ctx.OpFunctionCall(ctx.U32[1], ctx.increment_cas_ssbo, base_index, value, ssbo);
}
Id EmitStorageAtomicDec32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
return ctx.OpFunctionCall(ctx.U32[1], ctx.decrement_cas_ssbo, base_index, value, ssbo);
}
Id EmitStorageAtomicAnd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicAnd);
}
Id EmitStorageAtomicOr32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicOr);
}
Id EmitStorageAtomicXor32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicXor);
}
Id EmitStorageAtomicExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicExchange);
}
Id EmitStorageAtomicCompareExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id comparator, Id value) { // (hookshot: ATOMG.CAS)
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding,
offset, sizeof(u32))};
const auto [scope, semantics]{AtomicArgs(ctx)};
return ctx.OpAtomicCompareExchange(ctx.U32[1], pointer, scope, semantics, semantics, value, comparator);
}
Id EmitStorageAtomicIAdd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicIAdd,
&Sirit::Module::OpIAdd);
}
Id EmitStorageAtomicSMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMin,
&Sirit::Module::OpSMin);
}
Id EmitStorageAtomicUMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMin,
&Sirit::Module::OpUMin);
}
Id EmitStorageAtomicSMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMax,
&Sirit::Module::OpSMax);
}
Id EmitStorageAtomicUMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMax,
&Sirit::Module::OpUMax);
}
Id EmitStorageAtomicAnd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicAnd,
&Sirit::Module::OpBitwiseAnd);
}
Id EmitStorageAtomicOr64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicOr,
&Sirit::Module::OpBitwiseOr);
}
Id EmitStorageAtomicXor64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicXor,
&Sirit::Module::OpBitwiseXor);
}
Id EmitStorageAtomicExchange64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_int64_atomics) {
const Id pointer{StoragePointer(ctx, ctx.storage_types.U64, &StorageDefinitions::U64,
binding, offset, sizeof(u64))};
const auto [scope, semantics]{AtomicArgs(ctx)};
return ctx.OpAtomicExchange(ctx.U64, pointer, scope, semantics, value);
}
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
binding, offset, sizeof(u32[2]))};
const Id original{ctx.OpBitcast(ctx.U64, ctx.OpLoad(ctx.U32[2], pointer))};
ctx.OpStore(pointer, value);
return original;
}
Id EmitStorageAtomicIAdd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpIAdd);
}
Id EmitStorageAtomicSMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpSMin);
}
Id EmitStorageAtomicUMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpUMin);
}
Id EmitStorageAtomicSMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpSMax);
}
Id EmitStorageAtomicUMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpUMax);
}
Id EmitStorageAtomicAnd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpBitwiseAnd);
}
Id EmitStorageAtomicOr32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpBitwiseOr);
}
Id EmitStorageAtomicXor32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpBitwiseXor);
}
Id EmitStorageAtomicExchange32x2(EmitContext& ctx, const IR::Value& binding,
const IR::Value& offset, Id value) {
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
binding, offset, sizeof(u32[2]))};
const Id original{ctx.OpLoad(ctx.U32[2], pointer)};
ctx.OpStore(pointer, value);
return original;
}
Id EmitStorageAtomicAddF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
return ctx.OpFunctionCall(ctx.F32[1], ctx.f32_add_cas, base_index, value, ssbo);
}
Id EmitStorageAtomicAddF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_add_cas, base_index, value, ssbo)};
return ctx.OpBitcast(ctx.U32[1], result);
}
Id EmitStorageAtomicAddF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_add_cas, base_index, value, ssbo)};
return ctx.OpPackHalf2x16(ctx.U32[1], result);
}
Id EmitStorageAtomicMinF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_min_cas, base_index, value, ssbo)};
return ctx.OpBitcast(ctx.U32[1], result);
}
Id EmitStorageAtomicMinF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_min_cas, base_index, value, ssbo)};
return ctx.OpPackHalf2x16(ctx.U32[1], result);
}
Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_max_cas, base_index, value, ssbo)};
return ctx.OpBitcast(ctx.U32[1], result);
}
Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
const Id ssbo{ctx.ssbos[binding.U32()].U32};
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_max_cas, base_index, value, ssbo)};
return ctx.OpPackHalf2x16(ctx.U32[1], result);
}
Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[0])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[0], address, value);
}
Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[1])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[1], address, value);
}
Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[2])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[2], address, value);
}
Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[3])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[3], address, value);
}
Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[4])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[4], address, value);
}
Id EmitGlobalAtomicInc32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicDec32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[5])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[5], address, value);
}
Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[6])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[6], address, value);
}
Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[7])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[7], address, value);
}
Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_func_u32[8])) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[8], address, value);
}
Id EmitGlobalAtomicCompareExchange32(EmitContext& ctx, Id address, Id comparator, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_cas_func_u32)) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_cas_func_u32, address, comparator, value);
}
Id EmitGlobalAtomicIAdd64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicSMin64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicUMin64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicSMax64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicUMax64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicInc64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicDec64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicAnd64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicOr64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicXor64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicExchange64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicIAdd32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicSMin32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicUMin32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicSMax32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicUMax32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicInc32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicDec32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicAnd32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicOr32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicXor32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicExchange32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
if (!Sirit::ValidId(ctx.atomic_global_add_f32_func)) {
throw NotImplementedException("SPIR-V Instruction");
}
return ctx.OpFunctionCall(ctx.F32[1], ctx.atomic_global_add_f32_func, address, value);
}
Id EmitGlobalAtomicAddF16x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicAddF32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicMinF16x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicMinF32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicMaxF16x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitGlobalAtomicMaxF32x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,45 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
void MemoryBarrier(EmitContext& ctx, spv::Scope scope) {
const auto semantics{
spv::MemorySemanticsMask::AcquireRelease | spv::MemorySemanticsMask::UniformMemory |
spv::MemorySemanticsMask::WorkgroupMemory | spv::MemorySemanticsMask::AtomicCounterMemory |
spv::MemorySemanticsMask::ImageMemory};
ctx.OpMemoryBarrier(ctx.Const(static_cast<u32>(scope)), ctx.Const(static_cast<u32>(semantics)));
}
} // Anonymous namespace
void EmitBarrier(EmitContext& ctx) {
const auto execution{spv::Scope::Workgroup};
const auto memory{spv::Scope::Workgroup};
const auto memory_semantics{spv::MemorySemanticsMask::AcquireRelease |
spv::MemorySemanticsMask::WorkgroupMemory};
ctx.OpControlBarrier(ctx.Const(static_cast<u32>(execution)),
ctx.Const(static_cast<u32>(memory)),
ctx.Const(static_cast<u32>(memory_semantics)));
}
// (hookshot) The subgroup's invocations meet, and their workgroup (shared) memory accesses before it are visible to the ones after:
// what the Switch's in-order warp gives a warp's stores and LDSM's reads of them without a BAR
void EmitSubgroupBarrier(EmitContext& ctx) {
const auto scope{spv::Scope::Subgroup};
const auto memory_semantics{spv::MemorySemanticsMask::AcquireRelease | spv::MemorySemanticsMask::WorkgroupMemory};
ctx.OpControlBarrier(ctx.Const(static_cast<u32>(scope)), ctx.Const(static_cast<u32>(scope)),
ctx.Const(static_cast<u32>(memory_semantics)));
}
void EmitWorkgroupMemoryBarrier(EmitContext& ctx) {
MemoryBarrier(ctx, spv::Scope::Workgroup);
}
void EmitDeviceMemoryBarrier(EmitContext& ctx) {
MemoryBarrier(ctx, spv::Scope::Device);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,65 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
void EmitBitCastU16F16(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBitCastU32F32(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.U32[1], value);
}
void EmitBitCastU64F64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitBitCastF16U16(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBitCastF32U32(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.F32[1], value);
}
void EmitBitCastF64U64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitPackUint2x32(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.U64, value);
}
Id EmitUnpackUint2x32(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.U32[2], value);
}
Id EmitPackFloat2x16(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.U32[1], value);
}
Id EmitUnpackFloat2x16(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.F16[2], value);
}
Id EmitPackHalf2x16(EmitContext& ctx, Id value) {
return ctx.OpPackHalf2x16(ctx.U32[1], value);
}
Id EmitUnpackHalf2x16(EmitContext& ctx, Id value) {
return ctx.OpUnpackHalf2x16(ctx.F32[2], value);
}
Id EmitPackDouble2x32(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.F64[1], value);
}
Id EmitUnpackDouble2x32(EmitContext& ctx, Id value) {
return ctx.OpBitcast(ctx.U32[2], value);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,153 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
Id EmitCompositeConstructU32x2(EmitContext& ctx, Id e1, Id e2) {
return ctx.OpCompositeConstruct(ctx.U32[2], e1, e2);
}
Id EmitCompositeConstructU32x3(EmitContext& ctx, Id e1, Id e2, Id e3) {
return ctx.OpCompositeConstruct(ctx.U32[3], e1, e2, e3);
}
Id EmitCompositeConstructU32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4) {
return ctx.OpCompositeConstruct(ctx.U32[4], e1, e2, e3, e4);
}
Id EmitCompositeExtractU32x2(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.U32[1], composite, index);
}
Id EmitCompositeExtractU32x3(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.U32[1], composite, index);
}
Id EmitCompositeExtractU32x4(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.U32[1], composite, index);
}
Id EmitCompositeInsertU32x2(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.U32[2], object, composite, index);
}
Id EmitCompositeInsertU32x3(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.U32[3], object, composite, index);
}
Id EmitCompositeInsertU32x4(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.U32[4], object, composite, index);
}
Id EmitCompositeConstructF16x2(EmitContext& ctx, Id e1, Id e2) {
return ctx.OpCompositeConstruct(ctx.F16[2], e1, e2);
}
Id EmitCompositeConstructF16x3(EmitContext& ctx, Id e1, Id e2, Id e3) {
return ctx.OpCompositeConstruct(ctx.F16[3], e1, e2, e3);
}
Id EmitCompositeConstructF16x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4) {
return ctx.OpCompositeConstruct(ctx.F16[4], e1, e2, e3, e4);
}
Id EmitCompositeExtractF16x2(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.F16[1], composite, index);
}
Id EmitCompositeExtractF16x3(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.F16[1], composite, index);
}
Id EmitCompositeExtractF16x4(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.F16[1], composite, index);
}
Id EmitCompositeInsertF16x2(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F16[2], object, composite, index);
}
Id EmitCompositeInsertF16x3(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F16[3], object, composite, index);
}
Id EmitCompositeInsertF16x4(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F16[4], object, composite, index);
}
Id EmitCompositeConstructF32x2(EmitContext& ctx, Id e1, Id e2) {
return ctx.OpCompositeConstruct(ctx.F32[2], e1, e2);
}
Id EmitCompositeConstructF32x3(EmitContext& ctx, Id e1, Id e2, Id e3) {
return ctx.OpCompositeConstruct(ctx.F32[3], e1, e2, e3);
}
Id EmitCompositeConstructF32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4) {
return ctx.OpCompositeConstruct(ctx.F32[4], e1, e2, e3, e4);
}
Id EmitCompositeExtractF32x2(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.F32[1], composite, index);
}
Id EmitCompositeExtractF32x3(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.F32[1], composite, index);
}
Id EmitCompositeExtractF32x4(EmitContext& ctx, Id composite, u32 index) {
return ctx.OpCompositeExtract(ctx.F32[1], composite, index);
}
Id EmitCompositeInsertF32x2(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F32[2], object, composite, index);
}
Id EmitCompositeInsertF32x3(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F32[3], object, composite, index);
}
Id EmitCompositeInsertF32x4(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F32[4], object, composite, index);
}
void EmitCompositeConstructF64x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitCompositeConstructF64x3(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitCompositeConstructF64x4(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitCompositeExtractF64x2(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitCompositeExtractF64x3(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitCompositeExtractF64x4(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitCompositeInsertF64x2(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F64[2], object, composite, index);
}
Id EmitCompositeInsertF64x3(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F64[3], object, composite, index);
}
Id EmitCompositeInsertF64x4(EmitContext& ctx, Id composite, Id object, u32 index) {
return ctx.OpCompositeInsert(ctx.F64[4], object, composite, index);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,660 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <bit>
#include <tuple>
#include <utility>
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/mma_layout.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
template <typename... Args>
Id AttrPointer(EmitContext& ctx, Id pointer_type, Id vertex, Id base, Args&&... args) {
switch (ctx.stage) {
case Stage::TessellationControl:
case Stage::TessellationEval:
case Stage::Geometry:
return ctx.OpAccessChain(pointer_type, base, vertex, std::forward<Args>(args)...);
default:
return ctx.OpAccessChain(pointer_type, base, std::forward<Args>(args)...);
}
}
template <typename... Args>
Id OutputAccessChain(EmitContext& ctx, Id result_type, Id base, Args&&... args) {
if (ctx.stage == Stage::TessellationControl) {
const Id invocation_id{ctx.OpLoad(ctx.U32[1], ctx.invocation_id)};
return ctx.OpAccessChain(result_type, base, invocation_id, std::forward<Args>(args)...);
} else {
return ctx.OpAccessChain(result_type, base, std::forward<Args>(args)...);
}
}
struct OutAttr {
OutAttr(Id pointer_) : pointer{pointer_} {}
OutAttr(Id pointer_, Id type_) : pointer{pointer_}, type{type_} {}
Id pointer{};
Id type{};
};
std::optional<OutAttr> OutputAttrPointer(EmitContext& ctx, IR::Attribute attr) {
if (IR::IsGeneric(attr)) {
const u32 index{IR::GenericAttributeIndex(attr)};
const u32 element{IR::GenericAttributeElement(attr)};
const GenericElementInfo& info{ctx.output_generics.at(index).at(element)};
if (info.num_components == 1) {
return info.id;
} else {
const u32 index_element{element - info.first_element};
const Id index_id{ctx.Const(index_element)};
return OutputAccessChain(ctx, ctx.output_f32, info.id, index_id);
}
}
switch (attr) {
case IR::Attribute::PointSize:
return ctx.output_point_size;
case IR::Attribute::PositionX:
case IR::Attribute::PositionY:
case IR::Attribute::PositionZ:
case IR::Attribute::PositionW: {
const u32 element{static_cast<u32>(attr) % 4};
const Id element_id{ctx.Const(element)};
return OutputAccessChain(ctx, ctx.output_f32, ctx.output_position, element_id);
}
case IR::Attribute::ClipDistance0:
case IR::Attribute::ClipDistance1:
case IR::Attribute::ClipDistance2:
case IR::Attribute::ClipDistance3:
case IR::Attribute::ClipDistance4:
case IR::Attribute::ClipDistance5:
case IR::Attribute::ClipDistance6:
case IR::Attribute::ClipDistance7: {
const u32 base{static_cast<u32>(IR::Attribute::ClipDistance0)};
const u32 index{static_cast<u32>(attr) - base};
if (index >= ctx.profile.max_user_clip_distances) {
LOG_WARNING(Shader, "Ignoring clip distance store {} >= {} supported", index,
ctx.profile.max_user_clip_distances);
return std::nullopt;
}
const Id clip_num{ctx.Const(index)};
return OutputAccessChain(ctx, ctx.output_f32, ctx.clip_distances, clip_num);
}
case IR::Attribute::Layer:
if (ctx.profile.support_viewport_index_layer_non_geometry ||
ctx.stage == Shader::Stage::Geometry) {
return OutAttr{ctx.layer, ctx.U32[1]};
}
return std::nullopt;
case IR::Attribute::ViewportIndex:
if (!ctx.profile.support_multi_viewport) {
LOG_WARNING(Shader, "Ignoring viewport index store on non-supporting driver");
return std::nullopt;
}
if (ctx.profile.support_viewport_index_layer_non_geometry ||
ctx.stage == Shader::Stage::Geometry) {
return OutAttr{ctx.viewport_index, ctx.U32[1]};
}
return std::nullopt;
case IR::Attribute::ViewportMask:
if (!ctx.profile.support_viewport_mask) {
return std::nullopt;
}
return OutAttr{ctx.OpAccessChain(ctx.output_u32, ctx.viewport_mask, ctx.u32_zero_value),
ctx.U32[1]};
default:
throw NotImplementedException("Read attribute {}", attr);
}
}
Id GetCbuf(EmitContext& ctx, Id result_type, Id UniformDefinitions::*member_ptr, u32 element_size,
const IR::Value& binding, const IR::Value& offset, const Id indirect_func) {
Id buffer_offset;
const Id uniform_type{ctx.uniform_types.*member_ptr};
if (offset.IsImmediate()) {
// Hardware been proved to read the aligned offset (e.g. LDC.U32 at 6 will read offset 4)
const Id imm_offset{ctx.Const(offset.U32() / element_size)};
buffer_offset = imm_offset;
} else if (element_size > 1) {
const u32 log2_element_size{static_cast<u32>(std::countr_zero(element_size))};
const Id shift{ctx.Const(log2_element_size)};
buffer_offset = ctx.OpShiftRightLogical(ctx.U32[1], ctx.Def(offset), shift);
} else {
buffer_offset = ctx.Def(offset);
}
if (!binding.IsImmediate()) {
return ctx.OpFunctionCall(result_type, indirect_func, ctx.Def(binding), buffer_offset);
}
const Id cbuf{ctx.cbufs[binding.U32()].*member_ptr};
const Id access_chain{ctx.OpAccessChain(uniform_type, cbuf, ctx.u32_zero_value, buffer_offset)};
const Id val = ctx.OpLoad(result_type, access_chain);
if (offset.IsImmediate() || !ctx.profile.has_broken_robust) {
return val;
}
const Id cond = ctx.OpULessThanEqual(ctx.TypeBool(), buffer_offset, ctx.Const(0xFFFFu));
const Id zero = ctx.ConstantNull(result_type);
return ctx.OpSelect(result_type, cond, val, zero);
}
Id GetCbufU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
return GetCbuf(ctx, ctx.U32[1], &UniformDefinitions::U32, sizeof(u32), binding, offset,
ctx.load_const_func_u32);
}
Id GetCbufU32x4(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
return GetCbuf(ctx, ctx.U32[4], &UniformDefinitions::U32x4, sizeof(u32[4]), binding, offset,
ctx.load_const_func_u32x4);
}
Id GetCbufElement(EmitContext& ctx, Id vector, const IR::Value& offset, u32 index_offset) {
if (offset.IsImmediate()) {
const u32 element{(offset.U32() / 4) % 4 + index_offset};
return ctx.OpCompositeExtract(ctx.U32[1], vector, element);
}
const Id shift{ctx.OpShiftRightLogical(ctx.U32[1], ctx.Def(offset), ctx.Const(2u))};
Id element{ctx.OpBitwiseAnd(ctx.U32[1], shift, ctx.Const(3u))};
if (index_offset > 0) {
element = ctx.OpIAdd(ctx.U32[1], element, ctx.Const(index_offset));
}
return ctx.OpVectorExtractDynamic(ctx.U32[1], vector, element);
}
} // Anonymous namespace
void EmitGetRegister(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitSetRegister(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetPred(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitSetPred(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitSetGotoVariable(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetGotoVariable(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitSetIndirectBranchVariable(EmitContext&) {
throw LogicError("Unreachable instruction");
}
void EmitGetIndirectBranchVariable(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitGetCbufU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int8 &&
ctx.profile.support_uniform_and_storage_buffer_8bit) {
const Id load{GetCbuf(ctx, ctx.U8, &UniformDefinitions::U8, sizeof(u8), binding, offset,
ctx.load_const_func_u8)};
return ctx.OpUConvert(ctx.U32[1], load);
}
Id element{};
if (ctx.profile.support_descriptor_aliasing) {
element = GetCbufU32(ctx, binding, offset);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
element = GetCbufElement(ctx, vector, offset, 0u);
}
const Id bit_offset{ctx.BitOffset8(offset)};
return ctx.OpBitFieldUExtract(ctx.U32[1], element, bit_offset, ctx.Const(8u));
}
Id EmitGetCbufS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int8 &&
ctx.profile.support_uniform_and_storage_buffer_8bit) {
const Id load{GetCbuf(ctx, ctx.S8, &UniformDefinitions::S8, sizeof(s8), binding, offset,
ctx.load_const_func_u8)};
return ctx.OpSConvert(ctx.U32[1], load);
}
Id element{};
if (ctx.profile.support_descriptor_aliasing) {
element = GetCbufU32(ctx, binding, offset);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
element = GetCbufElement(ctx, vector, offset, 0u);
}
const Id bit_offset{ctx.BitOffset8(offset)};
return ctx.OpBitFieldSExtract(ctx.U32[1], element, bit_offset, ctx.Const(8u));
}
Id EmitGetCbufU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int16 &&
ctx.profile.support_uniform_and_storage_buffer_16bit) {
const Id load{GetCbuf(ctx, ctx.U16, &UniformDefinitions::U16, sizeof(u16), binding, offset,
ctx.load_const_func_u16)};
return ctx.OpUConvert(ctx.U32[1], load);
}
Id element{};
if (ctx.profile.support_descriptor_aliasing) {
element = GetCbufU32(ctx, binding, offset);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
element = GetCbufElement(ctx, vector, offset, 0u);
}
const Id bit_offset{ctx.BitOffset16(offset)};
return ctx.OpBitFieldUExtract(ctx.U32[1], element, bit_offset, ctx.Const(16u));
}
Id EmitGetCbufS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int16 &&
ctx.profile.support_uniform_and_storage_buffer_16bit) {
const Id load{GetCbuf(ctx, ctx.S16, &UniformDefinitions::S16, sizeof(s16), binding, offset,
ctx.load_const_func_u16)};
return ctx.OpSConvert(ctx.U32[1], load);
}
Id element{};
if (ctx.profile.support_descriptor_aliasing) {
element = GetCbufU32(ctx, binding, offset);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
element = GetCbufElement(ctx, vector, offset, 0u);
}
const Id bit_offset{ctx.BitOffset16(offset)};
return ctx.OpBitFieldSExtract(ctx.U32[1], element, bit_offset, ctx.Const(16u));
}
Id EmitGetCbufU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing) {
return GetCbufU32(ctx, binding, offset);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
return GetCbufElement(ctx, vector, offset, 0u);
}
}
Id EmitGetCbufF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing) {
return GetCbuf(ctx, ctx.F32[1], &UniformDefinitions::F32, sizeof(f32), binding, offset,
ctx.load_const_func_f32);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
return ctx.OpBitcast(ctx.F32[1], GetCbufElement(ctx, vector, offset, 0u));
}
}
Id EmitGetCbufU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing) {
return GetCbuf(ctx, ctx.U32[2], &UniformDefinitions::U32x2, sizeof(u32[2]), binding, offset,
ctx.load_const_func_u32x2);
} else {
const Id vector{GetCbufU32x4(ctx, binding, offset)};
return ctx.OpCompositeConstruct(ctx.U32[2], GetCbufElement(ctx, vector, offset, 0u),
GetCbufElement(ctx, vector, offset, 1u));
}
}
Id EmitGetAttribute(EmitContext& ctx, IR::Attribute attr, Id vertex) {
const u32 element{static_cast<u32>(attr) % 4};
if (IR::IsGeneric(attr)) {
const u32 index{IR::GenericAttributeIndex(attr)};
if (ctx.stage == Stage::Fragment && ctx.runtime_info.point_coord_generics.test(index)) {
// (hookshot) the point sprite's coordinate in place of the generic (RuntimeInfo::point_coord_generics)
if (element < 2) {
return ctx.OpLoad(ctx.F32[1],
ctx.OpAccessChain(ctx.input_f32, ctx.point_coord, ctx.Const(element)));
}
return ctx.Const(element == 3 ? 1.0f : 0.0f);
}
const auto& generic{ctx.input_generics.at(index)};
if (!ValidId(generic.id)) {
// Attribute is disabled or varying component is not written
return ctx.Const(element == 3 ? 1.0f : 0.0f);
}
const Id pointer{
AttrPointer(ctx, generic.pointer_type, vertex, generic.id, ctx.Const(element))};
const Id value{ctx.OpLoad(generic.component_type, pointer)};
return [&ctx, generic, value]() {
switch (generic.load_op) {
case InputGenericLoadOp::Bitcast:
return ctx.OpBitcast(ctx.F32[1], value);
case InputGenericLoadOp::SToF:
return ctx.OpConvertSToF(ctx.F32[1], value);
case InputGenericLoadOp::UToF:
return ctx.OpConvertUToF(ctx.F32[1], value);
default:
return value;
};
}();
}
switch (attr) {
case IR::Attribute::PrimitiveId:
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.primitive_id));
case IR::Attribute::Layer:
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.layer));
case IR::Attribute::PositionX:
case IR::Attribute::PositionY:
case IR::Attribute::PositionZ:
case IR::Attribute::PositionW:
return ctx.OpLoad(
ctx.F32[1],
ctx.need_input_position_indirect
? AttrPointer(ctx, ctx.input_f32, vertex, ctx.input_position, ctx.u32_zero_value,
ctx.Const(element))
: AttrPointer(ctx, ctx.input_f32, vertex, ctx.input_position, ctx.Const(element)));
case IR::Attribute::InstanceId:
if (ctx.profile.support_vertex_instance_id) {
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.instance_id));
} else {
const Id index{ctx.OpLoad(ctx.U32[1], ctx.instance_index)};
const Id base{ctx.OpLoad(ctx.U32[1], ctx.base_instance)};
return ctx.OpBitcast(ctx.F32[1], ctx.OpISub(ctx.U32[1], index, base));
}
case IR::Attribute::VertexId:
if (ctx.profile.support_vertex_instance_id) {
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.vertex_id));
} else {
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.vertex_index));
}
case IR::Attribute::BaseInstance:
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.base_instance));
case IR::Attribute::BaseVertex:
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.base_vertex));
case IR::Attribute::DrawID:
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.draw_index));
case IR::Attribute::FrontFace:
return ctx.OpSelect(ctx.F32[1], ctx.OpLoad(ctx.U1, ctx.front_face),
ctx.OpBitcast(ctx.F32[1], ctx.Const((std::numeric_limits<u32>::max)())),
ctx.f32_zero_value);
case IR::Attribute::PointSpriteS:
return ctx.OpLoad(ctx.F32[1],
ctx.OpAccessChain(ctx.input_f32, ctx.point_coord, ctx.u32_zero_value));
case IR::Attribute::PointSpriteT:
return ctx.OpLoad(ctx.F32[1],
ctx.OpAccessChain(ctx.input_f32, ctx.point_coord, ctx.Const(1U)));
case IR::Attribute::TessellationEvaluationPointU:
return ctx.OpLoad(ctx.F32[1],
ctx.OpAccessChain(ctx.input_f32, ctx.tess_coord, ctx.u32_zero_value));
case IR::Attribute::TessellationEvaluationPointV:
return ctx.OpLoad(ctx.F32[1],
ctx.OpAccessChain(ctx.input_f32, ctx.tess_coord, ctx.Const(1U)));
default:
throw NotImplementedException("Read attribute {}", attr);
}
}
Id EmitGetAttributeU32(EmitContext& ctx, IR::Attribute attr, Id) {
switch (attr) {
case IR::Attribute::PrimitiveId:
return ctx.OpLoad(ctx.U32[1], ctx.primitive_id);
case IR::Attribute::InstanceId:
if (ctx.profile.support_vertex_instance_id) {
return ctx.OpLoad(ctx.U32[1], ctx.instance_id);
} else {
const Id index{ctx.OpLoad(ctx.U32[1], ctx.instance_index)};
const Id base{ctx.OpLoad(ctx.U32[1], ctx.base_instance)};
return ctx.OpISub(ctx.U32[1], index, base);
}
case IR::Attribute::VertexId:
if (ctx.profile.support_vertex_instance_id) {
return ctx.OpLoad(ctx.U32[1], ctx.vertex_id);
} else {
return ctx.OpLoad(ctx.U32[1], ctx.vertex_index);
}
case IR::Attribute::BaseInstance:
return ctx.OpLoad(ctx.U32[1], ctx.base_instance);
case IR::Attribute::BaseVertex:
return ctx.OpLoad(ctx.U32[1], ctx.base_vertex);
case IR::Attribute::DrawID:
return ctx.OpLoad(ctx.U32[1], ctx.draw_index);
default:
throw NotImplementedException("Read U32 attribute {}", attr);
}
}
void EmitSetAttribute(EmitContext& ctx, IR::Attribute attr, Id value, [[maybe_unused]] Id vertex) {
const std::optional<OutAttr> output{OutputAttrPointer(ctx, attr)};
if (!output) {
return;
}
if (Sirit::ValidId(output->type)) {
value = ctx.OpBitcast(output->type, value);
}
static constexpr IR::Attribute cd0 = IR::Attribute::ClipDistance0;
static constexpr IR::Attribute cd7 = IR::Attribute::ClipDistance7;
if (attr >= cd0 && attr <= cd7) {
const u32 idx = (u32) attr - (u32) cd0;
clip_distance_written.set(idx);
}
ctx.OpStore(output->pointer, value);
}
Id EmitGetAttributeIndexed(EmitContext& ctx, Id offset, Id vertex) {
switch (ctx.stage) {
case Stage::TessellationControl:
case Stage::TessellationEval:
case Stage::Geometry:
return ctx.OpFunctionCall(ctx.F32[1], ctx.indexed_load_func, offset, vertex);
default:
return ctx.OpFunctionCall(ctx.F32[1], ctx.indexed_load_func, offset);
}
}
void EmitSetAttributeIndexed(EmitContext& ctx, Id offset, Id value, [[maybe_unused]] Id vertex) {
ctx.OpFunctionCall(ctx.void_id, ctx.indexed_store_func, offset, value);
}
Id EmitGetPatch(EmitContext& ctx, IR::Patch patch) {
if (!IR::IsGeneric(patch)) {
throw NotImplementedException("Non-generic patch load");
}
const u32 index{IR::GenericPatchIndex(patch)};
const Id element{ctx.Const(IR::GenericPatchElement(patch))};
const Id type{ctx.stage == Stage::TessellationControl ? ctx.output_f32 : ctx.input_f32};
const Id pointer{ctx.OpAccessChain(type, ctx.patches.at(index), element)};
return ctx.OpLoad(ctx.F32[1], pointer);
}
void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value) {
const Id pointer{[&] {
if (IR::IsGeneric(patch)) {
const u32 index{IR::GenericPatchIndex(patch)};
const Id element{ctx.Const(IR::GenericPatchElement(patch))};
return ctx.OpAccessChain(ctx.output_f32, ctx.patches.at(index), element);
}
switch (patch) {
case IR::Patch::TessellationLodLeft:
case IR::Patch::TessellationLodRight:
case IR::Patch::TessellationLodTop:
case IR::Patch::TessellationLodBottom: {
const u32 index{static_cast<u32>(patch) - u32(IR::Patch::TessellationLodLeft)};
const Id index_id{ctx.Const(index)};
return ctx.OpAccessChain(ctx.output_f32, ctx.output_tess_level_outer, index_id);
}
case IR::Patch::TessellationLodInteriorU:
return ctx.OpAccessChain(ctx.output_f32, ctx.output_tess_level_inner,
ctx.u32_zero_value);
case IR::Patch::TessellationLodInteriorV:
return ctx.OpAccessChain(ctx.output_f32, ctx.output_tess_level_inner, ctx.Const(1u));
default:
throw NotImplementedException("Patch {}", patch);
}
}()};
ctx.OpStore(pointer, value);
}
void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value) {
const Id component_id{ctx.Const(component)};
const AttributeType type{ctx.runtime_info.color_output_types[index]};
if (type == AttributeType::Float) {
const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
if (frag_nan_guard) { // (amd-fix, mma_layout.h) NaN -> 0, the rest clamped to +-1e30
const Id clamped{ctx.OpNMin(ctx.F32[1], ctx.OpNMax(ctx.F32[1], value, ctx.Const(-1e30f)), ctx.Const(1e30f))};
value = ctx.OpSelect(ctx.F32[1], ctx.OpIsNan(ctx.U1, value), ctx.Const(0.0f), clamped);
}
ctx.OpStore(pointer, value);
} else if (type == AttributeType::UnsignedInt) {
const Id pointer{ctx.OpAccessChain(ctx.output_u32, ctx.frag_color.at(index), component_id)};
ctx.OpStore(pointer, ctx.OpBitcast(ctx.U32[1], value));
} else if (type == AttributeType::SignedInt) {
const Id output_s32{ctx.TypePointer(spv::StorageClass::Output, ctx.S32[1])};
const Id pointer{ctx.OpAccessChain(output_s32, ctx.frag_color.at(index), component_id)};
ctx.OpStore(pointer, ctx.OpBitcast(ctx.S32[1], value));
} else {
// Disabled or unknown, treat as float
const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
ctx.OpStore(pointer, value);
}
}
void EmitSetSampleMask(EmitContext& ctx, Id value) {
const Id pointer{ctx.OpAccessChain(ctx.output_u32, ctx.sample_mask, ctx.u32_zero_value)};
ctx.OpStore(pointer, value);
}
void EmitSetFragDepth(EmitContext& ctx, Id value) {
if (!ctx.runtime_info.convert_depth_mode || ctx.profile.support_native_ndc) {
ctx.OpStore(ctx.frag_depth, value);
return;
}
const Id unit{ctx.Const(0.5f)};
const Id new_depth{ctx.OpFma(ctx.F32[1], value, unit, unit)};
ctx.OpStore(ctx.frag_depth, new_depth);
}
void EmitGetZFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitGetSFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitGetCFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitGetOFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitSetZFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitSetSFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitSetCFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
void EmitSetOFlag(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitWorkgroupId(EmitContext& ctx) {
const Id id{ctx.OpLoad(ctx.U32[3], ctx.workgroup_id)};
if (ctx.mesh_fallback && ctx.stage == Stage::Mesh && ctx.runtime_info.mesh_fallback_task) {
// (hookshot) a fallback mesh workgroup is a slot (its task's map gives it, EmitMeshPrologue): its id is its index in its
// task, as a task launches them from 0
const Id slot{Sirit::ValidId(ctx.mesh_fallback_slot) ? ctx.mesh_fallback_slot
: ctx.OpCompositeExtract(ctx.U32[1], id, 0U)};
const Id x{ctx.OpUMod(ctx.U32[1], slot, ctx.Const(ctx.runtime_info.mesh_fallback_task_threads))};
return ctx.OpCompositeConstruct(ctx.U32[3], x, ctx.u32_zero_value, ctx.u32_zero_value);
}
return id;
}
Id EmitLocalInvocationId(EmitContext& ctx) {
return ctx.OpLoad(ctx.U32[3], ctx.local_invocation_id);
}
Id EmitInvocationId(EmitContext& ctx) {
return ctx.OpLoad(ctx.U32[1], ctx.invocation_id);
}
Id EmitInvocationInfo(EmitContext& ctx) {
switch (ctx.stage) {
case Stage::TessellationControl:
case Stage::TessellationEval:
return ctx.OpShiftLeftLogical(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.patch_vertices_in), ctx.Const(16u));
case Stage::Geometry:
return ctx.OpShiftLeftLogical(ctx.U32[1], ctx.Const(InputTopologyVertices::vertices(ctx.runtime_info.input_topology)), ctx.Const(16u));
default:
LOG_WARNING(Shader, "(STUBBED) called");
return ctx.Const(0x00ff0000u);
}
}
Id EmitSampleId(EmitContext& ctx) {
return ctx.OpLoad(ctx.U32[1], ctx.sample_id);
}
Id EmitIsHelperInvocation(EmitContext& ctx) {
return ctx.OpLoad(ctx.U1, ctx.is_helper_invocation);
}
Id EmitSR_WScaleFactorXY(EmitContext& ctx) {
LOG_WARNING(Shader, "(STUBBED) called");
return ctx.Const(0x00ff0000u);
}
Id EmitSR_WScaleFactorZ(EmitContext& ctx) {
LOG_WARNING(Shader, "(STUBBED) called");
return ctx.Const(0x00ff0000u);
}
Id EmitYDirection(EmitContext& ctx) {
return ctx.Const(ctx.runtime_info.y_negate ? -1.0f : 1.0f);
}
Id EmitResolutionDownFactor(EmitContext& ctx) {
if (ctx.profile.unified_descriptor_binding) {
const Id pointer_type{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.F32[1])};
const Id index{ctx.Const(ctx.rescaling_downfactor_member_index)};
const Id pointer{ctx.OpAccessChain(pointer_type, ctx.rescaling_push_constants, index)};
return ctx.OpLoad(ctx.F32[1], pointer);
} else {
const Id composite{ctx.OpLoad(ctx.F32[4], ctx.rescaling_uniform_constant)};
return ctx.OpCompositeExtract(ctx.F32[1], composite, 2u);
}
}
Id EmitRenderArea(EmitContext& ctx) {
if (ctx.profile.unified_descriptor_binding) {
const Id pointer_type{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.F32[4])};
const Id index{ctx.Const(ctx.render_are_member_index)};
const Id pointer{ctx.OpAccessChain(pointer_type, ctx.render_area_push_constant, index)};
return ctx.OpLoad(ctx.F32[4], pointer);
} else {
throw NotImplementedException("SPIR-V Instruction");
}
}
Id EmitLoadLocal(EmitContext& ctx, Id word_offset) {
const Id pointer{
ctx.OpAccessChain(ctx.private_u32, ctx.local_memory, word_offset, ctx.Const(0U))};
return ctx.OpLoad(ctx.U32[1], pointer);
}
void EmitWriteLocal(EmitContext& ctx, Id word_offset, Id value) {
const Id pointer{
ctx.OpAccessChain(ctx.private_u32, ctx.local_memory, word_offset, ctx.Const(0U))};
ctx.OpStore(pointer, value);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,27 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
void EmitJoin(EmitContext&) {
throw NotImplementedException("Join shouldn't be emitted");
}
void EmitDemoteToHelperInvocation(EmitContext& ctx) {
if (ctx.profile.support_demote_to_helper_invocation) {
ctx.OpDemoteToHelperInvocationEXT();
} else {
const Id kill_label{ctx.OpLabel()};
const Id impossible_label{ctx.OpLabel()};
ctx.OpSelectionMerge(impossible_label, spv::SelectionControlMask::MaskNone);
ctx.OpBranchConditional(ctx.true_value, kill_label, impossible_label);
ctx.AddLabel(kill_label);
ctx.OpKill();
ctx.AddLabel(impossible_label);
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,326 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
Id ExtractU16(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpUConvert(ctx.U16, value);
} else {
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
}
}
Id ExtractS16(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpSConvert(ctx.S16, value);
} else {
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
}
}
Id ExtractU8(EmitContext& ctx, Id value) {
if (ctx.profile.support_int8) {
return ctx.OpUConvert(ctx.U8, value);
} else {
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
}
}
Id ExtractS8(EmitContext& ctx, Id value) {
if (ctx.profile.support_int8) {
return ctx.OpSConvert(ctx.S8, value);
} else {
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
}
}
} // Anonymous namespace
Id EmitConvertS16F16(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpSConvert(ctx.U32[1], ctx.OpConvertFToS(ctx.U16, value));
} else {
return ExtractS16(ctx, ctx.OpConvertFToS(ctx.U32[1], value));
}
}
Id EmitConvertS16F32(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpSConvert(ctx.U32[1], ctx.OpConvertFToS(ctx.U16, value));
} else {
return ExtractS16(ctx, ctx.OpConvertFToS(ctx.U32[1], value));
}
}
Id EmitConvertS16F64(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpSConvert(ctx.U32[1], ctx.OpConvertFToS(ctx.U16, value));
} else {
return ExtractS16(ctx, ctx.OpConvertFToS(ctx.U32[1], value));
}
}
Id EmitConvertS32F16(EmitContext& ctx, Id value) {
return ctx.OpConvertFToS(ctx.U32[1], value);
}
Id EmitConvertS32F32(EmitContext& ctx, Id value) {
if (ctx.profile.has_broken_signed_operations) {
return ctx.OpBitcast(ctx.U32[1], ctx.OpConvertFToS(ctx.S32[1], value));
} else {
return ctx.OpConvertFToS(ctx.U32[1], value);
}
}
Id EmitConvertS32F64(EmitContext& ctx, Id value) {
return ctx.OpConvertFToS(ctx.U32[1], value);
}
Id EmitConvertS64F16(EmitContext& ctx, Id value) {
return ctx.OpConvertFToS(ctx.U64, value);
}
Id EmitConvertS64F32(EmitContext& ctx, Id value) {
return ctx.OpConvertFToS(ctx.U64, value);
}
Id EmitConvertS64F64(EmitContext& ctx, Id value) {
return ctx.OpConvertFToS(ctx.U64, value);
}
Id EmitConvertU16F16(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpUConvert(ctx.U32[1], ctx.OpConvertFToU(ctx.U16, value));
} else {
return ExtractU16(ctx, ctx.OpConvertFToU(ctx.U32[1], value));
}
}
Id EmitConvertU16F32(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpUConvert(ctx.U32[1], ctx.OpConvertFToU(ctx.U16, value));
} else {
return ExtractU16(ctx, ctx.OpConvertFToU(ctx.U32[1], value));
}
}
Id EmitConvertU16F64(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpUConvert(ctx.U32[1], ctx.OpConvertFToU(ctx.U16, value));
} else {
return ExtractU16(ctx, ctx.OpConvertFToU(ctx.U32[1], value));
}
}
Id EmitConvertU32F16(EmitContext& ctx, Id value) {
return ctx.OpConvertFToU(ctx.U32[1], value);
}
Id EmitConvertU32F32(EmitContext& ctx, Id value) {
return ctx.OpConvertFToU(ctx.U32[1], value);
}
Id EmitConvertU32F64(EmitContext& ctx, Id value) {
return ctx.OpConvertFToU(ctx.U32[1], value);
}
Id EmitConvertU64F16(EmitContext& ctx, Id value) {
return ctx.OpConvertFToU(ctx.U64, value);
}
Id EmitConvertU64F32(EmitContext& ctx, Id value) {
return ctx.OpConvertFToU(ctx.U64, value);
}
Id EmitConvertU64F64(EmitContext& ctx, Id value) {
return ctx.OpConvertFToU(ctx.U64, value);
}
Id EmitConvertU64U32(EmitContext& ctx, Id value) {
return ctx.OpUConvert(ctx.U64, value);
}
Id EmitConvertU32U64(EmitContext& ctx, Id value) {
return ctx.OpUConvert(ctx.U32[1], value);
}
Id EmitConvertF16F32(EmitContext& ctx, Id value) {
#ifdef __ANDROID__
return ctx.OpFConvert(ctx.F16[1], value);
#else
const auto result = ctx.OpFConvert(ctx.F16[1], value);
const auto isOverflowing = ctx.OpIsNan(ctx.U1, result);
return ctx.OpSelect(ctx.F16[1], isOverflowing, ctx.Constant(ctx.F16[1], 0), result);
#endif
}
Id EmitConvertF32F16(EmitContext& ctx, Id value) {
return ctx.OpFConvert(ctx.F32[1], value);
}
Id EmitConvertF32F64(EmitContext& ctx, Id value) {
return ctx.OpFConvert(ctx.F32[1], value);
}
Id EmitConvertF64F32(EmitContext& ctx, Id value) {
return ctx.OpFConvert(ctx.F64[1], value);
}
Id EmitConvertF16S8(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F16[1], ExtractS8(ctx, value));
}
Id EmitConvertF16S16(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F16[1], ExtractS16(ctx, value));
}
Id EmitConvertF16S32(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F16[1], value);
}
Id EmitConvertF16S64(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F16[1], value);
}
Id EmitConvertF16U8(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F16[1], ExtractU8(ctx, value));
}
Id EmitConvertF16U16(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F16[1], ExtractU16(ctx, value));
}
Id EmitConvertF16U32(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F16[1], value);
}
Id EmitConvertF16U64(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F16[1], value);
}
Id EmitConvertF32S8(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F32[1], ExtractS8(ctx, value));
}
Id EmitConvertF32S16(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F32[1], ExtractS16(ctx, value));
}
Id EmitConvertF32S32(EmitContext& ctx, Id value) {
if (ctx.profile.has_broken_signed_operations) {
value = ctx.OpBitcast(ctx.S32[1], value);
}
return ctx.OpConvertSToF(ctx.F32[1], value);
}
Id EmitConvertF32S64(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F32[1], value);
}
Id EmitConvertF32U8(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F32[1], ExtractU8(ctx, value));
}
Id EmitConvertF32U16(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F32[1], ExtractU16(ctx, value));
}
Id EmitConvertF32U32(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F32[1], value);
}
Id EmitConvertF32U64(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F32[1], value);
}
Id EmitConvertF64S8(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F64[1], ExtractS8(ctx, value));
}
Id EmitConvertF64S16(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F64[1], ExtractS16(ctx, value));
}
Id EmitConvertF64S32(EmitContext& ctx, Id value) {
if (ctx.profile.has_broken_signed_operations) {
value = ctx.OpBitcast(ctx.S32[1], value);
}
return ctx.OpConvertSToF(ctx.F64[1], value);
}
Id EmitConvertF64S64(EmitContext& ctx, Id value) {
return ctx.OpConvertSToF(ctx.F64[1], value);
}
Id EmitConvertF64U8(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F64[1], ExtractU8(ctx, value));
}
Id EmitConvertF64U16(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F64[1], ExtractU16(ctx, value));
}
Id EmitConvertF64U32(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F64[1], value);
}
Id EmitConvertF64U64(EmitContext& ctx, Id value) {
return ctx.OpConvertUToF(ctx.F64[1], value);
}
Id EmitConvertU16U32(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpUConvert(ctx.U16, value);
} else {
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
}
}
Id EmitConvertU32U16(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpUConvert(ctx.U32[1], value);
} else {
return ExtractU16(ctx, value);
}
}
Id EmitConvertU8U32(EmitContext& ctx, Id value) {
if (ctx.profile.support_int8) {
return ctx.OpUConvert(ctx.U8, value);
} else {
return ExtractU8(ctx, value);
}
}
Id EmitConvertU32U8(EmitContext& ctx, Id value) {
if (ctx.profile.support_int8) {
return ctx.OpUConvert(ctx.U32[1], value);
} else {
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
}
}
// in signed
Id EmitConvertS32S8(EmitContext& ctx, Id value) {
if (ctx.profile.support_int8) {
return ctx.OpSConvert(ctx.U32[1], value);
} else {
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
}
}
Id EmitConvertS32S16(EmitContext& ctx, Id value) {
if (ctx.profile.support_int16) {
return ctx.OpSConvert(ctx.U32[1], value);
} else {
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,396 @@
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
#include "shader_recompiler/frontend/ir/modifiers.h"
namespace Shader::Backend::SPIRV {
namespace {
Id Decorate(EmitContext& ctx, IR::Inst* inst, Id op) {
const auto flags{inst->Flags<IR::FpControl>()};
if (flags.no_contraction) {
ctx.Decorate(op, spv::Decoration::NoContraction);
}
return op;
}
// (amd-fix, Lichii #432/#437) The guest's FMNMX returns the operand that is not NaN and saturation turns NaN into 0.
// FMin/FMax/FClamp leave NaN undefined: NVIDIA happens to return the other operand, AMD returns the NaN, and in OoT's
// bloom chain it fills the screen (the yellow-white haze). NMin/NMax return the operand that is not NaN.
Id Clamp(EmitContext& ctx, Id type, Id value, Id zero, Id one) {
return ctx.OpNMin(type, ctx.OpNMax(type, value, zero), one);
}
Id FPOrdNotEqual(EmitContext& ctx, Id lhs, Id rhs) {
if (ctx.profile.ignore_nan_fp_comparisons) {
const Id comp{ctx.OpFOrdEqual(ctx.U1, lhs, rhs)};
const Id lhs_not_nan{ctx.OpLogicalNot(ctx.U1, ctx.OpIsNan(ctx.U1, lhs))};
const Id rhs_not_nan{ctx.OpLogicalNot(ctx.U1, ctx.OpIsNan(ctx.U1, rhs))};
return ctx.OpLogicalAnd(ctx.U1, ctx.OpLogicalAnd(ctx.U1, comp, lhs_not_nan), rhs_not_nan);
} else {
return ctx.OpFOrdNotEqual(ctx.U1, lhs, rhs);
}
}
Id FPUnordCompare(Id (EmitContext::*comp_func)(Id, Id, Id), EmitContext& ctx, Id lhs, Id rhs) {
if (ctx.profile.ignore_nan_fp_comparisons) {
const Id lhs_nan{ctx.OpIsNan(ctx.U1, lhs)};
const Id rhs_nan{ctx.OpIsNan(ctx.U1, rhs)};
const Id comp{(ctx.*comp_func)(ctx.U1, lhs, rhs)};
return ctx.OpLogicalOr(ctx.U1, ctx.OpLogicalOr(ctx.U1, comp, lhs_nan), rhs_nan);
} else {
return (ctx.*comp_func)(ctx.U1, lhs, rhs);
}
}
} // Anonymous namespace
Id EmitFPAbs16(EmitContext& ctx, Id value) {
return ctx.OpFAbs(ctx.F16[1], value);
}
Id EmitFPAbs32(EmitContext& ctx, Id value) {
return ctx.OpFAbs(ctx.F32[1], value);
}
Id EmitFPAbs64(EmitContext& ctx, Id value) {
return ctx.OpFAbs(ctx.F64[1], value);
}
Id EmitFPAdd16(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
return Decorate(ctx, inst, ctx.OpFAdd(ctx.F16[1], a, b));
}
Id EmitFPAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
return Decorate(ctx, inst, ctx.OpFAdd(ctx.F32[1], a, b));
}
Id EmitFPAdd64(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
return Decorate(ctx, inst, ctx.OpFAdd(ctx.F64[1], a, b));
}
Id EmitFPFma16(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c) {
return Decorate(ctx, inst, ctx.OpFma(ctx.F16[1], a, b, c));
}
Id EmitFPFma32(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c) {
return Decorate(ctx, inst, ctx.OpFma(ctx.F32[1], a, b, c));
}
Id EmitFPFma64(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c) {
return Decorate(ctx, inst, ctx.OpFma(ctx.F64[1], a, b, c));
}
Id EmitFPMax32(EmitContext& ctx, Id a, Id b) {
return ctx.OpNMax(ctx.F32[1], a, b);
}
Id EmitFPMax64(EmitContext& ctx, Id a, Id b) {
return ctx.OpNMax(ctx.F64[1], a, b);
}
Id EmitFPMin32(EmitContext& ctx, Id a, Id b) {
return ctx.OpNMin(ctx.F32[1], a, b);
}
Id EmitFPMin64(EmitContext& ctx, Id a, Id b) {
return ctx.OpNMin(ctx.F64[1], a, b);
}
Id EmitFPMul16(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
return Decorate(ctx, inst, ctx.OpFMul(ctx.F16[1], a, b));
}
Id EmitFPMul32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
return Decorate(ctx, inst, ctx.OpFMul(ctx.F32[1], a, b));
}
Id EmitFPMul64(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
return Decorate(ctx, inst, ctx.OpFMul(ctx.F64[1], a, b));
}
Id EmitFPNeg16(EmitContext& ctx, Id value) {
return ctx.OpFNegate(ctx.F16[1], value);
}
Id EmitFPNeg32(EmitContext& ctx, Id value) {
return ctx.OpFNegate(ctx.F32[1], value);
}
Id EmitFPNeg64(EmitContext& ctx, Id value) {
return ctx.OpFNegate(ctx.F64[1], value);
}
Id EmitFPSin(EmitContext& ctx, Id value) {
return ctx.OpSin(ctx.F32[1], value);
}
Id EmitFPCos(EmitContext& ctx, Id value) {
return ctx.OpCos(ctx.F32[1], value);
}
Id EmitFPExp2(EmitContext& ctx, Id value) {
return ctx.OpExp2(ctx.F32[1], value);
}
Id EmitFPLog2(EmitContext& ctx, Id value) {
return ctx.OpLog2(ctx.F32[1], value);
}
Id EmitFPRecip32(EmitContext& ctx, Id value) {
return ctx.OpFDiv(ctx.F32[1], ctx.Const(1.0f), value);
}
Id EmitFPRecip64(EmitContext& ctx, Id value) {
return ctx.OpFDiv(ctx.F64[1], ctx.Constant(ctx.F64[1], 1.0f), value);
}
Id EmitFPRecipSqrt32(EmitContext& ctx, Id value) {
return ctx.OpInverseSqrt(ctx.F32[1], value);
}
Id EmitFPRecipSqrt64(EmitContext& ctx, Id value) {
return ctx.OpInverseSqrt(ctx.F64[1], value);
}
Id EmitFPSqrt(EmitContext& ctx, Id value) {
return ctx.OpSqrt(ctx.F32[1], value);
}
Id EmitFPSaturate16(EmitContext& ctx, Id value) {
const Id zero{ctx.Constant(ctx.F16[1], u16{0})};
const Id one{ctx.Constant(ctx.F16[1], u16{0x3c00})};
return Clamp(ctx, ctx.F16[1], value, zero, one);
}
Id EmitFPSaturate32(EmitContext& ctx, Id value) {
const Id zero{ctx.Const(f32{0.0f})};
const Id one{ctx.Const(f32{1.0f})};
return Clamp(ctx, ctx.F32[1], value, zero, one);
}
Id EmitFPSaturate64(EmitContext& ctx, Id value) {
const Id zero{ctx.Constant(ctx.F64[1], f64{0.0})};
const Id one{ctx.Constant(ctx.F64[1], f64{1.0})};
return Clamp(ctx, ctx.F64[1], value, zero, one);
}
Id EmitFPClamp16(EmitContext& ctx, Id value, Id min_value, Id max_value) {
return Clamp(ctx, ctx.F16[1], value, min_value, max_value);
}
Id EmitFPClamp32(EmitContext& ctx, Id value, Id min_value, Id max_value) {
return Clamp(ctx, ctx.F32[1], value, min_value, max_value);
}
Id EmitFPClamp64(EmitContext& ctx, Id value, Id min_value, Id max_value) {
return Clamp(ctx, ctx.F64[1], value, min_value, max_value);
}
Id EmitFPRoundEven16(EmitContext& ctx, Id value) {
return ctx.OpRoundEven(ctx.F16[1], value);
}
Id EmitFPRoundEven32(EmitContext& ctx, Id value) {
return ctx.OpRoundEven(ctx.F32[1], value);
}
Id EmitFPRoundEven64(EmitContext& ctx, Id value) {
return ctx.OpRoundEven(ctx.F64[1], value);
}
Id EmitFPFloor16(EmitContext& ctx, Id value) {
return ctx.OpFloor(ctx.F16[1], value);
}
Id EmitFPFloor32(EmitContext& ctx, Id value) {
return ctx.OpFloor(ctx.F32[1], value);
}
Id EmitFPFloor64(EmitContext& ctx, Id value) {
return ctx.OpFloor(ctx.F64[1], value);
}
Id EmitFPCeil16(EmitContext& ctx, Id value) {
return ctx.OpCeil(ctx.F16[1], value);
}
Id EmitFPCeil32(EmitContext& ctx, Id value) {
return ctx.OpCeil(ctx.F32[1], value);
}
Id EmitFPCeil64(EmitContext& ctx, Id value) {
return ctx.OpCeil(ctx.F64[1], value);
}
Id EmitFPTrunc16(EmitContext& ctx, Id value) {
return ctx.OpTrunc(ctx.F16[1], value);
}
Id EmitFPTrunc32(EmitContext& ctx, Id value) {
return ctx.OpTrunc(ctx.F32[1], value);
}
Id EmitFPTrunc64(EmitContext& ctx, Id value) {
return ctx.OpTrunc(ctx.F64[1], value);
}
Id EmitFPOrdEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdEqual(ctx.U1, lhs, rhs);
}
Id EmitFPUnordEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordEqual, ctx, lhs, rhs);
}
Id EmitFPUnordEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordEqual, ctx, lhs, rhs);
}
Id EmitFPUnordEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordEqual, ctx, lhs, rhs);
}
Id EmitFPOrdNotEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return FPOrdNotEqual(ctx, lhs, rhs);
}
Id EmitFPOrdNotEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return FPOrdNotEqual(ctx, lhs, rhs);
}
Id EmitFPOrdNotEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return FPOrdNotEqual(ctx, lhs, rhs);
}
Id EmitFPUnordNotEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFUnordNotEqual(ctx.U1, lhs, rhs);
}
Id EmitFPUnordNotEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFUnordNotEqual(ctx.U1, lhs, rhs);
}
Id EmitFPUnordNotEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFUnordNotEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdLessThan16(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdLessThan(ctx.U1, lhs, rhs);
}
Id EmitFPOrdLessThan32(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdLessThan(ctx.U1, lhs, rhs);
}
Id EmitFPOrdLessThan64(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdLessThan(ctx.U1, lhs, rhs);
}
Id EmitFPUnordLessThan16(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordLessThan, ctx, lhs, rhs);
}
Id EmitFPUnordLessThan32(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordLessThan, ctx, lhs, rhs);
}
Id EmitFPUnordLessThan64(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordLessThan, ctx, lhs, rhs);
}
Id EmitFPOrdGreaterThan16(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdGreaterThan(ctx.U1, lhs, rhs);
}
Id EmitFPOrdGreaterThan32(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdGreaterThan(ctx.U1, lhs, rhs);
}
Id EmitFPOrdGreaterThan64(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdGreaterThan(ctx.U1, lhs, rhs);
}
Id EmitFPUnordGreaterThan16(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordGreaterThan, ctx, lhs, rhs);
}
Id EmitFPUnordGreaterThan32(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordGreaterThan, ctx, lhs, rhs);
}
Id EmitFPUnordGreaterThan64(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordGreaterThan, ctx, lhs, rhs);
}
Id EmitFPOrdLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdLessThanEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdLessThanEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdLessThanEqual(ctx.U1, lhs, rhs);
}
Id EmitFPUnordLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordLessThanEqual, ctx, lhs, rhs);
}
Id EmitFPUnordLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordLessThanEqual, ctx, lhs, rhs);
}
Id EmitFPUnordLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordLessThanEqual, ctx, lhs, rhs);
}
Id EmitFPOrdGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdGreaterThanEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdGreaterThanEqual(ctx.U1, lhs, rhs);
}
Id EmitFPOrdGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpFOrdGreaterThanEqual(ctx.U1, lhs, rhs);
}
Id EmitFPUnordGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordGreaterThanEqual, ctx, lhs, rhs);
}
Id EmitFPUnordGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordGreaterThanEqual, ctx, lhs, rhs);
}
Id EmitFPUnordGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
return FPUnordCompare(&EmitContext::OpFUnordGreaterThanEqual, ctx, lhs, rhs);
}
Id EmitFPIsNan16(EmitContext& ctx, Id value) {
return ctx.OpIsNan(ctx.U1, value);
}
Id EmitFPIsNan32(EmitContext& ctx, Id value) {
return ctx.OpIsNan(ctx.U1, value);
}
Id EmitFPIsNan64(EmitContext& ctx, Id value) {
return ctx.OpIsNan(ctx.U1, value);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,838 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <boost/container/static_vector.hpp>
#include "common/settings.h"
#include "shader_recompiler/backend/spirv/emit_spirv.h"
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
#include "shader_recompiler/frontend/ir/modifiers.h"
namespace Shader::Backend::SPIRV {
namespace {
enum class NonUniformKind {
SampledImage,
StorageImage,
UniformTexelBuffer,
StorageTexelBuffer,
};
[[nodiscard]] bool IsNonUniformSupported(const Profile& profile, NonUniformKind kind) noexcept {
switch (kind) {
case NonUniformKind::SampledImage:
return profile.support_sampled_image_array_nonuniform_indexing;
case NonUniformKind::StorageImage:
return profile.support_storage_image_array_nonuniform_indexing;
case NonUniformKind::UniformTexelBuffer:
return profile.support_uniform_texel_buffer_array_nonuniform_indexing;
case NonUniformKind::StorageTexelBuffer:
return profile.support_storage_texel_buffer_array_nonuniform_indexing;
}
return false;
}
void DecorateNonUniform(EmitContext& ctx, Id object) {
if (ctx.non_uniform_ids.contains(object.value)) {
return;
}
ctx.Decorate(object, spv::Decoration::NonUniform);
ctx.non_uniform_ids.insert(object.value);
}
[[nodiscard]] bool MarkNonUniform(EmitContext& ctx, Id idx, const IR::Value& index,
NonUniformKind kind) {
if (index.IsImmediate() || !IsNonUniformSupported(ctx.profile, kind)) {
return false;
}
DecorateNonUniform(ctx, idx);
switch (kind) {
case NonUniformKind::SampledImage:
ctx.uses_nonuniform_sampled_image = true;
break;
case NonUniformKind::StorageImage:
ctx.uses_nonuniform_storage_image = true;
break;
case NonUniformKind::UniformTexelBuffer:
ctx.uses_nonuniform_uniform_texel_buffer = true;
break;
case NonUniformKind::StorageTexelBuffer:
ctx.uses_nonuniform_storage_texel_buffer = true;
break;
}
return true;
}
class ImageOperands {
public:
[[maybe_unused]] static constexpr bool ImageSampleOffsetAllowed = false;
[[maybe_unused]] static constexpr bool ImageGatherOffsetAllowed = true;
[[maybe_unused]] static constexpr bool ImageFetchOffsetAllowed = false;
[[maybe_unused]] static constexpr bool ImageGradientOffsetAllowed = false;
explicit ImageOperands(EmitContext& ctx, bool has_bias, bool has_lod, bool has_lod_clamp,
Id lod, const IR::Value& offset) {
if (has_bias) {
const Id bias{has_lod_clamp ? ctx.OpCompositeExtract(ctx.F32[1], lod, 0) : lod};
Add(spv::ImageOperandsMask::Bias, bias);
}
if (has_lod) {
const Id lod_value{has_lod_clamp ? ctx.OpCompositeExtract(ctx.F32[1], lod, 0) : lod};
Add(spv::ImageOperandsMask::Lod, lod_value);
}
AddOffset(ctx, offset, ImageSampleOffsetAllowed);
if (has_lod_clamp) {
const Id lod_clamp{has_bias ? ctx.OpCompositeExtract(ctx.F32[1], lod, 1) : lod};
Add(spv::ImageOperandsMask::MinLod, lod_clamp);
}
}
explicit ImageOperands(EmitContext& ctx, const IR::Value& offset, const IR::Value& offset2) {
if (offset2.IsEmpty()) {
AddOffset(ctx, offset, ImageGatherOffsetAllowed);
return;
}
const std::array values{offset.InstRecursive(), offset2.InstRecursive()};
if (!values[0]->AreAllArgsImmediates() || !values[1]->AreAllArgsImmediates()) {
LOG_WARNING(Shader_SPIRV, "Not all arguments in PTP are immediate, ignoring");
return;
}
const IR::Opcode opcode{values[0]->GetOpcode()};
if (opcode != values[1]->GetOpcode() || opcode != IR::Opcode::CompositeConstructU32x4) {
throw LogicError("Invalid PTP arguments");
}
auto read{[&](unsigned int a, unsigned int b) { return static_cast<s32>(values[a]->Arg(b).U32()); }};
const Id offsets{ctx.ConstantComposite(
ctx.TypeArray(ctx.S32[2], ctx.Const(4U)), ctx.SConst(read(0, 0), read(0, 1)),
ctx.SConst(read(0, 2), read(0, 3)), ctx.SConst(read(1, 0), read(1, 1)),
ctx.SConst(read(1, 2), read(1, 3)))};
Add(spv::ImageOperandsMask::ConstOffsets, offsets);
}
explicit ImageOperands(Id lod, Id ms) {
if (Sirit::ValidId(lod)) {
Add(spv::ImageOperandsMask::Lod, lod);
}
if (Sirit::ValidId(ms)) {
Add(spv::ImageOperandsMask::Sample, ms);
}
}
explicit ImageOperands(EmitContext& ctx, bool has_lod_clamp, Id derivatives,
u32 num_derivatives, const IR::Value& offset, Id lod_clamp) {
if (!Sirit::ValidId(derivatives)) {
throw LogicError("Derivatives must be present");
}
boost::container::static_vector<Id, 3> deriv_x_accum;
boost::container::static_vector<Id, 3> deriv_y_accum;
for (u32 i = 0; i < num_derivatives; ++i) {
deriv_x_accum.push_back(ctx.OpCompositeExtract(ctx.F32[1], derivatives, i * 2));
deriv_y_accum.push_back(ctx.OpCompositeExtract(ctx.F32[1], derivatives, i * 2 + 1));
}
const Id derivatives_X{ctx.OpCompositeConstruct(
ctx.F32[num_derivatives], std::span{deriv_x_accum.data(), deriv_x_accum.size()})};
const Id derivatives_Y{ctx.OpCompositeConstruct(
ctx.F32[num_derivatives], std::span{deriv_y_accum.data(), deriv_y_accum.size()})};
Add(spv::ImageOperandsMask::Grad, derivatives_X, derivatives_Y);
AddOffset(ctx, offset, ImageGradientOffsetAllowed);
if (has_lod_clamp) {
Add(spv::ImageOperandsMask::MinLod, lod_clamp);
}
}
explicit ImageOperands(EmitContext& ctx, bool has_lod_clamp, Id derivatives_1, Id derivatives_2,
const IR::Value& offset, Id lod_clamp) {
if (!Sirit::ValidId(derivatives_1) || !Sirit::ValidId(derivatives_2)) {
throw LogicError("Derivatives must be present");
}
boost::container::static_vector<Id, 3> deriv_1_accum{
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 0),
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 2),
ctx.OpCompositeExtract(ctx.F32[1], derivatives_2, 0),
};
boost::container::static_vector<Id, 3> deriv_2_accum{
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 1),
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 3),
ctx.OpCompositeExtract(ctx.F32[1], derivatives_2, 1),
};
const Id derivatives_id1{ctx.OpCompositeConstruct(
ctx.F32[3], std::span{deriv_1_accum.data(), deriv_1_accum.size()})};
const Id derivatives_id2{ctx.OpCompositeConstruct(
ctx.F32[3], std::span{deriv_2_accum.data(), deriv_2_accum.size()})};
Add(spv::ImageOperandsMask::Grad, derivatives_id1, derivatives_id2);
AddOffset(ctx, offset, ImageGradientOffsetAllowed);
if (has_lod_clamp) {
Add(spv::ImageOperandsMask::MinLod, lod_clamp);
}
}
std::span<const Id> Span() const noexcept {
return std::span{operands.data(), operands.size()};
}
std::optional<spv::ImageOperandsMask> MaskOptional() const noexcept {
return mask != spv::ImageOperandsMask{} ? std::make_optional(mask) : std::nullopt;
}
spv::ImageOperandsMask Mask() const noexcept {
return mask;
}
private:
void AddOffset(EmitContext& ctx, const IR::Value& offset, bool runtime_offset_allowed) {
if (offset.IsEmpty()) {
return;
}
if (offset.IsImmediate()) {
Add(spv::ImageOperandsMask::ConstOffset, ctx.SConst(static_cast<s32>(offset.U32())));
return;
}
IR::Inst* const inst{offset.InstRecursive()};
if (inst->AreAllArgsImmediates()) {
switch (inst->GetOpcode()) {
case IR::Opcode::CompositeConstructU32x2:
Add(spv::ImageOperandsMask::ConstOffset,
ctx.SConst(static_cast<s32>(inst->Arg(0).U32()),
static_cast<s32>(inst->Arg(1).U32())));
return;
case IR::Opcode::CompositeConstructU32x3:
Add(spv::ImageOperandsMask::ConstOffset,
ctx.SConst(static_cast<s32>(inst->Arg(0).U32()),
static_cast<s32>(inst->Arg(1).U32()),
static_cast<s32>(inst->Arg(2).U32())));
return;
case IR::Opcode::CompositeConstructU32x4:
Add(spv::ImageOperandsMask::ConstOffset,
ctx.SConst(static_cast<s32>(inst->Arg(0).U32()),
static_cast<s32>(inst->Arg(1).U32()),
static_cast<s32>(inst->Arg(2).U32()),
static_cast<s32>(inst->Arg(3).U32())));
return;
default:
break;
}
}
if (runtime_offset_allowed) {
Add(spv::ImageOperandsMask::Offset, ctx.Def(offset));
}
}
void Add(spv::ImageOperandsMask new_mask, Id value) {
mask = static_cast<spv::ImageOperandsMask>(static_cast<unsigned>(mask) |
static_cast<unsigned>(new_mask));
operands.push_back(value);
}
void Add(spv::ImageOperandsMask new_mask, Id value_1, Id value_2) {
mask = static_cast<spv::ImageOperandsMask>(static_cast<unsigned>(mask) |
static_cast<unsigned>(new_mask));
operands.push_back(value_1);
operands.push_back(value_2);
}
boost::container::static_vector<Id, 4> operands;
spv::ImageOperandsMask mask{};
};
Id Texture(EmitContext& ctx, IR::TextureInstInfo info, [[maybe_unused]] const IR::Value& index) {
const TextureDefinition& def{ctx.textures.at(info.descriptor_index)};
if (def.count > 1) {
auto const idx = index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index);
const bool non_uniform{MarkNonUniform(ctx, idx, index, NonUniformKind::SampledImage)};
const Id pointer{ctx.OpAccessChain(def.pointer_type, def.id, idx)};
const Id object{ctx.OpLoad(def.sampled_type, pointer)};
if (non_uniform) {
DecorateNonUniform(ctx, pointer);
DecorateNonUniform(ctx, object);
}
return object;
} else {
return ctx.OpLoad(def.sampled_type, def.id);
}
}
Id TextureImage(EmitContext& ctx, IR::TextureInstInfo info, const IR::Value& index) {
if (info.type == TextureType::Buffer) {
const TextureBufferDefinition& def{ctx.texture_buffers.at(info.descriptor_index)};
if (def.count > 1) {
const Id idx{index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index)};
const bool non_uniform{
MarkNonUniform(ctx, idx, index, NonUniformKind::UniformTexelBuffer)};
const Id ptr{ctx.OpAccessChain(ctx.image_buffer_type, def.id, idx)};
const Id object{ctx.OpLoad(ctx.image_buffer_type, ptr)};
if (non_uniform) {
DecorateNonUniform(ctx, ptr);
DecorateNonUniform(ctx, object);
}
return object;
}
return ctx.OpLoad(ctx.image_buffer_type, def.id);
} else {
const TextureDefinition& def{ctx.textures.at(info.descriptor_index)};
if (def.count > 1) {
auto const idx = index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index);
const bool non_uniform{MarkNonUniform(ctx, idx, index, NonUniformKind::SampledImage)};
const Id ptr = ctx.OpAccessChain(def.pointer_type, def.id, idx);
const Id object = ctx.OpLoad(def.sampled_type, ptr);
const Id image = ctx.OpImage(def.image_type, object);
if (non_uniform) {
DecorateNonUniform(ctx, ptr);
DecorateNonUniform(ctx, object);
DecorateNonUniform(ctx, image);
}
return image;
}
return ctx.OpImage(def.image_type, ctx.OpLoad(def.sampled_type, def.id));
}
}
std::pair<Id, bool> Image(EmitContext& ctx, const IR::Value& index, IR::TextureInstInfo info) {
if (info.type == TextureType::Buffer) {
const ImageBufferDefinition def{ctx.image_buffers.at(info.descriptor_index)};
if (def.count > 1) {
const Id idx{index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index)};
const bool non_uniform{
MarkNonUniform(ctx, idx, index, NonUniformKind::StorageTexelBuffer)};
const Id ptr{ctx.OpAccessChain(def.pointer_type, def.id, idx)};
const Id image{ctx.OpLoad(def.image_type, ptr)};
if (non_uniform) {
DecorateNonUniform(ctx, ptr);
DecorateNonUniform(ctx, image);
}
return {image, def.is_integer};
}
return {ctx.OpLoad(def.image_type, def.id), def.is_integer};
} else {
const ImageDefinition def{ctx.images.at(info.descriptor_index)};
if (def.count > 1) {
const Id idx{index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index)};
const bool non_uniform{MarkNonUniform(ctx, idx, index, NonUniformKind::StorageImage)};
const Id ptr{ctx.OpAccessChain(def.pointer_type, def.id, idx)};
const Id image{ctx.OpLoad(def.image_type, ptr)};
if (non_uniform) {
DecorateNonUniform(ctx, ptr);
DecorateNonUniform(ctx, image);
}
return {image, def.is_integer};
}
return {ctx.OpLoad(def.image_type, def.id), def.is_integer};
}
}
bool IsTextureMsaa(EmitContext& ctx, const IR::TextureInstInfo& info) {
if (info.type == TextureType::Buffer) {
return false;
}
return ctx.textures.at(info.descriptor_index).is_multisample;
}
bool IsTextureInteger(EmitContext& ctx, const IR::TextureInstInfo& info) {
if (info.type == TextureType::Buffer) {
return false;
}
return ctx.textures.at(info.descriptor_index).is_integer;
}
Id Decorate(EmitContext& ctx, IR::Inst* inst, Id sample) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
if (info.relaxed_precision != 0) {
ctx.Decorate(sample, spv::Decoration::RelaxedPrecision);
}
return sample;
}
template <typename MethodPtrType, typename... Args>
Id Emit(MethodPtrType sparse_ptr, MethodPtrType non_sparse_ptr, EmitContext& ctx, IR::Inst* inst,
Id result_type, Args&&... args) {
IR::Inst* const sparse{inst->GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)};
if (!sparse) {
return Decorate(ctx, inst, (ctx.*non_sparse_ptr)(result_type, std::forward<Args>(args)...));
}
const Id struct_type{ctx.TypeStruct(ctx.U32[1], result_type)};
const Id sample{(ctx.*sparse_ptr)(struct_type, std::forward<Args>(args)...)};
const Id resident_code{ctx.OpCompositeExtract(ctx.U32[1], sample, 0U)};
sparse->SetDefinition(ctx.OpImageSparseTexelsResident(ctx.U1, resident_code));
sparse->Invalidate();
Decorate(ctx, inst, sample);
return ctx.OpCompositeExtract(result_type, sample, 1U);
}
bool IsDynamicPtp(IR::Inst* inst, const IR::Value& offset, const IR::Value& offset2) {
if (offset2.IsEmpty() || offset.IsEmpty() ||
inst->GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)) {
return false;
}
const auto constant{[](const IR::Value& value) {
return value.IsImmediate() || value.InstRecursive()->AreAllArgsImmediates();
}};
return !constant(offset) || !constant(offset2);
}
// Per-tap offsets the IR could not fold to constants (ConstOffsets must be constant): one gather
// per tap with a runtime Offset, keeping the i0,j0 texel (w) of each footprint, which is the
// texel a per-tap gather returns for that tap.
template <typename GatherFn>
Id GatherDynamicPtp(EmitContext& ctx, bool is_integer, Id result_type, const IR::Value& offset,
const IR::Value& offset2, GatherFn&& gather) {
const std::array packed{ctx.Def(offset), ctx.Def(offset2)};
const Id component_type{is_integer ? ctx.U32[1] : ctx.F32[1]};
std::array<Id, 4> texels;
for (u32 tap = 0; tap < 4; ++tap) {
const Id vector{packed[tap / 2]};
const u32 first{(tap % 2) * 2};
const Id x{ctx.OpBitcast(ctx.S32[1], ctx.OpCompositeExtract(ctx.U32[1], vector, first))};
const Id y{ctx.OpBitcast(ctx.S32[1], ctx.OpCompositeExtract(ctx.U32[1], vector, first + 1))};
const Id tap_offset{ctx.OpCompositeConstruct(ctx.S32[2], x, y)};
texels[tap] = ctx.OpCompositeExtract(component_type, gather(tap_offset), 3U);
}
return ctx.OpCompositeConstruct(result_type, std::span<const Id>{texels.data(), texels.size()});
}
Id IsScaled(EmitContext& ctx, const IR::Value& index, Id member_index, u32 base_index) {
const Id push_constant_u32{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1])};
Id bit{};
if (index.IsImmediate()) {
// Use BitwiseAnd instead of BitfieldExtract for better codegen on Nvidia OpenGL.
// LOP32I.NZ is used to set the predicate rather than BFE+ISETP.
const u32 index_value{index.U32() + base_index};
const Id word_index{ctx.Const(index_value / 32)};
const Id bit_index_mask{ctx.Const(1u << (index_value % 32))};
const Id pointer{ctx.OpAccessChain(push_constant_u32, ctx.rescaling_push_constants,
member_index, word_index)};
const Id word{ctx.OpLoad(ctx.U32[1], pointer)};
bit = ctx.OpBitwiseAnd(ctx.U32[1], word, bit_index_mask);
} else {
Id index_value{ctx.Def(index)};
if (base_index != 0) {
index_value = ctx.OpIAdd(ctx.U32[1], index_value, ctx.Const(base_index));
}
const Id word_index{ctx.OpShiftRightLogical(ctx.U32[1], index_value, ctx.Const(5u))};
const Id bit_index{ctx.OpBitwiseAnd(ctx.U32[1], index_value, ctx.Const(31u))};
const Id pointer{ctx.OpAccessChain(push_constant_u32, ctx.rescaling_push_constants,
member_index, word_index)};
const Id word{ctx.OpLoad(ctx.U32[1], pointer)};
const Id bit_index_mask{ctx.OpShiftLeftLogical(ctx.U32[1], ctx.Const(1u), bit_index)};
bit = ctx.OpBitwiseAnd(ctx.U32[1], word, bit_index_mask);
}
return ctx.OpINotEqual(ctx.U1, bit, ctx.u32_zero_value);
}
Id BitTest(EmitContext& ctx, Id mask, Id bit) {
const Id shifted{ctx.OpShiftRightLogical(ctx.U32[1], mask, bit)};
const Id bit_value{ctx.OpBitwiseAnd(ctx.U32[1], shifted, ctx.Const(1u))};
return ctx.OpINotEqual(ctx.U1, bit_value, ctx.u32_zero_value);
}
Id ImageGatherSubpixelOffset(EmitContext& ctx, const IR::TextureInstInfo& info, Id texture,
Id coords) {
// Apply a subpixel offset of 1/512 the texel size of the texture to ensure same rounding on
// AMD hardware as on Maxwell or other Nvidia architectures.
const auto calculate_coords{[&](size_t dim) {
const Id nudge{ctx.Const(0x1p-9f)};
const Id image_size{ctx.OpImageQuerySizeLod(ctx.U32[dim], texture, ctx.u32_zero_value)};
Id offset{dim == 2 ? ctx.ConstantComposite(ctx.F32[dim], nudge, nudge)
: ctx.ConstantComposite(ctx.F32[dim], nudge, nudge, ctx.f32_zero_value)};
offset = ctx.OpFDiv(ctx.F32[dim], offset, ctx.OpConvertUToF(ctx.F32[dim], image_size));
return ctx.OpFAdd(ctx.F32[dim], coords, offset);
}};
switch (info.type) {
case TextureType::Color2D:
case TextureType::Color2DRect:
return calculate_coords(2);
case TextureType::ColorArray2D:
case TextureType::ColorCube:
return calculate_coords(3);
default:
return coords;
}
}
void AddOffsetToCoordinates(EmitContext& ctx, const IR::TextureInstInfo& info, Id& coords,
Id offset) {
if (!Sirit::ValidId(offset)) {
return;
}
Id result_type{};
switch (info.type) {
case TextureType::Buffer:
case TextureType::Color1D: {
result_type = ctx.U32[1];
break;
}
case TextureType::ColorArray1D:
offset = ctx.OpCompositeConstruct(ctx.U32[2], offset, ctx.u32_zero_value);
[[fallthrough]];
case TextureType::Color2D:
case TextureType::Color2DRect: {
result_type = ctx.U32[2];
break;
}
case TextureType::ColorArray2D:
offset = ctx.OpCompositeConstruct(ctx.U32[3], ctx.OpCompositeExtract(ctx.U32[1], offset, 0),
ctx.OpCompositeExtract(ctx.U32[1], offset, 1),
ctx.u32_zero_value);
[[fallthrough]];
case TextureType::Color3D: {
result_type = ctx.U32[3];
break;
}
case TextureType::ColorCube:
case TextureType::ColorArrayCube:
return;
}
coords = ctx.OpIAdd(result_type, coords, offset);
}
} // Anonymous namespace
Id EmitBindlessImageSampleImplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageSampleExplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageSampleDrefImplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageSampleDrefExplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageGather(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageGatherDref(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageFetch(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageQueryDimensions(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageQueryLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageGradient(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageRead(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBindlessImageWrite(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageSampleImplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageSampleExplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageSampleDrefImplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageSampleDrefExplicitLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageGather(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageGatherDref(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageFetch(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageQueryDimensions(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageQueryLod(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageGradient(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageRead(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitBoundImageWrite(EmitContext&) {
throw LogicError("Unreachable instruction");
}
Id EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id bias_lc, const IR::Value& offset) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const bool is_integer{IsTextureInteger(ctx, info)};
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
Id color;
if (ctx.stage == Stage::Fragment) {
const ImageOperands operands(ctx, info.has_bias != 0, false, info.has_lod_clamp != 0,
bias_lc, offset);
color = Emit(&EmitContext::OpImageSparseSampleImplicitLod,
&EmitContext::OpImageSampleImplicitLod, ctx, inst, result_type,
Texture(ctx, info, index), coords, operands.MaskOptional(), operands.Span());
} else {
// We can't use implicit lods on non-fragment stages on SPIR-V. Maxwell hardware behaves as
// if the lod was explicitly zero. This may change on Turing with implicit compute
// derivatives
const Id lod{ctx.Const(0.0f)};
const ImageOperands operands(ctx, false, true, info.has_lod_clamp != 0, lod, offset);
color = Emit(&EmitContext::OpImageSparseSampleExplicitLod,
&EmitContext::OpImageSampleExplicitLod, ctx, inst, result_type,
Texture(ctx, info, index), coords, operands.Mask(), operands.Span());
}
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
Id EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id lod, const IR::Value& offset) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const bool is_integer{IsTextureInteger(ctx, info)};
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
const ImageOperands operands(ctx, false, true, false, lod, offset);
Id result = Emit(&EmitContext::OpImageSparseSampleExplicitLod,
&EmitContext::OpImageSampleExplicitLod, ctx, inst, result_type,
Texture(ctx, info, index), coords, operands.Mask(), operands.Span());
#ifdef __ANDROID__
if (!is_integer && Settings::values.fix_bloom_effects.GetValue()) {
result = ctx.OpVectorTimesScalar(ctx.F32[4], result, ctx.Const(0.98f));
}
#endif
return is_integer ? ctx.OpBitcast(ctx.F32[4], result) : result;
}
Id EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
Id coords, Id dref, Id bias_lc, const IR::Value& offset) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
if (ctx.stage == Stage::Fragment) {
const ImageOperands operands(ctx, info.has_bias != 0, false, info.has_lod_clamp != 0,
bias_lc, offset);
return Emit(&EmitContext::OpImageSparseSampleDrefImplicitLod,
&EmitContext::OpImageSampleDrefImplicitLod, ctx, inst, ctx.F32[1],
Texture(ctx, info, index), coords, dref, operands.MaskOptional(),
operands.Span());
} else {
// Implicit lods in compute behave on hardware as if sampling from LOD 0.
// This check is to ensure all drivers behave this way.
const Id lod{ctx.Const(0.0f)};
const ImageOperands operands(ctx, false, true, false, lod, offset);
return Emit(&EmitContext::OpImageSparseSampleDrefExplicitLod,
&EmitContext::OpImageSampleDrefExplicitLod, ctx, inst, ctx.F32[1],
Texture(ctx, info, index), coords, dref, operands.Mask(), operands.Span());
}
}
Id EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
Id coords, Id dref, Id lod, const IR::Value& offset) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const ImageOperands operands(ctx, false, true, false, lod, offset);
return Emit(&EmitContext::OpImageSparseSampleDrefExplicitLod,
&EmitContext::OpImageSampleDrefExplicitLod, ctx, inst, ctx.F32[1],
Texture(ctx, info, index), coords, dref, operands.Mask(), operands.Span());
}
Id EmitImageGather(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
const IR::Value& offset, const IR::Value& offset2) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const bool is_integer{IsTextureInteger(ctx, info)};
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
if (ctx.profile.need_gather_subpixel_offset) {
coords = ImageGatherSubpixelOffset(ctx, info, TextureImage(ctx, info, index), coords);
}
if (IsDynamicPtp(inst, offset, offset2)) {
const Id texture{Texture(ctx, info, index)};
const Id component{ctx.Const(info.gather_component)};
const Id color{GatherDynamicPtp(ctx, is_integer, result_type, offset, offset2, [&](Id tap_offset) {
return Decorate(ctx, inst,
ctx.OpImageGather(result_type, texture, coords, component,
spv::ImageOperandsMask::Offset,
std::span<const Id>{&tap_offset, 1}));
})};
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
const ImageOperands operands(ctx, offset, offset2);
const Id color{Emit(&EmitContext::OpImageSparseGather, &EmitContext::OpImageGather, ctx, inst,
result_type, Texture(ctx, info, index), coords,
ctx.Const(info.gather_component), operands.MaskOptional(),
operands.Span())};
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
Id EmitImageGatherDref(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
const IR::Value& offset, const IR::Value& offset2, Id dref) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const bool is_integer{IsTextureInteger(ctx, info)};
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
if (ctx.profile.need_gather_subpixel_offset) {
coords = ImageGatherSubpixelOffset(ctx, info, TextureImage(ctx, info, index), coords);
}
if (IsDynamicPtp(inst, offset, offset2)) {
const Id texture{Texture(ctx, info, index)};
const Id color{GatherDynamicPtp(ctx, is_integer, result_type, offset, offset2, [&](Id tap_offset) {
return Decorate(ctx, inst,
ctx.OpImageDrefGather(result_type, texture, coords, dref,
spv::ImageOperandsMask::Offset,
std::span<const Id>{&tap_offset, 1}));
})};
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
const ImageOperands operands(ctx, offset, offset2);
const Id color{Emit(&EmitContext::OpImageSparseDrefGather, &EmitContext::OpImageDrefGather,
ctx, inst, result_type, Texture(ctx, info, index), coords, dref,
operands.MaskOptional(), operands.Span())};
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
Id EmitImageFetch(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id offset,
Id lod, Id ms) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const bool is_integer{IsTextureInteger(ctx, info)};
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
AddOffsetToCoordinates(ctx, info, coords, offset);
if (info.type == TextureType::Buffer) {
lod = Id{};
}
if (Sirit::ValidId(ms)) {
// This image is multisampled, lod must be implicit
lod = Id{};
}
const ImageOperands operands(lod, ms);
const Id color{Emit(&EmitContext::OpImageSparseFetch, &EmitContext::OpImageFetch, ctx, inst,
result_type, TextureImage(ctx, info, index), coords,
operands.MaskOptional(), operands.Span())};
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
Id EmitImageQueryDimensions(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id lod,
const IR::Value& skip_mips_val) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const Id image{TextureImage(ctx, info, index)};
const Id zero{ctx.u32_zero_value};
const bool skip_mips{skip_mips_val.U1()};
const auto mips{[&] { return skip_mips ? zero : ctx.OpImageQueryLevels(ctx.U32[1], image); }};
const bool is_msaa{IsTextureMsaa(ctx, info)};
const bool uses_lod{!is_msaa && info.type != TextureType::Buffer};
const auto query{[&](Id type) {
return uses_lod ? ctx.OpImageQuerySizeLod(type, image, lod)
: ctx.OpImageQuerySize(type, image);
}};
switch (info.type) {
case TextureType::Color1D:
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[1]), zero, zero, mips());
case TextureType::ColorArray1D:
case TextureType::Color2D:
case TextureType::ColorCube:
case TextureType::Color2DRect:
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[2]), zero, mips());
case TextureType::ColorArray2D:
case TextureType::Color3D:
case TextureType::ColorArrayCube:
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[3]), mips());
case TextureType::Buffer:
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[1]), zero, zero, mips());
}
throw LogicError("Unspecified image type {}", info.type.Value());
}
Id EmitImageQueryLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const Id zero{ctx.f32_zero_value};
const Id sampler{Texture(ctx, info, index)};
return ctx.OpCompositeConstruct(ctx.F32[4], ctx.OpImageQueryLod(ctx.F32[2], sampler, coords),
zero, zero);
}
Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id derivatives, const IR::Value& offset, Id lod_clamp) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const bool is_integer{IsTextureInteger(ctx, info)};
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
const auto operands = info.num_derivatives == 3
? ImageOperands(ctx, info.has_lod_clamp != 0, derivatives,
ctx.Def(offset), {}, lod_clamp)
: ImageOperands(ctx, info.has_lod_clamp != 0, derivatives,
info.num_derivatives, offset, lod_clamp);
const Id color{Emit(&EmitContext::OpImageSparseSampleExplicitLod,
&EmitContext::OpImageSampleExplicitLod, ctx, inst, result_type,
Texture(ctx, info, index), coords, operands.Mask(), operands.Span())};
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
}
Id EmitImageRead(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
if (info.image_format == ImageFormat::Typeless && !ctx.profile.support_typeless_image_loads) {
LOG_WARNING(Shader_SPIRV, "Typeless image read not supported by host");
return ctx.ConstantNull(ctx.U32[4]);
}
const auto [image, is_integer] = Image(ctx, index, info);
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
Id color{Emit(&EmitContext::OpImageSparseRead, &EmitContext::OpImageRead, ctx, inst,
result_type, image, coords, std::nullopt, std::span<const Id>{})};
if (!is_integer) {
color = ctx.OpBitcast(ctx.U32[4], color);
}
return color;
}
void EmitImageWrite(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id color) {
const auto info{inst->Flags<IR::TextureInstInfo>()};
const auto [image, is_integer] = Image(ctx, index, info);
if (!is_integer) {
color = ctx.OpBitcast(ctx.F32[4], color);
}
ctx.OpImageWrite(image, coords, color);
}
Id EmitIsTextureScaled(EmitContext& ctx, const IR::Value& index) {
if (ctx.profile.unified_descriptor_binding) {
const Id member_index{ctx.Const(ctx.rescaling_textures_member_index)};
return IsScaled(ctx, index, member_index, ctx.texture_rescaling_index);
} else {
const Id composite{ctx.OpLoad(ctx.F32[4], ctx.rescaling_uniform_constant)};
const Id mask_f32{ctx.OpCompositeExtract(ctx.F32[1], composite, 0u)};
const Id mask{ctx.OpBitcast(ctx.U32[1], mask_f32)};
return BitTest(ctx, mask, ctx.Def(index));
}
}
Id EmitIsImageScaled(EmitContext& ctx, const IR::Value& index) {
if (ctx.profile.unified_descriptor_binding) {
const Id member_index{ctx.Const(ctx.rescaling_images_member_index)};
return IsScaled(ctx, index, member_index, ctx.image_rescaling_index);
} else {
const Id composite{ctx.OpLoad(ctx.F32[4], ctx.rescaling_uniform_constant)};
const Id mask_f32{ctx.OpCompositeExtract(ctx.F32[1], composite, 1u)};
const Id mask{ctx.OpBitcast(ctx.U32[1], mask_f32)};
return BitTest(ctx, mask, ctx.Def(index));
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,183 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
#include "shader_recompiler/frontend/ir/modifiers.h"
namespace Shader::Backend::SPIRV {
namespace {
Id Image(EmitContext& ctx, IR::TextureInstInfo info) {
if (info.type == TextureType::Buffer) {
const ImageBufferDefinition def{ctx.image_buffers.at(info.descriptor_index)};
return def.id;
} else {
const ImageDefinition def{ctx.images.at(info.descriptor_index)};
return def.id;
}
}
std::pair<Id, Id> AtomicArgs(EmitContext& ctx) {
const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Device))};
const Id semantics{ctx.u32_zero_value};
return {scope, semantics};
}
Id ImageAtomicU32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id value,
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
if (!index.IsImmediate() || index.U32() != 0) {
// TODO: handle layers
throw NotImplementedException("Image indexing");
}
const auto info{inst->Flags<IR::TextureInstInfo>()};
const Id image{Image(ctx, info)};
const Id pointer{ctx.OpImageTexelPointer(ctx.image_u32, image, coords, ctx.Const(0U))};
const auto [scope, semantics]{AtomicArgs(ctx)};
return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
}
} // Anonymous namespace
Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicIAdd);
}
Id EmitImageAtomicSMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicSMin);
}
Id EmitImageAtomicUMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicUMin);
}
Id EmitImageAtomicSMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicSMax);
}
Id EmitImageAtomicUMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicUMax);
}
Id EmitImageAtomicInc32(EmitContext&, IR::Inst*, const IR::Value&, Id, Id) {
// TODO: This is not yet implemented
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitImageAtomicDec32(EmitContext&, IR::Inst*, const IR::Value&, Id, Id) {
// TODO: This is not yet implemented
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitImageAtomicAnd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicAnd);
}
Id EmitImageAtomicOr32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicOr);
}
Id EmitImageAtomicXor32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicXor);
}
Id EmitImageAtomicExchange32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value) {
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicExchange);
}
Id EmitBindlessImageAtomicIAdd32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicSMin32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicUMin32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicSMax32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicUMax32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicInc32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicDec32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicAnd32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicOr32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicXor32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBindlessImageAtomicExchange32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicIAdd32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicSMin32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicUMin32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicSMax32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicUMax32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicInc32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicDec32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicAnd32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicOr32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicXor32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitBoundImageAtomicExchange32(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,647 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <sirit/sirit.h>
#include "common/common_types.h"
namespace Shader::IR {
enum class Attribute : u64;
enum class Patch : u64;
class Inst;
class Value;
} // namespace Shader::IR
namespace Shader::Backend::SPIRV {
using Sirit::Id;
class EmitContext;
// Microinstruction emitters
Id EmitPhi(EmitContext& ctx, IR::Inst* inst);
void EmitVoid(EmitContext& ctx);
Id EmitIdentity(EmitContext& ctx, const IR::Value& value);
Id EmitConditionRef(EmitContext& ctx, const IR::Value& value);
void EmitReference(EmitContext&);
void EmitPhiMove(EmitContext&);
void EmitJoin(EmitContext& ctx);
void EmitDemoteToHelperInvocation(EmitContext& ctx);
void EmitBarrier(EmitContext& ctx);
void EmitSubgroupBarrier(EmitContext& ctx);
void EmitWorkgroupMemoryBarrier(EmitContext& ctx);
void EmitDeviceMemoryBarrier(EmitContext& ctx);
void EmitPrologue(EmitContext& ctx);
void EmitEpilogue(EmitContext& ctx);
void EmitEmitVertex(EmitContext& ctx, const IR::Value& stream);
void EmitEndPrimitive(EmitContext& ctx, const IR::Value& stream);
void EmitGetRegister(EmitContext& ctx);
void EmitSetRegister(EmitContext& ctx);
void EmitGetPred(EmitContext& ctx);
void EmitSetPred(EmitContext& ctx);
void EmitSetGotoVariable(EmitContext& ctx);
void EmitGetGotoVariable(EmitContext& ctx);
void EmitSetIndirectBranchVariable(EmitContext& ctx);
void EmitGetIndirectBranchVariable(EmitContext& ctx);
Id EmitGetCbufU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetCbufS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetCbufU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetCbufS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetCbufU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetCbufF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetCbufU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitGetAttribute(EmitContext& ctx, IR::Attribute attr, Id vertex);
Id EmitGetAttributeU32(EmitContext& ctx, IR::Attribute attr, Id vertex);
void EmitSetAttribute(EmitContext& ctx, IR::Attribute attr, Id value, Id vertex);
Id EmitGetAttributeIndexed(EmitContext& ctx, Id offset, Id vertex);
void EmitSetAttributeIndexed(EmitContext& ctx, Id offset, Id value, Id vertex);
Id EmitGetPatch(EmitContext& ctx, IR::Patch patch);
void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value);
Id EmitIsbeRead(EmitContext& ctx, Id address);
void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value);
void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value);
void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value);
void EmitMeshPrologue(EmitContext& ctx);
void EmitMeshEpilogue(EmitContext& ctx);
// (hookshot) a GPU without mesh shaders: the generated vertex stage that reads the mesh epilogue's records
void EmitMeshFallbackVertexBody(EmitContext& ctx);
void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value);
void EmitSetSampleMask(EmitContext& ctx, Id value);
void EmitSetFragDepth(EmitContext& ctx, Id value);
void EmitGetZFlag(EmitContext& ctx);
void EmitGetSFlag(EmitContext& ctx);
void EmitGetCFlag(EmitContext& ctx);
void EmitGetOFlag(EmitContext& ctx);
void EmitSetZFlag(EmitContext& ctx);
void EmitSetSFlag(EmitContext& ctx);
void EmitSetCFlag(EmitContext& ctx);
void EmitSetOFlag(EmitContext& ctx);
Id EmitWorkgroupId(EmitContext& ctx);
Id EmitLocalInvocationId(EmitContext& ctx);
Id EmitInvocationId(EmitContext& ctx);
Id EmitInvocationInfo(EmitContext& ctx);
Id EmitSampleId(EmitContext& ctx);
Id EmitIsHelperInvocation(EmitContext& ctx);
Id EmitSR_WScaleFactorXY(EmitContext& ctx);
Id EmitSR_WScaleFactorZ(EmitContext& ctx);
Id EmitYDirection(EmitContext& ctx);
Id EmitResolutionDownFactor(EmitContext& ctx);
Id EmitRenderArea(EmitContext& ctx);
Id EmitLoadLocal(EmitContext& ctx, Id word_offset);
void EmitWriteLocal(EmitContext& ctx, Id word_offset, Id value);
Id EmitUndefU1(EmitContext& ctx);
Id EmitUndefU8(EmitContext& ctx);
Id EmitUndefU16(EmitContext& ctx);
Id EmitUndefU32(EmitContext& ctx);
Id EmitUndefU64(EmitContext& ctx);
Id EmitLoadGlobalU8(EmitContext& ctx, Id address);
Id EmitLoadGlobalS8(EmitContext& ctx, Id address);
Id EmitLoadGlobalU16(EmitContext& ctx, Id address);
Id EmitLoadGlobalS16(EmitContext& ctx, Id address);
Id EmitLoadGlobal32(EmitContext& ctx, Id address);
Id EmitLoadGlobal64(EmitContext& ctx, Id address);
Id EmitLoadGlobal128(EmitContext& ctx, Id address);
void EmitWriteGlobalU8(EmitContext& ctx, Id address, Id value);
void EmitWriteGlobalS8(EmitContext& ctx, Id address, Id value);
void EmitWriteGlobalU16(EmitContext& ctx, Id address, Id value);
void EmitWriteGlobalS16(EmitContext& ctx, Id address, Id value);
void EmitWriteGlobal32(EmitContext& ctx, Id address, Id value);
void EmitWriteGlobal64(EmitContext& ctx, Id address, Id value);
void EmitWriteGlobal128(EmitContext& ctx, Id address, Id value);
Id EmitLoadStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitLoadStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitLoadStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitLoadStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitLoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitLoadStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
Id EmitLoadStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
void EmitWriteStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
void EmitWriteStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
void EmitWriteStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
void EmitWriteStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
void EmitWriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
void EmitWriteStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
void EmitWriteStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitLoadSharedU8(EmitContext& ctx, Id offset);
Id EmitLoadSharedS8(EmitContext& ctx, Id offset);
Id EmitLoadSharedU16(EmitContext& ctx, Id offset);
Id EmitLoadSharedS16(EmitContext& ctx, Id offset);
Id EmitLoadSharedU32(EmitContext& ctx, Id offset);
Id EmitLoadSharedU64(EmitContext& ctx, Id offset);
Id EmitLoadSharedU128(EmitContext& ctx, Id offset);
void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value);
void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value);
void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value);
void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value);
void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value);
Id EmitCompositeConstructU32x2(EmitContext& ctx, Id e1, Id e2);
Id EmitCompositeConstructU32x3(EmitContext& ctx, Id e1, Id e2, Id e3);
Id EmitCompositeConstructU32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4);
Id EmitCompositeExtractU32x2(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeExtractU32x3(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeExtractU32x4(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeInsertU32x2(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertU32x3(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertU32x4(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeConstructF16x2(EmitContext& ctx, Id e1, Id e2);
Id EmitCompositeConstructF16x3(EmitContext& ctx, Id e1, Id e2, Id e3);
Id EmitCompositeConstructF16x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4);
Id EmitCompositeExtractF16x2(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeExtractF16x3(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeExtractF16x4(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeInsertF16x2(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertF16x3(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertF16x4(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeConstructF32x2(EmitContext& ctx, Id e1, Id e2);
Id EmitCompositeConstructF32x3(EmitContext& ctx, Id e1, Id e2, Id e3);
Id EmitCompositeConstructF32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4);
Id EmitCompositeExtractF32x2(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeExtractF32x3(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeExtractF32x4(EmitContext& ctx, Id composite, u32 index);
Id EmitCompositeInsertF32x2(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertF32x3(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertF32x4(EmitContext& ctx, Id composite, Id object, u32 index);
void EmitCompositeConstructF64x2(EmitContext& ctx);
void EmitCompositeConstructF64x3(EmitContext& ctx);
void EmitCompositeConstructF64x4(EmitContext& ctx);
void EmitCompositeExtractF64x2(EmitContext& ctx);
void EmitCompositeExtractF64x3(EmitContext& ctx);
void EmitCompositeExtractF64x4(EmitContext& ctx);
Id EmitCompositeInsertF64x2(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertF64x3(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitCompositeInsertF64x4(EmitContext& ctx, Id composite, Id object, u32 index);
Id EmitSelectU1(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectU8(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectU16(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectU32(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectU64(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectF16(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectF32(EmitContext& ctx, Id cond, Id true_value, Id false_value);
Id EmitSelectF64(EmitContext& ctx, Id cond, Id true_value, Id false_value);
void EmitBitCastU16F16(EmitContext& ctx);
Id EmitBitCastU32F32(EmitContext& ctx, Id value);
void EmitBitCastU64F64(EmitContext& ctx);
void EmitBitCastF16U16(EmitContext&);
Id EmitBitCastF32U32(EmitContext& ctx, Id value);
void EmitBitCastF64U64(EmitContext& ctx);
Id EmitPackUint2x32(EmitContext& ctx, Id value);
Id EmitUnpackUint2x32(EmitContext& ctx, Id value);
Id EmitPackFloat2x16(EmitContext& ctx, Id value);
Id EmitUnpackFloat2x16(EmitContext& ctx, Id value);
Id EmitPackHalf2x16(EmitContext& ctx, Id value);
Id EmitUnpackHalf2x16(EmitContext& ctx, Id value);
Id EmitPackDouble2x32(EmitContext& ctx, Id value);
Id EmitUnpackDouble2x32(EmitContext& ctx, Id value);
void EmitGetZeroFromOp(EmitContext& ctx);
void EmitGetSignFromOp(EmitContext& ctx);
void EmitGetCarryFromOp(EmitContext& ctx);
void EmitGetOverflowFromOp(EmitContext& ctx);
void EmitGetSparseFromOp(EmitContext& ctx);
void EmitGetInBoundsFromOp(EmitContext& ctx);
Id EmitFPAbs16(EmitContext& ctx, Id value);
Id EmitFPAbs32(EmitContext& ctx, Id value);
Id EmitFPAbs64(EmitContext& ctx, Id value);
Id EmitFPAdd16(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitFPAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitFPAdd64(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitFPFma16(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c);
Id EmitFPFma32(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c);
Id EmitFPFma64(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c);
Id EmitFPMax32(EmitContext& ctx, Id a, Id b);
Id EmitFPMax64(EmitContext& ctx, Id a, Id b);
Id EmitFPMin32(EmitContext& ctx, Id a, Id b);
Id EmitFPMin64(EmitContext& ctx, Id a, Id b);
Id EmitFPMul16(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitFPMul32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitFPMul64(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitFPNeg16(EmitContext& ctx, Id value);
Id EmitFPNeg32(EmitContext& ctx, Id value);
Id EmitFPNeg64(EmitContext& ctx, Id value);
Id EmitFPSin(EmitContext& ctx, Id value);
Id EmitFPCos(EmitContext& ctx, Id value);
Id EmitFPExp2(EmitContext& ctx, Id value);
Id EmitFPLog2(EmitContext& ctx, Id value);
Id EmitFPRecip32(EmitContext& ctx, Id value);
Id EmitFPRecip64(EmitContext& ctx, Id value);
Id EmitFPRecipSqrt32(EmitContext& ctx, Id value);
Id EmitFPRecipSqrt64(EmitContext& ctx, Id value);
Id EmitFPSqrt(EmitContext& ctx, Id value);
Id EmitFPSaturate16(EmitContext& ctx, Id value);
Id EmitFPSaturate32(EmitContext& ctx, Id value);
Id EmitFPSaturate64(EmitContext& ctx, Id value);
Id EmitFPClamp16(EmitContext& ctx, Id value, Id min_value, Id max_value);
Id EmitFPClamp32(EmitContext& ctx, Id value, Id min_value, Id max_value);
Id EmitFPClamp64(EmitContext& ctx, Id value, Id min_value, Id max_value);
Id EmitFPRoundEven16(EmitContext& ctx, Id value);
Id EmitFPRoundEven32(EmitContext& ctx, Id value);
Id EmitFPRoundEven64(EmitContext& ctx, Id value);
Id EmitFPFloor16(EmitContext& ctx, Id value);
Id EmitFPFloor32(EmitContext& ctx, Id value);
Id EmitFPFloor64(EmitContext& ctx, Id value);
Id EmitFPCeil16(EmitContext& ctx, Id value);
Id EmitFPCeil32(EmitContext& ctx, Id value);
Id EmitFPCeil64(EmitContext& ctx, Id value);
Id EmitFPTrunc16(EmitContext& ctx, Id value);
Id EmitFPTrunc32(EmitContext& ctx, Id value);
Id EmitFPTrunc64(EmitContext& ctx, Id value);
Id EmitFPOrdEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdNotEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdNotEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdNotEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordNotEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordNotEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordNotEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdLessThan16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdLessThan32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdLessThan64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordLessThan16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordLessThan32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordLessThan64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdGreaterThan16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdGreaterThan32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdGreaterThan64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordGreaterThan16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordGreaterThan32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordGreaterThan64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPOrdGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPUnordGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
Id EmitFPIsNan16(EmitContext& ctx, Id value);
Id EmitFPIsNan32(EmitContext& ctx, Id value);
Id EmitFPIsNan64(EmitContext& ctx, Id value);
Id EmitIAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitIAdd64(EmitContext& ctx, Id a, Id b);
Id EmitISub32(EmitContext& ctx, Id a, Id b);
Id EmitISub64(EmitContext& ctx, Id a, Id b);
Id EmitIMul32(EmitContext& ctx, Id a, Id b);
Id EmitSDiv32(EmitContext& ctx, Id a, Id b);
Id EmitUDiv32(EmitContext& ctx, Id a, Id b);
Id EmitINeg32(EmitContext& ctx, Id value);
Id EmitINeg64(EmitContext& ctx, Id value);
Id EmitIAbs32(EmitContext& ctx, Id value);
Id EmitIAbs64(EmitContext& ctx, Id value);
Id EmitShiftLeftLogical32(EmitContext& ctx, Id base, Id shift);
Id EmitShiftLeftLogical64(EmitContext& ctx, Id base, Id shift);
Id EmitShiftRightLogical32(EmitContext& ctx, Id base, Id shift);
Id EmitShiftRightLogical64(EmitContext& ctx, Id base, Id shift);
Id EmitShiftRightArithmetic32(EmitContext& ctx, Id base, Id shift);
Id EmitShiftRightArithmetic64(EmitContext& ctx, Id base, Id shift);
Id EmitBitwiseAnd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitBitwiseOr32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitBitwiseXor32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
Id EmitBitFieldInsert(EmitContext& ctx, Id base, Id insert, Id offset, Id count);
Id EmitBitFieldSExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count);
Id EmitBitFieldUExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count);
Id EmitBitReverse32(EmitContext& ctx, Id value);
Id EmitBitCount32(EmitContext& ctx, Id value);
Id EmitBitwiseNot32(EmitContext& ctx, Id value);
Id EmitFindSMsb32(EmitContext& ctx, Id value);
Id EmitFindUMsb32(EmitContext& ctx, Id value);
Id EmitSMin32(EmitContext& ctx, Id a, Id b);
Id EmitUMin32(EmitContext& ctx, Id a, Id b);
Id EmitSMax32(EmitContext& ctx, Id a, Id b);
Id EmitUMax32(EmitContext& ctx, Id a, Id b);
Id EmitSClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max);
Id EmitUClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max);
Id EmitSLessThan(EmitContext& ctx, Id lhs, Id rhs);
Id EmitULessThan(EmitContext& ctx, Id lhs, Id rhs);
Id EmitIEqual(EmitContext& ctx, Id lhs, Id rhs);
Id EmitSLessThanEqual(EmitContext& ctx, Id lhs, Id rhs);
Id EmitULessThanEqual(EmitContext& ctx, Id lhs, Id rhs);
Id EmitSGreaterThan(EmitContext& ctx, Id lhs, Id rhs);
Id EmitUGreaterThan(EmitContext& ctx, Id lhs, Id rhs);
Id EmitINotEqual(EmitContext& ctx, Id lhs, Id rhs);
Id EmitSGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs);
Id EmitUGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs);
Id EmitSharedAtomicIAdd32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicSMin32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicUMin32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicSMax32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicUMax32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicInc32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicDec32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicAnd32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicOr32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicXor32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicExchange32(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicExchange64(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitSharedAtomicExchange32x2(EmitContext& ctx, Id pointer_offset, Id value);
Id EmitStorageAtomicIAdd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicSMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicUMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicSMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicUMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicInc32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicDec32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicAnd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicOr32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicXor32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicCompareExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id comparator, Id value);
Id EmitStorageAtomicIAdd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicSMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicUMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicSMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicUMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicAnd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicOr64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicXor64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicExchange64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicIAdd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicSMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicUMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicSMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicUMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicAnd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicOr32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicXor32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicExchange32x2(EmitContext& ctx, const IR::Value& binding,
const IR::Value& offset, Id value);
Id EmitStorageAtomicAddF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicAddF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicAddF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicMinF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicMinF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value);
Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicInc32(EmitContext& ctx);
Id EmitGlobalAtomicDec32(EmitContext& ctx);
Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicCompareExchange32(EmitContext& ctx, Id address, Id comparator, Id value);
Id EmitGlobalAtomicIAdd64(EmitContext& ctx);
Id EmitGlobalAtomicSMin64(EmitContext& ctx);
Id EmitGlobalAtomicUMin64(EmitContext& ctx);
Id EmitGlobalAtomicSMax64(EmitContext& ctx);
Id EmitGlobalAtomicUMax64(EmitContext& ctx);
Id EmitGlobalAtomicInc64(EmitContext& ctx);
Id EmitGlobalAtomicDec64(EmitContext& ctx);
Id EmitGlobalAtomicAnd64(EmitContext& ctx);
Id EmitGlobalAtomicOr64(EmitContext& ctx);
Id EmitGlobalAtomicXor64(EmitContext& ctx);
Id EmitGlobalAtomicExchange64(EmitContext& ctx);
Id EmitGlobalAtomicIAdd32x2(EmitContext& ctx);
Id EmitGlobalAtomicSMin32x2(EmitContext& ctx);
Id EmitGlobalAtomicUMin32x2(EmitContext& ctx);
Id EmitGlobalAtomicSMax32x2(EmitContext& ctx);
Id EmitGlobalAtomicUMax32x2(EmitContext& ctx);
Id EmitGlobalAtomicInc32x2(EmitContext& ctx);
Id EmitGlobalAtomicDec32x2(EmitContext& ctx);
Id EmitGlobalAtomicAnd32x2(EmitContext& ctx);
Id EmitGlobalAtomicOr32x2(EmitContext& ctx);
Id EmitGlobalAtomicXor32x2(EmitContext& ctx);
Id EmitGlobalAtomicExchange32x2(EmitContext& ctx);
Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value);
Id EmitGlobalAtomicAddF16x2(EmitContext& ctx);
Id EmitGlobalAtomicAddF32x2(EmitContext& ctx);
Id EmitGlobalAtomicMinF16x2(EmitContext& ctx);
Id EmitGlobalAtomicMinF32x2(EmitContext& ctx);
Id EmitGlobalAtomicMaxF16x2(EmitContext& ctx);
Id EmitGlobalAtomicMaxF32x2(EmitContext& ctx);
Id EmitLogicalOr(EmitContext& ctx, Id a, Id b);
Id EmitLogicalAnd(EmitContext& ctx, Id a, Id b);
Id EmitLogicalXor(EmitContext& ctx, Id a, Id b);
Id EmitLogicalNot(EmitContext& ctx, Id value);
Id EmitConvertS16F16(EmitContext& ctx, Id value);
Id EmitConvertS16F32(EmitContext& ctx, Id value);
Id EmitConvertS16F64(EmitContext& ctx, Id value);
Id EmitConvertS32F16(EmitContext& ctx, Id value);
Id EmitConvertS32F32(EmitContext& ctx, Id value);
Id EmitConvertS32F64(EmitContext& ctx, Id value);
Id EmitConvertS64F16(EmitContext& ctx, Id value);
Id EmitConvertS64F32(EmitContext& ctx, Id value);
Id EmitConvertS64F64(EmitContext& ctx, Id value);
Id EmitConvertU16F16(EmitContext& ctx, Id value);
Id EmitConvertU16F32(EmitContext& ctx, Id value);
Id EmitConvertU16F64(EmitContext& ctx, Id value);
Id EmitConvertU32F16(EmitContext& ctx, Id value);
Id EmitConvertU32F32(EmitContext& ctx, Id value);
Id EmitConvertU32F64(EmitContext& ctx, Id value);
Id EmitConvertU64F16(EmitContext& ctx, Id value);
Id EmitConvertU64F32(EmitContext& ctx, Id value);
Id EmitConvertU64F64(EmitContext& ctx, Id value);
Id EmitConvertU64U32(EmitContext& ctx, Id value);
Id EmitConvertU32U64(EmitContext& ctx, Id value);
Id EmitConvertF16F32(EmitContext& ctx, Id value);
Id EmitConvertF32F16(EmitContext& ctx, Id value);
Id EmitConvertF32F64(EmitContext& ctx, Id value);
Id EmitConvertF64F32(EmitContext& ctx, Id value);
Id EmitConvertF16S8(EmitContext& ctx, Id value);
Id EmitConvertF16S16(EmitContext& ctx, Id value);
Id EmitConvertF16S32(EmitContext& ctx, Id value);
Id EmitConvertF16S64(EmitContext& ctx, Id value);
Id EmitConvertF16U8(EmitContext& ctx, Id value);
Id EmitConvertF16U16(EmitContext& ctx, Id value);
Id EmitConvertF16U32(EmitContext& ctx, Id value);
Id EmitConvertF16U64(EmitContext& ctx, Id value);
Id EmitConvertF32S8(EmitContext& ctx, Id value);
Id EmitConvertF32S16(EmitContext& ctx, Id value);
Id EmitConvertF32S32(EmitContext& ctx, Id value);
Id EmitConvertF32S64(EmitContext& ctx, Id value);
Id EmitConvertF32U8(EmitContext& ctx, Id value);
Id EmitConvertF32U16(EmitContext& ctx, Id value);
Id EmitConvertF32U32(EmitContext& ctx, Id value);
Id EmitConvertF32U64(EmitContext& ctx, Id value);
Id EmitConvertF64S8(EmitContext& ctx, Id value);
Id EmitConvertF64S16(EmitContext& ctx, Id value);
Id EmitConvertF64S32(EmitContext& ctx, Id value);
Id EmitConvertF64S64(EmitContext& ctx, Id value);
Id EmitConvertF64U8(EmitContext& ctx, Id value);
Id EmitConvertF64U16(EmitContext& ctx, Id value);
Id EmitConvertF64U32(EmitContext& ctx, Id value);
Id EmitConvertF64U64(EmitContext& ctx, Id value);
Id EmitConvertU16U32(EmitContext& ctx, Id value);
Id EmitConvertU32U16(EmitContext& ctx, Id value);
Id EmitConvertU8U32(EmitContext& ctx, Id value);
Id EmitConvertU32U8(EmitContext& ctx, Id value);
Id EmitConvertS32S8(EmitContext& ctx, Id value);
Id EmitConvertS32S16(EmitContext& ctx, Id value);
Id EmitBindlessImageSampleImplicitLod(EmitContext&);
Id EmitBindlessImageSampleExplicitLod(EmitContext&);
Id EmitBindlessImageSampleDrefImplicitLod(EmitContext&);
Id EmitBindlessImageSampleDrefExplicitLod(EmitContext&);
Id EmitBindlessImageGather(EmitContext&);
Id EmitBindlessImageGatherDref(EmitContext&);
Id EmitBindlessImageFetch(EmitContext&);
Id EmitBindlessImageQueryDimensions(EmitContext&);
Id EmitBindlessImageQueryLod(EmitContext&);
Id EmitBindlessImageGradient(EmitContext&);
Id EmitBindlessImageRead(EmitContext&);
Id EmitBindlessImageWrite(EmitContext&);
Id EmitBoundImageSampleImplicitLod(EmitContext&);
Id EmitBoundImageSampleExplicitLod(EmitContext&);
Id EmitBoundImageSampleDrefImplicitLod(EmitContext&);
Id EmitBoundImageSampleDrefExplicitLod(EmitContext&);
Id EmitBoundImageGather(EmitContext&);
Id EmitBoundImageGatherDref(EmitContext&);
Id EmitBoundImageFetch(EmitContext&);
Id EmitBoundImageQueryDimensions(EmitContext&);
Id EmitBoundImageQueryLod(EmitContext&);
Id EmitBoundImageGradient(EmitContext&);
Id EmitBoundImageRead(EmitContext&);
Id EmitBoundImageWrite(EmitContext&);
Id EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id bias_lc, const IR::Value& offset);
Id EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id lod, const IR::Value& offset);
Id EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
Id coords, Id dref, Id bias_lc, const IR::Value& offset);
Id EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
Id coords, Id dref, Id lod, const IR::Value& offset);
Id EmitImageGather(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
const IR::Value& offset, const IR::Value& offset2);
Id EmitImageGatherDref(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
const IR::Value& offset, const IR::Value& offset2, Id dref);
Id EmitImageFetch(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id offset,
Id lod, Id ms);
Id EmitImageQueryDimensions(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id lod,
const IR::Value& skip_mips);
Id EmitImageQueryLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id derivatives, const IR::Value& offset, Id lod_clamp);
Id EmitImageRead(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
void EmitImageWrite(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id color);
Id EmitIsTextureScaled(EmitContext& ctx, const IR::Value& index);
Id EmitIsImageScaled(EmitContext& ctx, const IR::Value& index);
Id EmitBindlessImageAtomicIAdd32(EmitContext&);
Id EmitBindlessImageAtomicSMin32(EmitContext&);
Id EmitBindlessImageAtomicUMin32(EmitContext&);
Id EmitBindlessImageAtomicSMax32(EmitContext&);
Id EmitBindlessImageAtomicUMax32(EmitContext&);
Id EmitBindlessImageAtomicInc32(EmitContext&);
Id EmitBindlessImageAtomicDec32(EmitContext&);
Id EmitBindlessImageAtomicAnd32(EmitContext&);
Id EmitBindlessImageAtomicOr32(EmitContext&);
Id EmitBindlessImageAtomicXor32(EmitContext&);
Id EmitBindlessImageAtomicExchange32(EmitContext&);
Id EmitBoundImageAtomicIAdd32(EmitContext&);
Id EmitBoundImageAtomicSMin32(EmitContext&);
Id EmitBoundImageAtomicUMin32(EmitContext&);
Id EmitBoundImageAtomicSMax32(EmitContext&);
Id EmitBoundImageAtomicUMax32(EmitContext&);
Id EmitBoundImageAtomicInc32(EmitContext&);
Id EmitBoundImageAtomicDec32(EmitContext&);
Id EmitBoundImageAtomicAnd32(EmitContext&);
Id EmitBoundImageAtomicOr32(EmitContext&);
Id EmitBoundImageAtomicXor32(EmitContext&);
Id EmitBoundImageAtomicExchange32(EmitContext&);
Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicSMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicUMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicSMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicUMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicInc32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicDec32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicAnd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicOr32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicXor32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitImageAtomicExchange32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
Id value);
Id EmitLaneId(EmitContext& ctx);
Id EmitVoteAll(EmitContext& ctx, Id pred);
Id EmitVoteAny(EmitContext& ctx, Id pred);
Id EmitVoteEqual(EmitContext& ctx, Id pred);
Id EmitSubgroupBallot(EmitContext& ctx, Id pred);
Id EmitSubgroupEqMask(EmitContext& ctx);
Id EmitSubgroupLtMask(EmitContext& ctx);
Id EmitSubgroupLeMask(EmitContext& ctx);
Id EmitSubgroupGtMask(EmitContext& ctx);
Id EmitSubgroupGeMask(EmitContext& ctx);
Id EmitShuffleIndex(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask);
Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask);
Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask);
Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask);
Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane);
Id EmitMatrixMulAddF16(EmitContext& ctx, Id a, Id b, Id c);
void DefineSoftwareMatrixFunction(EmitContext& ctx);
Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction);
Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle);
Id EmitDPdxFine(EmitContext& ctx, Id op_a);
Id EmitDPdyFine(EmitContext& ctx, Id op_a);
Id EmitDPdxCoarse(EmitContext& ctx, Id op_a);
Id EmitDPdyCoarse(EmitContext& ctx, Id op_a);
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,284 @@
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
void SetZeroFlag(EmitContext& ctx, IR::Inst* inst, Id result) {
IR::Inst* const zero{inst->GetAssociatedPseudoOperation(IR::Opcode::GetZeroFromOp)};
if (!zero) {
return;
}
zero->SetDefinition(ctx.OpIEqual(ctx.U1, result, ctx.u32_zero_value));
zero->Invalidate();
}
void SetSignFlag(EmitContext& ctx, IR::Inst* inst, Id result) {
IR::Inst* const sign{inst->GetAssociatedPseudoOperation(IR::Opcode::GetSignFromOp)};
if (!sign) {
return;
}
sign->SetDefinition(ctx.OpSLessThan(ctx.U1, result, ctx.u32_zero_value));
sign->Invalidate();
}
} // Anonymous namespace
Id EmitIAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
Id result{};
if (IR::Inst* const carry{inst->GetAssociatedPseudoOperation(IR::Opcode::GetCarryFromOp)}) {
const Id carry_type{ctx.TypeStruct(ctx.U32[1], ctx.U32[1])};
const Id carry_result{ctx.OpIAddCarry(carry_type, a, b)};
result = ctx.OpCompositeExtract(ctx.U32[1], carry_result, 0U);
const Id carry_value{ctx.OpCompositeExtract(ctx.U32[1], carry_result, 1U)};
carry->SetDefinition(ctx.OpINotEqual(ctx.U1, carry_value, ctx.u32_zero_value));
carry->Invalidate();
} else {
result = ctx.OpIAdd(ctx.U32[1], a, b);
}
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
if (IR::Inst * overflow{inst->GetAssociatedPseudoOperation(IR::Opcode::GetOverflowFromOp)}) {
// https://stackoverflow.com/questions/55468823/how-to-detect-integer-overflow-in-c
constexpr u32 s32_max{static_cast<u32>((std::numeric_limits<s32>::max)())};
const Id is_positive{ctx.OpSGreaterThanEqual(ctx.U1, a, ctx.u32_zero_value)};
const Id sub_a{ctx.OpISub(ctx.U32[1], ctx.Const(s32_max), a)};
const Id positive_test{ctx.OpSGreaterThan(ctx.U1, b, sub_a)};
const Id negative_test{ctx.OpSLessThan(ctx.U1, b, sub_a)};
const Id carry_flag{ctx.OpSelect(ctx.U1, is_positive, positive_test, negative_test)};
overflow->SetDefinition(carry_flag);
overflow->Invalidate();
}
return result;
}
Id EmitIAdd64(EmitContext& ctx, Id a, Id b) {
return ctx.OpIAdd(ctx.U64, a, b);
}
Id EmitISub32(EmitContext& ctx, Id a, Id b) {
return ctx.OpISub(ctx.U32[1], a, b);
}
Id EmitISub64(EmitContext& ctx, Id a, Id b) {
return ctx.OpISub(ctx.U64, a, b);
}
Id EmitIMul32(EmitContext& ctx, Id a, Id b) {
return ctx.OpIMul(ctx.U32[1], a, b);
}
Id EmitSDiv32(EmitContext& ctx, Id a, Id b) {
return ctx.OpSDiv(ctx.U32[1], a, b);
}
Id EmitUDiv32(EmitContext& ctx, Id a, Id b) {
return ctx.OpUDiv(ctx.U32[1], a, b);
}
Id EmitINeg32(EmitContext& ctx, Id value) {
return ctx.OpSNegate(ctx.U32[1], value);
}
Id EmitINeg64(EmitContext& ctx, Id value) {
return ctx.OpSNegate(ctx.U64, value);
}
Id EmitIAbs32(EmitContext& ctx, Id value) {
return ctx.OpSAbs(ctx.U32[1], value);
}
Id EmitIAbs64(EmitContext& ctx, Id value) {
return ctx.OpSAbs(ctx.U64, value);
}
Id EmitShiftLeftLogical32(EmitContext& ctx, Id base, Id shift) {
return ctx.OpShiftLeftLogical(ctx.U32[1], base, shift);
}
Id EmitShiftLeftLogical64(EmitContext& ctx, Id base, Id shift) {
return ctx.OpShiftLeftLogical(ctx.U64, base, shift);
}
Id EmitShiftRightLogical32(EmitContext& ctx, Id base, Id shift) {
return ctx.OpShiftRightLogical(ctx.U32[1], base, shift);
}
Id EmitShiftRightLogical64(EmitContext& ctx, Id base, Id shift) {
return ctx.OpShiftRightLogical(ctx.U64, base, shift);
}
Id EmitShiftRightArithmetic32(EmitContext& ctx, Id base, Id shift) {
return ctx.OpShiftRightArithmetic(ctx.U32[1], base, shift);
}
Id EmitShiftRightArithmetic64(EmitContext& ctx, Id base, Id shift) {
return ctx.OpShiftRightArithmetic(ctx.U64, base, shift);
}
Id EmitBitwiseAnd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
const Id result{ctx.OpBitwiseAnd(ctx.U32[1], a, b)};
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitBitwiseOr32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
const Id result{ctx.OpBitwiseOr(ctx.U32[1], a, b)};
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitBitwiseXor32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
const Id result{ctx.OpBitwiseXor(ctx.U32[1], a, b)};
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitBitFieldInsert(EmitContext& ctx, Id base, Id insert, Id offset, Id count) {
return ctx.OpBitFieldInsert(ctx.U32[1], base, insert, offset, count);
}
Id EmitBitFieldSExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count) {
const Id result{ctx.OpBitFieldSExtract(ctx.U32[1], base, offset, count)};
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitBitFieldUExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count) {
const Id result{ctx.OpBitFieldUExtract(ctx.U32[1], base, offset, count)};
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitBitReverse32(EmitContext& ctx, Id value) {
return ctx.OpBitReverse(ctx.U32[1], value);
}
Id EmitBitCount32(EmitContext& ctx, Id value) {
return ctx.OpBitCount(ctx.U32[1], value);
}
Id EmitBitwiseNot32(EmitContext& ctx, Id value) {
return ctx.OpNot(ctx.U32[1], value);
}
Id EmitFindSMsb32(EmitContext& ctx, Id value) {
return ctx.OpFindSMsb(ctx.U32[1], value);
}
Id EmitFindUMsb32(EmitContext& ctx, Id value) {
return ctx.OpFindUMsb(ctx.U32[1], value);
}
Id EmitSMin32(EmitContext& ctx, Id a, Id b) {
const bool is_broken{ctx.profile.has_broken_signed_operations};
if (is_broken) {
a = ctx.OpBitcast(ctx.S32[1], a);
b = ctx.OpBitcast(ctx.S32[1], b);
}
const Id result{ctx.OpSMin(ctx.U32[1], a, b)};
return is_broken ? ctx.OpBitcast(ctx.U32[1], result) : result;
}
Id EmitUMin32(EmitContext& ctx, Id a, Id b) {
return ctx.OpUMin(ctx.U32[1], a, b);
}
Id EmitSMax32(EmitContext& ctx, Id a, Id b) {
const bool is_broken{ctx.profile.has_broken_signed_operations};
if (is_broken) {
a = ctx.OpBitcast(ctx.S32[1], a);
b = ctx.OpBitcast(ctx.S32[1], b);
}
const Id result{ctx.OpSMax(ctx.U32[1], a, b)};
return is_broken ? ctx.OpBitcast(ctx.U32[1], result) : result;
}
Id EmitUMax32(EmitContext& ctx, Id a, Id b) {
return ctx.OpUMax(ctx.U32[1], a, b);
}
Id EmitSClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max) {
Id result{};
if (ctx.profile.has_broken_signed_operations || ctx.profile.has_broken_spirv_clamp) {
value = ctx.OpBitcast(ctx.S32[1], value);
min = ctx.OpBitcast(ctx.S32[1], min);
max = ctx.OpBitcast(ctx.S32[1], max);
if (ctx.profile.has_broken_spirv_clamp) {
result = ctx.OpSMax(ctx.S32[1], ctx.OpSMin(ctx.S32[1], value, max), min);
} else {
result = ctx.OpSClamp(ctx.S32[1], value, min, max);
}
result = ctx.OpBitcast(ctx.U32[1], result);
} else {
result = ctx.OpSClamp(ctx.U32[1], value, min, max);
}
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitUClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max) {
Id result{};
if (ctx.profile.has_broken_spirv_clamp) {
result = ctx.OpUMax(ctx.U32[1], ctx.OpUMin(ctx.U32[1], value, max), min);
} else {
result = ctx.OpUClamp(ctx.U32[1], value, min, max);
}
SetZeroFlag(ctx, inst, result);
SetSignFlag(ctx, inst, result);
return result;
}
Id EmitSLessThan(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpSLessThan(ctx.U1, lhs, rhs);
}
Id EmitULessThan(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpULessThan(ctx.U1, lhs, rhs);
}
Id EmitIEqual(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpIEqual(ctx.U1, lhs, rhs);
}
Id EmitSLessThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpSLessThanEqual(ctx.U1, lhs, rhs);
}
Id EmitULessThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpULessThanEqual(ctx.U1, lhs, rhs);
}
Id EmitSGreaterThan(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpSGreaterThan(ctx.U1, lhs, rhs);
}
Id EmitUGreaterThan(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpUGreaterThan(ctx.U1, lhs, rhs);
}
Id EmitINotEqual(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpINotEqual(ctx.U1, lhs, rhs);
}
Id EmitSGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpSGreaterThanEqual(ctx.U1, lhs, rhs);
}
Id EmitUGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
return ctx.OpUGreaterThanEqual(ctx.U1, lhs, rhs);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,25 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
Id EmitLogicalOr(EmitContext& ctx, Id a, Id b) {
return ctx.OpLogicalOr(ctx.U1, a, b);
}
Id EmitLogicalAnd(EmitContext& ctx, Id a, Id b) {
return ctx.OpLogicalAnd(ctx.U1, a, b);
}
Id EmitLogicalXor(EmitContext& ctx, Id a, Id b) {
return ctx.OpLogicalNotEqual(ctx.U1, a, b);
}
Id EmitLogicalNot(EmitContext& ctx, Id value) {
return ctx.OpLogicalNot(ctx.U1, value);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,343 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <bit>
#include "shader_recompiler/backend/spirv/emit_spirv.h"
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size,
u32 index_offset = 0) {
if (offset.IsImmediate()) {
const u32 imm_offset{static_cast<u32>(offset.U32() / element_size) + index_offset};
return ctx.Const(imm_offset);
}
const u32 shift{static_cast<u32>(std::countr_zero(element_size))};
Id index{ctx.Def(offset)};
if (shift != 0) {
const Id shift_id{ctx.Const(shift)};
index = ctx.OpShiftRightLogical(ctx.U32[1], index, shift_id);
}
if (index_offset != 0) {
index = ctx.OpIAdd(ctx.U32[1], index, ctx.Const(index_offset));
}
return index;
}
Id StoragePointer(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
const StorageTypeDefinition& type_def, size_t element_size,
Id StorageDefinitions::*member_ptr, u32 index_offset = 0) {
if (!binding.IsImmediate()) {
throw NotImplementedException("Dynamic storage buffer indexing");
}
const Id ssbo{ctx.ssbos[binding.U32()].*member_ptr};
const Id index{StorageIndex(ctx, offset, element_size, index_offset)};
return ctx.OpAccessChain(type_def.element, ssbo, ctx.u32_zero_value, index);
}
Id LoadStorage(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id result_type,
const StorageTypeDefinition& type_def, size_t element_size,
Id StorageDefinitions::*member_ptr, u32 index_offset = 0) {
const Id pointer{
StoragePointer(ctx, binding, offset, type_def, element_size, member_ptr, index_offset)};
return ctx.OpLoad(result_type, pointer);
}
Id LoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
u32 index_offset = 0) {
return LoadStorage(ctx, binding, offset, ctx.U32[1], ctx.storage_types.U32, sizeof(u32),
&StorageDefinitions::U32, index_offset);
}
void WriteStorage(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
const StorageTypeDefinition& type_def, size_t element_size,
Id StorageDefinitions::*member_ptr, u32 index_offset = 0) {
const Id pointer{
StoragePointer(ctx, binding, offset, type_def, element_size, member_ptr, index_offset)};
ctx.OpStore(pointer, value);
}
void WriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
u32 index_offset = 0) {
WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32, sizeof(u32),
&StorageDefinitions::U32, index_offset);
}
void WriteStorageByCasLoop(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value, Id bit_offset, Id bit_count) {
const Id pointer{StoragePointer(ctx, binding, offset, ctx.storage_types.U32, sizeof(u32),
&StorageDefinitions::U32)};
ctx.OpFunctionCall(ctx.TypeVoid(), ctx.write_storage_cas_loop_func, pointer, value, bit_offset,
bit_count);
}
} // Anonymous namespace
// (hookshot) 8- and 16-bit loads through a pointer no storage buffer was found for: the aligned 32-bit word by the same
// search as LoadGlobal32, and the byte or half of it the address names (the console's loads are naturally aligned). They
// threw "not implemented": 30 of OoT's mesh programs (LoadGlobalS16) were left untranslated on every card, issue 351
namespace {
Id LoadGlobalNarrow(EmitContext& ctx, Id address, u32 bits, bool is_signed) {
if (!ctx.profile.support_int64) {
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
return ctx.Const(0u);
}
const Id aligned{ctx.OpBitwiseAnd(ctx.U64, address, ctx.Constant(ctx.U64, u64{~u64{3}}))};
const Id word{ctx.OpFunctionCall(ctx.U32[1], ctx.load_global_func_u32, aligned)};
const Id low{ctx.OpUConvert(ctx.U32[1], ctx.OpBitwiseAnd(ctx.U64, address, ctx.Constant(ctx.U64, u64{3})))};
const Id shift{ctx.OpShiftLeftLogical(ctx.U32[1], low, ctx.Const(3u))};
return is_signed ? ctx.OpBitFieldSExtract(ctx.U32[1], word, shift, ctx.Const(bits))
: ctx.OpBitFieldUExtract(ctx.U32[1], word, shift, ctx.Const(bits));
}
} // Anonymous namespace
Id EmitLoadGlobalU8(EmitContext& ctx, Id address) {
return LoadGlobalNarrow(ctx, address, 8, false);
}
Id EmitLoadGlobalS8(EmitContext& ctx, Id address) {
return LoadGlobalNarrow(ctx, address, 8, true);
}
Id EmitLoadGlobalU16(EmitContext& ctx, Id address) {
return LoadGlobalNarrow(ctx, address, 16, false);
}
Id EmitLoadGlobalS16(EmitContext& ctx, Id address) {
return LoadGlobalNarrow(ctx, address, 16, true);
}
Id EmitLoadGlobal32(EmitContext& ctx, Id address) {
if (ctx.profile.support_int64) {
return ctx.OpFunctionCall(ctx.U32[1], ctx.load_global_func_u32, address);
}
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
return ctx.Const(0u);
}
Id EmitLoadGlobal64(EmitContext& ctx, Id address) {
if (ctx.profile.support_int64) {
return ctx.OpFunctionCall(ctx.U32[2], ctx.load_global_func_u32x2, address);
}
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
return ctx.Const(0u, 0u);
}
Id EmitLoadGlobal128(EmitContext& ctx, Id address) {
if (ctx.profile.support_int64) {
return ctx.OpFunctionCall(ctx.U32[4], ctx.load_global_func_u32x4, address);
}
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
return ctx.Const(0u, 0u, 0u, 0u);
}
// (hookshot) 8- and 16-bit stores through a pointer no storage buffer was found for: the aligned 32-bit word by the same
// search as LoadGlobal32, changed in place with a compare-exchange loop (spirv_emit_context.cpp). They threw "not
// implemented", the same gap as the narrow loads (issue 192)
namespace {
void WriteGlobalNarrow(EmitContext& ctx, Id address, Id value, u32 bits) {
if (!ctx.profile.support_int64) {
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
return;
}
const Id low{ctx.OpUConvert(ctx.U32[1], ctx.OpBitwiseAnd(ctx.U64, address, ctx.Constant(ctx.U64, u64{3})))};
const Id bit_offset{ctx.OpShiftLeftLogical(ctx.U32[1], low, ctx.Const(3u))};
ctx.OpFunctionCall(ctx.TypeVoid(), ctx.write_global_cas_loop_func, address, value, bit_offset, ctx.Const(bits));
}
} // Anonymous namespace
void EmitWriteGlobalU8(EmitContext& ctx, Id address, Id value) {
WriteGlobalNarrow(ctx, address, value, 8);
}
void EmitWriteGlobalS8(EmitContext& ctx, Id address, Id value) {
WriteGlobalNarrow(ctx, address, value, 8);
}
void EmitWriteGlobalU16(EmitContext& ctx, Id address, Id value) {
WriteGlobalNarrow(ctx, address, value, 16);
}
void EmitWriteGlobalS16(EmitContext& ctx, Id address, Id value) {
WriteGlobalNarrow(ctx, address, value, 16);
}
void EmitWriteGlobal32(EmitContext& ctx, Id address, Id value) {
if (ctx.profile.support_int64) {
ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32, address, value);
return;
}
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
}
void EmitWriteGlobal64(EmitContext& ctx, Id address, Id value) {
if (ctx.profile.support_int64) {
ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32x2, address, value);
return;
}
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
}
void EmitWriteGlobal128(EmitContext& ctx, Id address, Id value) {
if (ctx.profile.support_int64) {
ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32x4, address, value);
return;
}
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
}
Id EmitLoadStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
ctx.profile.support_descriptor_aliasing) {
return ctx.OpUConvert(ctx.U32[1],
LoadStorage(ctx, binding, offset, ctx.U8, ctx.storage_types.U8,
sizeof(u8), &StorageDefinitions::U8));
} else {
return ctx.OpBitFieldUExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
ctx.BitOffset8(offset), ctx.Const(8u));
}
}
Id EmitLoadStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
ctx.profile.support_descriptor_aliasing) {
return ctx.OpSConvert(ctx.U32[1],
LoadStorage(ctx, binding, offset, ctx.S8, ctx.storage_types.S8,
sizeof(s8), &StorageDefinitions::S8));
} else {
return ctx.OpBitFieldSExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
ctx.BitOffset8(offset), ctx.Const(8u));
}
}
Id EmitLoadStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
ctx.profile.support_descriptor_aliasing) {
return ctx.OpUConvert(ctx.U32[1],
LoadStorage(ctx, binding, offset, ctx.U16, ctx.storage_types.U16,
sizeof(u16), &StorageDefinitions::U16));
} else {
return ctx.OpBitFieldUExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
ctx.BitOffset16(offset), ctx.Const(16u));
}
}
Id EmitLoadStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
ctx.profile.support_descriptor_aliasing) {
return ctx.OpSConvert(ctx.U32[1],
LoadStorage(ctx, binding, offset, ctx.S16, ctx.storage_types.S16,
sizeof(s16), &StorageDefinitions::S16));
} else {
return ctx.OpBitFieldSExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
ctx.BitOffset16(offset), ctx.Const(16u));
}
}
Id EmitLoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
return LoadStorage32(ctx, binding, offset);
}
Id EmitLoadStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing) {
return LoadStorage(ctx, binding, offset, ctx.U32[2], ctx.storage_types.U32x2,
sizeof(u32[2]), &StorageDefinitions::U32x2);
} else {
return ctx.OpCompositeConstruct(ctx.U32[2], LoadStorage32(ctx, binding, offset, 0),
LoadStorage32(ctx, binding, offset, 1));
}
}
Id EmitLoadStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
if (ctx.profile.support_descriptor_aliasing) {
return LoadStorage(ctx, binding, offset, ctx.U32[4], ctx.storage_types.U32x4,
sizeof(u32[4]), &StorageDefinitions::U32x4);
} else {
return ctx.OpCompositeConstruct(ctx.U32[4], LoadStorage32(ctx, binding, offset, 0),
LoadStorage32(ctx, binding, offset, 1),
LoadStorage32(ctx, binding, offset, 2),
LoadStorage32(ctx, binding, offset, 3));
}
}
void EmitWriteStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
ctx.profile.support_descriptor_aliasing) {
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U8, value), ctx.storage_types.U8,
sizeof(u8), &StorageDefinitions::U8);
} else {
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u));
}
}
void EmitWriteStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
ctx.profile.support_descriptor_aliasing) {
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S8, value), ctx.storage_types.S8,
sizeof(s8), &StorageDefinitions::S8);
} else {
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u));
}
}
void EmitWriteStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
ctx.profile.support_descriptor_aliasing) {
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U16, value), ctx.storage_types.U16,
sizeof(u16), &StorageDefinitions::U16);
} else {
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u));
}
}
void EmitWriteStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
ctx.profile.support_descriptor_aliasing) {
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S16, value), ctx.storage_types.S16,
sizeof(s16), &StorageDefinitions::S16);
} else {
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u));
}
}
void EmitWriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
WriteStorage32(ctx, binding, offset, value);
}
void EmitWriteStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_descriptor_aliasing) {
WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32x2, sizeof(u32[2]),
&StorageDefinitions::U32x2);
} else {
for (u32 index = 0; index < 2; ++index) {
const Id element{ctx.OpCompositeExtract(ctx.U32[1], value, index)};
WriteStorage32(ctx, binding, offset, element, index);
}
}
}
void EmitWriteStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
Id value) {
if (ctx.profile.support_descriptor_aliasing) {
WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32x4, sizeof(u32[4]),
&StorageDefinitions::U32x4);
} else {
for (u32 index = 0; index < 4; ++index) {
const Id element{ctx.OpCompositeExtract(ctx.U32[1], value, index)};
WriteStorage32(ctx, binding, offset, element, index);
}
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,656 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// Switch 2 (NVN) task and mesh programs. They talk to the hardware through the stage buffer
// (ISBE): a task program writes the number of mesh workgroups at byte 4 of its output region and
// its payload after it; each mesh workgroup reads that region (its own index at byte 0) and
// writes its vertices and primitives to an attribute region in groups of 32, one 32-word line per
// attribute, plus the primitive count and 8-bit vertex indices to a map region. Here the task's
// output region is the task payload, the mesh output regions live in workgroup memory, and the
// epilogue (reached by every invocation, see structured_control_flow.cpp) hands them to Vulkan.
#include <algorithm>
#include <cstdlib>
#include <span>
#include "shader_recompiler/backend/spirv/emit_spirv.h"
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
void Barrier(EmitContext& ctx) {
ctx.OpControlBarrier(ctx.Const(static_cast<u32>(spv::Scope::Workgroup)),
ctx.Const(static_cast<u32>(spv::Scope::Workgroup)),
ctx.Const(static_cast<u32>(spv::MemorySemanticsMask::AcquireRelease |
spv::MemorySemanticsMask::WorkgroupMemory)));
}
Id LocalInvocation(EmitContext& ctx) {
return ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.local_invocation_id), 0U);
}
Id Clamped(EmitContext& ctx, Id index, u32 count) {
return ctx.OpUMin(ctx.U32[1], index, ctx.Const(count - 1));
}
Id PushU64(EmitContext& ctx, u32 member);
Id PushU32(EmitContext& ctx, u32 member) {
const Id pointer{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1])};
return ctx.OpLoad(ctx.U32[1], ctx.OpAccessChain(pointer, ctx.rescaling_push_constants, ctx.Const(member)));
}
Id WorkgroupId(EmitContext& ctx);
Id FallbackSlot(EmitContext& ctx);
Id BufferWordAt(EmitContext& ctx, Id base, Id word);
Id BufferWord(EmitContext& ctx, u32 member, Id word);
Id PayloadWord(EmitContext& ctx, Id index) {
if (ctx.mesh_fallback && ctx.stage == Stage::Mesh && ctx.runtime_info.mesh_fallback_task) {
// (the task wrote it out: the mesh workgroups are a later dispatch, and read their task workgroup's payload, a slot
// being task workgroup * threads + its index in the task)
const Id task{ctx.OpUDiv(ctx.U32[1], FallbackSlot(ctx), ctx.Const(ctx.runtime_info.mesh_fallback_task_threads))};
const Id first{ctx.OpIAdd(
ctx.U32[1], ctx.Const(MESH_FALLBACK_CAPACITY),
ctx.OpIMul(ctx.U32[1], task, ctx.Const(ctx.mesh_info.payload_words)))};
return BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index),
ctx.OpIAdd(ctx.U32[1], first,
Clamped(ctx, index, ctx.mesh_info.payload_words)));
}
return ctx.OpAccessChain(ctx.mesh_payload_u32, ctx.mesh_payload,
Clamped(ctx, index, ctx.mesh_info.payload_words));
}
Id SharedWord(EmitContext& ctx, Id array, Id index) {
return ctx.OpAccessChain(ctx.mesh_shared_u32, array, index);
}
// Runs body under `if (condition)`.
template <typename Func>
void If(EmitContext& ctx, Id condition, Func&& body) {
const Id then_label{ctx.OpLabel()};
const Id merge_label{ctx.OpLabel()};
ctx.OpSelectionMerge(merge_label, spv::SelectionControlMask::MaskNone);
ctx.OpBranchConditional(condition, then_label, merge_label);
ctx.AddLabel(then_label);
body();
ctx.OpBranch(merge_label);
ctx.AddLabel(merge_label);
}
// Copies the lines of element `index` (a vertex or a primitive) from the attribute region.
void CopyLines(EmitContext& ctx, Id index, u32 base, u32 num_lines,
const std::vector<EmitContext::MeshLineTarget>& targets) {
// Word of line 0: base + (index / 32) * num_lines * 32 + index % 32.
const Id group{ctx.OpShiftRightLogical(ctx.U32[1], index, ctx.Const(5U))};
const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U))};
const Id first{ctx.OpIAdd(ctx.U32[1],
ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)),
ctx.OpIAdd(ctx.U32[1], slot, ctx.Const(base)))};
for (u32 line = 0; line < num_lines; ++line) {
const EmitContext::MeshLineTarget& target{targets[line]};
if (!Sirit::ValidId(target.variable)) {
continue;
}
const Id word{ctx.OpLoad(
ctx.U32[1],
SharedWord(ctx, ctx.mesh_attributes, ctx.OpIAdd(ctx.U32[1], first, ctx.Const(line * 32))))};
const Id value{target.is_float ? ctx.OpBitcast(ctx.F32[1], word) : word};
const Id pointer_type{target.is_float ? ctx.output_f32 : ctx.output_u32};
const Id pointer{target.element == ~0U
? ctx.OpAccessChain(pointer_type, target.variable, index)
: ctx.OpAccessChain(pointer_type, target.variable, index,
ctx.Const(target.element))};
ctx.OpStore(pointer, value);
}
}
// OpenGL-style clip space: z = (z + w) / 2, from the stage buffer's position lines.
void ConvertDepth(EmitContext& ctx, Id vertex) {
const auto& lines{ctx.mesh_info.vertex_lines};
const auto z_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionZ))};
const auto w_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionW))};
if (z_line == lines.end() || w_line == lines.end()) {
return;
}
const u32 num_lines{static_cast<u32>(lines.size())};
const Id group{ctx.OpShiftRightLogical(ctx.U32[1], vertex, ctx.Const(5U))};
const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], vertex, ctx.Const(31U))};
const Id first{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)),
slot)};
const auto read = [&](auto line) {
const u32 offset{static_cast<u32>(line - lines.begin()) * 32};
return ctx.OpBitcast(ctx.F32[1],
ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_attributes,
ctx.OpIAdd(ctx.U32[1], first,
ctx.Const(offset)))));
};
const Id z{ctx.OpFMul(ctx.F32[1], ctx.OpFAdd(ctx.F32[1], read(z_line), read(w_line)),
ctx.Const(0.5f))};
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, ctx.output_position, vertex, ctx.Const(2U)), z);
}
/// (hookshot) The fallback's records: one per primitive corner, its validity word and the stage buffer's vertex and
/// primitive lines. A mesh workgroup writes its workgroup index and a fixed number of records for it; the generated
/// vertex stage reads one record per vertex.
u32 FallbackRecordWords(const MeshInfo& mesh) {
return 1 + static_cast<u32>(mesh.vertex_lines.size()) +
static_cast<u32>(mesh.primitive_lines.size());
}
Id PushU64(EmitContext& ctx, u32 member) {
const Id pointer_type{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U64)};
return ctx.OpLoad(
ctx.U64, ctx.OpAccessChain(pointer_type, ctx.rescaling_push_constants, ctx.Const(member)));
}
/// The word at a device address.
Id BufferWordAt(EmitContext& ctx, Id base, Id word) {
const Id address{
ctx.OpIAdd(ctx.U64, base,
ctx.OpUConvert(ctx.U64, ctx.OpIMul(ctx.U32[1], word, ctx.Const(4U))))};
return ctx.MeshFallbackWord(address);
}
/// The word at a pushed device address.
Id BufferWord(EmitContext& ctx, u32 member, Id word) {
return BufferWordAt(ctx, PushU64(ctx, member), word);
}
/// A load and a store of such a word. (PhysicalStorageBuffer accesses must say their alignment: spirv-val rejects them
/// without it.) Workgroup words take the same, which is valid there too.
Id LoadWord(EmitContext& ctx, Id pointer) {
return ctx.OpLoad(ctx.U32[1], pointer, spv::MemoryAccessMask::Aligned, 4U);
}
void StoreWord(EmitContext& ctx, Id pointer, Id value) {
ctx.OpStore(pointer, value, spv::MemoryAccessMask::Aligned, 4U);
}
Id WorkgroupId(EmitContext& ctx) {
return ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.workgroup_id), 0U);
}
/// A fallback mesh workgroup's slot: the one its task's map gave it (EmitMeshPrologue), else its workgroup id
Id FallbackSlot(EmitContext& ctx) {
return Sirit::ValidId(ctx.mesh_fallback_slot) ? ctx.mesh_fallback_slot : WorkgroupId(ctx);
}
/// Runs body once for each index invocation, invocation + threads, ... below count. (Unrolled: count and threads are
/// known here. A loop's index variable declared at the call, mid-function, was invalid SPIR-V: a function's OpVariables
/// must open its first block)
template <typename Func>
void Strided(EmitContext& ctx, u32 count, u32 threads, Id invocation, Func&& body) {
for (u32 first = 0; first < count; first += std::max(threads, 1U)) {
const Id index{first ? ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first)) : invocation};
If(ctx, ctx.OpULessThan(ctx.U1, index, ctx.Const(count)), [&] { body(index); });
}
}
/// The raw stage buffer word of line `line` for element `element` (a vertex or a primitive).
Id LineWord(EmitContext& ctx, Id array, Id element, u32 base, u32 num_lines, u32 line) {
const Id group{ctx.OpShiftRightLogical(ctx.U32[1], element, ctx.Const(5U))};
const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], element, ctx.Const(31U))};
const Id first{ctx.OpIAdd(
ctx.U32[1], ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)),
ctx.OpIAdd(ctx.U32[1], slot, ctx.Const(base)))};
return ctx.OpLoad(
ctx.U32[1],
SharedWord(ctx, array, ctx.OpIAdd(ctx.U32[1], first, ctx.Const(line * 32))));
}
/// A mesh workgroup takes records for its primitives' corners from the records buffer's allocator and writes them; the vertex
/// stage's indirect draw covers the records taken (MESH_FALLBACK_DRAW_WORD). A task's flag first bounds its count.
void MeshFallbackEpilogue(EmitContext& ctx) {
const MeshInfo& mesh{ctx.mesh_info};
const u32 corners{mesh.Corners()};
const u32 words{FallbackRecordWords(mesh)};
const u32 fit{MESH_FALLBACK_RECORD_WORDS / words}; // (this program's records the buffer holds)
const Id none{ctx.Const(0xFFFFFFFFU)};
Barrier(ctx);
// (with a task program every workgroup dispatched was started by it: the dispatch is the map's, TaskEpilogue)
const Id slot{FallbackSlot(ctx)};
const Id in_range{ctx.OpULessThan(ctx.U1, slot, ctx.Const(MESH_FALLBACK_CAPACITY))};
Id count{ctx.OpUMin(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count),
ctx.Const(mesh.max_primitives))};
count = ctx.OpSelect(ctx.U32[1], in_range, count, ctx.u32_zero_value);
const Id taken{ctx.OpIMul(ctx.U32[1], count, ctx.Const(corners))};
const auto atomic_add = [&](u32 word, Id value) {
return ctx.OpAtomicIAdd(ctx.U32[1], BufferWord(ctx, ctx.mesh_fallback_records_member_index, ctx.Const(word)),
ctx.Const(static_cast<u32>(spv::Scope::Device)), ctx.u32_zero_value, value);
};
// (every thread has read the primitive count above: thread 0 hands the others its first record in that word. Records
// that do not fit are not taken; taking only grows, so the ones that fit are the first, and the draw's count is theirs)
Barrier(ctx);
If(ctx, ctx.OpIEqual(ctx.U1, LocalInvocation(ctx), ctx.u32_zero_value), [&] {
ctx.OpStore(ctx.mesh_primitive_count, none);
If(ctx, ctx.OpINotEqual(ctx.U1, taken, ctx.u32_zero_value), [&] {
const Id base{atomic_add(MESH_FALLBACK_ALLOC_WORD, taken)};
const Id fits{ctx.OpLogicalAnd(ctx.U1, ctx.OpULessThanEqual(ctx.U1, base, ctx.Const(fit)),
ctx.OpULessThanEqual(ctx.U1, taken,
ctx.OpISub(ctx.U32[1], ctx.Const(fit), base)))};
If(ctx, fits, [&] {
atomic_add(MESH_FALLBACK_DRAW_WORD, taken);
ctx.OpStore(ctx.mesh_primitive_count, base);
});
});
});
Barrier(ctx);
const Id base{ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count)};
If(ctx, ctx.OpINotEqual(ctx.U1, base, none), [&] {
const Id invocation{LocalInvocation(ctx)};
Strided(ctx, mesh.max_primitives, ctx.mesh_threads, invocation, [&](Id primitive) {
If(ctx, ctx.OpULessThan(ctx.U1, primitive, count), [&] {
for (u32 corner = 0; corner < corners; ++corner) {
const Id byte{ctx.OpIAdd(
ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(corners)),
ctx.Const(4U + corner))};
// (an index byte the program left unwritten is held to the vertex count)
const Id vertex{Clamped(
ctx, ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_map, byte)),
mesh.max_vertices)};
const Id record{ctx.OpIAdd(
ctx.U32[1], base,
ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(corners)),
ctx.Const(corner)))};
const Id word{ctx.OpIAdd(ctx.U32[1], ctx.Const(MESH_FALLBACK_HEADER_WORDS),
ctx.OpIMul(ctx.U32[1], record, ctx.Const(words)))};
StoreWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index, word),
ctx.Const(1U));
for (u32 line = 0; line < mesh.vertex_lines.size(); ++line) {
const Id value{LineWord(ctx, ctx.mesh_attributes, vertex, 0,
static_cast<u32>(mesh.vertex_lines.size()), line)};
StoreWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index,
ctx.OpIAdd(ctx.U32[1], word, ctx.Const(1U + line))),
value);
}
for (u32 line = 0; line < mesh.primitive_lines.size(); ++line) {
const Id value{LineWord(ctx, ctx.mesh_attributes, primitive,
ctx.mesh_primitive_base,
static_cast<u32>(mesh.primitive_lines.size()), line)};
StoreWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index,
ctx.OpIAdd(ctx.U32[1], word,
ctx.Const(static_cast<u32>(1U + mesh.vertex_lines.size()) +
line))),
value);
}
}
});
});
});
}
void StorePosition(EmitContext& ctx, u32 element, Id value) {
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, ctx.output_position, ctx.Const(element)), value);
}
void StoreFallbackOutput(EmitContext& ctx, IR::Attribute attribute, Id value) {
if (IR::IsGeneric(attribute)) {
const u32 index{IR::GenericAttributeIndex(attribute)};
const u32 element{IR::GenericAttributeElement(attribute)};
const GenericElementInfo& info{ctx.output_generics.at(index).at(element)};
if (!Sirit::ValidId(info.id)) {
return;
}
if (info.num_components == 1) {
ctx.OpStore(info.id, value);
} else {
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, info.id,
ctx.Const(element - info.first_element)),
value);
}
return;
}
if (attribute == IR::Attribute::PointSize && Sirit::ValidId(ctx.output_point_size)) {
ctx.OpStore(ctx.output_point_size, value);
}
}
u32 task_cap(u32 threads) { // (RX_VK_TASK_CAP=<n>: mesh workgroups a task may start; 0: one a thread)
static const char* e = getenv("RX_VK_TASK_CAP");
const u32 n = e ? static_cast<u32>(atoi(e)) : 0U;
return n ? n : threads;
}
void TaskEpilogue(EmitContext& ctx) {
Barrier(ctx);
// (hookshot) at most one mesh workgroup a task thread: each thread keeps or culls one meshlet (the programs compact them by subgroup
// ballot). It was 65535, and a payload word a task left unwritten is anything: when the task culled everything, tens of thousands of
// mesh workgroups for a draw held the GPU past Windows' two seconds, and the device was lost, on about half the first starts
const Id count{ctx.OpUMin(ctx.U32[1], LoadWord(ctx, PayloadWord(ctx, ctx.Const(1U))),
ctx.Const(task_cap(ctx.mesh_threads)))};
if (ctx.mesh_fallback) {
// (no OpEmitMeshTasksEXT: each thread that started a mesh workgroup takes an index of the mesh dispatch
// (MESH_FALLBACK_DISPATCH_WORD) and writes its slot, task workgroup * threads + its index, at that index of the
// payload buffer's map. The mesh dispatch is those indices: no more, whatever the CPU knew of the task count)
const Id slot{ctx.OpIAdd(ctx.U32[1],
ctx.OpIMul(ctx.U32[1], WorkgroupId(ctx), ctx.Const(ctx.mesh_threads)),
LocalInvocation(ctx))};
// (the count is how many meshlets this task workgroup started: the first that many threads, not all of them)
const Id started{ctx.OpLogicalAnd(ctx.U1, ctx.OpULessThan(ctx.U1, LocalInvocation(ctx), count),
ctx.OpULessThan(ctx.U1, slot, ctx.Const(MESH_FALLBACK_CAPACITY)))};
If(ctx, started, [&] {
const Id index{ctx.OpAtomicIAdd(
ctx.U32[1],
BufferWord(ctx, ctx.mesh_fallback_records_member_index, ctx.Const(MESH_FALLBACK_DISPATCH_WORD)),
ctx.Const(static_cast<u32>(spv::Scope::Device)), ctx.u32_zero_value, ctx.Const(1U))};
// (fewer than MESH_FALLBACK_CAPACITY: a started slot is below it)
StoreWord(ctx, BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index), index), slot);
});
// (the payload once a task workgroup, its threads sharing the words, where its mesh workgroups read it (PayloadWord).
// A copy a slot was 32 times the writes and the memory: 52 MB a draw at 16384 slots)
const Id task{WorkgroupId(ctx)};
const Id first_slot{ctx.OpIMul(ctx.U32[1], task, ctx.Const(ctx.mesh_threads))};
If(ctx, ctx.OpULessThan(ctx.U1, first_slot, ctx.Const(MESH_FALLBACK_CAPACITY)), [&] {
const Id first{ctx.OpIAdd(
ctx.U32[1], ctx.Const(MESH_FALLBACK_CAPACITY),
ctx.OpIMul(ctx.U32[1], task, ctx.Const(ctx.mesh_info.payload_words)))};
Strided(ctx, ctx.mesh_info.payload_words, ctx.mesh_threads, LocalInvocation(ctx), [&](Id index) {
StoreWord(ctx, BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index),
ctx.OpIAdd(ctx.U32[1], first, index)),
LoadWord(ctx, PayloadWord(ctx, index)));
});
});
return;
}
const Id one{ctx.Const(1U)};
// Marker for OpEmitMeshTasksEXT count, 1, 1, payload (PatchMeshInstructions).
ctx.OpStore(count, one);
ctx.OpStore(one, ctx.mesh_payload);
ctx.mesh_tasks_marker = {count, one, one, ctx.mesh_payload};
}
void MeshEpilogue(EmitContext& ctx) {
const MeshInfo& mesh{ctx.mesh_info};
if (ctx.mesh_fallback) {
MeshFallbackEpilogue(ctx);
return;
}
Barrier(ctx);
const Id count{ctx.OpUMin(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count),
ctx.Const(mesh.max_primitives))};
const Id max_vertices{ctx.Const(mesh.max_vertices)};
// Marker for OpSetMeshOutputsEXT max_vertices, count (PatchMeshInstructions).
ctx.OpStore(max_vertices, count);
ctx.mesh_outputs_marker = {max_vertices, count};
const Id invocation{LocalInvocation(ctx)};
const u32 threads{ctx.mesh_threads};
const bool convert_depth{ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc &&
Sirit::ValidId(ctx.output_position)};
const u32 vertex_lines{static_cast<u32>(mesh.vertex_lines.size())};
for (u32 first = 0; first < mesh.max_vertices; first += threads) {
const Id vertex{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))};
const auto copy = [&] {
for (const auto& d : ctx.mesh_vertex_defaults) {
for (u32 c = 0; c < 4; ++c) {
if (d.missing & (1U << c)) {
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, d.variable, vertex, ctx.Const(c)),
ctx.Const(c == 3 ? 1.0f : 0.0f));
}
}
}
CopyLines(ctx, vertex, 0, vertex_lines, ctx.mesh_vertex_targets);
if (convert_depth) {
ConvertDepth(ctx, vertex);
}
};
if (first + threads > mesh.max_vertices) {
If(ctx, ctx.OpULessThan(ctx.U1, vertex, max_vertices), copy);
} else {
copy();
}
}
const u32 primitive_lines{static_cast<u32>(mesh.primitive_lines.size())};
for (u32 first = 0; first < mesh.max_primitives; first += threads) {
const Id primitive{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))};
If(ctx, ctx.OpULessThan(ctx.U1, primitive, count), [&] {
// Vertex indices: map bytes 4 + 3 * primitive (lines: 4 + 2 * primitive, points: 4 + primitive).
const u32 corners{mesh.Corners()};
const Id byte{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(corners)),
ctx.Const(4U))};
std::array<Id, 3> indices;
for (u32 element = 0; element < corners; ++element) {
const Id address{ctx.OpIAdd(ctx.U32[1], byte, ctx.Const(element))};
// (hookshot) within the vertices: an index byte the program left unwritten is anything, and an index
// past the vertex count is undefined (a device fault on NVIDIA)
indices[element] = Clamped(ctx, ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_map, address)),
mesh.max_vertices);
}
const Id output_indices{ctx.TypePointer(spv::StorageClass::Output, ctx.U32[corners])};
ctx.OpStore(ctx.OpAccessChain(output_indices, ctx.mesh_primitive_indices, primitive),
corners == 1 ? indices[0] // (a point's index is a scalar)
: ctx.OpCompositeConstruct(ctx.U32[corners],
std::span<const Id>(indices.data(), corners)));
CopyLines(ctx, primitive, ctx.mesh_primitive_base, primitive_lines,
ctx.mesh_primitive_targets);
});
}
}
} // Anonymous namespace
void EmitMeshFallbackVertexBody(EmitContext& ctx) {
// (the fragment stage's inputs for the mesh program's per-primitive generics are Flat: the generated vertex
// stage's outputs must match, and DefineOutputs only knows the stores, not which of them are per-primitive)
for (u32 index = 0; index < IR::NUM_GENERICS; ++index) {
if (!ctx.runtime_info.per_primitive_generics.test(index)) {
continue;
}
for (u32 element = 0; element < 4; ++element) {
const Id id{ctx.output_generics[index][element].id};
if (!Sirit::ValidId(id)) {
continue;
}
bool seen{};
for (u32 before = 0; before < element; ++before) {
seen |= ctx.output_generics[index][before].id.value == id.value;
}
if (!seen) {
ctx.Decorate(id, spv::Decoration::Flat);
}
}
}
// (every output component starts at (0,0,0,1), as the native mesh stage's do (6dcf6f1) and a vertex stage's; the record's
// lines below go over the ones the mesh program wrote. Left undefined, AMD's driver gave the fragment stage whatever its
// registers held: specks of colour in Kokiri Forest's bushes on an RX 5500M)
for (u32 index = 0; index < IR::NUM_GENERICS; ++index) {
for (u32 element = 0; element < 4; ++element) {
const GenericElementInfo& info{ctx.output_generics[index][element]};
if (!Sirit::ValidId(info.id)) {
continue;
}
const Id value{element == 3 ? ctx.Const(1.0f) : ctx.f32_zero_value};
if (info.num_components == 1) {
ctx.OpStore(info.id, value);
} else {
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, info.id, ctx.Const(element - info.first_element)), value);
}
}
}
if (Sirit::ValidId(ctx.output_point_size)) {
ctx.OpStore(ctx.output_point_size, ctx.Const(1.0f));
}
const MeshInfo& mesh{ctx.mesh_info};
const u32 vertex_lines{static_cast<u32>(mesh.vertex_lines.size())};
const u32 primitive_lines{static_cast<u32>(mesh.primitive_lines.size())};
const u32 words{FallbackRecordWords(mesh)};
// (a record a vertex, in the order the mesh workgroups took them (MeshFallbackEpilogue): the indirect draw's count is the
// records taken, so every one it reaches was written)
const Id vertex{ctx.OpLoad(ctx.U32[1], ctx.vertex_index)};
const Id first{ctx.OpIAdd(ctx.U32[1], ctx.Const(MESH_FALLBACK_HEADER_WORDS),
ctx.OpIMul(ctx.U32[1], vertex, ctx.Const(words)))};
const Id valid{LoadWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index, first))};
const auto read = [&](u32 word) {
return ctx.OpBitcast(
ctx.F32[1],
LoadWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index,
ctx.OpIAdd(ctx.U32[1], first, ctx.Const(word)))));
};
std::array<Id, 4> position{};
for (u32 line = 0; line < vertex_lines; ++line) {
const IR::Attribute attribute{mesh.vertex_lines[line]};
const Id value{read(1U + line)};
const u32 raw{static_cast<u32>(attribute)};
if (raw >= static_cast<u32>(IR::Attribute::PositionX) &&
raw <= static_cast<u32>(IR::Attribute::PositionW)) {
position[raw - static_cast<u32>(IR::Attribute::PositionX)] = value;
} else {
StoreFallbackOutput(ctx, attribute, value);
}
}
for (u32 line = 0; line < primitive_lines; ++line) {
StoreFallbackOutput(ctx, IR::Attribute{mesh.primitive_lines[line]},
read(1U + vertex_lines + line));
}
// (OpenGL-style clip space, as the mesh epilogue's ConvertDepth)
if (ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc &&
Sirit::ValidId(position[2]) && Sirit::ValidId(position[3])) {
position[2] = ctx.OpFMul(ctx.F32[1], ctx.OpFAdd(ctx.F32[1], position[2], position[3]),
ctx.Const(0.5f));
}
for (u32 element = 0; element < 4; ++element) {
if (Sirit::ValidId(position[element])) {
StorePosition(ctx, element, position[element]);
}
}
// (a record no mesh workgroup wrote, which the draw's count should never reach: outside the clip volume)
If(ctx, ctx.OpIEqual(ctx.U1, valid, ctx.u32_zero_value),
[&] {
StorePosition(ctx, 0, ctx.f32_zero_value);
StorePosition(ctx, 1, ctx.f32_zero_value);
StorePosition(ctx, 2, ctx.f32_zero_value);
StorePosition(ctx, 3, ctx.Const(-1.0f));
});
}
Id EmitIsbeRead(EmitContext& ctx, Id address) {
const Id native_workgroup{
ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.workgroup_id), 0U)};
if (ctx.stage == Stage::Task) {
const Id workgroup{native_workgroup};
if (ctx.mesh_fallback) {
const Id input{PushU64(ctx, ctx.mesh_fallback_indirect_input_member_index)};
const Id indirect{ctx.OpINotEqual(ctx.U1, input, ctx.Constant(ctx.U64, u64{0}))};
// On a direct draw the header's instance count is a valid, read-only word for the unused load.
const Id base{ctx.OpSelect(ctx.U64, indirect, input, PushU64(ctx, ctx.mesh_fallback_records_member_index))};
const Id gpu_first{LoadWord(ctx, BufferWordAt(ctx, base, ctx.Const(1U)))};
const Id first{ctx.OpSelect(ctx.U32[1], indirect, gpu_first, PushU32(ctx, ctx.mesh_first_member_index))};
return ctx.OpIAdd(ctx.U32[1], workgroup, first);
}
if (ctx.wants_mesh_first && Sirit::ValidId(ctx.rescaling_push_constants)) {
// (hookshot) + the draw's first workgroup: NVN draws [first, first + count). Without it every draw began at 0: the
// foliage's draws (first up to ~1900) drew the first records again and again, and the forest had no leaves
const Id table{ctx.OpLoad(ctx.U64, ctx.OpAccessChain(ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U64),
ctx.rescaling_push_constants, ctx.Const(ctx.mesh_first_member_index)))};
const Id draw{ctx.OpLoad(ctx.U32[1], ctx.draw_index)};
const Id at{ctx.OpIAdd(ctx.U64, table, ctx.OpUConvert(ctx.U64, ctx.OpIMul(ctx.U32[1], draw, ctx.Const(16U))))};
const Id member{ctx.OpAccessChain(ctx.mesh_first_member_pointer, ctx.OpConvertUToPtr(ctx.mesh_first_block_pointer, at),
ctx.Const(0U))};
const Id first{ctx.OpLoad(ctx.U32[1], member, spv::MemoryAccessMask::Aligned, 4U)};
return ctx.OpIAdd(ctx.U32[1], workgroup, first);
}
return workgroup; // the input region holds the workgroup index (VERTEX_ID)
}
const Id workgroup{native_workgroup};
const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))};
const Id pointer{PayloadWord(ctx, index)};
const Id word{ctx.mesh_fallback ? LoadWord(ctx, pointer) : ctx.OpLoad(ctx.U32[1], pointer)}; // (the fallback's: a device address)
// (hookshot) a fallback mesh workgroup is a slot, task workgroup * threads + its index: address 0 is that index in its task,
// from 0 as OpEmitMeshTasksEXT numbers them. It was the slot, which picked a meshlet past the end of every task's list
const Id within{ctx.mesh_fallback && ctx.runtime_info.mesh_fallback_task
? ctx.OpUMod(ctx.U32[1], FallbackSlot(ctx), ctx.Const(ctx.runtime_info.mesh_fallback_task_threads))
: workgroup};
return ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, address, ctx.u32_zero_value), within,
word);
}
void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value) {
const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))};
if (ctx.stage == Stage::Task) {
StoreWord(ctx, PayloadWord(ctx, index), value);
return;
}
ctx.OpStore(SharedWord(ctx, ctx.mesh_attributes, Clamped(ctx, index, ctx.mesh_attribute_words)),
value);
}
void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value) {
if (ctx.stage != Stage::Mesh) {
throw NotImplementedException("Stage buffer map write outside mesh programs");
}
ctx.OpStore(SharedWord(ctx, ctx.mesh_map, Clamped(ctx, address, ctx.mesh_map_bytes)),
ctx.OpBitwiseAnd(ctx.U32[1], value, ctx.Const(0xffU)));
}
void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value) {
// The primitive count: the map is written from its end, so its first word is at byte 3.
if (ctx.stage != Stage::Mesh || !address.IsImmediate() || address.U32() != 3) {
throw NotImplementedException("32-bit stage buffer map write");
}
ctx.OpStore(ctx.mesh_primitive_count, value);
}
void EmitMeshPrologue(EmitContext& ctx) {
if (ctx.stage == Stage::Task) {
// (hookshot) the count starts at nothing: task payload memory holds whatever the last task left, and a task that keeps no meshlet
// never writes it. The barrier: every thread has done this before any of them goes on to write the real count
StoreWord(ctx, PayloadWord(ctx, ctx.Const(1U)), ctx.u32_zero_value);
Barrier(ctx);
}
if (ctx.stage != Stage::Mesh) {
return;
}
if (ctx.mesh_fallback && ctx.runtime_info.mesh_fallback_task) {
// (hookshot) the slot its task gave this workgroup, from the map at its index of the dispatch (TaskEpilogue). Read here,
// in the entry block, it serves every use; a load, no branch (see below)
ctx.mesh_fallback_slot =
LoadWord(ctx, BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index), WorkgroupId(ctx)));
}
// (local) The stage buffer's map and attribute regions start at zero, as they come out on NVIDIA: workgroup memory holds
// whatever the last workgroup on that unit left, and a vertex or an index byte the program left unwritten was that. On an
// RX 7700S the foliage drawn into the 2400 x 2400 shadow map around Link came out with hundreds of long thin triangles
// across the whole map (vertices from other meshlets' leftovers), dark lines on the ground, the water and Link that
// changed every frame. Zeroed, such a primitive is (0,0,0): no area. Straight-line stores, each clamped into its array
// (no branch: see below). RX_VK_MESH_NO_ZERO=1: as before
static const bool no_zero{std::getenv("RX_VK_MESH_NO_ZERO") != nullptr};
if (!no_zero && ctx.mesh_threads) {
const Id invocation{LocalInvocation(ctx)};
const auto zero_all = [&](Id array, u32 count) {
if (!Sirit::ValidId(array) || count == 0) {
return;
}
for (u32 first = 0; first < count; first += ctx.mesh_threads) {
const Id at{Clamped(ctx, ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first)), count)};
ctx.OpStore(SharedWord(ctx, array, at), ctx.u32_zero_value);
}
};
zero_all(ctx.mesh_map, ctx.mesh_map_bytes);
zero_all(ctx.mesh_attributes, ctx.mesh_attribute_words);
}
if (ctx.mesh_fallback && !ctx.runtime_info.mesh_fallback_task && ctx.mesh_threads) {
// (a mesh program with no task reads its workgroup index from the payload: NVN's hardware puts it at byte 0)
const Id invocation{LocalInvocation(ctx)};
for (u32 first = 0; first < ctx.mesh_info.payload_words; first += ctx.mesh_threads) {
const Id at{Clamped(ctx, ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first)),
ctx.mesh_info.payload_words)};
ctx.OpStore(SharedWord(ctx, ctx.mesh_payload, at), ctx.u32_zero_value);
}
if (ctx.mesh_info.payload_words) {
ctx.OpStore(SharedWord(ctx, ctx.mesh_payload, ctx.u32_zero_value), WorkgroupId(ctx));
}
Barrier(ctx);
}
// Every invocation stores the zero: no branch here, the entry block's label is a phi
// predecessor for later blocks.
ctx.OpStore(ctx.mesh_primitive_count, ctx.u32_zero_value);
Barrier(ctx);
}
void EmitMeshEpilogue(EmitContext& ctx) {
if (ctx.stage == Stage::Task) {
TaskEpilogue(ctx);
} else if (ctx.stage == Stage::Mesh) {
MeshEpilogue(ctx);
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,41 @@
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
Id EmitSelectU1(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.U1, cond, true_value, false_value);
}
Id EmitSelectU8(EmitContext&, Id, Id, Id) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitSelectU16(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.U16, cond, true_value, false_value);
}
Id EmitSelectU32(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.U32[1], cond, true_value, false_value);
}
Id EmitSelectU64(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.U64, cond, true_value, false_value);
}
Id EmitSelectF16(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.F16[1], cond, true_value, false_value);
}
Id EmitSelectF32(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.F32[1], cond, true_value, false_value);
}
Id EmitSelectF64(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
return ctx.OpSelect(ctx.F64[1], cond, true_value, false_value);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,176 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
Id Pointer(EmitContext& ctx, Id pointer_type, Id array, Id offset, u32 shift) {
const Id shift_id{ctx.Const(shift)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
return ctx.OpAccessChain(pointer_type, array, ctx.u32_zero_value, index);
}
Id Word(EmitContext& ctx, Id offset) {
const Id shift_id{ctx.Const(2U)};
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)};
return ctx.OpLoad(ctx.U32[1], pointer);
}
std::pair<Id, Id> ExtractArgs(EmitContext& ctx, Id offset, u32 mask, u32 count) {
const Id shift{ctx.OpShiftLeftLogical(ctx.U32[1], offset, ctx.Const(3U))};
const Id bit{ctx.OpBitwiseAnd(ctx.U32[1], shift, ctx.Const(mask))};
const Id count_id{ctx.Const(count)};
return {bit, count_id};
}
} // Anonymous namespace
Id EmitLoadSharedU8(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U8, pointer));
} else {
const auto [bit, count]{ExtractArgs(ctx, offset, 24, 8)};
return ctx.OpBitFieldUExtract(ctx.U32[1], Word(ctx, offset), bit, count);
}
}
Id EmitLoadSharedS8(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U8, pointer));
} else {
const auto [bit, count]{ExtractArgs(ctx, offset, 24, 8)};
return ctx.OpBitFieldSExtract(ctx.U32[1], Word(ctx, offset), bit, count);
}
}
Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
} else {
const auto [bit, count]{ExtractArgs(ctx, offset, 16, 16)};
return ctx.OpBitFieldUExtract(ctx.U32[1], Word(ctx, offset), bit, count);
}
}
Id EmitLoadSharedS16(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
} else {
const auto [bit, count]{ExtractArgs(ctx, offset, 16, 16)};
return ctx.OpBitFieldSExtract(ctx.U32[1], Word(ctx, offset), bit, count);
}
}
Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2)};
return ctx.OpLoad(ctx.U32[1], pointer);
} else {
return Word(ctx, offset);
}
}
Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
return ctx.OpLoad(ctx.U32[2], pointer);
} else {
const Id shift_id{ctx.Const(2U)};
const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
const Id next_index{ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(1U))};
const Id lhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, base_index)};
const Id rhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, next_index)};
return ctx.OpCompositeConstruct(ctx.U32[2], ctx.OpLoad(ctx.U32[1], lhs_pointer),
ctx.OpLoad(ctx.U32[1], rhs_pointer));
}
}
Id EmitLoadSharedU128(EmitContext& ctx, Id offset) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
return ctx.OpLoad(ctx.U32[4], pointer);
}
const Id shift_id{ctx.Const(2U)};
const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
std::array<Id, 4> values{};
for (u32 i = 0; i < 4; ++i) {
const Id index{i == 0 ? base_index : ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(i))};
const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)};
values[i] = ctx.OpLoad(ctx.U32[1], pointer);
}
return ctx.OpCompositeConstruct(ctx.U32[4], values);
}
void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
ctx.OpStore(pointer, ctx.OpUConvert(ctx.U8, value));
} else {
ctx.OpFunctionCall(ctx.void_id, ctx.shared_store_u8_func, offset, value);
}
}
void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
ctx.OpStore(pointer, ctx.OpUConvert(ctx.U16, value));
} else {
ctx.OpFunctionCall(ctx.void_id, ctx.shared_store_u16_func, offset, value);
}
}
void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
Id pointer{};
if (ctx.uses_explicit_workgroup_layout) {
pointer = Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2);
} else {
const Id shift{ctx.Const(2U)};
const Id word_offset{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)};
pointer = ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, word_offset);
}
ctx.OpStore(pointer, value);
}
void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
ctx.OpStore(pointer, value);
return;
}
const Id shift{ctx.Const(2U)};
const Id word_offset{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)};
const Id next_offset{ctx.OpIAdd(ctx.U32[1], word_offset, ctx.Const(1U))};
const Id lhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, word_offset)};
const Id rhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, next_offset)};
ctx.OpStore(lhs_pointer, ctx.OpCompositeExtract(ctx.U32[1], value, 0U));
ctx.OpStore(rhs_pointer, ctx.OpCompositeExtract(ctx.U32[1], value, 1U));
}
void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value) {
if (ctx.uses_explicit_workgroup_layout) {
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
ctx.OpStore(pointer, value);
return;
}
const Id shift{ctx.Const(2U)};
const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)};
for (u32 i = 0; i < 4; ++i) {
const Id index{i == 0 ? base_index : ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(i))};
const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)};
ctx.OpStore(pointer, ctx.OpCompositeExtract(ctx.U32[1], value, i));
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,186 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv.h"
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
void ConvertDepthMode(EmitContext& ctx) {
const Id type{ctx.F32[1]};
const Id position{ctx.OpLoad(ctx.F32[4], ctx.output_position)};
const Id z{ctx.OpCompositeExtract(type, position, 2u)};
const Id w{ctx.OpCompositeExtract(type, position, 3u)};
const Id screen_depth{ctx.OpFMul(type, ctx.OpFAdd(type, z, w), ctx.Constant(type, 0.5f))};
const Id vector{ctx.OpCompositeInsert(ctx.F32[4], screen_depth, position, 2u)};
ctx.OpStore(ctx.output_position, vector);
}
void SetFixedPipelinePointSize(EmitContext& ctx) {
if (ctx.runtime_info.fixed_state_point_size) {
const float point_size{*ctx.runtime_info.fixed_state_point_size};
ctx.OpStore(ctx.output_point_size, ctx.Const(point_size));
}
}
Id DefaultVarying(EmitContext& ctx, u32 num_components, u32 element, Id zero, Id one,
Id default_vector) {
switch (num_components) {
case 1:
return element == 3 ? one : zero;
case 2:
return ctx.ConstantComposite(ctx.F32[2], zero, element + 1 == 3 ? one : zero);
case 3:
return ctx.ConstantComposite(ctx.F32[3], zero, zero, element + 2 == 3 ? one : zero);
case 4:
return default_vector;
}
throw InvalidArgument("Bad element");
}
Id ComparisonFunction(EmitContext& ctx, CompareFunction comparison, Id operand_1, Id operand_2) {
switch (comparison) {
case CompareFunction::Never:
return ctx.false_value;
case CompareFunction::Less:
return ctx.OpFOrdLessThan(ctx.U1, operand_1, operand_2);
case CompareFunction::Equal:
return ctx.OpFOrdEqual(ctx.U1, operand_1, operand_2);
case CompareFunction::LessThanEqual:
return ctx.OpFOrdLessThanEqual(ctx.U1, operand_1, operand_2);
case CompareFunction::Greater:
return ctx.OpFOrdGreaterThan(ctx.U1, operand_1, operand_2);
case CompareFunction::NotEqual:
return ctx.OpFOrdNotEqual(ctx.U1, operand_1, operand_2);
case CompareFunction::GreaterThanEqual:
return ctx.OpFOrdGreaterThanEqual(ctx.U1, operand_1, operand_2);
case CompareFunction::Always:
return ctx.true_value;
}
throw InvalidArgument("Comparison function {}", comparison);
}
void AlphaTest(EmitContext& ctx) {
if (!ctx.runtime_info.alpha_test_func) {
return;
}
const auto comparison{*ctx.runtime_info.alpha_test_func};
if (comparison == CompareFunction::Always) {
return;
}
if (!Sirit::ValidId(ctx.frag_color[0])) {
return;
}
const Id type{ctx.F32[1]};
const Id rt0_color{ctx.OpLoad(ctx.F32[4], ctx.frag_color[0])};
const Id alpha{ctx.OpCompositeExtract(type, rt0_color, 3u)};
const Id true_label{ctx.OpLabel()};
const Id discard_label{ctx.OpLabel()};
const Id alpha_reference{ctx.Const(ctx.runtime_info.alpha_test_reference)};
const Id condition{ComparisonFunction(ctx, comparison, alpha, alpha_reference)};
ctx.OpSelectionMerge(true_label, spv::SelectionControlMask::MaskNone);
ctx.OpBranchConditional(condition, true_label, discard_label);
ctx.AddLabel(discard_label);
ctx.OpKill();
ctx.AddLabel(true_label);
}
} // Anonymous namespace
void EmitPrologue(EmitContext& ctx) {
if (ctx.stage == Stage::Fragment && ctx.runtime_info.dual_source_blend) {
// Initialize dual-source blending outputs - prevents MoltenVK crash.
const Id zero{ctx.Const(0.0f)};
const Id one{ctx.Const(1.0f)};
const Id default_color{ctx.ConstantComposite(ctx.F32[4], zero, zero, zero, one)};
for (u32 i = 0; i < 2; ++i) {
if (Sirit::ValidId(ctx.frag_color[i])) {
ctx.OpStore(ctx.frag_color[i], default_color);
}
}
}
if (ctx.stage == Stage::VertexB) {
const Id zero{ctx.Const(0.0f)};
const Id one{ctx.Const(1.0f)};
const Id default_vector{ctx.ConstantComposite(ctx.F32[4], zero, zero, zero, one)};
ctx.OpStore(ctx.output_position, default_vector);
for (const auto& info : ctx.output_generics) {
if (info[0].num_components == 0) {
continue;
}
u32 element{0};
while (element < 4) {
const auto& element_info{info[element]};
const u32 num{element_info.num_components};
const Id value{DefaultVarying(ctx, num, element, zero, one, default_vector)};
ctx.OpStore(element_info.id, value);
element += num;
}
}
if (Sirit::ValidId(ctx.clip_distances)) {
for (u32 i = 0; i < ctx.profile.max_user_clip_distances; ++i) {
if (!clip_distance_written.test(i)) {
const Id idx = ctx.Const(i);
const Id element = ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, idx);
ctx.OpStore(element, ctx.Const(0.0f));
}
}
}
}
if (ctx.stage == Stage::VertexB || ctx.stage == Stage::Geometry) {
SetFixedPipelinePointSize(ctx);
}
if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) {
EmitMeshPrologue(ctx);
}
}
void EmitEpilogue(EmitContext& ctx) {
if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) {
EmitMeshEpilogue(ctx);
return;
}
if (ctx.stage == Stage::VertexB && ctx.runtime_info.convert_depth_mode &&
!ctx.profile.support_native_ndc) {
ConvertDepthMode(ctx);
}
if (ctx.stage == Stage::Fragment) {
AlphaTest(ctx);
}
}
void EmitEmitVertex(EmitContext& ctx, const IR::Value& stream) {
if (ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc) {
ConvertDepthMode(ctx);
}
if (!ctx.profile.support_geometry_streams) {
throw NotImplementedException("Geometry streams");
} else if (stream.IsImmediate()) {
ctx.OpEmitStreamVertex(ctx.Def(stream));
} else {
LOG_WARNING(Shader_SPIRV, "Stream is not immediate");
ctx.OpEmitStreamVertex(ctx.u32_zero_value);
}
// Restore fixed pipeline point size after emitting the vertex
SetFixedPipelinePointSize(ctx);
}
void EmitEndPrimitive(EmitContext& ctx, const IR::Value& stream) {
if (!ctx.profile.support_geometry_streams) {
throw NotImplementedException("Geometry streams");
} else if (stream.IsImmediate()) {
ctx.OpEndStreamPrimitive(ctx.Def(stream));
} else {
LOG_WARNING(Shader_SPIRV, "Stream is not immediate");
ctx.OpEndStreamPrimitive(ctx.u32_zero_value);
}
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,47 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <cstdlib>
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
namespace Shader::Backend::SPIRV {
namespace {
// Registers read before any write are zero rather than OpUndef, as NVIDIA hardware and drivers
// give them; AMD's OpUndef is whatever the register held. Title 1's glow pass exits before
// writing its colour on some pixels, which AMD blended in as green patches on the road.
// EDEN_NX2_UNDEF_ZERO=0 emits OpUndef again.
bool UndefIsZero() {
static const bool zero = [] {
const char* const value = std::getenv("EDEN_NX2_UNDEF_ZERO");
return value == nullptr || value[0] != '0';
}();
return zero;
}
} // Anonymous namespace
Id EmitUndefU1(EmitContext& ctx) {
return UndefIsZero() ? ctx.false_value : ctx.OpUndef(ctx.U1);
}
Id EmitUndefU8(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitUndefU16(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
Id EmitUndefU32(EmitContext& ctx) {
return UndefIsZero() ? ctx.u32_zero_value : ctx.OpUndef(ctx.U32[1]);
}
Id EmitUndefU64(EmitContext&) {
throw NotImplementedException("SPIR-V Instruction");
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,540 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
#include "shader_recompiler/backend/spirv/mma_layout.h"
namespace Shader::Backend::SPIRV {
namespace {
Id SubgroupScope(EmitContext& ctx) {
return ctx.Const(static_cast<u32>(spv::Scope::Subgroup));
}
bool StageSupportsSubgroups(EmitContext& ctx) {
return ctx.profile.SupportsSubgroupStage(ctx.stage);
}
Id GetThreadId(EmitContext& ctx) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.u32_zero_value;
}
return ctx.OpLoad(ctx.U32[1], ctx.subgroup_local_invocation_id);
}
Id WarpExtract(EmitContext& ctx, Id value) {
const Id thread_id{GetThreadId(ctx)};
const Id local_index{ctx.OpShiftRightArithmetic(ctx.U32[1], thread_id, ctx.Const(5U))};
if (ctx.profile.has_broken_spirv_subgroup_mask_vector_extract_dynamic) {
const Id c0_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(0U)),
ctx.OpCompositeExtract(ctx.U32[1], value, 0U), ctx.Const(0U))};
const Id c1_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(1U)),
ctx.OpCompositeExtract(ctx.U32[1], value, 1U), ctx.Const(0U))};
const Id c2_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(2U)),
ctx.OpCompositeExtract(ctx.U32[1], value, 2U), ctx.Const(0U))};
const Id c3_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(3U)),
ctx.OpCompositeExtract(ctx.U32[1], value, 3U), ctx.Const(0U))};
const Id c0_or_c1{ctx.OpBitwiseOr(ctx.U32[1], c0_sel, c1_sel)};
const Id c2_or_c3{ctx.OpBitwiseOr(ctx.U32[1], c2_sel, c3_sel)};
const Id c0_or_c1_or_c2_or_c3{ctx.OpBitwiseOr(ctx.U32[1], c0_or_c1, c2_or_c3)};
return c0_or_c1_or_c2_or_c3;
} else {
return ctx.OpVectorExtractDynamic(ctx.U32[1], value, local_index);
}
}
Id LoadMask(EmitContext& ctx, Id mask) {
const Id value{ctx.OpLoad(ctx.U32[4], mask)};
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return ctx.OpCompositeExtract(ctx.U32[1], value, 0U);
}
return WarpExtract(ctx, value);
}
void SetInBoundsFlag(IR::Inst* inst, Id result) {
IR::Inst* const in_bounds{inst->GetAssociatedPseudoOperation(IR::Opcode::GetInBoundsFromOp)};
if (!in_bounds) {
return;
}
in_bounds->SetDefinition(result);
in_bounds->Invalidate();
}
Id ComputeMinThreadId(EmitContext& ctx, Id thread_id, Id segmentation_mask) {
return ctx.OpBitwiseAnd(ctx.U32[1], thread_id, segmentation_mask);
}
Id ComputeMaxThreadId(EmitContext& ctx, Id min_thread_id, Id clamp, Id not_seg_mask) {
return ctx.OpBitwiseOr(ctx.U32[1], min_thread_id,
ctx.OpBitwiseAnd(ctx.U32[1], clamp, not_seg_mask));
}
Id GetMaxThreadId(EmitContext& ctx, Id thread_id, Id clamp, Id segmentation_mask) {
const Id not_seg_mask{ctx.OpNot(ctx.U32[1], segmentation_mask)};
const Id min_thread_id{ComputeMinThreadId(ctx, thread_id, segmentation_mask)};
return ComputeMaxThreadId(ctx, min_thread_id, clamp, not_seg_mask);
}
Id HostThreadId(EmitContext& ctx, Id thread_id) {
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return thread_id;
}
const Id partition_idx{ctx.OpShiftRightLogical(ctx.U32[1], GetThreadId(ctx), ctx.Const(5u))};
const Id partition_base{ctx.OpShiftLeftLogical(ctx.U32[1], partition_idx, ctx.Const(5u))};
return ctx.OpIAdd(ctx.U32[1], thread_id, partition_base);
}
Id GuestLane(EmitContext& ctx, Id index) {
return ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U));
}
bool UseShuffleFallback(EmitContext& ctx) {
return ctx.profile.has_broken_spirv_subgroup_shuffle ||
(ctx.profile.has_broken_spirv_fragment_shuffle && ctx.stage == Stage::Fragment);
}
Id ShuffleAbsolute(EmitContext& ctx, Id value, Id src_thread_id) {
if (!UseShuffleFallback(ctx)) {
return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id);
}
Id result{ctx.u32_zero_value};
for (u32 lane = 0; lane < ctx.profile.max_subgroup_size; ++lane) {
const Id read{
ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, ctx.Const(lane))};
const Id matches{ctx.OpIEqual(ctx.U1, src_thread_id, ctx.Const(lane))};
result = ctx.OpSelect(ctx.U32[1], matches, read, result);
}
return result;
}
Id ShuffleRelative(EmitContext& ctx, Id value, Id delta, Id src_thread_id, spv::Op op) {
if (!ctx.profile.support_shuffle_relative || UseShuffleFallback(ctx)) {
return ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id));
}
const Id result{ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, delta)};
ctx.opcode_rewrites.emplace_back(result.value, op);
return result;
}
Id BroadcastLane(EmitContext& ctx, Id value, u32 lane) {
Id result{
ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, ctx.Const(lane))};
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return result;
}
const Id partition_idx{ctx.OpShiftRightLogical(ctx.U32[1], GetThreadId(ctx), ctx.Const(5u))};
for (u32 base = 32; base < ctx.profile.max_subgroup_size; base += 32) {
const Id read{ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value,
ctx.Const(base + lane))};
const Id matches{ctx.OpIEqual(ctx.U1, partition_idx, ctx.Const(base >> 5))};
result = ctx.OpSelect(ctx.U32[1], matches, read, result);
}
return result;
}
} // Anonymous namespace
Id EmitLaneId(EmitContext& ctx) {
const Id id{GetThreadId(ctx)};
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return id;
}
return ctx.OpBitwiseAnd(ctx.U32[1], id, ctx.Const(31U));
}
Id EmitVoteAll(EmitContext& ctx, Id pred) {
if (!StageSupportsSubgroups(ctx)) {
return pred;
}
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return ctx.OpGroupNonUniformAll(ctx.U1, SubgroupScope(ctx), pred);
}
const Id mask_ballot{
ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), ctx.true_value)};
const Id active_mask{WarpExtract(ctx, mask_ballot)};
const Id ballot{
WarpExtract(ctx, ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred))};
const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], ballot, active_mask)};
return ctx.OpIEqual(ctx.U1, lhs, active_mask);
}
Id EmitVoteAny(EmitContext& ctx, Id pred) {
if (!StageSupportsSubgroups(ctx)) {
return pred;
}
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return ctx.OpGroupNonUniformAny(ctx.U1, SubgroupScope(ctx), pred);
}
const Id mask_ballot{
ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), ctx.true_value)};
const Id active_mask{WarpExtract(ctx, mask_ballot)};
const Id ballot{
WarpExtract(ctx, ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred))};
const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], ballot, active_mask)};
return ctx.OpINotEqual(ctx.U1, lhs, ctx.u32_zero_value);
}
Id EmitVoteEqual(EmitContext& ctx, Id pred) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.true_value;
}
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return ctx.OpGroupNonUniformAllEqual(ctx.U1, SubgroupScope(ctx), pred);
}
const Id mask_ballot{
ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), ctx.true_value)};
const Id active_mask{WarpExtract(ctx, mask_ballot)};
const Id ballot{
WarpExtract(ctx, ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred))};
const Id lhs{ctx.OpBitwiseXor(ctx.U32[1], ballot, active_mask)};
return ctx.OpLogicalOr(ctx.U1, ctx.OpIEqual(ctx.U1, lhs, ctx.u32_zero_value),
ctx.OpIEqual(ctx.U1, lhs, active_mask));
}
Id EmitSubgroupBallot(EmitContext& ctx, Id pred) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.OpSelect(ctx.U32[1], pred, ctx.Const(1u), ctx.u32_zero_value);
}
const Id ballot{ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred)};
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
return ctx.OpCompositeExtract(ctx.U32[1], ballot, 0U);
}
return WarpExtract(ctx, ballot);
}
Id EmitSubgroupEqMask(EmitContext& ctx) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.Const(1u);
}
return LoadMask(ctx, ctx.subgroup_mask_eq);
}
Id EmitSubgroupLtMask(EmitContext& ctx) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.u32_zero_value;
}
return LoadMask(ctx, ctx.subgroup_mask_lt);
}
Id EmitSubgroupLeMask(EmitContext& ctx) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.Const(1u);
}
return LoadMask(ctx, ctx.subgroup_mask_le);
}
Id EmitSubgroupGtMask(EmitContext& ctx) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.u32_zero_value;
}
return LoadMask(ctx, ctx.subgroup_mask_gt);
}
Id EmitSubgroupGeMask(EmitContext& ctx) {
if (!StageSupportsSubgroups(ctx)) {
return ctx.Const(1u);
}
return LoadMask(ctx, ctx.subgroup_mask_ge);
}
Id EmitShuffleIndex(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask) {
const Id not_seg_mask{ctx.OpNot(ctx.U32[1], segmentation_mask)};
const Id thread_id{EmitLaneId(ctx)};
const Id min_thread_id{ComputeMinThreadId(ctx, thread_id, segmentation_mask)};
const Id max_thread_id{ComputeMaxThreadId(ctx, min_thread_id, clamp, not_seg_mask)};
const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], GuestLane(ctx, index), not_seg_mask)};
const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], lhs, min_thread_id)};
const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)};
SetInBoundsFlag(inst, in_range);
if (!StageSupportsSubgroups(ctx)) {
return value;
}
const IR::Value lane{inst->Arg(1).Resolve()};
const IR::Value segment{inst->Arg(3).Resolve()};
if (lane.IsImmediate() && segment.IsImmediate() && segment.U32() == 0) {
return ctx.OpSelect(ctx.U32[1], in_range, BroadcastLane(ctx, value, lane.U32() & 31),
value);
}
const Id shuffled{ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id))};
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
}
Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask) {
if (!StageSupportsSubgroups(ctx)) {
SetInBoundsFlag(inst, ctx.false_value);
return value;
}
const Id delta{GuestLane(ctx, index)};
const Id thread_id{EmitLaneId(ctx)};
const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
const Id src_thread_id{ctx.OpISub(ctx.U32[1], thread_id, delta)};
const Id in_range{ctx.OpSGreaterThanEqual(ctx.U1, src_thread_id, max_thread_id)};
SetInBoundsFlag(inst, in_range);
const Id shuffled{ShuffleRelative(ctx, value, delta, src_thread_id,
spv::Op::OpGroupNonUniformShuffleUp)};
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
}
Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask) {
if (!StageSupportsSubgroups(ctx)) {
SetInBoundsFlag(inst, ctx.false_value);
return value;
}
const Id delta{GuestLane(ctx, index)};
const Id thread_id{EmitLaneId(ctx)};
const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
const Id src_thread_id{ctx.OpIAdd(ctx.U32[1], thread_id, delta)};
const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)};
SetInBoundsFlag(inst, in_range);
const Id shuffled{ShuffleRelative(ctx, value, delta, src_thread_id,
spv::Op::OpGroupNonUniformShuffleDown)};
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
}
Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
Id segmentation_mask) {
if (!StageSupportsSubgroups(ctx)) {
SetInBoundsFlag(inst, ctx.false_value);
return value;
}
const Id mask{GuestLane(ctx, index)};
const Id thread_id{EmitLaneId(ctx)};
const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
const Id src_thread_id{ctx.OpBitwiseXor(ctx.U32[1], thread_id, mask)};
const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)};
SetInBoundsFlag(inst, in_range);
// (hookshot) through the broadcasts where shuffles are broken (UseShuffleFallback: AMD's fragment stages, Lichii #273),
// as the absolute and relative shuffles are; #273 left SHFL.BFLY native, and some of Kokiri Forest's plants use it
const Id shuffled{UseShuffleFallback(ctx)
? ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id))
: ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, mask)};
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
}
Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) {
if (ctx.profile.support_quad_shuffles) {
return ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, lane);
}
const Id base{ctx.OpBitwiseAnd(ctx.U32[1], GetThreadId(ctx), ctx.Const(~3u))};
const Id local_lane{ctx.OpBitwiseAnd(ctx.U32[1], lane, ctx.Const(3u))};
const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], base, local_lane)};
return ShuffleAbsolute(ctx, value, src_thread_id);
}
Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction) {
if (ctx.profile.support_quad_shuffles) {
const Id result{
ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, direction)};
ctx.opcode_rewrites.emplace_back(result.value, spv::Op::OpGroupNonUniformQuadSwap);
return result;
}
const Id xor_mask{ctx.OpIAdd(ctx.U32[1], direction, ctx.Const(1u))};
// (hookshot) not through UseShuffleFallback's broadcasts: a program with quad shuffles alone (uses_quad_shuffles) has
// neither the lane id input nor the ballot capability they need, and the SPIR-V made with them crashed the start
return ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, xor_mask);
}
Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle) {
const Id three{ctx.Const(3U)};
Id mask{GetThreadId(ctx)};
mask = ctx.OpBitwiseAnd(ctx.U32[1], mask, three);
mask = ctx.OpShiftLeftLogical(ctx.U32[1], mask, ctx.Const(1U));
mask = ctx.OpShiftRightLogical(ctx.U32[1], swizzle, mask);
mask = ctx.OpBitwiseAnd(ctx.U32[1], mask, three);
const Id modifier_a{ctx.OpVectorExtractDynamic(ctx.F32[1], ctx.fswzadd_lut_a, mask)};
const Id modifier_b{ctx.OpVectorExtractDynamic(ctx.F32[1], ctx.fswzadd_lut_b, mask)};
const Id result_a{ctx.OpFMul(ctx.F32[1], op_a, modifier_a)};
const Id result_b{ctx.OpFMul(ctx.F32[1], op_b, modifier_b)};
return ctx.OpFAdd(ctx.F32[1], result_a, result_b);
}
Id EmitDPdxFine(EmitContext& ctx, Id op_a) {
return ctx.OpDPdxFine(ctx.F32[1], op_a);
}
Id EmitDPdyFine(EmitContext& ctx, Id op_a) {
return ctx.OpDPdyFine(ctx.F32[1], op_a);
}
Id EmitDPdxCoarse(EmitContext& ctx, Id op_a) {
return ctx.OpDPdxCoarse(ctx.F32[1], op_a);
}
Id EmitDPdyCoarse(EmitContext& ctx, Id op_a) {
return ctx.OpDPdyCoarse(ctx.F32[1], op_a);
}
// (hookshot) HMMA in software, from Lichii (zortos293/Lichii, emit_spirv_tensor.cpp, GPL-3.0): the 32 lanes' fragments
// through subgroup shuffles in one helper function, accumulated with FMA in f32 and rounded to f16 after every 8
// columns of K, as HMMA.F16 does. It needs no cooperative matrices (RX_VK_MMA_SOFTWARE=1, vk_coopmat.cpp).
void DefineSoftwareMatrixFunction(EmitContext& ctx) {
{
constexpr bool variant = true;
ctx.AddCapability(spv::Capability::GroupNonUniform);
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
if (!ctx.profile.SupportsSubgroupStage(ctx.stage) ||
ctx.profile.has_broken_spirv_subgroup_shuffle)
throw NotImplementedException("Software HMMA requires subgroup shuffle support");
const Id u32ty = ctx.U32[1], f32ty = ctx.F32[1];
const auto imm = [&](u32 v) { return ctx.Const(v); };
const auto add = [&](Id a, Id b) { return ctx.OpIAdd(u32ty, a, b); };
const auto and_ = [&](Id a, u32 b) { return ctx.OpBitwiseAnd(u32ty, a, imm(b)); };
const auto unpack = [&](Id v) { return ctx.OpUnpackHalf2x16(ctx.F32[2], v); };
const auto round_half = [&](Id value) {
// Do not express the intermediate rounding as unpack(pack(value)): drivers may
// eliminate that round trip. Round normal significands explicitly, ties to even;
// half subnormals lie on the exactly representable 2^-24 grid.
const Id bits = ctx.OpBitcast(u32ty, value);
const Id magnitude = and_(bits, 0x7fffffffU);
const Id tie = and_(ctx.OpShiftRightLogical(u32ty, magnitude, imm(13)), 1);
const Id rounded = and_(add(add(magnitude, imm(0xfff)), tie), ~0x1fffU);
const Id normal = ctx.OpSelect(u32ty,
ctx.OpUGreaterThanEqual(ctx.U1, rounded, imm(0x47800000)), imm(0x7f800000), rounded);
const Id scaled = ctx.OpFMul(f32ty, ctx.OpBitcast(f32ty, magnitude), ctx.Const(16777216.0f));
const Id subnormal = ctx.OpBitcast(u32ty,
ctx.OpFMul(f32ty, ctx.OpRoundEven(f32ty, scaled), ctx.Const(0x1p-24f)));
const Id finite = ctx.OpSelect(u32ty,
ctx.OpUGreaterThanEqual(ctx.U1, magnitude, imm(0x38800000)), normal, subnormal);
const Id result = ctx.OpSelect(u32ty,
ctx.OpUGreaterThanEqual(ctx.U1, magnitude, imm(0x7f800000)), magnitude, finite);
return ctx.OpBitcast(f32ty, ctx.OpBitwiseOr(u32ty, result, and_(bits, 0x80000000)));
};
const Id type = ctx.TypeFunction(ctx.U32[2], ctx.U32[4], ctx.U32[2], ctx.U32[2]);
const Id fn = ctx.OpFunction(ctx.U32[2], spv::FunctionControlMask::DontInline, type);
ctx.software_matrix_function = fn;
const Id matrix_a = ctx.OpFunctionParameter(ctx.U32[4]);
const Id matrix_b = ctx.OpFunctionParameter(ctx.U32[2]);
const Id matrix_c = ctx.OpFunctionParameter(ctx.U32[2]);
ctx.AddLabel();
const Id counter = ctx.AddLocalVariable(ctx.TypePointer(spv::StorageClass::Function, u32ty),
spv::StorageClass::Function);
std::array<Id, 4> accumulators;
for (auto& acc : accumulators)
acc = ctx.AddLocalVariable(ctx.TypePointer(spv::StorageClass::Function, f32ty),
spv::StorageClass::Function);
std::array<Id, 8> inputs;
for (u32 i = 0; i < 4; ++i)
inputs[i] = ctx.OpCompositeExtract(u32ty, matrix_a, i);
for (u32 i = 0; i < 2; ++i) {
inputs[4 + i] = ctx.OpCompositeExtract(u32ty, matrix_b, i);
inputs[6 + i] = ctx.OpCompositeExtract(u32ty, matrix_c, i);
}
for (u32 row = 0; row < 2; ++row) {
const Id c = unpack(inputs[6 + row]);
for (u32 col = 0; col < 2; ++col)
ctx.OpStore(accumulators[row * 2 + col], ctx.OpCompositeExtract(f32ty, c, col));
}
ctx.OpStore(counter, imm(0));
const Id lane = ctx.OpLoad(u32ty, ctx.subgroup_local_invocation_id);
const Id warp_base = and_(lane, ~31U);
const Id row_base = and_(lane, 28);
const Id col_base = ctx.OpShiftLeftLogical(u32ty, and_(lane, 3), imm(3));
const Id header = ctx.OpLabel(), body = ctx.OpLabel(), next = ctx.OpLabel(), end = ctx.OpLabel();
ctx.OpBranch(header);
ctx.AddLabel(header);
const Id pair = ctx.OpLoad(u32ty, counter);
const Id more = ctx.OpULessThan(ctx.U1, pair, imm(variant ? 8 : 4));
ctx.OpLoopMerge(end, next, spv::LoopControlMask::DontUnroll);
ctx.OpBranchConditional(more, body, end);
ctx.AddLabel(body);
const Id upper = ctx.OpUGreaterThanEqual(ctx.U1, pair, imm(4));
const Id component = and_(pair, 3);
const Id a_lane = add(warp_base, add(row_base, component));
const Id b_lane = add(warp_base, add(col_base, component));
const Id b = ctx.OpSelect(u32ty, upper, inputs[5], inputs[4]);
std::array<Id, 2> bs;
for (u32 col = 0; col < 2; ++col)
bs[col] = unpack(ctx.OpGroupNonUniformShuffle(u32ty, imm(3), b,
add(b_lane, imm(col * 4))));
const Id round_now = ctx.OpIEqual(ctx.U1, component, imm(3));
for (u32 row = 0; row < 2; ++row) {
const Id a = ctx.OpSelect(u32ty, upper, inputs[row + 2], inputs[row]);
const Id as = unpack(ctx.OpGroupNonUniformShuffle(u32ty, imm(3), a, a_lane));
for (u32 col = 0; col < 2; ++col) {
Id value = ctx.OpLoad(f32ty, accumulators[row * 2 + col]);
for (u32 half = 0; half < 2; ++half) {
value = ctx.OpFma(f32ty, ctx.OpCompositeExtract(f32ty, as, half),
ctx.OpCompositeExtract(f32ty, bs[col], half), value);
ctx.Decorate(value, spv::Decoration::NoContraction);
}
ctx.OpStore(accumulators[row * 2 + col], value);
}
}
// HMMA.F16 rounds after each K=8 group, including the midpoint of K=16.
// OpSelect evaluates both operands: keep the expensive half conversion in
// a branch so the other three pairs do no rounding work. The counter is
// identical in every participating lane and all shuffles precede this branch.
const Id round_block = ctx.OpLabel(), round_merge = ctx.OpLabel();
ctx.OpSelectionMerge(round_merge, spv::SelectionControlMask::DontFlatten);
ctx.OpBranchConditional(round_now, round_block, round_merge);
ctx.AddLabel(round_block);
for (const Id accumulator : accumulators) {
ctx.OpStore(accumulator, round_half(ctx.OpLoad(f32ty, accumulator)));
}
ctx.OpBranch(round_merge);
ctx.AddLabel(round_merge);
ctx.OpBranch(next);
ctx.AddLabel(next);
ctx.OpStore(counter, add(pair, imm(1)));
ctx.OpBranch(header);
ctx.AddLabel(end);
std::array<Id, 2> packed;
for (u32 row = 0; row < 2; ++row)
packed[row] = ctx.OpPackHalf2x16(u32ty, ctx.OpCompositeConstruct(ctx.F32[2],
ctx.OpLoad(f32ty, accumulators[row * 2]), ctx.OpLoad(f32ty, accumulators[row * 2 + 1])));
ctx.OpReturnValue(ctx.OpCompositeConstruct(ctx.U32[2], packed[0], packed[1]));
ctx.OpFunctionEnd();
}
}
// (hookshot) HMMA_nx2: D(16x8) = A(16x16) * B(16x8) + C on the subgroup's cooperative matrices. Each
// lane's fragment registers hold its elements in the console's order (two f16 a word); they go to and come
// from the device's elements in its own order (mma_layout.h, measured by vk_coopmat.cpp).
Id EmitMatrixMulAddF16(EmitContext& ctx, Id a, Id b, Id c) {
if (mma_software) return ctx.OpFunctionCall(ctx.U32[2], ctx.software_matrix_function, a, b, c);
const Id f16{ctx.F16[1]};
const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Subgroup))};
const auto type = [&](u32 rows, u32 columns, spv::CooperativeMatrixUse use) {
return ctx.TypeCooperativeMatrixKHR(f16, scope, ctx.Const(rows), ctx.Const(columns),
ctx.Const(static_cast<u32>(use)));
};
const Id a_type{type(16, 16, spv::CooperativeMatrixUse::MatrixAKHR)};
const Id b_type{type(16, 8, spv::CooperativeMatrixUse::MatrixBKHR)};
const Id c_type{type(16, 8, spv::CooperativeMatrixUse::MatrixAccumulatorKHR)};
const Id zero{ctx.Constant(f16, 0U)};
const auto load = [&](Id matrix_type, Id words, u32 count, const auto& order) {
Id m{ctx.OpCompositeConstruct(matrix_type, zero)};
for (u32 w = 0; w < count; ++w) {
const Id word{count == 1 ? words : ctx.OpCompositeExtract(ctx.U32[1], words, w)};
const Id pair{ctx.OpBitcast(ctx.F16[2], word)};
for (u32 h = 0; h < 2; ++h) {
m = ctx.OpCompositeInsert(matrix_type, ctx.OpCompositeExtract(f16, pair, h), m, order[w * 2 + h]);
}
}
return m;
};
const Id d{ctx.OpCooperativeMatrixMulAddKHR(c_type, load(a_type, a, 4, mma_order_a), load(b_type, b, 2, mma_order_b),
load(c_type, c, 2, mma_order_c))};
Id out[2];
for (u32 w = 0; w < 2; ++w) {
const Id pair{ctx.OpCompositeConstruct(ctx.F16[2], ctx.OpCompositeExtract(f16, d, mma_order_c[w * 2]),
ctx.OpCompositeExtract(f16, d, mma_order_c[w * 2 + 1]))};
out[w] = ctx.OpBitcast(ctx.U32[1], pair);
}
return ctx.OpCompositeConstruct(ctx.U32[2], out[0], out[1]);
}
} // namespace Shader::Backend::SPIRV

View File

@@ -0,0 +1,29 @@
// (hookshot) The device's element order of the 16x8x16 f16 cooperative matrices HMMA_nx2 runs on.
//
// A lane's HMMA fragment registers hold its matrix elements in the console's order (PTX mma.m16n8k16's: two f16
// a register, low half first). The host's cooperative matrices hold each lane's share in an order of the
// implementation's own; vk_coopmat.cpp measures it at start, and only when every lane holds the same positions in
// both does the back end turn HMMA_nx2 on. Element e of the console's fragment is then the device's element
// mma_order_x[e].
#pragma once
#include <array>
#include <cstdint>
namespace Shader::Backend::SPIRV {
inline std::array<uint32_t, 8> mma_order_a{0, 1, 2, 3, 4, 5, 6, 7};
inline std::array<uint32_t, 4> mma_order_b{0, 1, 2, 3};
inline std::array<uint32_t, 4> mma_order_c{0, 1, 2, 3};
inline bool mma_software = false; // (HMMA_nx2 in software: emit_spirv_warp.cpp, RX_VK_MMA_SOFTWARE=1)
// (amd-fix) float colour outputs carry no NaN or infinity to their target (EmitSetFragColor): one NaN texel spreads
// through bloom/blur/downsample averages into glowing blocks (OoT's flowers on AMD, Hookshot issue 606). On for every
// card but NVIDIA (vk_coopmat.cpp coopmat_init); RX_VK_NO_NAN_GUARD=1 turns it off. Idea from NX2navi patch 0027.
inline bool frag_nan_guard = false;
// (amd-fix) gl_Position decorated Invariant (vertex and mesh outputs): the game draws a depth pre-pass, then the colour
// pass again with depth EQUAL; Maxwell computes the same position in both, AMD's compiler may not without Invariant, and
// the colour pass is then rejected in spots: holes showing the bright sky, blown up by bloom (the glowing flowers).
// On for every card but NVIDIA (vk_coopmat.cpp); RX_VK_NO_INVARIANT=1 off. From NX2navi patch 0007.
inline bool position_invariant = false;
} // namespace Shader::Backend::SPIRV

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,477 @@
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <array>
#include <sirit/sirit.h>
#include "common/container/unordered_set.h"
#include "shader_recompiler/backend/bindings.h"
#include "shader_recompiler/frontend/ir/program.h"
#include "shader_recompiler/profile.h"
#include "shader_recompiler/runtime_info.h"
#include "shader_recompiler/shader_info.h"
namespace Shader::Backend::SPIRV {
static std::bitset<8> clip_distance_written;
using Sirit::Id;
class VectorTypes {
public:
void Define(Sirit::Module& sirit_ctx, Id base_type, std::string_view name);
[[nodiscard]] Id operator[](size_t size) const noexcept {
return defs[size - 1];
}
private:
std::array<Id, 4> defs{};
};
struct TextureDefinition {
Id id;
Id sampled_type;
Id pointer_type;
Id image_type;
u32 count;
bool is_multisample;
bool is_integer;
};
struct TextureBufferDefinition {
Id id;
u32 count;
};
struct ImageBufferDefinition {
Id id;
Id image_type;
Id pointer_type;
u32 count;
bool is_integer;
};
struct ImageDefinition {
Id id;
Id image_type;
Id pointer_type;
u32 count;
bool is_integer;
};
struct UniformDefinitions {
Id U8{};
Id S8{};
Id U16{};
Id S16{};
Id U32{};
Id F32{};
Id U32x2{};
Id U32x4{};
constexpr static size_t NumElements(Id UniformDefinitions::*member_ptr) {
if (member_ptr == &UniformDefinitions::U8) {
return 1;
}
if (member_ptr == &UniformDefinitions::S8) {
return 1;
}
if (member_ptr == &UniformDefinitions::U16) {
return 1;
}
if (member_ptr == &UniformDefinitions::S16) {
return 1;
}
if (member_ptr == &UniformDefinitions::U32) {
return 1;
}
if (member_ptr == &UniformDefinitions::F32) {
return 1;
}
if (member_ptr == &UniformDefinitions::U32x2) {
return 2;
}
if (member_ptr == &UniformDefinitions::U32x4) {
return 4;
}
ASSERT(false);
return 1;
}
constexpr static bool IsFloat(Id UniformDefinitions::*member_ptr) {
if (member_ptr == &UniformDefinitions::F32) {
return true;
}
return false;
}
};
struct StorageTypeDefinition {
Id array{};
Id element{};
};
struct StorageTypeDefinitions {
StorageTypeDefinition U8{};
StorageTypeDefinition S8{};
StorageTypeDefinition U16{};
StorageTypeDefinition S16{};
StorageTypeDefinition U32{};
StorageTypeDefinition U64{};
StorageTypeDefinition F32{};
StorageTypeDefinition U32x2{};
StorageTypeDefinition U32x4{};
};
struct StorageDefinitions {
Id U8{};
Id S8{};
Id U16{};
Id S16{};
Id U32{};
Id F32{};
Id U64{};
Id U32x2{};
Id U32x4{};
};
enum class InputGenericLoadOp {
None,
Bitcast,
SToF,
UToF,
};
struct InputGenericInfo {
Id id;
Id pointer_type;
Id component_type;
InputGenericLoadOp load_op;
};
struct GenericElementInfo {
Id id{};
u32 first_element{};
u32 num_components{};
};
class EmitContext final : public Sirit::Module {
public:
explicit EmitContext(const Profile& profile, const RuntimeInfo& runtime_info,
IR::Program& program, Bindings& binding);
~EmitContext();
[[nodiscard]] Id Def(const IR::Value& value);
[[nodiscard]] Id BitOffset8(const IR::Value& offset);
[[nodiscard]] Id BitOffset16(const IR::Value& offset);
Id Const(u32 value) {
return Constant(U32[1], value);
}
Id Const(u32 element_1, u32 element_2) {
return ConstantComposite(U32[2], Const(element_1), Const(element_2));
}
Id Const(u32 element_1, u32 element_2, u32 element_3) {
return ConstantComposite(U32[3], Const(element_1), Const(element_2), Const(element_3));
}
Id Const(u32 element_1, u32 element_2, u32 element_3, u32 element_4) {
return ConstantComposite(U32[4], Const(element_1), Const(element_2), Const(element_3),
Const(element_4));
}
Id SConst(s32 value) {
return Constant(S32[1], value);
}
Id SConst(s32 element_1, s32 element_2) {
return ConstantComposite(S32[2], SConst(element_1), SConst(element_2));
}
Id SConst(s32 element_1, s32 element_2, s32 element_3) {
return ConstantComposite(S32[3], SConst(element_1), SConst(element_2), SConst(element_3));
}
Id SConst(s32 element_1, s32 element_2, s32 element_3, s32 element_4) {
return ConstantComposite(S32[4], SConst(element_1), SConst(element_2), SConst(element_3),
SConst(element_4));
}
Id Const(f32 value) {
return Constant(F32[1], value);
}
const Profile& profile;
const RuntimeInfo& runtime_info;
Stage stage{};
Id void_id{};
Id U1{};
Id U8{};
Id S8{};
Id U16{};
Id S16{};
Id U64{};
VectorTypes F32;
VectorTypes U32;
VectorTypes S32;
VectorTypes F16;
VectorTypes F64;
Id true_value{};
Id false_value{};
Id u32_zero_value{};
Id f32_zero_value{};
UniformDefinitions uniform_types;
StorageTypeDefinitions storage_types;
Id private_u32{};
Id shared_u8{};
Id shared_u16{};
Id shared_u32{};
Id shared_u64{};
Id shared_u32x2{};
Id shared_u32x4{};
Id input_f32{};
Id input_u32{};
Id input_s32{};
Id output_f32{};
Id output_u32{};
Id image_buffer_type{};
Id image_u32{};
std::array<UniformDefinitions, Info::MAX_CBUFS> cbufs{};
std::array<StorageDefinitions, Info::MAX_SSBOS> ssbos{};
std::vector<TextureBufferDefinition> texture_buffers;
std::vector<ImageBufferDefinition> image_buffers;
std::vector<TextureDefinition> textures;
std::vector<ImageDefinition> images;
Id workgroup_id{};
Id local_invocation_id{};
Id invocation_id{};
Id patch_vertices_in{};
Id sample_id{};
Id is_helper_invocation{};
Id subgroup_local_invocation_id{};
Id software_matrix_function{}; // (hookshot: emit_spirv_warp.cpp)
Id subgroup_mask_eq{};
Id subgroup_mask_lt{};
Id subgroup_mask_le{};
Id subgroup_mask_gt{};
Id subgroup_mask_ge{};
Id instance_id{};
Id instance_index{};
Id base_instance{};
Id vertex_id{};
Id vertex_index{};
Id draw_index{};
Id base_vertex{};
Id front_face{};
Id point_coord{};
Id tess_coord{};
Id clip_distances{};
Id layer{};
Id viewport_index{};
Id viewport_mask{};
Id primitive_id{};
Id fswzadd_lut_a{};
Id fswzadd_lut_b{};
Id indexed_load_func{};
Id indexed_store_func{};
Id rescaling_uniform_constant{};
Id rescaling_push_constants{};
Id rescaling_textures_type{};
Id rescaling_images_type{};
u32 rescaling_textures_member_index{};
u32 rescaling_images_member_index{};
u32 rescaling_downfactor_member_index{};
u32 guest_pages_member_index{}; // (hookshot: GUEST_PAGES_PUSH_OFFSET's member, compute only)
bool wants_guest_pages{};
Id probe_log{}; // (EvilOOT, RX_VK_BLOCK_PROBE: the miss log's U32 storage buffer, per-block entry counters at word 128+)
u32 mesh_first_member_index{}; // (hookshot: MESH_FIRST_PUSH_OFFSET's member, task programs)
bool wants_mesh_first{};
Id mesh_first_block_pointer{}; // (a PhysicalStorageBuffer pointer to a block of one u32, and to its member)
Id mesh_first_member_pointer{};
u32 texture_rescaling_index{};
u32 image_rescaling_index{};
Id render_area_push_constant{};
u32 render_are_member_index{};
Id local_memory{};
bool uses_explicit_workgroup_layout{};
Id shared_memory_u8{};
Id shared_memory_u16{};
Id shared_memory_u32{};
Id shared_memory_u64{};
Id shared_memory_u32x2{};
Id shared_memory_u32x4{};
Id shared_memory_u32_type{};
Id shared_store_u8_func{};
Id shared_store_u16_func{};
Id increment_cas_shared{};
Id increment_cas_ssbo{};
Id decrement_cas_shared{};
Id decrement_cas_ssbo{};
Id f32_add_cas{};
Id f16x2_add_cas{};
Id f16x2_min_cas{};
Id f16x2_max_cas{};
Id f32x2_add_cas{};
Id f32x2_min_cas{};
Id f32x2_max_cas{};
Id write_storage_cas_loop_func{};
// (hookshot) storage buffer tables (Info::global_tables): one array variable a type, which only the global
// memory fallbacks use; and those fallbacks for the 32-bit atomics (IAdd SMin UMin SMax UMax And Or Xor Exchange)
struct GlobalTableVars {
u32 cbuf_index{};
u32 cbuf_offset{};
u32 count{};
StorageDefinitions vars{};
};
std::vector<GlobalTableVars> ssbo_tables;
std::array<Id, 9> atomic_global_func_u32{};
Id atomic_global_cas_func_u32{}; // (hookshot) (address, comparator, value) -> the old value
Id load_global_func_u32{};
Id load_global_func_u32x2{};
Id load_global_func_u32x4{};
Id write_global_func_u32{};
Id write_global_func_u32x2{};
Id write_global_func_u32x4{};
Id write_global_cas_loop_func{};
Id atomic_global_add_f32_func{}; // (hookshot) (address, value) -> the old value
bool need_input_position_indirect{};
Id input_position{};
std::array<InputGenericInfo, 32> input_generics{};
Id output_point_size{};
Id output_position{};
std::array<std::array<GenericElementInfo, 4>, 32> output_generics{};
Id output_tess_level_outer{};
Id output_tess_level_inner{};
std::array<Id, 30> patches{};
std::array<Id, 8> frag_color{};
Id sample_mask{};
Id frag_depth{};
std::vector<Id> interfaces;
std::vector<std::pair<u32, spv::Op>> opcode_rewrites;
// Task and mesh programs. Their stage buffer: the task's output region is the task payload
// (the mesh stage's input region); the mesh output regions live in workgroup memory until
// the epilogue copies them to the outputs. The two mesh instructions Sirit lacks are emitted
// as OpStore markers with these operands and replaced after assembly (PatchMeshInstructions).
struct MeshLineTarget {
Id variable; ///< arrayed output
u32 element; ///< vector element, or ~0 for a scalar output
bool is_float; ///< bitcast the stage buffer word to F32
};
Id mesh_payload{};
Id mesh_payload_u32{}; ///< pointer type to one payload word
Id mesh_attributes{};
Id mesh_map{};
Id mesh_primitive_count{};
/// (hookshot) a fallback mesh workgroup's slot (task workgroup * threads + its index), read from the task's map in the
/// entry block (EmitMeshPrologue); invalid without a task program, where the slot is the workgroup id
Id mesh_fallback_slot{};
Id mesh_shared_u32{}; ///< pointer type to one workgroup word
Id mesh_primitive_indices{};
MeshInfo mesh_info{};
u32 mesh_threads{};
u32 mesh_attribute_words{};
u32 mesh_map_bytes{};
u32 mesh_primitive_base{}; ///< first word of the per-primitive groups
std::vector<MeshLineTarget> mesh_vertex_targets;
// (hookshot) a vertex generic output and the components no vertex line writes: they get (0,0,0,1) as in the vertex stage
struct MeshDefault {
Id variable;
u32 missing;
};
std::vector<MeshDefault> mesh_vertex_defaults;
std::vector<MeshLineTarget> mesh_primitive_targets;
std::array<Id, 2> mesh_outputs_marker{};
bool mesh_zero_outputs_marker{}; ///< (hookshot) OpStore(zero, zero): OpSetMeshOutputsEXT 0 0 where a branch has no target
std::array<Id, 4> mesh_tasks_marker{};
// (hookshot) a GPU without VK_EXT_mesh_shader: Task and Mesh are compute shaders. Their stage buffer outputs go to
// the payload, map and record buffers, pushed as device addresses; a generated vertex stage reads the records.
bool mesh_fallback{};
bool wants_mesh_fallback{};
Id mesh_fallback_word_ptr{}; ///< PhysicalStorageBuffer pointer to a block of one u32
u32 mesh_fallback_payload_member_index{};
u32 mesh_fallback_records_member_index{};
u32 mesh_fallback_indirect_input_member_index{};
/// A pointer to the u32 at a PhysicalStorageBuffer device address
Id MeshFallbackWord(Id address);
Id load_const_func_u8{};
Id load_const_func_u16{};
Id load_const_func_u32{};
Id load_const_func_f32{};
Id load_const_func_u32x2{};
Id load_const_func_u32x4{};
// Sirit::Id doesn't play nice with *::set<>
::Common::unordered_set<u32> non_uniform_ids;
bool uses_nonuniform_sampled_image{};
bool uses_nonuniform_storage_image{};
bool uses_nonuniform_uniform_texel_buffer{};
bool uses_nonuniform_storage_texel_buffer{};
private:
void DefineCommonTypes(const Info& info);
void DefineCommonConstants();
void DefineInterfaces(const IR::Program& program);
void DefineLocalMemory(const IR::Program& program);
void DefineSharedMemory(const IR::Program& program);
void DefineSharedMemoryFunctions(const IR::Program& program);
void DefineConstantBuffers(const Info& info, u32& binding);
void DefineConstantBufferIndirectFunctions(const Info& info);
void DefineStorageBuffers(const Info& info, u32& binding);
void DefineTextureBuffers(const Info& info, u32& binding);
void DefineImageBuffers(const Info& info, u32& binding);
void DefineTextures(const Info& info, u32& binding, u32& scaling_index);
void DefineImages(const Info& info, u32& binding, u32& scaling_index);
void DefineAttributeMemAccess(const Info& info);
void DefineWriteStorageCasLoopFunction(const Info& info);
void DefineGlobalMemoryFunctions(const Info& info);
void DefineRescalingInput(const Info& info);
void DefineRescalingInputPushConstant();
void DefineRescalingInputUniformConstant();
void DefineRenderArea(const Info& info);
void DefineInputs(const IR::Program& program);
void DefineOutputs(const IR::Program& program);
void DefineMesh(const IR::Program& program);
};
} // namespace Shader::Backend::SPIRV