mirror of
https://git.triforceproject.com/triforce/source
synced 2026-11-20 16:49:57 -04:00
Triforce v0.2.0
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
20
gpu/vk/eden/shader_recompiler/backend/bindings.h
Normal file
20
gpu/vk/eden/shader_recompiler/backend/bindings.h
Normal file
@@ -0,0 +1,20 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "common/common_types.h"
|
||||
|
||||
namespace Shader::Backend {
|
||||
|
||||
struct Bindings {
|
||||
u32 unified{};
|
||||
u32 uniform_buffer{};
|
||||
u32 storage_buffer{};
|
||||
u32 texture{};
|
||||
u32 image{};
|
||||
u32 texture_scaling_index{};
|
||||
u32 image_scaling_index{};
|
||||
};
|
||||
|
||||
} // namespace Shader::Backend
|
||||
764
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv.cpp
Normal file
764
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv.cpp
Normal file
@@ -0,0 +1,764 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <algorithm>
|
||||
#include <span>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "common/settings.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
#include "shader_recompiler/frontend/ir/basic_block.h"
|
||||
#include "shader_recompiler/frontend/ir/program.h"
|
||||
#include "shader_recompiler/backend/spirv/mma_layout.h" // (hookshot)
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
template <class Func>
|
||||
struct FuncTraits {};
|
||||
|
||||
template <class ReturnType_, class... Args>
|
||||
struct FuncTraits<ReturnType_ (*)(Args...)> {
|
||||
using ReturnType = ReturnType_;
|
||||
|
||||
static constexpr size_t NUM_ARGS = sizeof...(Args);
|
||||
|
||||
template <size_t I>
|
||||
using ArgType = std::tuple_element_t<I, std::tuple<Args...>>;
|
||||
};
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable : 4702) // Ignore unreachable code warning
|
||||
#endif
|
||||
|
||||
template <auto func, typename... Args>
|
||||
void SetDefinition(EmitContext& ctx, IR::Inst* inst, Args... args) {
|
||||
inst->SetDefinition<Id>(func(ctx, std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
||||
template <typename ArgType>
|
||||
ArgType Arg(EmitContext& ctx, const IR::Value& arg) {
|
||||
if constexpr (std::is_same_v<ArgType, Id>) {
|
||||
return ctx.Def(arg);
|
||||
} else if constexpr (std::is_same_v<ArgType, const IR::Value&>) {
|
||||
return arg;
|
||||
} else if constexpr (std::is_same_v<ArgType, u32>) {
|
||||
return arg.U32();
|
||||
} else if constexpr (std::is_same_v<ArgType, IR::Attribute>) {
|
||||
return arg.Attribute();
|
||||
} else if constexpr (std::is_same_v<ArgType, IR::Patch>) {
|
||||
return arg.Patch();
|
||||
} else if constexpr (std::is_same_v<ArgType, IR::Reg>) {
|
||||
return arg.Reg();
|
||||
}
|
||||
}
|
||||
|
||||
template <auto func, bool is_first_arg_inst, size_t... I>
|
||||
void Invoke(EmitContext& ctx, IR::Inst* inst, std::index_sequence<I...>) {
|
||||
using Traits = FuncTraits<decltype(func)>;
|
||||
if constexpr (std::is_same_v<typename Traits::ReturnType, Id>) {
|
||||
if constexpr (is_first_arg_inst) {
|
||||
SetDefinition<func>(
|
||||
ctx, inst, inst,
|
||||
Arg<typename Traits::template ArgType<I + 2>>(ctx, inst->Arg(I))...);
|
||||
} else {
|
||||
SetDefinition<func>(
|
||||
ctx, inst, Arg<typename Traits::template ArgType<I + 1>>(ctx, inst->Arg(I))...);
|
||||
}
|
||||
} else {
|
||||
if constexpr (is_first_arg_inst) {
|
||||
func(ctx, inst, Arg<typename Traits::template ArgType<I + 2>>(ctx, inst->Arg(I))...);
|
||||
} else {
|
||||
func(ctx, Arg<typename Traits::template ArgType<I + 1>>(ctx, inst->Arg(I))...);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <auto func>
|
||||
void Invoke(EmitContext& ctx, IR::Inst* inst) {
|
||||
using Traits = FuncTraits<decltype(func)>;
|
||||
static_assert(Traits::NUM_ARGS >= 1, "Insufficient arguments");
|
||||
if constexpr (Traits::NUM_ARGS == 1) {
|
||||
Invoke<func, false>(ctx, inst, std::make_index_sequence<0>{});
|
||||
} else {
|
||||
using FirstArgType = typename Traits::template ArgType<1>;
|
||||
static constexpr bool is_first_arg_inst = std::is_same_v<FirstArgType, IR::Inst*>;
|
||||
using Indices = std::make_index_sequence<Traits::NUM_ARGS - (is_first_arg_inst ? 2 : 1)>;
|
||||
Invoke<func, is_first_arg_inst>(ctx, inst, Indices{});
|
||||
}
|
||||
}
|
||||
|
||||
void EmitInstOf(EmitContext& ctx, IR::Inst* inst) {
|
||||
switch (inst->GetOpcode()) {
|
||||
#define OPCODE(name, result_type, ...) \
|
||||
case IR::Opcode::name: \
|
||||
return Invoke<&Emit##name>(ctx, inst);
|
||||
#include "shader_recompiler/frontend/ir/opcodes.inc"
|
||||
#undef OPCODE
|
||||
}
|
||||
throw LogicError("Invalid opcode {}", inst->GetOpcode());
|
||||
}
|
||||
|
||||
void EmitInst(EmitContext& ctx, IR::Inst* inst) {
|
||||
// (hookshot) the emitters that are left unimplemented say only "SPIR-V Instruction": name the IR op
|
||||
try {
|
||||
EmitInstOf(ctx, inst);
|
||||
} catch (const NotImplementedException& e) {
|
||||
throw NotImplementedException("{} ({})", e.what(), IR::NameOf(inst->GetOpcode()));
|
||||
}
|
||||
}
|
||||
|
||||
Id TypeId(const EmitContext& ctx, IR::Type type) {
|
||||
switch (type) {
|
||||
case IR::Type::U1:
|
||||
return ctx.U1;
|
||||
case IR::Type::U32:
|
||||
return ctx.U32[1];
|
||||
default:
|
||||
throw NotImplementedException("Phi node type {}", type);
|
||||
}
|
||||
}
|
||||
|
||||
void Traverse(EmitContext& ctx, IR::Program& program) {
|
||||
IR::Block* current_block{};
|
||||
// (EvilOOT, RX_VK_BLOCK_PROBE=1 with RX_VK_MISS_LOG=1 and RX_VK_MISS_LOG_PROG=<key>: each block's entries counted, by
|
||||
// invocation, in the miss log's words 128 + the block's index in program.blocks: IR::DumpProgram's "Block $n")
|
||||
static const bool probe_env{std::getenv("RX_VK_BLOCK_PROBE") != nullptr};
|
||||
const bool probe{probe_env && Sirit::ValidId(ctx.probe_log)};
|
||||
const auto block_index{[&](const IR::Block* b) -> u32 {
|
||||
const auto it{std::find(program.blocks.begin(), program.blocks.end(), b)};
|
||||
return static_cast<u32>(it - program.blocks.begin());
|
||||
}};
|
||||
for (const IR::AbstractSyntaxNode& node : program.syntax_list) {
|
||||
switch (node.type) {
|
||||
case IR::AbstractSyntaxNode::Type::Block: {
|
||||
const Id label{node.data.block->Definition<Id>()};
|
||||
if (current_block) {
|
||||
ctx.OpBranch(label);
|
||||
}
|
||||
current_block = node.data.block;
|
||||
ctx.AddLabel(label);
|
||||
bool probed{!probe};
|
||||
for (IR::Inst& inst : node.data.block->Instructions()) {
|
||||
if (!probed && inst.GetOpcode() != IR::Opcode::Phi) { // (after the phis: they must lead the block)
|
||||
probed = true;
|
||||
const u32 at{128U + std::min<u32>(block_index(node.data.block), 4000U)};
|
||||
const Id counter{ctx.OpAccessChain(ctx.storage_types.U32.element, ctx.probe_log, ctx.u32_zero_value,
|
||||
ctx.Const(at))};
|
||||
ctx.OpAtomicIAdd(ctx.U32[1], counter, ctx.Const(static_cast<u32>(spv::Scope::Device)), ctx.u32_zero_value,
|
||||
ctx.Const(1U));
|
||||
}
|
||||
EmitInst(ctx, &inst);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case IR::AbstractSyntaxNode::Type::If: {
|
||||
const Id if_label{node.data.if_node.body->Definition<Id>()};
|
||||
const Id endif_label{node.data.if_node.merge->Definition<Id>()};
|
||||
ctx.OpSelectionMerge(endif_label, spv::SelectionControlMask::MaskNone);
|
||||
ctx.OpBranchConditional(ctx.Def(node.data.if_node.cond), if_label, endif_label);
|
||||
break;
|
||||
}
|
||||
case IR::AbstractSyntaxNode::Type::Loop: {
|
||||
const Id body_label{node.data.loop.body->Definition<Id>()};
|
||||
const Id continue_label{node.data.loop.continue_block->Definition<Id>()};
|
||||
const Id endloop_label{node.data.loop.merge->Definition<Id>()};
|
||||
|
||||
ctx.OpLoopMerge(endloop_label, continue_label, spv::LoopControlMask::MaskNone);
|
||||
ctx.OpBranch(body_label);
|
||||
break;
|
||||
}
|
||||
case IR::AbstractSyntaxNode::Type::Break: {
|
||||
const Id break_label{node.data.break_node.merge->Definition<Id>()};
|
||||
const Id skip_label{node.data.break_node.skip->Definition<Id>()};
|
||||
ctx.OpBranchConditional(ctx.Def(node.data.break_node.cond), break_label, skip_label);
|
||||
break;
|
||||
}
|
||||
case IR::AbstractSyntaxNode::Type::EndIf:
|
||||
if (current_block) {
|
||||
ctx.OpBranch(node.data.end_if.merge->Definition<Id>());
|
||||
}
|
||||
break;
|
||||
case IR::AbstractSyntaxNode::Type::Repeat: {
|
||||
Id cond{ctx.Def(node.data.repeat.cond)};
|
||||
if (!Settings::values.disable_shader_loop_safety_checks) {
|
||||
const Id pointer_type{ctx.TypePointer(spv::StorageClass::Private, ctx.U32[1])};
|
||||
const Id safety_counter{ctx.AddGlobalVariable(
|
||||
pointer_type, spv::StorageClass::Private, ctx.Const(0x2000u))};
|
||||
if (ctx.profile.supported_spirv >= 0x00010400) {
|
||||
ctx.interfaces.push_back(safety_counter);
|
||||
}
|
||||
const Id old_counter{ctx.OpLoad(ctx.U32[1], safety_counter)};
|
||||
const Id new_counter{ctx.OpISub(ctx.U32[1], old_counter, ctx.Const(1u))};
|
||||
ctx.OpStore(safety_counter, new_counter);
|
||||
|
||||
const Id safety_cond{
|
||||
ctx.OpSGreaterThanEqual(ctx.U1, new_counter, ctx.u32_zero_value)};
|
||||
cond = ctx.OpLogicalAnd(ctx.U1, cond, safety_cond);
|
||||
}
|
||||
const Id loop_header_label{node.data.repeat.loop_header->Definition<Id>()};
|
||||
const Id merge_label{node.data.repeat.merge->Definition<Id>()};
|
||||
ctx.OpBranchConditional(cond, loop_header_label, merge_label);
|
||||
break;
|
||||
}
|
||||
case IR::AbstractSyntaxNode::Type::Return:
|
||||
ctx.OpReturn();
|
||||
break;
|
||||
case IR::AbstractSyntaxNode::Type::Unreachable:
|
||||
if (ctx.stage == Stage::Mesh && !ctx.mesh_fallback) {
|
||||
// (hookshot) an indirect branch to none of its table's targets: the mesh shader leaves with no outputs.
|
||||
// RADV makes OpUnreachable a return that leaves its output counts undefined, and GS_ALLOC_REQ then asked
|
||||
// for a garbage count and hung the GPU (issue 1000). Marker for OpSetMeshOutputsEXT 0 0
|
||||
// (PatchMeshInstructions); the epilogue writes the outputs, so none were written yet
|
||||
ctx.OpStore(ctx.u32_zero_value, ctx.u32_zero_value);
|
||||
ctx.mesh_zero_outputs_marker = true;
|
||||
ctx.OpReturn();
|
||||
} else {
|
||||
ctx.OpUnreachable();
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (node.type != IR::AbstractSyntaxNode::Type::Block) {
|
||||
current_block = nullptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Id DefineMain(EmitContext& ctx, IR::Program& program) {
|
||||
const Id void_function{ctx.TypeFunction(ctx.void_id)};
|
||||
const Id main{ctx.OpFunction(ctx.void_id, spv::FunctionControlMask::MaskNone, void_function)};
|
||||
for (IR::Block* const block : program.blocks) {
|
||||
block->SetDefinition(ctx.OpLabel());
|
||||
}
|
||||
Traverse(ctx, program);
|
||||
ctx.OpFunctionEnd();
|
||||
return main;
|
||||
}
|
||||
|
||||
spv::ExecutionMode ExecutionMode(TessPrimitive primitive) {
|
||||
switch (primitive) {
|
||||
case TessPrimitive::Isolines:
|
||||
return spv::ExecutionMode::Isolines;
|
||||
case TessPrimitive::Triangles:
|
||||
return spv::ExecutionMode::Triangles;
|
||||
case TessPrimitive::Quads:
|
||||
return spv::ExecutionMode::Quads;
|
||||
}
|
||||
throw InvalidArgument("Tessellation primitive {}", primitive);
|
||||
}
|
||||
|
||||
spv::ExecutionMode ExecutionMode(TessSpacing spacing) {
|
||||
switch (spacing) {
|
||||
case TessSpacing::Equal:
|
||||
return spv::ExecutionMode::SpacingEqual;
|
||||
case TessSpacing::FractionalOdd:
|
||||
return spv::ExecutionMode::SpacingFractionalOdd;
|
||||
case TessSpacing::FractionalEven:
|
||||
return spv::ExecutionMode::SpacingFractionalEven;
|
||||
}
|
||||
throw InvalidArgument("Tessellation spacing {}", spacing);
|
||||
}
|
||||
|
||||
void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) {
|
||||
const std::span interfaces(ctx.interfaces.data(), ctx.interfaces.size());
|
||||
if (ctx.mesh_fallback && (program.stage == Stage::Task || program.stage == Stage::Mesh)) {
|
||||
// (hookshot, issue 640: on a GPU without mesh shaders, a task or mesh program is a compute shader)
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U, 1U);
|
||||
ctx.AddEntryPoint(spv::ExecutionModel::GLCompute, main, "main", interfaces);
|
||||
return;
|
||||
}
|
||||
spv::ExecutionModel execution_model{};
|
||||
switch (program.stage) {
|
||||
case Stage::Compute: {
|
||||
const std::array<u32, 3> workgroup_size{program.workgroup_size};
|
||||
execution_model = spv::ExecutionModel::GLCompute;
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, workgroup_size[0],
|
||||
workgroup_size[1], workgroup_size[2]);
|
||||
break;
|
||||
}
|
||||
case Stage::VertexB:
|
||||
execution_model = spv::ExecutionModel::Vertex;
|
||||
break;
|
||||
case Stage::Task:
|
||||
execution_model = spv::ExecutionModel::TaskEXT;
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U,
|
||||
1U);
|
||||
break;
|
||||
case Stage::Mesh:
|
||||
execution_model = spv::ExecutionModel::MeshEXT;
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::LocalSize, program.workgroup_size[0], 1U,
|
||||
1U);
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.mesh.max_vertices);
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputPrimitivesEXT,
|
||||
program.mesh.max_primitives);
|
||||
ctx.AddExecutionMode(main, program.mesh.Corners() == 1 ? spv::ExecutionMode::OutputPoints
|
||||
: program.mesh.Corners() == 2 ? spv::ExecutionMode::OutputLinesEXT
|
||||
: spv::ExecutionMode::OutputTrianglesEXT);
|
||||
break;
|
||||
case Stage::TessellationControl:
|
||||
execution_model = spv::ExecutionModel::TessellationControl;
|
||||
ctx.AddCapability(spv::Capability::Tessellation);
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.invocations);
|
||||
break;
|
||||
case Stage::TessellationEval:
|
||||
execution_model = spv::ExecutionModel::TessellationEvaluation;
|
||||
ctx.AddCapability(spv::Capability::Tessellation);
|
||||
ctx.AddExecutionMode(main, ExecutionMode(ctx.runtime_info.tess_primitive));
|
||||
ctx.AddExecutionMode(main, ExecutionMode(ctx.runtime_info.tess_spacing));
|
||||
ctx.AddExecutionMode(main, ctx.runtime_info.tess_clockwise
|
||||
? spv::ExecutionMode::VertexOrderCw
|
||||
: spv::ExecutionMode::VertexOrderCcw);
|
||||
break;
|
||||
case Stage::Geometry:
|
||||
execution_model = spv::ExecutionModel::Geometry;
|
||||
ctx.AddCapability(spv::Capability::Geometry);
|
||||
if (ctx.profile.support_geometry_streams) {
|
||||
ctx.AddCapability(spv::Capability::GeometryStreams);
|
||||
}
|
||||
switch (ctx.runtime_info.input_topology) {
|
||||
case InputTopology::Points:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::InputPoints);
|
||||
break;
|
||||
case InputTopology::Lines:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::InputLines);
|
||||
break;
|
||||
case InputTopology::LinesAdjacency:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::InputLinesAdjacency);
|
||||
break;
|
||||
case InputTopology::Triangles:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::Triangles);
|
||||
break;
|
||||
case InputTopology::TrianglesAdjacency:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::InputTrianglesAdjacency);
|
||||
break;
|
||||
}
|
||||
switch (program.output_topology) {
|
||||
case OutputTopology::PointList:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputPoints);
|
||||
break;
|
||||
case OutputTopology::LineStrip:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputLineStrip);
|
||||
break;
|
||||
case OutputTopology::TriangleStrip:
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputTriangleStrip);
|
||||
break;
|
||||
}
|
||||
if (program.info.stores[IR::Attribute::PointSize]) {
|
||||
ctx.AddCapability(spv::Capability::GeometryPointSize);
|
||||
}
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OutputVertices, program.output_vertices);
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::Invocations, program.invocations);
|
||||
if (program.is_geometry_passthrough) {
|
||||
if (ctx.profile.support_geometry_shader_passthrough) {
|
||||
ctx.AddExtension("SPV_NV_geometry_shader_passthrough");
|
||||
ctx.AddCapability(spv::Capability::GeometryShaderPassthroughNV);
|
||||
} else {
|
||||
LOG_WARNING(Shader_SPIRV, "Geometry shader passthrough used with no support");
|
||||
}
|
||||
}
|
||||
break;
|
||||
case Stage::Fragment:
|
||||
execution_model = spv::ExecutionModel::Fragment;
|
||||
if (ctx.profile.lower_left_origin_mode) {
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OriginLowerLeft);
|
||||
} else {
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::OriginUpperLeft);
|
||||
}
|
||||
if (program.info.stores_frag_depth) {
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::DepthReplacing);
|
||||
}
|
||||
if (ctx.runtime_info.force_early_z) {
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::EarlyFragmentTests);
|
||||
}
|
||||
if (ctx.profile.support_shader_quad_control && program.info.uses_quad_shuffles) {
|
||||
ctx.AddExtension("SPV_KHR_quad_control");
|
||||
ctx.AddCapability(spv::Capability::QuadControlKHR);
|
||||
ctx.AddExecutionMode(main, spv::ExecutionMode::RequireFullQuadsKHR);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw NotImplementedException("Stage {}", program.stage);
|
||||
}
|
||||
ctx.AddEntryPoint(execution_model, main, "main", interfaces);
|
||||
}
|
||||
|
||||
void SetupDenormControl(const Profile& profile, const IR::Program& program, EmitContext& ctx,
|
||||
Id main_func) {
|
||||
const Info& info{program.info};
|
||||
if (info.uses_fp32_denorms_flush && info.uses_fp32_denorms_preserve) {
|
||||
LOG_DEBUG(Shader_SPIRV, "Fp32 denorm flush and preserve on the same shader");
|
||||
} else if (info.uses_fp32_denorms_flush) {
|
||||
if (profile.support_fp32_denorm_flush && !profile.has_broken_fp32_denorm_flush) {
|
||||
ctx.AddCapability(spv::Capability::DenormFlushToZero);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormFlushToZero, 32U);
|
||||
} else {
|
||||
// Drivers will most likely flush denorms by default, no need to warn
|
||||
}
|
||||
} else if (info.uses_fp32_denorms_preserve) {
|
||||
if (profile.support_fp32_denorm_preserve) {
|
||||
ctx.AddCapability(spv::Capability::DenormPreserve);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormPreserve, 32U);
|
||||
} else {
|
||||
LOG_DEBUG(Shader_SPIRV, "Fp32 denorm preserve used in shader without host support");
|
||||
}
|
||||
}
|
||||
if (!profile.support_separate_denorm_behavior || profile.has_broken_fp16_float_controls) {
|
||||
// No separate denorm behavior
|
||||
return;
|
||||
}
|
||||
if (info.uses_fp16_denorms_flush && info.uses_fp16_denorms_preserve) {
|
||||
LOG_DEBUG(Shader_SPIRV, "Fp16 denorm flush and preserve on the same shader");
|
||||
} else if (info.uses_fp16_denorms_flush) {
|
||||
if (profile.support_fp16_denorm_flush) {
|
||||
ctx.AddCapability(spv::Capability::DenormFlushToZero);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormFlushToZero, 16U);
|
||||
} else {
|
||||
// Same as fp32, no need to warn as most drivers will flush by default
|
||||
}
|
||||
} else if (info.uses_fp16_denorms_preserve) {
|
||||
if (profile.support_fp16_denorm_preserve) {
|
||||
ctx.AddCapability(spv::Capability::DenormPreserve);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::DenormPreserve, 16U);
|
||||
} else {
|
||||
LOG_DEBUG(Shader_SPIRV, "Fp16 denorm preserve used in shader without host support");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SetupSignedNanCapabilities(const Profile& profile, const IR::Program& program,
|
||||
EmitContext& ctx, Id main_func) {
|
||||
if (profile.has_broken_fp16_float_controls && program.info.uses_fp16) {
|
||||
return;
|
||||
}
|
||||
if (program.info.uses_fp16 && profile.support_fp16_signed_zero_nan_preserve) {
|
||||
ctx.AddCapability(spv::Capability::SignedZeroInfNanPreserve);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::SignedZeroInfNanPreserve, 16U);
|
||||
}
|
||||
if (profile.support_fp32_signed_zero_nan_preserve) {
|
||||
ctx.AddCapability(spv::Capability::SignedZeroInfNanPreserve);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::SignedZeroInfNanPreserve, 32U);
|
||||
}
|
||||
if (program.info.uses_fp64 && profile.support_fp64_signed_zero_nan_preserve) {
|
||||
ctx.AddCapability(spv::Capability::SignedZeroInfNanPreserve);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::SignedZeroInfNanPreserve, 64U);
|
||||
}
|
||||
}
|
||||
|
||||
void SetupTransformFeedbackCapabilities(EmitContext& ctx, Id main_func) {
|
||||
if (ctx.runtime_info.xfb_count == 0) {
|
||||
return;
|
||||
}
|
||||
ctx.AddCapability(spv::Capability::TransformFeedback);
|
||||
ctx.AddExecutionMode(main_func, spv::ExecutionMode::Xfb);
|
||||
}
|
||||
|
||||
void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ctx) {
|
||||
if (info.uses_sampled_1d) {
|
||||
ctx.AddCapability(spv::Capability::Sampled1D);
|
||||
}
|
||||
if (info.uses_image_1d) {
|
||||
ctx.AddCapability(spv::Capability::Image1D);
|
||||
}
|
||||
if (info.uses_sparse_residency) {
|
||||
ctx.AddCapability(spv::Capability::SparseResidency);
|
||||
}
|
||||
if (info.uses_demote_to_helper_invocation && profile.support_demote_to_helper_invocation) {
|
||||
if (profile.supported_spirv < 0x00010600) {
|
||||
ctx.AddExtension("SPV_EXT_demote_to_helper_invocation");
|
||||
}
|
||||
ctx.AddCapability(spv::Capability::DemoteToHelperInvocation);
|
||||
}
|
||||
if (info.stores[IR::Attribute::ViewportIndex] && profile.support_multi_viewport) {
|
||||
ctx.AddCapability(spv::Capability::MultiViewport);
|
||||
}
|
||||
if (info.stores[IR::Attribute::ViewportMask] && profile.support_viewport_mask) {
|
||||
ctx.AddExtension("SPV_NV_viewport_array2");
|
||||
ctx.AddCapability(spv::Capability::ShaderViewportMaskNV);
|
||||
}
|
||||
if (info.stores[IR::Attribute::Layer] || info.stores[IR::Attribute::ViewportIndex]) {
|
||||
if (profile.support_viewport_index_layer_non_geometry && ctx.stage != Stage::Geometry) {
|
||||
ctx.AddExtension("SPV_EXT_shader_viewport_index_layer");
|
||||
ctx.AddCapability(spv::Capability::ShaderViewportIndexLayerEXT);
|
||||
}
|
||||
}
|
||||
if ((!profile.support_vertex_instance_id &&
|
||||
(info.loads[IR::Attribute::InstanceId] || info.loads[IR::Attribute::VertexId])) ||
|
||||
info.loads[IR::Attribute::BaseInstance] || info.loads[IR::Attribute::BaseVertex] ||
|
||||
info.loads[IR::Attribute::DrawID] || (ctx.wants_mesh_first && !ctx.mesh_fallback)) {
|
||||
ctx.AddExtension("SPV_KHR_shader_draw_parameters");
|
||||
ctx.AddCapability(spv::Capability::DrawParameters);
|
||||
}
|
||||
if ((info.uses_subgroup_vote || info.uses_subgroup_invocation_id ||
|
||||
info.uses_subgroup_shuffles || info.uses_subgroup_mask) &&
|
||||
profile.support_vote && profile.SupportsSubgroupStage(ctx.stage)) {
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformBallot);
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
|
||||
if (info.uses_subgroup_shuffles && profile.support_shuffle_relative) {
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformShuffleRelative);
|
||||
}
|
||||
if (!profile.warp_size_potentially_larger_than_guest) {
|
||||
// vote ops are only used when not taking the long path
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformVote);
|
||||
}
|
||||
}
|
||||
if (info.uses_cooperative_matrix && !mma_software) {
|
||||
ctx.AddExtension("SPV_KHR_cooperative_matrix");
|
||||
ctx.AddCapability(spv::Capability::CooperativeMatrixKHR);
|
||||
ctx.AddExtension("SPV_KHR_vulkan_memory_model");
|
||||
ctx.AddCapability(spv::Capability::VulkanMemoryModel);
|
||||
ctx.AddCapability(spv::Capability::VulkanMemoryModelDeviceScope);
|
||||
ctx.SetMemoryModel(spv::MemoryModel::VulkanKHR);
|
||||
}
|
||||
if (info.uses_cooperative_matrix && mma_software) { // (hookshot: DefineSoftwareMatrixFunction's)
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniform);
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
|
||||
}
|
||||
if (info.uses_quad_shuffles) {
|
||||
if (profile.support_quad_shuffles) {
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformQuad);
|
||||
}
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
|
||||
}
|
||||
if (info.uses_int64_bit_atomics && profile.support_int64_atomics) {
|
||||
ctx.AddCapability(spv::Capability::Int64Atomics);
|
||||
}
|
||||
if (info.uses_typeless_image_reads && profile.support_typeless_image_loads) {
|
||||
ctx.AddCapability(spv::Capability::StorageImageReadWithoutFormat);
|
||||
}
|
||||
if (info.uses_typeless_image_writes) {
|
||||
ctx.AddCapability(spv::Capability::StorageImageWriteWithoutFormat);
|
||||
}
|
||||
if (info.uses_image_buffers) {
|
||||
ctx.AddCapability(spv::Capability::ImageBuffer);
|
||||
}
|
||||
if (info.uses_sample_id) {
|
||||
ctx.AddCapability(spv::Capability::SampleRateShading);
|
||||
}
|
||||
if (info.uses_derivatives) {
|
||||
ctx.AddCapability(spv::Capability::DerivativeControl);
|
||||
}
|
||||
// TODO: Track this usage
|
||||
ctx.AddCapability(spv::Capability::ImageGatherExtended);
|
||||
ctx.AddCapability(spv::Capability::ImageQuery);
|
||||
ctx.AddCapability(spv::Capability::SampledBuffer);
|
||||
if (!ctx.non_uniform_ids.empty()) {
|
||||
if (ctx.profile.supported_spirv < 0x00010500)
|
||||
ctx.AddExtension("SPV_EXT_descriptor_indexing");
|
||||
ctx.AddCapability(spv::Capability::ShaderNonUniform);
|
||||
if (ctx.uses_nonuniform_sampled_image) {
|
||||
ctx.AddCapability(spv::Capability::SampledImageArrayNonUniformIndexing);
|
||||
}
|
||||
if (ctx.uses_nonuniform_storage_image) {
|
||||
ctx.AddCapability(spv::Capability::StorageImageArrayNonUniformIndexing);
|
||||
}
|
||||
if (ctx.uses_nonuniform_uniform_texel_buffer) {
|
||||
ctx.AddCapability(spv::Capability::UniformTexelBufferArrayNonUniformIndexing);
|
||||
}
|
||||
if (ctx.uses_nonuniform_storage_texel_buffer) {
|
||||
ctx.AddCapability(spv::Capability::StorageTexelBufferArrayNonUniformIndexing);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void PatchPhiNodes(IR::Program& program, EmitContext& ctx) {
|
||||
// Flatten all leading PHIs from each block into a vector
|
||||
std::vector<IR::Inst*> phi_instructions;
|
||||
for (IR::Block* block : program.blocks) {
|
||||
for (auto it = block->begin(); it != block->end(); ++it) {
|
||||
if (it->GetOpcode() != IR::Opcode::Phi)
|
||||
break;
|
||||
phi_instructions.push_back(&*it);
|
||||
}
|
||||
}
|
||||
|
||||
if (phi_instructions.empty()) {
|
||||
return; // nothing to patch
|
||||
}
|
||||
|
||||
// Start "before" first PHI; advance on phi_arg == 0
|
||||
size_t phi_index = static_cast<size_t>(-1);
|
||||
|
||||
ctx.PatchDeferredPhi([&](size_t phi_arg, Id parent) -> std::pair<Id, Id> {
|
||||
if (phi_arg == 0) {
|
||||
++phi_index;
|
||||
}
|
||||
IR::Inst* phi = phi_instructions[phi_index];
|
||||
return { ctx.Def(phi->Arg(phi_arg)), parent };
|
||||
});
|
||||
}
|
||||
|
||||
void RewriteOpcodes(std::vector<u32>& code, std::span<const std::pair<u32, spv::Op>> rewrites) {
|
||||
if (rewrites.empty()) {
|
||||
return;
|
||||
}
|
||||
size_t offset = 5;
|
||||
while (offset + 2 < code.size()) {
|
||||
const u32 word_count{code[offset] >> 16};
|
||||
const auto opcode{static_cast<spv::Op>(code[offset] & 0xFFFFu)};
|
||||
if (word_count == 0) {
|
||||
return;
|
||||
}
|
||||
if (opcode == spv::Op::OpGroupNonUniformShuffleXor ||
|
||||
opcode == spv::Op::OpGroupNonUniformQuadBroadcast) {
|
||||
const auto it{std::ranges::find(rewrites, code[offset + 2],
|
||||
&std::pair<u32, spv::Op>::first)};
|
||||
if (it != rewrites.end()) {
|
||||
code[offset] = (code[offset] & 0xFFFF0000u) | static_cast<u32>(it->second);
|
||||
}
|
||||
}
|
||||
offset += word_count;
|
||||
}
|
||||
}
|
||||
|
||||
// Replaces the OpStore markers of the mesh instructions Sirit cannot emit: OpStore(vertices,
|
||||
// primitives) becomes OpSetMeshOutputsEXT, and OpStore(x, y), OpStore(z, payload), OpReturn
|
||||
// becomes the OpEmitMeshTasksEXT terminator.
|
||||
void PatchMeshInstructions(std::vector<u32>& code, const EmitContext& ctx) {
|
||||
const bool set_outputs{Sirit::ValidId(ctx.mesh_outputs_marker[0])};
|
||||
const bool emit_tasks{Sirit::ValidId(ctx.mesh_tasks_marker[0])};
|
||||
const bool zero_outputs{ctx.mesh_zero_outputs_marker};
|
||||
if (!set_outputs && !emit_tasks && !zero_outputs) {
|
||||
return;
|
||||
}
|
||||
const auto store_at = [&](size_t offset, Id pointer, Id object) {
|
||||
return offset + 2 < code.size() &&
|
||||
code[offset] == ((3U << 16) | static_cast<u32>(spv::Op::OpStore)) &&
|
||||
code[offset + 1] == pointer.value && code[offset + 2] == object.value;
|
||||
};
|
||||
const auto& tasks{ctx.mesh_tasks_marker};
|
||||
size_t offset = 5;
|
||||
bool patched{};
|
||||
while (offset < code.size()) {
|
||||
const u32 word_count{code[offset] >> 16};
|
||||
if (word_count == 0) {
|
||||
break;
|
||||
}
|
||||
if (set_outputs &&
|
||||
store_at(offset, ctx.mesh_outputs_marker[0], ctx.mesh_outputs_marker[1])) {
|
||||
code[offset] = (3U << 16) | static_cast<u32>(spv::Op::OpSetMeshOutputsEXT);
|
||||
patched = true;
|
||||
} else if (zero_outputs && store_at(offset, ctx.u32_zero_value, ctx.u32_zero_value)) {
|
||||
code[offset] = (3U << 16) | static_cast<u32>(spv::Op::OpSetMeshOutputsEXT);
|
||||
patched = true;
|
||||
} else if (emit_tasks && store_at(offset, tasks[0], tasks[1]) &&
|
||||
store_at(offset + 3, tasks[2], tasks[3]) && offset + 6 < code.size() &&
|
||||
code[offset + 6] == ((1U << 16) | static_cast<u32>(spv::Op::OpReturn))) {
|
||||
const std::array<u32, 5> emit{(5U << 16) | static_cast<u32>(spv::Op::OpEmitMeshTasksEXT),
|
||||
tasks[0].value, tasks[1].value, tasks[2].value,
|
||||
tasks[3].value};
|
||||
code.erase(code.begin() + offset, code.begin() + offset + 7);
|
||||
code.insert(code.begin() + offset, emit.begin(), emit.end());
|
||||
patched = true;
|
||||
offset += emit.size();
|
||||
continue;
|
||||
}
|
||||
offset += word_count;
|
||||
}
|
||||
if (!patched) {
|
||||
throw LogicError("Mesh instruction markers not found");
|
||||
}
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings) {
|
||||
EmitContext ctx{profile, runtime_info, program, bindings};
|
||||
if (program.info.uses_cooperative_matrix && mma_software) DefineSoftwareMatrixFunction(ctx);
|
||||
Id main{};
|
||||
if (runtime_info.mesh_fallback_vertex) { // (hookshot: the fallback's generated vertex stage, which reads the records)
|
||||
const Id void_function{ctx.TypeFunction(ctx.void_id)};
|
||||
main = ctx.OpFunction(ctx.void_id, spv::FunctionControlMask::MaskNone, void_function);
|
||||
ctx.AddLabel();
|
||||
EmitMeshFallbackVertexBody(ctx);
|
||||
ctx.OpReturn();
|
||||
ctx.OpFunctionEnd();
|
||||
} else {
|
||||
main = DefineMain(ctx, program);
|
||||
}
|
||||
DefineEntryPoint(program, ctx, main);
|
||||
if (profile.support_float_controls) {
|
||||
ctx.AddExtension("SPV_KHR_float_controls");
|
||||
SetupDenormControl(profile, program, ctx, main);
|
||||
SetupSignedNanCapabilities(profile, program, ctx, main);
|
||||
}
|
||||
SetupCapabilities(profile, program.info, ctx);
|
||||
SetupTransformFeedbackCapabilities(ctx, main);
|
||||
PatchPhiNodes(program, ctx);
|
||||
std::vector<u32> code{ctx.Assemble()};
|
||||
RewriteOpcodes(code, ctx.opcode_rewrites);
|
||||
PatchMeshInstructions(code, ctx);
|
||||
return code;
|
||||
}
|
||||
|
||||
Id EmitPhi(EmitContext& ctx, IR::Inst* inst) {
|
||||
const size_t num_args{inst->NumArgs()};
|
||||
boost::container::small_vector<Id, 32> blocks;
|
||||
blocks.reserve(num_args);
|
||||
for (size_t index = 0; index < num_args; ++index) {
|
||||
blocks.push_back(inst->PhiBlock(index)->Definition<Id>());
|
||||
}
|
||||
// The type of a phi instruction is stored in its flags
|
||||
const Id result_type{TypeId(ctx, inst->Flags<IR::Type>())};
|
||||
return ctx.DeferredOpPhi(result_type, std::span(blocks.data(), blocks.size()));
|
||||
}
|
||||
|
||||
void EmitVoid(EmitContext&) {}
|
||||
|
||||
Id EmitIdentity(EmitContext& ctx, const IR::Value& value) {
|
||||
const Id id{ctx.Def(value)};
|
||||
if (!Sirit::ValidId(id)) {
|
||||
throw NotImplementedException("Forward identity declaration");
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
Id EmitConditionRef(EmitContext& ctx, const IR::Value& value) {
|
||||
const Id id{ctx.Def(value)};
|
||||
if (!Sirit::ValidId(id)) {
|
||||
throw NotImplementedException("Forward identity declaration");
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
void EmitReference(EmitContext&) {}
|
||||
|
||||
void EmitPhiMove(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetZeroFromOp(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetSignFromOp(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetCarryFromOp(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetOverflowFromOp(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetSparseFromOp(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetInBoundsFromOp(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
72
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv.h
Normal file
72
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv.h
Normal file
@@ -0,0 +1,72 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "common/common_types.h"
|
||||
#include "shader_recompiler/backend/bindings.h"
|
||||
#include "shader_recompiler/frontend/ir/program.h"
|
||||
#include "shader_recompiler/profile.h"
|
||||
#include "shader_recompiler/runtime_info.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
constexpr u32 NUM_TEXTURE_SCALING_WORDS = 4;
|
||||
constexpr u32 NUM_IMAGE_SCALING_WORDS = 2;
|
||||
constexpr u32 NUM_TEXTURE_AND_IMAGE_SCALING_WORDS =
|
||||
NUM_TEXTURE_SCALING_WORDS + NUM_IMAGE_SCALING_WORDS;
|
||||
|
||||
struct RescalingLayout {
|
||||
alignas(16) std::array<u32, NUM_TEXTURE_SCALING_WORDS> rescaling_textures;
|
||||
alignas(16) std::array<u32, NUM_IMAGE_SCALING_WORDS> rescaling_images;
|
||||
u32 down_factor;
|
||||
};
|
||||
struct RenderAreaLayout {
|
||||
std::array<f32, 4> render_area;
|
||||
};
|
||||
constexpr u32 RESCALING_LAYOUT_WORDS_OFFSET = offsetof(RescalingLayout, rescaling_textures);
|
||||
constexpr u32 RESCALING_LAYOUT_DOWN_FACTOR_OFFSET = offsetof(RescalingLayout, down_factor);
|
||||
constexpr u32 RENDERAREA_LAYOUT_OFFSET = offsetof(RenderAreaLayout, render_area);
|
||||
// (hookshot) guest memory's page table, a u64 device address pushed to every compute program that reads global memory, after the blocks
|
||||
// above. It was a constant in the SPIR-V, and the address changes from run to run: the drivers' pipeline caches never hit
|
||||
constexpr u32 GUEST_PAGES_PUSH_OFFSET = sizeof(RescalingLayout) + sizeof(RenderAreaLayout);
|
||||
// (hookshot) a task program's first workgroup: NVN draws tasks [first, first + count), Vulkan from 0. The address of the draw's first
|
||||
// (a u32, 16 bytes per draw of an indirect draw, read at + DrawIndex * 16) is pushed here
|
||||
constexpr u32 MESH_FIRST_PUSH_OFFSET = GUEST_PAGES_PUSH_OFFSET + 8;
|
||||
// (hookshot) the mesh shader fallback's buffers, after it: the task payloads and the vertex records (u64 device addresses).
|
||||
// Paths without the fallback never push them.
|
||||
constexpr u32 MESH_FALLBACK_PUSH_OFFSET = MESH_FIRST_PUSH_OFFSET + 8;
|
||||
constexpr u32 MESH_FALLBACK_PAYLOAD_PUSH_OFFSET = MESH_FALLBACK_PUSH_OFFSET;
|
||||
constexpr u32 MESH_FALLBACK_RECORDS_PUSH_OFFSET = MESH_FALLBACK_PUSH_OFFSET + 8;
|
||||
constexpr u32 MESH_FALLBACK_INDIRECT_INPUT_PUSH_OFFSET = MESH_FALLBACK_PUSH_OFFSET + 16;
|
||||
constexpr u32 MESH_FALLBACK_PUSH_END = MESH_FALLBACK_PUSH_OFFSET + 24;
|
||||
// (hookshot) the fallback's mesh workgroups a draw can run: each a slot, a word of the payload buffer saying whether its task
|
||||
// started it; then each task workgroup's payload. It was 1024, a record block each, and Ocarina's floor grass (433 tasks of
|
||||
// 32 slots) lost 13 of 14 meshlets. At 16384, Hyrule Field's far trees (3cb76827/4f0c0b40: 721 tasks of 32) lost the tasks
|
||||
// past 512, a different set each frame as the culling's list order changes: they flickered. 65535: Vulkan's least
|
||||
// maxComputeWorkGroupCount, which the mesh dispatch can reach; the payload buffer is ~6.5 MB a recording thread at 800 words.
|
||||
constexpr u32 MESH_FALLBACK_CAPACITY = 65535;
|
||||
// (hookshot) the records buffer: a VkDrawIndirectCommand the vertex stage's draw reads (its vertex count the records written),
|
||||
// the allocator's word, then the records. A mesh workgroup takes records for its primitives' corners only, so a draw covers
|
||||
// what was drawn, not every slot's maximum
|
||||
constexpr u32 MESH_FALLBACK_DRAW_WORD = 0; // (vertexCount; instanceCount at word 1, set to 1 by the host)
|
||||
constexpr u32 MESH_FALLBACK_ALLOC_WORD = 4; // (records taken, past the end too: those that do not fit are not drawn)
|
||||
// (a VkDispatchIndirectCommand for the mesh workgroups a task program started (y and z 1, set by the host): each started task
|
||||
// thread takes an index of it and writes its slot there in the payload buffer's map, so the mesh dispatch runs only those. With
|
||||
// a count from the GPU (DrawMeshTasksIndirect, the floor grass) the host knew none, and dispatched no mesh workgroup)
|
||||
constexpr u32 MESH_FALLBACK_DISPATCH_WORD = 5;
|
||||
constexpr u32 MESH_FALLBACK_HEADER_WORDS = 8;
|
||||
constexpr u32 MESH_FALLBACK_RECORD_WORDS = 16u << 20; // (64 MB: ~47k grass primitives of 31 words are 17 MB)
|
||||
|
||||
[[nodiscard]] std::vector<u32> EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings);
|
||||
[[nodiscard]] inline std::vector<u32> EmitSPIRV(const Profile& profile, IR::Program& program) {
|
||||
Bindings binding;
|
||||
return EmitSPIRV(profile, {}, program, binding);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,625 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <bit>
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id SharedPointer(EmitContext& ctx, Id offset, u32 index_offset = 0) {
|
||||
const Id shift_id{ctx.Const(2U)};
|
||||
Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
if (index_offset > 0) {
|
||||
index = ctx.OpIAdd(ctx.U32[1], index, ctx.Const(index_offset));
|
||||
}
|
||||
return ctx.uses_explicit_workgroup_layout
|
||||
? ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index)
|
||||
: ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index);
|
||||
}
|
||||
|
||||
Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size) {
|
||||
if (offset.IsImmediate()) {
|
||||
const u32 imm_offset{static_cast<u32>(offset.U32() / element_size)};
|
||||
return ctx.Const(imm_offset);
|
||||
}
|
||||
const u32 shift{static_cast<u32>(std::countr_zero(element_size))};
|
||||
const Id index{ctx.Def(offset)};
|
||||
if (shift == 0) {
|
||||
return index;
|
||||
}
|
||||
const Id shift_id{ctx.Const(shift)};
|
||||
return ctx.OpShiftRightLogical(ctx.U32[1], index, shift_id);
|
||||
}
|
||||
|
||||
Id StoragePointer(EmitContext& ctx, const StorageTypeDefinition& type_def,
|
||||
Id StorageDefinitions::*member_ptr, const IR::Value& binding,
|
||||
const IR::Value& offset, size_t element_size) {
|
||||
if (!binding.IsImmediate()) {
|
||||
throw NotImplementedException("Dynamic storage buffer indexing");
|
||||
}
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].*member_ptr};
|
||||
const Id index{StorageIndex(ctx, offset, element_size)};
|
||||
return ctx.OpAccessChain(type_def.element, ssbo, ctx.u32_zero_value, index);
|
||||
}
|
||||
|
||||
std::pair<Id, Id> AtomicArgs(EmitContext& ctx) {
|
||||
const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Device))};
|
||||
const Id semantics{ctx.u32_zero_value};
|
||||
return {scope, semantics};
|
||||
}
|
||||
|
||||
Id SharedAtomicU32(EmitContext& ctx, Id offset, Id value,
|
||||
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
|
||||
const Id pointer{SharedPointer(ctx, offset)};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
|
||||
}
|
||||
|
||||
Id StorageAtomicU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
|
||||
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding,
|
||||
offset, sizeof(u32))};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
|
||||
}
|
||||
|
||||
Id StorageAtomicU64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
|
||||
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id),
|
||||
Id (Sirit::Module::*non_atomic_func)(Id, Id, Id)) {
|
||||
if (!ctx.profile.support_descriptor_aliasing) {
|
||||
LOG_WARNING(Shader_SPIRV, "Descriptor aliasing not supported, this cannot be atomic.");
|
||||
return ctx.ConstantNull(ctx.U64);
|
||||
}
|
||||
|
||||
if (ctx.profile.support_int64_atomics) {
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U64, &StorageDefinitions::U64,
|
||||
binding, offset, sizeof(u64))};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return (ctx.*atomic_func)(ctx.U64, pointer, scope, semantics, value);
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
|
||||
binding, offset, sizeof(u32[2]))};
|
||||
const Id original_value{ctx.OpBitcast(ctx.U64, ctx.OpLoad(ctx.U32[2], pointer))};
|
||||
const Id result{(ctx.*non_atomic_func)(ctx.U64, value, original_value)};
|
||||
ctx.OpStore(pointer, ctx.OpBitcast(ctx.U32[2], result));
|
||||
return original_value;
|
||||
}
|
||||
|
||||
Id StorageAtomicU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
|
||||
Id (Sirit::Module::*non_atomic_func)(Id, Id, Id)) {
|
||||
if (!ctx.profile.support_descriptor_aliasing) {
|
||||
LOG_WARNING(Shader_SPIRV, "Descriptor aliasing not supported, this cannot be atomic.");
|
||||
return ctx.ConstantNull(ctx.U32[2]);
|
||||
}
|
||||
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
|
||||
binding, offset, sizeof(u32[2]))};
|
||||
const Id original_value{ctx.OpLoad(ctx.U32[2], pointer)};
|
||||
const Id result{(ctx.*non_atomic_func)(ctx.U32[2], value, original_value)};
|
||||
ctx.OpStore(pointer, result);
|
||||
return original_value;
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitSharedAtomicIAdd32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicIAdd);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicSMin32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicSMin);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicUMin32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicUMin);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicSMax32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicSMax);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicUMax32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicUMax);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicInc32(EmitContext& ctx, Id offset, Id value) {
|
||||
const Id shift_id{ctx.Const(2U)};
|
||||
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.increment_cas_shared, index, value);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicDec32(EmitContext& ctx, Id offset, Id value) {
|
||||
const Id shift_id{ctx.Const(2U)};
|
||||
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.decrement_cas_shared, index, value);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicAnd32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicAnd);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicOr32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicOr);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicXor32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicXor);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicExchange32(EmitContext& ctx, Id offset, Id value) {
|
||||
return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicExchange);
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicExchange64(EmitContext& ctx, Id offset, Id value) {
|
||||
if (ctx.profile.support_shared_int64_atomics && ctx.uses_explicit_workgroup_layout) {
|
||||
const Id shift_id{ctx.Const(3U)};
|
||||
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
const Id pointer{
|
||||
ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return ctx.OpAtomicExchange(ctx.U64, pointer, scope, semantics, value);
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
|
||||
const Id pointer_1{SharedPointer(ctx, offset, 0)};
|
||||
const Id pointer_2{SharedPointer(ctx, offset, 1)};
|
||||
const Id value_1{ctx.OpLoad(ctx.U32[1], pointer_1)};
|
||||
const Id value_2{ctx.OpLoad(ctx.U32[1], pointer_2)};
|
||||
const Id new_vector{ctx.OpBitcast(ctx.U32[2], value)};
|
||||
ctx.OpStore(pointer_1, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 0U));
|
||||
ctx.OpStore(pointer_2, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 1U));
|
||||
return ctx.OpBitcast(ctx.U64, ctx.OpCompositeConstruct(ctx.U32[2], value_1, value_2));
|
||||
}
|
||||
|
||||
Id EmitSharedAtomicExchange32x2(EmitContext& ctx, Id offset, Id value) {
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
|
||||
const Id pointer_1{SharedPointer(ctx, offset, 0)};
|
||||
const Id pointer_2{SharedPointer(ctx, offset, 1)};
|
||||
const Id value_1{ctx.OpLoad(ctx.U32[1], pointer_1)};
|
||||
const Id value_2{ctx.OpLoad(ctx.U32[1], pointer_2)};
|
||||
const Id new_vector{ctx.OpBitcast(ctx.U32[2], value)};
|
||||
ctx.OpStore(pointer_1, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 0U));
|
||||
ctx.OpStore(pointer_2, ctx.OpCompositeExtract(ctx.U32[1], new_vector, 1U));
|
||||
return ctx.OpCompositeConstruct(ctx.U32[2], value_1, value_2);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicIAdd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicIAdd);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicSMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMin);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicUMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMin);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicSMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMax);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicUMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMax);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicInc32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.increment_cas_ssbo, base_index, value, ssbo);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicDec32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.decrement_cas_ssbo, base_index, value, ssbo);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicAnd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicAnd);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicOr32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicOr);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicXor32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicXor);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32(ctx, binding, offset, value, &Sirit::Module::OpAtomicExchange);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicCompareExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id comparator, Id value) { // (hookshot: ATOMG.CAS)
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32, &StorageDefinitions::U32, binding,
|
||||
offset, sizeof(u32))};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return ctx.OpAtomicCompareExchange(ctx.U32[1], pointer, scope, semantics, semantics, value, comparator);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicIAdd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicIAdd,
|
||||
&Sirit::Module::OpIAdd);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicSMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMin,
|
||||
&Sirit::Module::OpSMin);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicUMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMin,
|
||||
&Sirit::Module::OpUMin);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicSMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicSMax,
|
||||
&Sirit::Module::OpSMax);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicUMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicUMax,
|
||||
&Sirit::Module::OpUMax);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicAnd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicAnd,
|
||||
&Sirit::Module::OpBitwiseAnd);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicOr64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicOr,
|
||||
&Sirit::Module::OpBitwiseOr);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicXor64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU64(ctx, binding, offset, value, &Sirit::Module::OpAtomicXor,
|
||||
&Sirit::Module::OpBitwiseXor);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicExchange64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_int64_atomics) {
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U64, &StorageDefinitions::U64,
|
||||
binding, offset, sizeof(u64))};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return ctx.OpAtomicExchange(ctx.U64, pointer, scope, semantics, value);
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
|
||||
binding, offset, sizeof(u32[2]))};
|
||||
const Id original{ctx.OpBitcast(ctx.U64, ctx.OpLoad(ctx.U32[2], pointer))};
|
||||
ctx.OpStore(pointer, value);
|
||||
return original;
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicIAdd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpIAdd);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicSMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpSMin);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicUMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpUMin);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicSMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpSMax);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicUMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpUMax);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicAnd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpBitwiseAnd);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicOr32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpBitwiseOr);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicXor32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
return StorageAtomicU32x2(ctx, binding, offset, value, &Sirit::Module::OpBitwiseXor);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicExchange32x2(EmitContext& ctx, const IR::Value& binding,
|
||||
const IR::Value& offset, Id value) {
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 atomics not supported, fallback to non-atomic");
|
||||
const Id pointer{StoragePointer(ctx, ctx.storage_types.U32x2, &StorageDefinitions::U32x2,
|
||||
binding, offset, sizeof(u32[2]))};
|
||||
const Id original{ctx.OpLoad(ctx.U32[2], pointer)};
|
||||
ctx.OpStore(pointer, value);
|
||||
return original;
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicAddF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
return ctx.OpFunctionCall(ctx.F32[1], ctx.f32_add_cas, base_index, value, ssbo);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicAddF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_add_cas, base_index, value, ssbo)};
|
||||
return ctx.OpBitcast(ctx.U32[1], result);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicAddF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_add_cas, base_index, value, ssbo)};
|
||||
return ctx.OpPackHalf2x16(ctx.U32[1], result);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicMinF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_min_cas, base_index, value, ssbo)};
|
||||
return ctx.OpBitcast(ctx.U32[1], result);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicMinF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_min_cas, base_index, value, ssbo)};
|
||||
return ctx.OpPackHalf2x16(ctx.U32[1], result);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
const Id result{ctx.OpFunctionCall(ctx.F16[2], ctx.f16x2_max_cas, base_index, value, ssbo)};
|
||||
return ctx.OpBitcast(ctx.U32[1], result);
|
||||
}
|
||||
|
||||
Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].U32};
|
||||
const Id base_index{StorageIndex(ctx, offset, sizeof(u32))};
|
||||
const Id result{ctx.OpFunctionCall(ctx.F32[2], ctx.f32x2_max_cas, base_index, value, ssbo)};
|
||||
return ctx.OpPackHalf2x16(ctx.U32[1], result);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[0])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[0], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[1])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[1], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[2])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[2], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[3])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[3], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[4])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[4], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicInc32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicDec32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[5])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[5], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[6])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[6], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[7])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[7], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_func_u32[8])) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_func_u32[8], address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicCompareExchange32(EmitContext& ctx, Id address, Id comparator, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_cas_func_u32)) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.atomic_global_cas_func_u32, address, comparator, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicIAdd64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicSMin64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicUMin64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicSMax64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicUMax64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicInc64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicDec64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicAnd64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicOr64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicXor64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicExchange64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicIAdd32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicSMin32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicUMin32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicSMax32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicUMax32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicInc32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicDec32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicAnd32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicOr32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicXor32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicExchange32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value) { // (hookshot: the fallback)
|
||||
if (!Sirit::ValidId(ctx.atomic_global_add_f32_func)) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
return ctx.OpFunctionCall(ctx.F32[1], ctx.atomic_global_add_f32_func, address, value);
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicAddF16x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicAddF32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicMinF16x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicMinF32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicMaxF16x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitGlobalAtomicMaxF32x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,45 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
void MemoryBarrier(EmitContext& ctx, spv::Scope scope) {
|
||||
const auto semantics{
|
||||
spv::MemorySemanticsMask::AcquireRelease | spv::MemorySemanticsMask::UniformMemory |
|
||||
spv::MemorySemanticsMask::WorkgroupMemory | spv::MemorySemanticsMask::AtomicCounterMemory |
|
||||
spv::MemorySemanticsMask::ImageMemory};
|
||||
ctx.OpMemoryBarrier(ctx.Const(static_cast<u32>(scope)), ctx.Const(static_cast<u32>(semantics)));
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
void EmitBarrier(EmitContext& ctx) {
|
||||
const auto execution{spv::Scope::Workgroup};
|
||||
const auto memory{spv::Scope::Workgroup};
|
||||
const auto memory_semantics{spv::MemorySemanticsMask::AcquireRelease |
|
||||
spv::MemorySemanticsMask::WorkgroupMemory};
|
||||
ctx.OpControlBarrier(ctx.Const(static_cast<u32>(execution)),
|
||||
ctx.Const(static_cast<u32>(memory)),
|
||||
ctx.Const(static_cast<u32>(memory_semantics)));
|
||||
}
|
||||
|
||||
// (hookshot) The subgroup's invocations meet, and their workgroup (shared) memory accesses before it are visible to the ones after:
|
||||
// what the Switch's in-order warp gives a warp's stores and LDSM's reads of them without a BAR
|
||||
void EmitSubgroupBarrier(EmitContext& ctx) {
|
||||
const auto scope{spv::Scope::Subgroup};
|
||||
const auto memory_semantics{spv::MemorySemanticsMask::AcquireRelease | spv::MemorySemanticsMask::WorkgroupMemory};
|
||||
ctx.OpControlBarrier(ctx.Const(static_cast<u32>(scope)), ctx.Const(static_cast<u32>(scope)),
|
||||
ctx.Const(static_cast<u32>(memory_semantics)));
|
||||
}
|
||||
|
||||
void EmitWorkgroupMemoryBarrier(EmitContext& ctx) {
|
||||
MemoryBarrier(ctx, spv::Scope::Workgroup);
|
||||
}
|
||||
|
||||
void EmitDeviceMemoryBarrier(EmitContext& ctx) {
|
||||
MemoryBarrier(ctx, spv::Scope::Device);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,65 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
void EmitBitCastU16F16(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBitCastU32F32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
void EmitBitCastU64F64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitBitCastF16U16(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBitCastF32U32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
void EmitBitCastF64U64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitPackUint2x32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitUnpackUint2x32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.U32[2], value);
|
||||
}
|
||||
|
||||
Id EmitPackFloat2x16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitUnpackFloat2x16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.F16[2], value);
|
||||
}
|
||||
|
||||
Id EmitPackHalf2x16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpPackHalf2x16(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitUnpackHalf2x16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpUnpackHalf2x16(ctx.F32[2], value);
|
||||
}
|
||||
|
||||
Id EmitPackDouble2x32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitUnpackDouble2x32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitcast(ctx.U32[2], value);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,153 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
Id EmitCompositeConstructU32x2(EmitContext& ctx, Id e1, Id e2) {
|
||||
return ctx.OpCompositeConstruct(ctx.U32[2], e1, e2);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructU32x3(EmitContext& ctx, Id e1, Id e2, Id e3) {
|
||||
return ctx.OpCompositeConstruct(ctx.U32[3], e1, e2, e3);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructU32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4) {
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], e1, e2, e3, e4);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractU32x2(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractU32x3(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractU32x4(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertU32x2(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.U32[2], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertU32x3(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.U32[3], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertU32x4(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.U32[4], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructF16x2(EmitContext& ctx, Id e1, Id e2) {
|
||||
return ctx.OpCompositeConstruct(ctx.F16[2], e1, e2);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructF16x3(EmitContext& ctx, Id e1, Id e2, Id e3) {
|
||||
return ctx.OpCompositeConstruct(ctx.F16[3], e1, e2, e3);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructF16x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4) {
|
||||
return ctx.OpCompositeConstruct(ctx.F16[4], e1, e2, e3, e4);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractF16x2(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.F16[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractF16x3(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.F16[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractF16x4(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.F16[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF16x2(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F16[2], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF16x3(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F16[3], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF16x4(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F16[4], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructF32x2(EmitContext& ctx, Id e1, Id e2) {
|
||||
return ctx.OpCompositeConstruct(ctx.F32[2], e1, e2);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructF32x3(EmitContext& ctx, Id e1, Id e2, Id e3) {
|
||||
return ctx.OpCompositeConstruct(ctx.F32[3], e1, e2, e3);
|
||||
}
|
||||
|
||||
Id EmitCompositeConstructF32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4) {
|
||||
return ctx.OpCompositeConstruct(ctx.F32[4], e1, e2, e3, e4);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractF32x2(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.F32[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractF32x3(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.F32[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeExtractF32x4(EmitContext& ctx, Id composite, u32 index) {
|
||||
return ctx.OpCompositeExtract(ctx.F32[1], composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF32x2(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F32[2], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF32x3(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F32[3], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF32x4(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F32[4], object, composite, index);
|
||||
}
|
||||
|
||||
void EmitCompositeConstructF64x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitCompositeConstructF64x3(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitCompositeConstructF64x4(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitCompositeExtractF64x2(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitCompositeExtractF64x3(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitCompositeExtractF64x4(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF64x2(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F64[2], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF64x3(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F64[3], object, composite, index);
|
||||
}
|
||||
|
||||
Id EmitCompositeInsertF64x4(EmitContext& ctx, Id composite, Id object, u32 index) {
|
||||
return ctx.OpCompositeInsert(ctx.F64[4], object, composite, index);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,660 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <bit>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/mma_layout.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
template <typename... Args>
|
||||
Id AttrPointer(EmitContext& ctx, Id pointer_type, Id vertex, Id base, Args&&... args) {
|
||||
switch (ctx.stage) {
|
||||
case Stage::TessellationControl:
|
||||
case Stage::TessellationEval:
|
||||
case Stage::Geometry:
|
||||
return ctx.OpAccessChain(pointer_type, base, vertex, std::forward<Args>(args)...);
|
||||
default:
|
||||
return ctx.OpAccessChain(pointer_type, base, std::forward<Args>(args)...);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
Id OutputAccessChain(EmitContext& ctx, Id result_type, Id base, Args&&... args) {
|
||||
if (ctx.stage == Stage::TessellationControl) {
|
||||
const Id invocation_id{ctx.OpLoad(ctx.U32[1], ctx.invocation_id)};
|
||||
return ctx.OpAccessChain(result_type, base, invocation_id, std::forward<Args>(args)...);
|
||||
} else {
|
||||
return ctx.OpAccessChain(result_type, base, std::forward<Args>(args)...);
|
||||
}
|
||||
}
|
||||
|
||||
struct OutAttr {
|
||||
OutAttr(Id pointer_) : pointer{pointer_} {}
|
||||
OutAttr(Id pointer_, Id type_) : pointer{pointer_}, type{type_} {}
|
||||
|
||||
Id pointer{};
|
||||
Id type{};
|
||||
};
|
||||
|
||||
std::optional<OutAttr> OutputAttrPointer(EmitContext& ctx, IR::Attribute attr) {
|
||||
if (IR::IsGeneric(attr)) {
|
||||
const u32 index{IR::GenericAttributeIndex(attr)};
|
||||
const u32 element{IR::GenericAttributeElement(attr)};
|
||||
const GenericElementInfo& info{ctx.output_generics.at(index).at(element)};
|
||||
if (info.num_components == 1) {
|
||||
return info.id;
|
||||
} else {
|
||||
const u32 index_element{element - info.first_element};
|
||||
const Id index_id{ctx.Const(index_element)};
|
||||
return OutputAccessChain(ctx, ctx.output_f32, info.id, index_id);
|
||||
}
|
||||
}
|
||||
|
||||
switch (attr) {
|
||||
case IR::Attribute::PointSize:
|
||||
return ctx.output_point_size;
|
||||
case IR::Attribute::PositionX:
|
||||
case IR::Attribute::PositionY:
|
||||
case IR::Attribute::PositionZ:
|
||||
case IR::Attribute::PositionW: {
|
||||
const u32 element{static_cast<u32>(attr) % 4};
|
||||
const Id element_id{ctx.Const(element)};
|
||||
return OutputAccessChain(ctx, ctx.output_f32, ctx.output_position, element_id);
|
||||
}
|
||||
case IR::Attribute::ClipDistance0:
|
||||
case IR::Attribute::ClipDistance1:
|
||||
case IR::Attribute::ClipDistance2:
|
||||
case IR::Attribute::ClipDistance3:
|
||||
case IR::Attribute::ClipDistance4:
|
||||
case IR::Attribute::ClipDistance5:
|
||||
case IR::Attribute::ClipDistance6:
|
||||
case IR::Attribute::ClipDistance7: {
|
||||
const u32 base{static_cast<u32>(IR::Attribute::ClipDistance0)};
|
||||
const u32 index{static_cast<u32>(attr) - base};
|
||||
if (index >= ctx.profile.max_user_clip_distances) {
|
||||
LOG_WARNING(Shader, "Ignoring clip distance store {} >= {} supported", index,
|
||||
ctx.profile.max_user_clip_distances);
|
||||
return std::nullopt;
|
||||
}
|
||||
const Id clip_num{ctx.Const(index)};
|
||||
return OutputAccessChain(ctx, ctx.output_f32, ctx.clip_distances, clip_num);
|
||||
}
|
||||
case IR::Attribute::Layer:
|
||||
if (ctx.profile.support_viewport_index_layer_non_geometry ||
|
||||
ctx.stage == Shader::Stage::Geometry) {
|
||||
return OutAttr{ctx.layer, ctx.U32[1]};
|
||||
}
|
||||
return std::nullopt;
|
||||
case IR::Attribute::ViewportIndex:
|
||||
if (!ctx.profile.support_multi_viewport) {
|
||||
LOG_WARNING(Shader, "Ignoring viewport index store on non-supporting driver");
|
||||
return std::nullopt;
|
||||
}
|
||||
if (ctx.profile.support_viewport_index_layer_non_geometry ||
|
||||
ctx.stage == Shader::Stage::Geometry) {
|
||||
return OutAttr{ctx.viewport_index, ctx.U32[1]};
|
||||
}
|
||||
return std::nullopt;
|
||||
case IR::Attribute::ViewportMask:
|
||||
if (!ctx.profile.support_viewport_mask) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return OutAttr{ctx.OpAccessChain(ctx.output_u32, ctx.viewport_mask, ctx.u32_zero_value),
|
||||
ctx.U32[1]};
|
||||
default:
|
||||
throw NotImplementedException("Read attribute {}", attr);
|
||||
}
|
||||
}
|
||||
|
||||
Id GetCbuf(EmitContext& ctx, Id result_type, Id UniformDefinitions::*member_ptr, u32 element_size,
|
||||
const IR::Value& binding, const IR::Value& offset, const Id indirect_func) {
|
||||
Id buffer_offset;
|
||||
const Id uniform_type{ctx.uniform_types.*member_ptr};
|
||||
if (offset.IsImmediate()) {
|
||||
// Hardware been proved to read the aligned offset (e.g. LDC.U32 at 6 will read offset 4)
|
||||
const Id imm_offset{ctx.Const(offset.U32() / element_size)};
|
||||
buffer_offset = imm_offset;
|
||||
} else if (element_size > 1) {
|
||||
const u32 log2_element_size{static_cast<u32>(std::countr_zero(element_size))};
|
||||
const Id shift{ctx.Const(log2_element_size)};
|
||||
buffer_offset = ctx.OpShiftRightLogical(ctx.U32[1], ctx.Def(offset), shift);
|
||||
} else {
|
||||
buffer_offset = ctx.Def(offset);
|
||||
}
|
||||
if (!binding.IsImmediate()) {
|
||||
return ctx.OpFunctionCall(result_type, indirect_func, ctx.Def(binding), buffer_offset);
|
||||
}
|
||||
|
||||
const Id cbuf{ctx.cbufs[binding.U32()].*member_ptr};
|
||||
const Id access_chain{ctx.OpAccessChain(uniform_type, cbuf, ctx.u32_zero_value, buffer_offset)};
|
||||
const Id val = ctx.OpLoad(result_type, access_chain);
|
||||
|
||||
if (offset.IsImmediate() || !ctx.profile.has_broken_robust) {
|
||||
return val;
|
||||
}
|
||||
|
||||
const Id cond = ctx.OpULessThanEqual(ctx.TypeBool(), buffer_offset, ctx.Const(0xFFFFu));
|
||||
const Id zero = ctx.ConstantNull(result_type);
|
||||
return ctx.OpSelect(result_type, cond, val, zero);
|
||||
}
|
||||
|
||||
Id GetCbufU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
return GetCbuf(ctx, ctx.U32[1], &UniformDefinitions::U32, sizeof(u32), binding, offset,
|
||||
ctx.load_const_func_u32);
|
||||
}
|
||||
|
||||
Id GetCbufU32x4(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
return GetCbuf(ctx, ctx.U32[4], &UniformDefinitions::U32x4, sizeof(u32[4]), binding, offset,
|
||||
ctx.load_const_func_u32x4);
|
||||
}
|
||||
|
||||
Id GetCbufElement(EmitContext& ctx, Id vector, const IR::Value& offset, u32 index_offset) {
|
||||
if (offset.IsImmediate()) {
|
||||
const u32 element{(offset.U32() / 4) % 4 + index_offset};
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], vector, element);
|
||||
}
|
||||
const Id shift{ctx.OpShiftRightLogical(ctx.U32[1], ctx.Def(offset), ctx.Const(2u))};
|
||||
Id element{ctx.OpBitwiseAnd(ctx.U32[1], shift, ctx.Const(3u))};
|
||||
if (index_offset > 0) {
|
||||
element = ctx.OpIAdd(ctx.U32[1], element, ctx.Const(index_offset));
|
||||
}
|
||||
return ctx.OpVectorExtractDynamic(ctx.U32[1], vector, element);
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
void EmitGetRegister(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitSetRegister(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetPred(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitSetPred(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitSetGotoVariable(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetGotoVariable(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitSetIndirectBranchVariable(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
void EmitGetIndirectBranchVariable(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitGetCbufU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int8 &&
|
||||
ctx.profile.support_uniform_and_storage_buffer_8bit) {
|
||||
const Id load{GetCbuf(ctx, ctx.U8, &UniformDefinitions::U8, sizeof(u8), binding, offset,
|
||||
ctx.load_const_func_u8)};
|
||||
return ctx.OpUConvert(ctx.U32[1], load);
|
||||
}
|
||||
Id element{};
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
element = GetCbufU32(ctx, binding, offset);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
element = GetCbufElement(ctx, vector, offset, 0u);
|
||||
}
|
||||
const Id bit_offset{ctx.BitOffset8(offset)};
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], element, bit_offset, ctx.Const(8u));
|
||||
}
|
||||
|
||||
Id EmitGetCbufS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int8 &&
|
||||
ctx.profile.support_uniform_and_storage_buffer_8bit) {
|
||||
const Id load{GetCbuf(ctx, ctx.S8, &UniformDefinitions::S8, sizeof(s8), binding, offset,
|
||||
ctx.load_const_func_u8)};
|
||||
return ctx.OpSConvert(ctx.U32[1], load);
|
||||
}
|
||||
Id element{};
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
element = GetCbufU32(ctx, binding, offset);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
element = GetCbufElement(ctx, vector, offset, 0u);
|
||||
}
|
||||
const Id bit_offset{ctx.BitOffset8(offset)};
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], element, bit_offset, ctx.Const(8u));
|
||||
}
|
||||
|
||||
Id EmitGetCbufU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int16 &&
|
||||
ctx.profile.support_uniform_and_storage_buffer_16bit) {
|
||||
const Id load{GetCbuf(ctx, ctx.U16, &UniformDefinitions::U16, sizeof(u16), binding, offset,
|
||||
ctx.load_const_func_u16)};
|
||||
return ctx.OpUConvert(ctx.U32[1], load);
|
||||
}
|
||||
Id element{};
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
element = GetCbufU32(ctx, binding, offset);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
element = GetCbufElement(ctx, vector, offset, 0u);
|
||||
}
|
||||
const Id bit_offset{ctx.BitOffset16(offset)};
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], element, bit_offset, ctx.Const(16u));
|
||||
}
|
||||
|
||||
Id EmitGetCbufS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing && ctx.profile.support_int16 &&
|
||||
ctx.profile.support_uniform_and_storage_buffer_16bit) {
|
||||
const Id load{GetCbuf(ctx, ctx.S16, &UniformDefinitions::S16, sizeof(s16), binding, offset,
|
||||
ctx.load_const_func_u16)};
|
||||
return ctx.OpSConvert(ctx.U32[1], load);
|
||||
}
|
||||
Id element{};
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
element = GetCbufU32(ctx, binding, offset);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
element = GetCbufElement(ctx, vector, offset, 0u);
|
||||
}
|
||||
const Id bit_offset{ctx.BitOffset16(offset)};
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], element, bit_offset, ctx.Const(16u));
|
||||
}
|
||||
|
||||
Id EmitGetCbufU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
return GetCbufU32(ctx, binding, offset);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
return GetCbufElement(ctx, vector, offset, 0u);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitGetCbufF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
return GetCbuf(ctx, ctx.F32[1], &UniformDefinitions::F32, sizeof(f32), binding, offset,
|
||||
ctx.load_const_func_f32);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
return ctx.OpBitcast(ctx.F32[1], GetCbufElement(ctx, vector, offset, 0u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitGetCbufU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
return GetCbuf(ctx, ctx.U32[2], &UniformDefinitions::U32x2, sizeof(u32[2]), binding, offset,
|
||||
ctx.load_const_func_u32x2);
|
||||
} else {
|
||||
const Id vector{GetCbufU32x4(ctx, binding, offset)};
|
||||
return ctx.OpCompositeConstruct(ctx.U32[2], GetCbufElement(ctx, vector, offset, 0u),
|
||||
GetCbufElement(ctx, vector, offset, 1u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitGetAttribute(EmitContext& ctx, IR::Attribute attr, Id vertex) {
|
||||
const u32 element{static_cast<u32>(attr) % 4};
|
||||
if (IR::IsGeneric(attr)) {
|
||||
const u32 index{IR::GenericAttributeIndex(attr)};
|
||||
if (ctx.stage == Stage::Fragment && ctx.runtime_info.point_coord_generics.test(index)) {
|
||||
// (hookshot) the point sprite's coordinate in place of the generic (RuntimeInfo::point_coord_generics)
|
||||
if (element < 2) {
|
||||
return ctx.OpLoad(ctx.F32[1],
|
||||
ctx.OpAccessChain(ctx.input_f32, ctx.point_coord, ctx.Const(element)));
|
||||
}
|
||||
return ctx.Const(element == 3 ? 1.0f : 0.0f);
|
||||
}
|
||||
const auto& generic{ctx.input_generics.at(index)};
|
||||
if (!ValidId(generic.id)) {
|
||||
// Attribute is disabled or varying component is not written
|
||||
return ctx.Const(element == 3 ? 1.0f : 0.0f);
|
||||
}
|
||||
const Id pointer{
|
||||
AttrPointer(ctx, generic.pointer_type, vertex, generic.id, ctx.Const(element))};
|
||||
const Id value{ctx.OpLoad(generic.component_type, pointer)};
|
||||
return [&ctx, generic, value]() {
|
||||
switch (generic.load_op) {
|
||||
case InputGenericLoadOp::Bitcast:
|
||||
return ctx.OpBitcast(ctx.F32[1], value);
|
||||
case InputGenericLoadOp::SToF:
|
||||
return ctx.OpConvertSToF(ctx.F32[1], value);
|
||||
case InputGenericLoadOp::UToF:
|
||||
return ctx.OpConvertUToF(ctx.F32[1], value);
|
||||
default:
|
||||
return value;
|
||||
};
|
||||
}();
|
||||
}
|
||||
switch (attr) {
|
||||
case IR::Attribute::PrimitiveId:
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.primitive_id));
|
||||
case IR::Attribute::Layer:
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.layer));
|
||||
case IR::Attribute::PositionX:
|
||||
case IR::Attribute::PositionY:
|
||||
case IR::Attribute::PositionZ:
|
||||
case IR::Attribute::PositionW:
|
||||
return ctx.OpLoad(
|
||||
ctx.F32[1],
|
||||
ctx.need_input_position_indirect
|
||||
? AttrPointer(ctx, ctx.input_f32, vertex, ctx.input_position, ctx.u32_zero_value,
|
||||
ctx.Const(element))
|
||||
: AttrPointer(ctx, ctx.input_f32, vertex, ctx.input_position, ctx.Const(element)));
|
||||
case IR::Attribute::InstanceId:
|
||||
if (ctx.profile.support_vertex_instance_id) {
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.instance_id));
|
||||
} else {
|
||||
const Id index{ctx.OpLoad(ctx.U32[1], ctx.instance_index)};
|
||||
const Id base{ctx.OpLoad(ctx.U32[1], ctx.base_instance)};
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpISub(ctx.U32[1], index, base));
|
||||
}
|
||||
case IR::Attribute::VertexId:
|
||||
if (ctx.profile.support_vertex_instance_id) {
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.vertex_id));
|
||||
} else {
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.vertex_index));
|
||||
}
|
||||
case IR::Attribute::BaseInstance:
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.base_instance));
|
||||
case IR::Attribute::BaseVertex:
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.base_vertex));
|
||||
case IR::Attribute::DrawID:
|
||||
return ctx.OpBitcast(ctx.F32[1], ctx.OpLoad(ctx.U32[1], ctx.draw_index));
|
||||
case IR::Attribute::FrontFace:
|
||||
return ctx.OpSelect(ctx.F32[1], ctx.OpLoad(ctx.U1, ctx.front_face),
|
||||
ctx.OpBitcast(ctx.F32[1], ctx.Const((std::numeric_limits<u32>::max)())),
|
||||
ctx.f32_zero_value);
|
||||
case IR::Attribute::PointSpriteS:
|
||||
return ctx.OpLoad(ctx.F32[1],
|
||||
ctx.OpAccessChain(ctx.input_f32, ctx.point_coord, ctx.u32_zero_value));
|
||||
case IR::Attribute::PointSpriteT:
|
||||
return ctx.OpLoad(ctx.F32[1],
|
||||
ctx.OpAccessChain(ctx.input_f32, ctx.point_coord, ctx.Const(1U)));
|
||||
case IR::Attribute::TessellationEvaluationPointU:
|
||||
return ctx.OpLoad(ctx.F32[1],
|
||||
ctx.OpAccessChain(ctx.input_f32, ctx.tess_coord, ctx.u32_zero_value));
|
||||
case IR::Attribute::TessellationEvaluationPointV:
|
||||
return ctx.OpLoad(ctx.F32[1],
|
||||
ctx.OpAccessChain(ctx.input_f32, ctx.tess_coord, ctx.Const(1U)));
|
||||
default:
|
||||
throw NotImplementedException("Read attribute {}", attr);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitGetAttributeU32(EmitContext& ctx, IR::Attribute attr, Id) {
|
||||
switch (attr) {
|
||||
case IR::Attribute::PrimitiveId:
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.primitive_id);
|
||||
case IR::Attribute::InstanceId:
|
||||
if (ctx.profile.support_vertex_instance_id) {
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.instance_id);
|
||||
} else {
|
||||
const Id index{ctx.OpLoad(ctx.U32[1], ctx.instance_index)};
|
||||
const Id base{ctx.OpLoad(ctx.U32[1], ctx.base_instance)};
|
||||
return ctx.OpISub(ctx.U32[1], index, base);
|
||||
}
|
||||
case IR::Attribute::VertexId:
|
||||
if (ctx.profile.support_vertex_instance_id) {
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.vertex_id);
|
||||
} else {
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.vertex_index);
|
||||
}
|
||||
case IR::Attribute::BaseInstance:
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.base_instance);
|
||||
case IR::Attribute::BaseVertex:
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.base_vertex);
|
||||
case IR::Attribute::DrawID:
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.draw_index);
|
||||
default:
|
||||
throw NotImplementedException("Read U32 attribute {}", attr);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSetAttribute(EmitContext& ctx, IR::Attribute attr, Id value, [[maybe_unused]] Id vertex) {
|
||||
const std::optional<OutAttr> output{OutputAttrPointer(ctx, attr)};
|
||||
if (!output) {
|
||||
return;
|
||||
}
|
||||
if (Sirit::ValidId(output->type)) {
|
||||
value = ctx.OpBitcast(output->type, value);
|
||||
}
|
||||
|
||||
static constexpr IR::Attribute cd0 = IR::Attribute::ClipDistance0;
|
||||
static constexpr IR::Attribute cd7 = IR::Attribute::ClipDistance7;
|
||||
|
||||
if (attr >= cd0 && attr <= cd7) {
|
||||
const u32 idx = (u32) attr - (u32) cd0;
|
||||
clip_distance_written.set(idx);
|
||||
}
|
||||
ctx.OpStore(output->pointer, value);
|
||||
}
|
||||
|
||||
Id EmitGetAttributeIndexed(EmitContext& ctx, Id offset, Id vertex) {
|
||||
switch (ctx.stage) {
|
||||
case Stage::TessellationControl:
|
||||
case Stage::TessellationEval:
|
||||
case Stage::Geometry:
|
||||
return ctx.OpFunctionCall(ctx.F32[1], ctx.indexed_load_func, offset, vertex);
|
||||
default:
|
||||
return ctx.OpFunctionCall(ctx.F32[1], ctx.indexed_load_func, offset);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSetAttributeIndexed(EmitContext& ctx, Id offset, Id value, [[maybe_unused]] Id vertex) {
|
||||
ctx.OpFunctionCall(ctx.void_id, ctx.indexed_store_func, offset, value);
|
||||
}
|
||||
|
||||
Id EmitGetPatch(EmitContext& ctx, IR::Patch patch) {
|
||||
if (!IR::IsGeneric(patch)) {
|
||||
throw NotImplementedException("Non-generic patch load");
|
||||
}
|
||||
const u32 index{IR::GenericPatchIndex(patch)};
|
||||
const Id element{ctx.Const(IR::GenericPatchElement(patch))};
|
||||
const Id type{ctx.stage == Stage::TessellationControl ? ctx.output_f32 : ctx.input_f32};
|
||||
const Id pointer{ctx.OpAccessChain(type, ctx.patches.at(index), element)};
|
||||
return ctx.OpLoad(ctx.F32[1], pointer);
|
||||
}
|
||||
|
||||
void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value) {
|
||||
const Id pointer{[&] {
|
||||
if (IR::IsGeneric(patch)) {
|
||||
const u32 index{IR::GenericPatchIndex(patch)};
|
||||
const Id element{ctx.Const(IR::GenericPatchElement(patch))};
|
||||
return ctx.OpAccessChain(ctx.output_f32, ctx.patches.at(index), element);
|
||||
}
|
||||
switch (patch) {
|
||||
case IR::Patch::TessellationLodLeft:
|
||||
case IR::Patch::TessellationLodRight:
|
||||
case IR::Patch::TessellationLodTop:
|
||||
case IR::Patch::TessellationLodBottom: {
|
||||
const u32 index{static_cast<u32>(patch) - u32(IR::Patch::TessellationLodLeft)};
|
||||
const Id index_id{ctx.Const(index)};
|
||||
return ctx.OpAccessChain(ctx.output_f32, ctx.output_tess_level_outer, index_id);
|
||||
}
|
||||
case IR::Patch::TessellationLodInteriorU:
|
||||
return ctx.OpAccessChain(ctx.output_f32, ctx.output_tess_level_inner,
|
||||
ctx.u32_zero_value);
|
||||
case IR::Patch::TessellationLodInteriorV:
|
||||
return ctx.OpAccessChain(ctx.output_f32, ctx.output_tess_level_inner, ctx.Const(1u));
|
||||
default:
|
||||
throw NotImplementedException("Patch {}", patch);
|
||||
}
|
||||
}()};
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
|
||||
void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value) {
|
||||
const Id component_id{ctx.Const(component)};
|
||||
const AttributeType type{ctx.runtime_info.color_output_types[index]};
|
||||
if (type == AttributeType::Float) {
|
||||
const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
|
||||
if (frag_nan_guard) { // (amd-fix, mma_layout.h) NaN -> 0, the rest clamped to +-1e30
|
||||
const Id clamped{ctx.OpNMin(ctx.F32[1], ctx.OpNMax(ctx.F32[1], value, ctx.Const(-1e30f)), ctx.Const(1e30f))};
|
||||
value = ctx.OpSelect(ctx.F32[1], ctx.OpIsNan(ctx.U1, value), ctx.Const(0.0f), clamped);
|
||||
}
|
||||
ctx.OpStore(pointer, value);
|
||||
} else if (type == AttributeType::UnsignedInt) {
|
||||
const Id pointer{ctx.OpAccessChain(ctx.output_u32, ctx.frag_color.at(index), component_id)};
|
||||
ctx.OpStore(pointer, ctx.OpBitcast(ctx.U32[1], value));
|
||||
} else if (type == AttributeType::SignedInt) {
|
||||
const Id output_s32{ctx.TypePointer(spv::StorageClass::Output, ctx.S32[1])};
|
||||
const Id pointer{ctx.OpAccessChain(output_s32, ctx.frag_color.at(index), component_id)};
|
||||
ctx.OpStore(pointer, ctx.OpBitcast(ctx.S32[1], value));
|
||||
} else {
|
||||
// Disabled or unknown, treat as float
|
||||
const Id pointer{ctx.OpAccessChain(ctx.output_f32, ctx.frag_color.at(index), component_id)};
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitSetSampleMask(EmitContext& ctx, Id value) {
|
||||
const Id pointer{ctx.OpAccessChain(ctx.output_u32, ctx.sample_mask, ctx.u32_zero_value)};
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
|
||||
void EmitSetFragDepth(EmitContext& ctx, Id value) {
|
||||
if (!ctx.runtime_info.convert_depth_mode || ctx.profile.support_native_ndc) {
|
||||
ctx.OpStore(ctx.frag_depth, value);
|
||||
return;
|
||||
}
|
||||
const Id unit{ctx.Const(0.5f)};
|
||||
const Id new_depth{ctx.OpFma(ctx.F32[1], value, unit, unit)};
|
||||
ctx.OpStore(ctx.frag_depth, new_depth);
|
||||
}
|
||||
|
||||
void EmitGetZFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitGetSFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitGetCFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitGetOFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitSetZFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitSetSFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitSetCFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
void EmitSetOFlag(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitWorkgroupId(EmitContext& ctx) {
|
||||
const Id id{ctx.OpLoad(ctx.U32[3], ctx.workgroup_id)};
|
||||
if (ctx.mesh_fallback && ctx.stage == Stage::Mesh && ctx.runtime_info.mesh_fallback_task) {
|
||||
// (hookshot) a fallback mesh workgroup is a slot (its task's map gives it, EmitMeshPrologue): its id is its index in its
|
||||
// task, as a task launches them from 0
|
||||
const Id slot{Sirit::ValidId(ctx.mesh_fallback_slot) ? ctx.mesh_fallback_slot
|
||||
: ctx.OpCompositeExtract(ctx.U32[1], id, 0U)};
|
||||
const Id x{ctx.OpUMod(ctx.U32[1], slot, ctx.Const(ctx.runtime_info.mesh_fallback_task_threads))};
|
||||
return ctx.OpCompositeConstruct(ctx.U32[3], x, ctx.u32_zero_value, ctx.u32_zero_value);
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
Id EmitLocalInvocationId(EmitContext& ctx) {
|
||||
return ctx.OpLoad(ctx.U32[3], ctx.local_invocation_id);
|
||||
}
|
||||
|
||||
Id EmitInvocationId(EmitContext& ctx) {
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.invocation_id);
|
||||
}
|
||||
|
||||
Id EmitInvocationInfo(EmitContext& ctx) {
|
||||
switch (ctx.stage) {
|
||||
case Stage::TessellationControl:
|
||||
case Stage::TessellationEval:
|
||||
return ctx.OpShiftLeftLogical(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.patch_vertices_in), ctx.Const(16u));
|
||||
case Stage::Geometry:
|
||||
return ctx.OpShiftLeftLogical(ctx.U32[1], ctx.Const(InputTopologyVertices::vertices(ctx.runtime_info.input_topology)), ctx.Const(16u));
|
||||
default:
|
||||
LOG_WARNING(Shader, "(STUBBED) called");
|
||||
return ctx.Const(0x00ff0000u);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitSampleId(EmitContext& ctx) {
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.sample_id);
|
||||
}
|
||||
|
||||
Id EmitIsHelperInvocation(EmitContext& ctx) {
|
||||
return ctx.OpLoad(ctx.U1, ctx.is_helper_invocation);
|
||||
}
|
||||
|
||||
Id EmitSR_WScaleFactorXY(EmitContext& ctx) {
|
||||
LOG_WARNING(Shader, "(STUBBED) called");
|
||||
return ctx.Const(0x00ff0000u);
|
||||
}
|
||||
|
||||
Id EmitSR_WScaleFactorZ(EmitContext& ctx) {
|
||||
LOG_WARNING(Shader, "(STUBBED) called");
|
||||
return ctx.Const(0x00ff0000u);
|
||||
}
|
||||
|
||||
Id EmitYDirection(EmitContext& ctx) {
|
||||
return ctx.Const(ctx.runtime_info.y_negate ? -1.0f : 1.0f);
|
||||
}
|
||||
|
||||
Id EmitResolutionDownFactor(EmitContext& ctx) {
|
||||
if (ctx.profile.unified_descriptor_binding) {
|
||||
const Id pointer_type{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.F32[1])};
|
||||
const Id index{ctx.Const(ctx.rescaling_downfactor_member_index)};
|
||||
const Id pointer{ctx.OpAccessChain(pointer_type, ctx.rescaling_push_constants, index)};
|
||||
return ctx.OpLoad(ctx.F32[1], pointer);
|
||||
} else {
|
||||
const Id composite{ctx.OpLoad(ctx.F32[4], ctx.rescaling_uniform_constant)};
|
||||
return ctx.OpCompositeExtract(ctx.F32[1], composite, 2u);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitRenderArea(EmitContext& ctx) {
|
||||
if (ctx.profile.unified_descriptor_binding) {
|
||||
const Id pointer_type{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.F32[4])};
|
||||
const Id index{ctx.Const(ctx.render_are_member_index)};
|
||||
const Id pointer{ctx.OpAccessChain(pointer_type, ctx.render_area_push_constant, index)};
|
||||
return ctx.OpLoad(ctx.F32[4], pointer);
|
||||
} else {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadLocal(EmitContext& ctx, Id word_offset) {
|
||||
const Id pointer{
|
||||
ctx.OpAccessChain(ctx.private_u32, ctx.local_memory, word_offset, ctx.Const(0U))};
|
||||
return ctx.OpLoad(ctx.U32[1], pointer);
|
||||
}
|
||||
|
||||
void EmitWriteLocal(EmitContext& ctx, Id word_offset, Id value) {
|
||||
const Id pointer{
|
||||
ctx.OpAccessChain(ctx.private_u32, ctx.local_memory, word_offset, ctx.Const(0U))};
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,27 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
void EmitJoin(EmitContext&) {
|
||||
throw NotImplementedException("Join shouldn't be emitted");
|
||||
}
|
||||
|
||||
void EmitDemoteToHelperInvocation(EmitContext& ctx) {
|
||||
if (ctx.profile.support_demote_to_helper_invocation) {
|
||||
ctx.OpDemoteToHelperInvocationEXT();
|
||||
} else {
|
||||
const Id kill_label{ctx.OpLabel()};
|
||||
const Id impossible_label{ctx.OpLabel()};
|
||||
ctx.OpSelectionMerge(impossible_label, spv::SelectionControlMask::MaskNone);
|
||||
ctx.OpBranchConditional(ctx.true_value, kill_label, impossible_label);
|
||||
ctx.AddLabel(kill_label);
|
||||
ctx.OpKill();
|
||||
ctx.AddLabel(impossible_label);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,326 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id ExtractU16(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpUConvert(ctx.U16, value);
|
||||
} else {
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
Id ExtractS16(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpSConvert(ctx.S16, value);
|
||||
} else {
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
Id ExtractU8(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int8) {
|
||||
return ctx.OpUConvert(ctx.U8, value);
|
||||
} else {
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
Id ExtractS8(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int8) {
|
||||
return ctx.OpSConvert(ctx.S8, value);
|
||||
} else {
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitConvertS16F16(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpSConvert(ctx.U32[1], ctx.OpConvertFToS(ctx.U16, value));
|
||||
} else {
|
||||
return ExtractS16(ctx, ctx.OpConvertFToS(ctx.U32[1], value));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertS16F32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpSConvert(ctx.U32[1], ctx.OpConvertFToS(ctx.U16, value));
|
||||
} else {
|
||||
return ExtractS16(ctx, ctx.OpConvertFToS(ctx.U32[1], value));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertS16F64(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpSConvert(ctx.U32[1], ctx.OpConvertFToS(ctx.U16, value));
|
||||
} else {
|
||||
return ExtractS16(ctx, ctx.OpConvertFToS(ctx.U32[1], value));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertS32F16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToS(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertS32F32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.has_broken_signed_operations) {
|
||||
return ctx.OpBitcast(ctx.U32[1], ctx.OpConvertFToS(ctx.S32[1], value));
|
||||
} else {
|
||||
return ctx.OpConvertFToS(ctx.U32[1], value);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertS32F64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToS(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertS64F16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToS(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertS64F32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToS(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertS64F64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToS(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertU16F16(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpUConvert(ctx.U32[1], ctx.OpConvertFToU(ctx.U16, value));
|
||||
} else {
|
||||
return ExtractU16(ctx, ctx.OpConvertFToU(ctx.U32[1], value));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertU16F32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpUConvert(ctx.U32[1], ctx.OpConvertFToU(ctx.U16, value));
|
||||
} else {
|
||||
return ExtractU16(ctx, ctx.OpConvertFToU(ctx.U32[1], value));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertU16F64(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpUConvert(ctx.U32[1], ctx.OpConvertFToU(ctx.U16, value));
|
||||
} else {
|
||||
return ExtractU16(ctx, ctx.OpConvertFToU(ctx.U32[1], value));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertU32F16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToU(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertU32F32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToU(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertU32F64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToU(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertU64F16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToU(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertU64F32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToU(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertU64F64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertFToU(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertU64U32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpUConvert(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitConvertU32U64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpUConvert(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF16F32(EmitContext& ctx, Id value) {
|
||||
#ifdef __ANDROID__
|
||||
return ctx.OpFConvert(ctx.F16[1], value);
|
||||
#else
|
||||
const auto result = ctx.OpFConvert(ctx.F16[1], value);
|
||||
const auto isOverflowing = ctx.OpIsNan(ctx.U1, result);
|
||||
return ctx.OpSelect(ctx.F16[1], isOverflowing, ctx.Constant(ctx.F16[1], 0), result);
|
||||
#endif
|
||||
}
|
||||
|
||||
Id EmitConvertF32F16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFConvert(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF32F64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFConvert(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF64F32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFConvert(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF16S8(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F16[1], ExtractS8(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF16S16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F16[1], ExtractS16(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF16S32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF16S64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF16U8(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F16[1], ExtractU8(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF16U16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F16[1], ExtractU16(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF16U32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF16U64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF32S8(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F32[1], ExtractS8(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF32S16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F32[1], ExtractS16(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF32S32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.has_broken_signed_operations) {
|
||||
value = ctx.OpBitcast(ctx.S32[1], value);
|
||||
}
|
||||
return ctx.OpConvertSToF(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF32S64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF32U8(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F32[1], ExtractU8(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF32U16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F32[1], ExtractU16(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF32U32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF32U64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF64S8(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F64[1], ExtractS8(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF64S16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F64[1], ExtractS16(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF64S32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.has_broken_signed_operations) {
|
||||
value = ctx.OpBitcast(ctx.S32[1], value);
|
||||
}
|
||||
return ctx.OpConvertSToF(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF64S64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertSToF(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF64U8(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F64[1], ExtractU8(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF64U16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F64[1], ExtractU16(ctx, value));
|
||||
}
|
||||
|
||||
Id EmitConvertF64U32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertF64U64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpConvertUToF(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitConvertU16U32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpUConvert(ctx.U16, value);
|
||||
} else {
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertU32U16(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpUConvert(ctx.U32[1], value);
|
||||
} else {
|
||||
return ExtractU16(ctx, value);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertU8U32(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int8) {
|
||||
return ctx.OpUConvert(ctx.U8, value);
|
||||
} else {
|
||||
return ExtractU8(ctx, value);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertU32U8(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int8) {
|
||||
return ctx.OpUConvert(ctx.U32[1], value);
|
||||
} else {
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
// in signed
|
||||
Id EmitConvertS32S8(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int8) {
|
||||
return ctx.OpSConvert(ctx.U32[1], value);
|
||||
} else {
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitConvertS32S16(EmitContext& ctx, Id value) {
|
||||
if (ctx.profile.support_int16) {
|
||||
return ctx.OpSConvert(ctx.U32[1], value);
|
||||
} else {
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], value, ctx.u32_zero_value, ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,396 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
#include "shader_recompiler/frontend/ir/modifiers.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id Decorate(EmitContext& ctx, IR::Inst* inst, Id op) {
|
||||
const auto flags{inst->Flags<IR::FpControl>()};
|
||||
if (flags.no_contraction) {
|
||||
ctx.Decorate(op, spv::Decoration::NoContraction);
|
||||
}
|
||||
return op;
|
||||
}
|
||||
|
||||
// (amd-fix, Lichii #432/#437) The guest's FMNMX returns the operand that is not NaN and saturation turns NaN into 0.
|
||||
// FMin/FMax/FClamp leave NaN undefined: NVIDIA happens to return the other operand, AMD returns the NaN, and in OoT's
|
||||
// bloom chain it fills the screen (the yellow-white haze). NMin/NMax return the operand that is not NaN.
|
||||
Id Clamp(EmitContext& ctx, Id type, Id value, Id zero, Id one) {
|
||||
return ctx.OpNMin(type, ctx.OpNMax(type, value, zero), one);
|
||||
}
|
||||
|
||||
Id FPOrdNotEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
if (ctx.profile.ignore_nan_fp_comparisons) {
|
||||
const Id comp{ctx.OpFOrdEqual(ctx.U1, lhs, rhs)};
|
||||
const Id lhs_not_nan{ctx.OpLogicalNot(ctx.U1, ctx.OpIsNan(ctx.U1, lhs))};
|
||||
const Id rhs_not_nan{ctx.OpLogicalNot(ctx.U1, ctx.OpIsNan(ctx.U1, rhs))};
|
||||
return ctx.OpLogicalAnd(ctx.U1, ctx.OpLogicalAnd(ctx.U1, comp, lhs_not_nan), rhs_not_nan);
|
||||
} else {
|
||||
return ctx.OpFOrdNotEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
}
|
||||
|
||||
Id FPUnordCompare(Id (EmitContext::*comp_func)(Id, Id, Id), EmitContext& ctx, Id lhs, Id rhs) {
|
||||
if (ctx.profile.ignore_nan_fp_comparisons) {
|
||||
const Id lhs_nan{ctx.OpIsNan(ctx.U1, lhs)};
|
||||
const Id rhs_nan{ctx.OpIsNan(ctx.U1, rhs)};
|
||||
const Id comp{(ctx.*comp_func)(ctx.U1, lhs, rhs)};
|
||||
return ctx.OpLogicalOr(ctx.U1, ctx.OpLogicalOr(ctx.U1, comp, lhs_nan), rhs_nan);
|
||||
} else {
|
||||
return (ctx.*comp_func)(ctx.U1, lhs, rhs);
|
||||
}
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitFPAbs16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFAbs(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPAbs32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFAbs(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPAbs64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFAbs(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPAdd16(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
return Decorate(ctx, inst, ctx.OpFAdd(ctx.F16[1], a, b));
|
||||
}
|
||||
|
||||
Id EmitFPAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
return Decorate(ctx, inst, ctx.OpFAdd(ctx.F32[1], a, b));
|
||||
}
|
||||
|
||||
Id EmitFPAdd64(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
return Decorate(ctx, inst, ctx.OpFAdd(ctx.F64[1], a, b));
|
||||
}
|
||||
|
||||
Id EmitFPFma16(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c) {
|
||||
return Decorate(ctx, inst, ctx.OpFma(ctx.F16[1], a, b, c));
|
||||
}
|
||||
|
||||
Id EmitFPFma32(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c) {
|
||||
return Decorate(ctx, inst, ctx.OpFma(ctx.F32[1], a, b, c));
|
||||
}
|
||||
|
||||
Id EmitFPFma64(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c) {
|
||||
return Decorate(ctx, inst, ctx.OpFma(ctx.F64[1], a, b, c));
|
||||
}
|
||||
|
||||
Id EmitFPMax32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpNMax(ctx.F32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitFPMax64(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpNMax(ctx.F64[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitFPMin32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpNMin(ctx.F32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitFPMin64(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpNMin(ctx.F64[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitFPMul16(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
return Decorate(ctx, inst, ctx.OpFMul(ctx.F16[1], a, b));
|
||||
}
|
||||
|
||||
Id EmitFPMul32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
return Decorate(ctx, inst, ctx.OpFMul(ctx.F32[1], a, b));
|
||||
}
|
||||
|
||||
Id EmitFPMul64(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
return Decorate(ctx, inst, ctx.OpFMul(ctx.F64[1], a, b));
|
||||
}
|
||||
|
||||
Id EmitFPNeg16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFNegate(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPNeg32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFNegate(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPNeg64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFNegate(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPSin(EmitContext& ctx, Id value) {
|
||||
return ctx.OpSin(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPCos(EmitContext& ctx, Id value) {
|
||||
return ctx.OpCos(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPExp2(EmitContext& ctx, Id value) {
|
||||
return ctx.OpExp2(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPLog2(EmitContext& ctx, Id value) {
|
||||
return ctx.OpLog2(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPRecip32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFDiv(ctx.F32[1], ctx.Const(1.0f), value);
|
||||
}
|
||||
|
||||
Id EmitFPRecip64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFDiv(ctx.F64[1], ctx.Constant(ctx.F64[1], 1.0f), value);
|
||||
}
|
||||
|
||||
Id EmitFPRecipSqrt32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpInverseSqrt(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPRecipSqrt64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpInverseSqrt(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPSqrt(EmitContext& ctx, Id value) {
|
||||
return ctx.OpSqrt(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPSaturate16(EmitContext& ctx, Id value) {
|
||||
const Id zero{ctx.Constant(ctx.F16[1], u16{0})};
|
||||
const Id one{ctx.Constant(ctx.F16[1], u16{0x3c00})};
|
||||
return Clamp(ctx, ctx.F16[1], value, zero, one);
|
||||
}
|
||||
|
||||
Id EmitFPSaturate32(EmitContext& ctx, Id value) {
|
||||
const Id zero{ctx.Const(f32{0.0f})};
|
||||
const Id one{ctx.Const(f32{1.0f})};
|
||||
return Clamp(ctx, ctx.F32[1], value, zero, one);
|
||||
}
|
||||
|
||||
Id EmitFPSaturate64(EmitContext& ctx, Id value) {
|
||||
const Id zero{ctx.Constant(ctx.F64[1], f64{0.0})};
|
||||
const Id one{ctx.Constant(ctx.F64[1], f64{1.0})};
|
||||
return Clamp(ctx, ctx.F64[1], value, zero, one);
|
||||
}
|
||||
|
||||
Id EmitFPClamp16(EmitContext& ctx, Id value, Id min_value, Id max_value) {
|
||||
return Clamp(ctx, ctx.F16[1], value, min_value, max_value);
|
||||
}
|
||||
|
||||
Id EmitFPClamp32(EmitContext& ctx, Id value, Id min_value, Id max_value) {
|
||||
return Clamp(ctx, ctx.F32[1], value, min_value, max_value);
|
||||
}
|
||||
|
||||
Id EmitFPClamp64(EmitContext& ctx, Id value, Id min_value, Id max_value) {
|
||||
return Clamp(ctx, ctx.F64[1], value, min_value, max_value);
|
||||
}
|
||||
|
||||
Id EmitFPRoundEven16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpRoundEven(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPRoundEven32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpRoundEven(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPRoundEven64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpRoundEven(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPFloor16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFloor(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPFloor32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFloor(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPFloor64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFloor(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPCeil16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpCeil(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPCeil32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpCeil(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPCeil64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpCeil(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPTrunc16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpTrunc(ctx.F16[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPTrunc32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpTrunc(ctx.F32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPTrunc64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpTrunc(ctx.F64[1], value);
|
||||
}
|
||||
|
||||
Id EmitFPOrdEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdNotEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPOrdNotEqual(ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdNotEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPOrdNotEqual(ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdNotEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPOrdNotEqual(ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordNotEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFUnordNotEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordNotEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFUnordNotEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordNotEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFUnordNotEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdLessThan16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdLessThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdLessThan32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdLessThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdLessThan64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdLessThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordLessThan16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordLessThan, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordLessThan32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordLessThan, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordLessThan64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordLessThan, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdGreaterThan16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdGreaterThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdGreaterThan32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdGreaterThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdGreaterThan64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdGreaterThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordGreaterThan16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordGreaterThan, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordGreaterThan32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordGreaterThan, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordGreaterThan64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordGreaterThan, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdLessThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdLessThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdLessThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordLessThanEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordLessThanEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordLessThanEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdGreaterThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdGreaterThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPOrdGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpFOrdGreaterThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordGreaterThanEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordGreaterThanEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPUnordGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return FPUnordCompare(&EmitContext::OpFUnordGreaterThanEqual, ctx, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitFPIsNan16(EmitContext& ctx, Id value) {
|
||||
return ctx.OpIsNan(ctx.U1, value);
|
||||
}
|
||||
|
||||
Id EmitFPIsNan32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpIsNan(ctx.U1, value);
|
||||
}
|
||||
|
||||
Id EmitFPIsNan64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpIsNan(ctx.U1, value);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
838
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_image.cpp
Normal file
838
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_image.cpp
Normal file
@@ -0,0 +1,838 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <boost/container/static_vector.hpp>
|
||||
|
||||
#include "common/settings.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
#include "shader_recompiler/frontend/ir/modifiers.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
|
||||
enum class NonUniformKind {
|
||||
SampledImage,
|
||||
StorageImage,
|
||||
UniformTexelBuffer,
|
||||
StorageTexelBuffer,
|
||||
};
|
||||
|
||||
[[nodiscard]] bool IsNonUniformSupported(const Profile& profile, NonUniformKind kind) noexcept {
|
||||
switch (kind) {
|
||||
case NonUniformKind::SampledImage:
|
||||
return profile.support_sampled_image_array_nonuniform_indexing;
|
||||
case NonUniformKind::StorageImage:
|
||||
return profile.support_storage_image_array_nonuniform_indexing;
|
||||
case NonUniformKind::UniformTexelBuffer:
|
||||
return profile.support_uniform_texel_buffer_array_nonuniform_indexing;
|
||||
case NonUniformKind::StorageTexelBuffer:
|
||||
return profile.support_storage_texel_buffer_array_nonuniform_indexing;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void DecorateNonUniform(EmitContext& ctx, Id object) {
|
||||
if (ctx.non_uniform_ids.contains(object.value)) {
|
||||
return;
|
||||
}
|
||||
ctx.Decorate(object, spv::Decoration::NonUniform);
|
||||
ctx.non_uniform_ids.insert(object.value);
|
||||
}
|
||||
|
||||
[[nodiscard]] bool MarkNonUniform(EmitContext& ctx, Id idx, const IR::Value& index,
|
||||
NonUniformKind kind) {
|
||||
if (index.IsImmediate() || !IsNonUniformSupported(ctx.profile, kind)) {
|
||||
return false;
|
||||
}
|
||||
DecorateNonUniform(ctx, idx);
|
||||
switch (kind) {
|
||||
case NonUniformKind::SampledImage:
|
||||
ctx.uses_nonuniform_sampled_image = true;
|
||||
break;
|
||||
case NonUniformKind::StorageImage:
|
||||
ctx.uses_nonuniform_storage_image = true;
|
||||
break;
|
||||
case NonUniformKind::UniformTexelBuffer:
|
||||
ctx.uses_nonuniform_uniform_texel_buffer = true;
|
||||
break;
|
||||
case NonUniformKind::StorageTexelBuffer:
|
||||
ctx.uses_nonuniform_storage_texel_buffer = true;
|
||||
break;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
class ImageOperands {
|
||||
public:
|
||||
[[maybe_unused]] static constexpr bool ImageSampleOffsetAllowed = false;
|
||||
[[maybe_unused]] static constexpr bool ImageGatherOffsetAllowed = true;
|
||||
[[maybe_unused]] static constexpr bool ImageFetchOffsetAllowed = false;
|
||||
[[maybe_unused]] static constexpr bool ImageGradientOffsetAllowed = false;
|
||||
|
||||
explicit ImageOperands(EmitContext& ctx, bool has_bias, bool has_lod, bool has_lod_clamp,
|
||||
Id lod, const IR::Value& offset) {
|
||||
if (has_bias) {
|
||||
const Id bias{has_lod_clamp ? ctx.OpCompositeExtract(ctx.F32[1], lod, 0) : lod};
|
||||
Add(spv::ImageOperandsMask::Bias, bias);
|
||||
}
|
||||
if (has_lod) {
|
||||
const Id lod_value{has_lod_clamp ? ctx.OpCompositeExtract(ctx.F32[1], lod, 0) : lod};
|
||||
Add(spv::ImageOperandsMask::Lod, lod_value);
|
||||
}
|
||||
AddOffset(ctx, offset, ImageSampleOffsetAllowed);
|
||||
if (has_lod_clamp) {
|
||||
const Id lod_clamp{has_bias ? ctx.OpCompositeExtract(ctx.F32[1], lod, 1) : lod};
|
||||
Add(spv::ImageOperandsMask::MinLod, lod_clamp);
|
||||
}
|
||||
}
|
||||
|
||||
explicit ImageOperands(EmitContext& ctx, const IR::Value& offset, const IR::Value& offset2) {
|
||||
if (offset2.IsEmpty()) {
|
||||
AddOffset(ctx, offset, ImageGatherOffsetAllowed);
|
||||
return;
|
||||
}
|
||||
const std::array values{offset.InstRecursive(), offset2.InstRecursive()};
|
||||
if (!values[0]->AreAllArgsImmediates() || !values[1]->AreAllArgsImmediates()) {
|
||||
LOG_WARNING(Shader_SPIRV, "Not all arguments in PTP are immediate, ignoring");
|
||||
return;
|
||||
}
|
||||
const IR::Opcode opcode{values[0]->GetOpcode()};
|
||||
if (opcode != values[1]->GetOpcode() || opcode != IR::Opcode::CompositeConstructU32x4) {
|
||||
throw LogicError("Invalid PTP arguments");
|
||||
}
|
||||
auto read{[&](unsigned int a, unsigned int b) { return static_cast<s32>(values[a]->Arg(b).U32()); }};
|
||||
|
||||
const Id offsets{ctx.ConstantComposite(
|
||||
ctx.TypeArray(ctx.S32[2], ctx.Const(4U)), ctx.SConst(read(0, 0), read(0, 1)),
|
||||
ctx.SConst(read(0, 2), read(0, 3)), ctx.SConst(read(1, 0), read(1, 1)),
|
||||
ctx.SConst(read(1, 2), read(1, 3)))};
|
||||
Add(spv::ImageOperandsMask::ConstOffsets, offsets);
|
||||
}
|
||||
|
||||
explicit ImageOperands(Id lod, Id ms) {
|
||||
if (Sirit::ValidId(lod)) {
|
||||
Add(spv::ImageOperandsMask::Lod, lod);
|
||||
}
|
||||
if (Sirit::ValidId(ms)) {
|
||||
Add(spv::ImageOperandsMask::Sample, ms);
|
||||
}
|
||||
}
|
||||
|
||||
explicit ImageOperands(EmitContext& ctx, bool has_lod_clamp, Id derivatives,
|
||||
u32 num_derivatives, const IR::Value& offset, Id lod_clamp) {
|
||||
if (!Sirit::ValidId(derivatives)) {
|
||||
throw LogicError("Derivatives must be present");
|
||||
}
|
||||
boost::container::static_vector<Id, 3> deriv_x_accum;
|
||||
boost::container::static_vector<Id, 3> deriv_y_accum;
|
||||
for (u32 i = 0; i < num_derivatives; ++i) {
|
||||
deriv_x_accum.push_back(ctx.OpCompositeExtract(ctx.F32[1], derivatives, i * 2));
|
||||
deriv_y_accum.push_back(ctx.OpCompositeExtract(ctx.F32[1], derivatives, i * 2 + 1));
|
||||
}
|
||||
const Id derivatives_X{ctx.OpCompositeConstruct(
|
||||
ctx.F32[num_derivatives], std::span{deriv_x_accum.data(), deriv_x_accum.size()})};
|
||||
const Id derivatives_Y{ctx.OpCompositeConstruct(
|
||||
ctx.F32[num_derivatives], std::span{deriv_y_accum.data(), deriv_y_accum.size()})};
|
||||
Add(spv::ImageOperandsMask::Grad, derivatives_X, derivatives_Y);
|
||||
AddOffset(ctx, offset, ImageGradientOffsetAllowed);
|
||||
if (has_lod_clamp) {
|
||||
Add(spv::ImageOperandsMask::MinLod, lod_clamp);
|
||||
}
|
||||
}
|
||||
|
||||
explicit ImageOperands(EmitContext& ctx, bool has_lod_clamp, Id derivatives_1, Id derivatives_2,
|
||||
const IR::Value& offset, Id lod_clamp) {
|
||||
if (!Sirit::ValidId(derivatives_1) || !Sirit::ValidId(derivatives_2)) {
|
||||
throw LogicError("Derivatives must be present");
|
||||
}
|
||||
boost::container::static_vector<Id, 3> deriv_1_accum{
|
||||
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 0),
|
||||
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 2),
|
||||
ctx.OpCompositeExtract(ctx.F32[1], derivatives_2, 0),
|
||||
};
|
||||
boost::container::static_vector<Id, 3> deriv_2_accum{
|
||||
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 1),
|
||||
ctx.OpCompositeExtract(ctx.F32[1], derivatives_1, 3),
|
||||
ctx.OpCompositeExtract(ctx.F32[1], derivatives_2, 1),
|
||||
};
|
||||
const Id derivatives_id1{ctx.OpCompositeConstruct(
|
||||
ctx.F32[3], std::span{deriv_1_accum.data(), deriv_1_accum.size()})};
|
||||
const Id derivatives_id2{ctx.OpCompositeConstruct(
|
||||
ctx.F32[3], std::span{deriv_2_accum.data(), deriv_2_accum.size()})};
|
||||
Add(spv::ImageOperandsMask::Grad, derivatives_id1, derivatives_id2);
|
||||
AddOffset(ctx, offset, ImageGradientOffsetAllowed);
|
||||
if (has_lod_clamp) {
|
||||
Add(spv::ImageOperandsMask::MinLod, lod_clamp);
|
||||
}
|
||||
}
|
||||
|
||||
std::span<const Id> Span() const noexcept {
|
||||
return std::span{operands.data(), operands.size()};
|
||||
}
|
||||
|
||||
std::optional<spv::ImageOperandsMask> MaskOptional() const noexcept {
|
||||
return mask != spv::ImageOperandsMask{} ? std::make_optional(mask) : std::nullopt;
|
||||
}
|
||||
|
||||
spv::ImageOperandsMask Mask() const noexcept {
|
||||
return mask;
|
||||
}
|
||||
|
||||
private:
|
||||
void AddOffset(EmitContext& ctx, const IR::Value& offset, bool runtime_offset_allowed) {
|
||||
if (offset.IsEmpty()) {
|
||||
return;
|
||||
}
|
||||
if (offset.IsImmediate()) {
|
||||
Add(spv::ImageOperandsMask::ConstOffset, ctx.SConst(static_cast<s32>(offset.U32())));
|
||||
return;
|
||||
}
|
||||
IR::Inst* const inst{offset.InstRecursive()};
|
||||
if (inst->AreAllArgsImmediates()) {
|
||||
switch (inst->GetOpcode()) {
|
||||
case IR::Opcode::CompositeConstructU32x2:
|
||||
Add(spv::ImageOperandsMask::ConstOffset,
|
||||
ctx.SConst(static_cast<s32>(inst->Arg(0).U32()),
|
||||
static_cast<s32>(inst->Arg(1).U32())));
|
||||
return;
|
||||
case IR::Opcode::CompositeConstructU32x3:
|
||||
Add(spv::ImageOperandsMask::ConstOffset,
|
||||
ctx.SConst(static_cast<s32>(inst->Arg(0).U32()),
|
||||
static_cast<s32>(inst->Arg(1).U32()),
|
||||
static_cast<s32>(inst->Arg(2).U32())));
|
||||
return;
|
||||
case IR::Opcode::CompositeConstructU32x4:
|
||||
Add(spv::ImageOperandsMask::ConstOffset,
|
||||
ctx.SConst(static_cast<s32>(inst->Arg(0).U32()),
|
||||
static_cast<s32>(inst->Arg(1).U32()),
|
||||
static_cast<s32>(inst->Arg(2).U32()),
|
||||
static_cast<s32>(inst->Arg(3).U32())));
|
||||
return;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (runtime_offset_allowed) {
|
||||
Add(spv::ImageOperandsMask::Offset, ctx.Def(offset));
|
||||
}
|
||||
}
|
||||
|
||||
void Add(spv::ImageOperandsMask new_mask, Id value) {
|
||||
mask = static_cast<spv::ImageOperandsMask>(static_cast<unsigned>(mask) |
|
||||
static_cast<unsigned>(new_mask));
|
||||
operands.push_back(value);
|
||||
}
|
||||
|
||||
void Add(spv::ImageOperandsMask new_mask, Id value_1, Id value_2) {
|
||||
mask = static_cast<spv::ImageOperandsMask>(static_cast<unsigned>(mask) |
|
||||
static_cast<unsigned>(new_mask));
|
||||
operands.push_back(value_1);
|
||||
operands.push_back(value_2);
|
||||
}
|
||||
|
||||
boost::container::static_vector<Id, 4> operands;
|
||||
spv::ImageOperandsMask mask{};
|
||||
};
|
||||
|
||||
Id Texture(EmitContext& ctx, IR::TextureInstInfo info, [[maybe_unused]] const IR::Value& index) {
|
||||
const TextureDefinition& def{ctx.textures.at(info.descriptor_index)};
|
||||
if (def.count > 1) {
|
||||
auto const idx = index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index);
|
||||
const bool non_uniform{MarkNonUniform(ctx, idx, index, NonUniformKind::SampledImage)};
|
||||
const Id pointer{ctx.OpAccessChain(def.pointer_type, def.id, idx)};
|
||||
const Id object{ctx.OpLoad(def.sampled_type, pointer)};
|
||||
if (non_uniform) {
|
||||
DecorateNonUniform(ctx, pointer);
|
||||
DecorateNonUniform(ctx, object);
|
||||
}
|
||||
return object;
|
||||
} else {
|
||||
return ctx.OpLoad(def.sampled_type, def.id);
|
||||
}
|
||||
}
|
||||
|
||||
Id TextureImage(EmitContext& ctx, IR::TextureInstInfo info, const IR::Value& index) {
|
||||
if (info.type == TextureType::Buffer) {
|
||||
const TextureBufferDefinition& def{ctx.texture_buffers.at(info.descriptor_index)};
|
||||
if (def.count > 1) {
|
||||
const Id idx{index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index)};
|
||||
const bool non_uniform{
|
||||
MarkNonUniform(ctx, idx, index, NonUniformKind::UniformTexelBuffer)};
|
||||
const Id ptr{ctx.OpAccessChain(ctx.image_buffer_type, def.id, idx)};
|
||||
const Id object{ctx.OpLoad(ctx.image_buffer_type, ptr)};
|
||||
if (non_uniform) {
|
||||
DecorateNonUniform(ctx, ptr);
|
||||
DecorateNonUniform(ctx, object);
|
||||
}
|
||||
return object;
|
||||
}
|
||||
return ctx.OpLoad(ctx.image_buffer_type, def.id);
|
||||
} else {
|
||||
const TextureDefinition& def{ctx.textures.at(info.descriptor_index)};
|
||||
if (def.count > 1) {
|
||||
auto const idx = index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index);
|
||||
const bool non_uniform{MarkNonUniform(ctx, idx, index, NonUniformKind::SampledImage)};
|
||||
const Id ptr = ctx.OpAccessChain(def.pointer_type, def.id, idx);
|
||||
const Id object = ctx.OpLoad(def.sampled_type, ptr);
|
||||
const Id image = ctx.OpImage(def.image_type, object);
|
||||
if (non_uniform) {
|
||||
DecorateNonUniform(ctx, ptr);
|
||||
DecorateNonUniform(ctx, object);
|
||||
DecorateNonUniform(ctx, image);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
return ctx.OpImage(def.image_type, ctx.OpLoad(def.sampled_type, def.id));
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<Id, bool> Image(EmitContext& ctx, const IR::Value& index, IR::TextureInstInfo info) {
|
||||
if (info.type == TextureType::Buffer) {
|
||||
const ImageBufferDefinition def{ctx.image_buffers.at(info.descriptor_index)};
|
||||
if (def.count > 1) {
|
||||
const Id idx{index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index)};
|
||||
const bool non_uniform{
|
||||
MarkNonUniform(ctx, idx, index, NonUniformKind::StorageTexelBuffer)};
|
||||
const Id ptr{ctx.OpAccessChain(def.pointer_type, def.id, idx)};
|
||||
const Id image{ctx.OpLoad(def.image_type, ptr)};
|
||||
if (non_uniform) {
|
||||
DecorateNonUniform(ctx, ptr);
|
||||
DecorateNonUniform(ctx, image);
|
||||
}
|
||||
return {image, def.is_integer};
|
||||
}
|
||||
return {ctx.OpLoad(def.image_type, def.id), def.is_integer};
|
||||
} else {
|
||||
const ImageDefinition def{ctx.images.at(info.descriptor_index)};
|
||||
if (def.count > 1) {
|
||||
const Id idx{index.IsImmediate() ? ctx.Const(index.U32()) : ctx.Def(index)};
|
||||
const bool non_uniform{MarkNonUniform(ctx, idx, index, NonUniformKind::StorageImage)};
|
||||
const Id ptr{ctx.OpAccessChain(def.pointer_type, def.id, idx)};
|
||||
const Id image{ctx.OpLoad(def.image_type, ptr)};
|
||||
if (non_uniform) {
|
||||
DecorateNonUniform(ctx, ptr);
|
||||
DecorateNonUniform(ctx, image);
|
||||
}
|
||||
return {image, def.is_integer};
|
||||
}
|
||||
return {ctx.OpLoad(def.image_type, def.id), def.is_integer};
|
||||
}
|
||||
}
|
||||
|
||||
bool IsTextureMsaa(EmitContext& ctx, const IR::TextureInstInfo& info) {
|
||||
if (info.type == TextureType::Buffer) {
|
||||
return false;
|
||||
}
|
||||
return ctx.textures.at(info.descriptor_index).is_multisample;
|
||||
}
|
||||
|
||||
bool IsTextureInteger(EmitContext& ctx, const IR::TextureInstInfo& info) {
|
||||
if (info.type == TextureType::Buffer) {
|
||||
return false;
|
||||
}
|
||||
return ctx.textures.at(info.descriptor_index).is_integer;
|
||||
}
|
||||
|
||||
Id Decorate(EmitContext& ctx, IR::Inst* inst, Id sample) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
if (info.relaxed_precision != 0) {
|
||||
ctx.Decorate(sample, spv::Decoration::RelaxedPrecision);
|
||||
}
|
||||
return sample;
|
||||
}
|
||||
|
||||
template <typename MethodPtrType, typename... Args>
|
||||
Id Emit(MethodPtrType sparse_ptr, MethodPtrType non_sparse_ptr, EmitContext& ctx, IR::Inst* inst,
|
||||
Id result_type, Args&&... args) {
|
||||
IR::Inst* const sparse{inst->GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)};
|
||||
if (!sparse) {
|
||||
return Decorate(ctx, inst, (ctx.*non_sparse_ptr)(result_type, std::forward<Args>(args)...));
|
||||
}
|
||||
const Id struct_type{ctx.TypeStruct(ctx.U32[1], result_type)};
|
||||
const Id sample{(ctx.*sparse_ptr)(struct_type, std::forward<Args>(args)...)};
|
||||
const Id resident_code{ctx.OpCompositeExtract(ctx.U32[1], sample, 0U)};
|
||||
sparse->SetDefinition(ctx.OpImageSparseTexelsResident(ctx.U1, resident_code));
|
||||
sparse->Invalidate();
|
||||
Decorate(ctx, inst, sample);
|
||||
return ctx.OpCompositeExtract(result_type, sample, 1U);
|
||||
}
|
||||
|
||||
bool IsDynamicPtp(IR::Inst* inst, const IR::Value& offset, const IR::Value& offset2) {
|
||||
if (offset2.IsEmpty() || offset.IsEmpty() ||
|
||||
inst->GetAssociatedPseudoOperation(IR::Opcode::GetSparseFromOp)) {
|
||||
return false;
|
||||
}
|
||||
const auto constant{[](const IR::Value& value) {
|
||||
return value.IsImmediate() || value.InstRecursive()->AreAllArgsImmediates();
|
||||
}};
|
||||
return !constant(offset) || !constant(offset2);
|
||||
}
|
||||
|
||||
// Per-tap offsets the IR could not fold to constants (ConstOffsets must be constant): one gather
|
||||
// per tap with a runtime Offset, keeping the i0,j0 texel (w) of each footprint, which is the
|
||||
// texel a per-tap gather returns for that tap.
|
||||
template <typename GatherFn>
|
||||
Id GatherDynamicPtp(EmitContext& ctx, bool is_integer, Id result_type, const IR::Value& offset,
|
||||
const IR::Value& offset2, GatherFn&& gather) {
|
||||
const std::array packed{ctx.Def(offset), ctx.Def(offset2)};
|
||||
const Id component_type{is_integer ? ctx.U32[1] : ctx.F32[1]};
|
||||
std::array<Id, 4> texels;
|
||||
for (u32 tap = 0; tap < 4; ++tap) {
|
||||
const Id vector{packed[tap / 2]};
|
||||
const u32 first{(tap % 2) * 2};
|
||||
const Id x{ctx.OpBitcast(ctx.S32[1], ctx.OpCompositeExtract(ctx.U32[1], vector, first))};
|
||||
const Id y{ctx.OpBitcast(ctx.S32[1], ctx.OpCompositeExtract(ctx.U32[1], vector, first + 1))};
|
||||
const Id tap_offset{ctx.OpCompositeConstruct(ctx.S32[2], x, y)};
|
||||
texels[tap] = ctx.OpCompositeExtract(component_type, gather(tap_offset), 3U);
|
||||
}
|
||||
return ctx.OpCompositeConstruct(result_type, std::span<const Id>{texels.data(), texels.size()});
|
||||
}
|
||||
|
||||
Id IsScaled(EmitContext& ctx, const IR::Value& index, Id member_index, u32 base_index) {
|
||||
const Id push_constant_u32{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1])};
|
||||
Id bit{};
|
||||
if (index.IsImmediate()) {
|
||||
// Use BitwiseAnd instead of BitfieldExtract for better codegen on Nvidia OpenGL.
|
||||
// LOP32I.NZ is used to set the predicate rather than BFE+ISETP.
|
||||
const u32 index_value{index.U32() + base_index};
|
||||
const Id word_index{ctx.Const(index_value / 32)};
|
||||
const Id bit_index_mask{ctx.Const(1u << (index_value % 32))};
|
||||
const Id pointer{ctx.OpAccessChain(push_constant_u32, ctx.rescaling_push_constants,
|
||||
member_index, word_index)};
|
||||
const Id word{ctx.OpLoad(ctx.U32[1], pointer)};
|
||||
bit = ctx.OpBitwiseAnd(ctx.U32[1], word, bit_index_mask);
|
||||
} else {
|
||||
Id index_value{ctx.Def(index)};
|
||||
if (base_index != 0) {
|
||||
index_value = ctx.OpIAdd(ctx.U32[1], index_value, ctx.Const(base_index));
|
||||
}
|
||||
const Id word_index{ctx.OpShiftRightLogical(ctx.U32[1], index_value, ctx.Const(5u))};
|
||||
const Id bit_index{ctx.OpBitwiseAnd(ctx.U32[1], index_value, ctx.Const(31u))};
|
||||
const Id pointer{ctx.OpAccessChain(push_constant_u32, ctx.rescaling_push_constants,
|
||||
member_index, word_index)};
|
||||
const Id word{ctx.OpLoad(ctx.U32[1], pointer)};
|
||||
const Id bit_index_mask{ctx.OpShiftLeftLogical(ctx.U32[1], ctx.Const(1u), bit_index)};
|
||||
bit = ctx.OpBitwiseAnd(ctx.U32[1], word, bit_index_mask);
|
||||
}
|
||||
return ctx.OpINotEqual(ctx.U1, bit, ctx.u32_zero_value);
|
||||
}
|
||||
|
||||
Id BitTest(EmitContext& ctx, Id mask, Id bit) {
|
||||
const Id shifted{ctx.OpShiftRightLogical(ctx.U32[1], mask, bit)};
|
||||
const Id bit_value{ctx.OpBitwiseAnd(ctx.U32[1], shifted, ctx.Const(1u))};
|
||||
return ctx.OpINotEqual(ctx.U1, bit_value, ctx.u32_zero_value);
|
||||
}
|
||||
|
||||
Id ImageGatherSubpixelOffset(EmitContext& ctx, const IR::TextureInstInfo& info, Id texture,
|
||||
Id coords) {
|
||||
// Apply a subpixel offset of 1/512 the texel size of the texture to ensure same rounding on
|
||||
// AMD hardware as on Maxwell or other Nvidia architectures.
|
||||
const auto calculate_coords{[&](size_t dim) {
|
||||
const Id nudge{ctx.Const(0x1p-9f)};
|
||||
const Id image_size{ctx.OpImageQuerySizeLod(ctx.U32[dim], texture, ctx.u32_zero_value)};
|
||||
Id offset{dim == 2 ? ctx.ConstantComposite(ctx.F32[dim], nudge, nudge)
|
||||
: ctx.ConstantComposite(ctx.F32[dim], nudge, nudge, ctx.f32_zero_value)};
|
||||
offset = ctx.OpFDiv(ctx.F32[dim], offset, ctx.OpConvertUToF(ctx.F32[dim], image_size));
|
||||
return ctx.OpFAdd(ctx.F32[dim], coords, offset);
|
||||
}};
|
||||
switch (info.type) {
|
||||
case TextureType::Color2D:
|
||||
case TextureType::Color2DRect:
|
||||
return calculate_coords(2);
|
||||
case TextureType::ColorArray2D:
|
||||
case TextureType::ColorCube:
|
||||
return calculate_coords(3);
|
||||
default:
|
||||
return coords;
|
||||
}
|
||||
}
|
||||
|
||||
void AddOffsetToCoordinates(EmitContext& ctx, const IR::TextureInstInfo& info, Id& coords,
|
||||
Id offset) {
|
||||
if (!Sirit::ValidId(offset)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Id result_type{};
|
||||
switch (info.type) {
|
||||
case TextureType::Buffer:
|
||||
case TextureType::Color1D: {
|
||||
result_type = ctx.U32[1];
|
||||
break;
|
||||
}
|
||||
case TextureType::ColorArray1D:
|
||||
offset = ctx.OpCompositeConstruct(ctx.U32[2], offset, ctx.u32_zero_value);
|
||||
[[fallthrough]];
|
||||
case TextureType::Color2D:
|
||||
case TextureType::Color2DRect: {
|
||||
result_type = ctx.U32[2];
|
||||
break;
|
||||
}
|
||||
case TextureType::ColorArray2D:
|
||||
offset = ctx.OpCompositeConstruct(ctx.U32[3], ctx.OpCompositeExtract(ctx.U32[1], offset, 0),
|
||||
ctx.OpCompositeExtract(ctx.U32[1], offset, 1),
|
||||
ctx.u32_zero_value);
|
||||
[[fallthrough]];
|
||||
case TextureType::Color3D: {
|
||||
result_type = ctx.U32[3];
|
||||
break;
|
||||
}
|
||||
case TextureType::ColorCube:
|
||||
case TextureType::ColorArrayCube:
|
||||
return;
|
||||
}
|
||||
coords = ctx.OpIAdd(result_type, coords, offset);
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitBindlessImageSampleImplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageSampleExplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageSampleDrefImplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageSampleDrefExplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageGather(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageGatherDref(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageFetch(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageQueryDimensions(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageQueryLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageGradient(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageRead(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageWrite(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageSampleImplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageSampleExplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageSampleDrefImplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageSampleDrefExplicitLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageGather(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageGatherDref(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageFetch(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageQueryDimensions(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageQueryLod(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageGradient(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageRead(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageWrite(EmitContext&) {
|
||||
throw LogicError("Unreachable instruction");
|
||||
}
|
||||
|
||||
Id EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id bias_lc, const IR::Value& offset) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const bool is_integer{IsTextureInteger(ctx, info)};
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
Id color;
|
||||
if (ctx.stage == Stage::Fragment) {
|
||||
const ImageOperands operands(ctx, info.has_bias != 0, false, info.has_lod_clamp != 0,
|
||||
bias_lc, offset);
|
||||
color = Emit(&EmitContext::OpImageSparseSampleImplicitLod,
|
||||
&EmitContext::OpImageSampleImplicitLod, ctx, inst, result_type,
|
||||
Texture(ctx, info, index), coords, operands.MaskOptional(), operands.Span());
|
||||
} else {
|
||||
// We can't use implicit lods on non-fragment stages on SPIR-V. Maxwell hardware behaves as
|
||||
// if the lod was explicitly zero. This may change on Turing with implicit compute
|
||||
// derivatives
|
||||
const Id lod{ctx.Const(0.0f)};
|
||||
const ImageOperands operands(ctx, false, true, info.has_lod_clamp != 0, lod, offset);
|
||||
color = Emit(&EmitContext::OpImageSparseSampleExplicitLod,
|
||||
&EmitContext::OpImageSampleExplicitLod, ctx, inst, result_type,
|
||||
Texture(ctx, info, index), coords, operands.Mask(), operands.Span());
|
||||
}
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
|
||||
Id EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id lod, const IR::Value& offset) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const bool is_integer{IsTextureInteger(ctx, info)};
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
const ImageOperands operands(ctx, false, true, false, lod, offset);
|
||||
|
||||
Id result = Emit(&EmitContext::OpImageSparseSampleExplicitLod,
|
||||
&EmitContext::OpImageSampleExplicitLod, ctx, inst, result_type,
|
||||
Texture(ctx, info, index), coords, operands.Mask(), operands.Span());
|
||||
#ifdef __ANDROID__
|
||||
if (!is_integer && Settings::values.fix_bloom_effects.GetValue()) {
|
||||
result = ctx.OpVectorTimesScalar(ctx.F32[4], result, ctx.Const(0.98f));
|
||||
}
|
||||
#endif
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], result) : result;
|
||||
}
|
||||
|
||||
Id EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
|
||||
Id coords, Id dref, Id bias_lc, const IR::Value& offset) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
if (ctx.stage == Stage::Fragment) {
|
||||
const ImageOperands operands(ctx, info.has_bias != 0, false, info.has_lod_clamp != 0,
|
||||
bias_lc, offset);
|
||||
return Emit(&EmitContext::OpImageSparseSampleDrefImplicitLod,
|
||||
&EmitContext::OpImageSampleDrefImplicitLod, ctx, inst, ctx.F32[1],
|
||||
Texture(ctx, info, index), coords, dref, operands.MaskOptional(),
|
||||
operands.Span());
|
||||
} else {
|
||||
// Implicit lods in compute behave on hardware as if sampling from LOD 0.
|
||||
// This check is to ensure all drivers behave this way.
|
||||
const Id lod{ctx.Const(0.0f)};
|
||||
const ImageOperands operands(ctx, false, true, false, lod, offset);
|
||||
return Emit(&EmitContext::OpImageSparseSampleDrefExplicitLod,
|
||||
&EmitContext::OpImageSampleDrefExplicitLod, ctx, inst, ctx.F32[1],
|
||||
Texture(ctx, info, index), coords, dref, operands.Mask(), operands.Span());
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
|
||||
Id coords, Id dref, Id lod, const IR::Value& offset) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const ImageOperands operands(ctx, false, true, false, lod, offset);
|
||||
return Emit(&EmitContext::OpImageSparseSampleDrefExplicitLod,
|
||||
&EmitContext::OpImageSampleDrefExplicitLod, ctx, inst, ctx.F32[1],
|
||||
Texture(ctx, info, index), coords, dref, operands.Mask(), operands.Span());
|
||||
}
|
||||
|
||||
Id EmitImageGather(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
const IR::Value& offset, const IR::Value& offset2) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const bool is_integer{IsTextureInteger(ctx, info)};
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
if (ctx.profile.need_gather_subpixel_offset) {
|
||||
coords = ImageGatherSubpixelOffset(ctx, info, TextureImage(ctx, info, index), coords);
|
||||
}
|
||||
if (IsDynamicPtp(inst, offset, offset2)) {
|
||||
const Id texture{Texture(ctx, info, index)};
|
||||
const Id component{ctx.Const(info.gather_component)};
|
||||
const Id color{GatherDynamicPtp(ctx, is_integer, result_type, offset, offset2, [&](Id tap_offset) {
|
||||
return Decorate(ctx, inst,
|
||||
ctx.OpImageGather(result_type, texture, coords, component,
|
||||
spv::ImageOperandsMask::Offset,
|
||||
std::span<const Id>{&tap_offset, 1}));
|
||||
})};
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
const ImageOperands operands(ctx, offset, offset2);
|
||||
const Id color{Emit(&EmitContext::OpImageSparseGather, &EmitContext::OpImageGather, ctx, inst,
|
||||
result_type, Texture(ctx, info, index), coords,
|
||||
ctx.Const(info.gather_component), operands.MaskOptional(),
|
||||
operands.Span())};
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
|
||||
Id EmitImageGatherDref(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
const IR::Value& offset, const IR::Value& offset2, Id dref) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const bool is_integer{IsTextureInteger(ctx, info)};
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
if (ctx.profile.need_gather_subpixel_offset) {
|
||||
coords = ImageGatherSubpixelOffset(ctx, info, TextureImage(ctx, info, index), coords);
|
||||
}
|
||||
if (IsDynamicPtp(inst, offset, offset2)) {
|
||||
const Id texture{Texture(ctx, info, index)};
|
||||
const Id color{GatherDynamicPtp(ctx, is_integer, result_type, offset, offset2, [&](Id tap_offset) {
|
||||
return Decorate(ctx, inst,
|
||||
ctx.OpImageDrefGather(result_type, texture, coords, dref,
|
||||
spv::ImageOperandsMask::Offset,
|
||||
std::span<const Id>{&tap_offset, 1}));
|
||||
})};
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
const ImageOperands operands(ctx, offset, offset2);
|
||||
const Id color{Emit(&EmitContext::OpImageSparseDrefGather, &EmitContext::OpImageDrefGather,
|
||||
ctx, inst, result_type, Texture(ctx, info, index), coords, dref,
|
||||
operands.MaskOptional(), operands.Span())};
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
|
||||
Id EmitImageFetch(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id offset,
|
||||
Id lod, Id ms) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const bool is_integer{IsTextureInteger(ctx, info)};
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
AddOffsetToCoordinates(ctx, info, coords, offset);
|
||||
if (info.type == TextureType::Buffer) {
|
||||
lod = Id{};
|
||||
}
|
||||
if (Sirit::ValidId(ms)) {
|
||||
// This image is multisampled, lod must be implicit
|
||||
lod = Id{};
|
||||
}
|
||||
const ImageOperands operands(lod, ms);
|
||||
const Id color{Emit(&EmitContext::OpImageSparseFetch, &EmitContext::OpImageFetch, ctx, inst,
|
||||
result_type, TextureImage(ctx, info, index), coords,
|
||||
operands.MaskOptional(), operands.Span())};
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
|
||||
Id EmitImageQueryDimensions(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id lod,
|
||||
const IR::Value& skip_mips_val) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const Id image{TextureImage(ctx, info, index)};
|
||||
const Id zero{ctx.u32_zero_value};
|
||||
const bool skip_mips{skip_mips_val.U1()};
|
||||
const auto mips{[&] { return skip_mips ? zero : ctx.OpImageQueryLevels(ctx.U32[1], image); }};
|
||||
const bool is_msaa{IsTextureMsaa(ctx, info)};
|
||||
const bool uses_lod{!is_msaa && info.type != TextureType::Buffer};
|
||||
const auto query{[&](Id type) {
|
||||
return uses_lod ? ctx.OpImageQuerySizeLod(type, image, lod)
|
||||
: ctx.OpImageQuerySize(type, image);
|
||||
}};
|
||||
switch (info.type) {
|
||||
case TextureType::Color1D:
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[1]), zero, zero, mips());
|
||||
case TextureType::ColorArray1D:
|
||||
case TextureType::Color2D:
|
||||
case TextureType::ColorCube:
|
||||
case TextureType::Color2DRect:
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[2]), zero, mips());
|
||||
case TextureType::ColorArray2D:
|
||||
case TextureType::Color3D:
|
||||
case TextureType::ColorArrayCube:
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[3]), mips());
|
||||
case TextureType::Buffer:
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], query(ctx.U32[1]), zero, zero, mips());
|
||||
}
|
||||
throw LogicError("Unspecified image type {}", info.type.Value());
|
||||
}
|
||||
|
||||
Id EmitImageQueryLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const Id zero{ctx.f32_zero_value};
|
||||
const Id sampler{Texture(ctx, info, index)};
|
||||
return ctx.OpCompositeConstruct(ctx.F32[4], ctx.OpImageQueryLod(ctx.F32[2], sampler, coords),
|
||||
zero, zero);
|
||||
}
|
||||
|
||||
Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id derivatives, const IR::Value& offset, Id lod_clamp) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const bool is_integer{IsTextureInteger(ctx, info)};
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
const auto operands = info.num_derivatives == 3
|
||||
? ImageOperands(ctx, info.has_lod_clamp != 0, derivatives,
|
||||
ctx.Def(offset), {}, lod_clamp)
|
||||
: ImageOperands(ctx, info.has_lod_clamp != 0, derivatives,
|
||||
info.num_derivatives, offset, lod_clamp);
|
||||
const Id color{Emit(&EmitContext::OpImageSparseSampleExplicitLod,
|
||||
&EmitContext::OpImageSampleExplicitLod, ctx, inst, result_type,
|
||||
Texture(ctx, info, index), coords, operands.Mask(), operands.Span())};
|
||||
return is_integer ? ctx.OpBitcast(ctx.F32[4], color) : color;
|
||||
}
|
||||
|
||||
Id EmitImageRead(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
if (info.image_format == ImageFormat::Typeless && !ctx.profile.support_typeless_image_loads) {
|
||||
LOG_WARNING(Shader_SPIRV, "Typeless image read not supported by host");
|
||||
return ctx.ConstantNull(ctx.U32[4]);
|
||||
}
|
||||
const auto [image, is_integer] = Image(ctx, index, info);
|
||||
const Id result_type{is_integer ? ctx.U32[4] : ctx.F32[4]};
|
||||
Id color{Emit(&EmitContext::OpImageSparseRead, &EmitContext::OpImageRead, ctx, inst,
|
||||
result_type, image, coords, std::nullopt, std::span<const Id>{})};
|
||||
if (!is_integer) {
|
||||
color = ctx.OpBitcast(ctx.U32[4], color);
|
||||
}
|
||||
return color;
|
||||
}
|
||||
|
||||
void EmitImageWrite(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id color) {
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const auto [image, is_integer] = Image(ctx, index, info);
|
||||
if (!is_integer) {
|
||||
color = ctx.OpBitcast(ctx.F32[4], color);
|
||||
}
|
||||
ctx.OpImageWrite(image, coords, color);
|
||||
}
|
||||
|
||||
Id EmitIsTextureScaled(EmitContext& ctx, const IR::Value& index) {
|
||||
if (ctx.profile.unified_descriptor_binding) {
|
||||
const Id member_index{ctx.Const(ctx.rescaling_textures_member_index)};
|
||||
return IsScaled(ctx, index, member_index, ctx.texture_rescaling_index);
|
||||
} else {
|
||||
const Id composite{ctx.OpLoad(ctx.F32[4], ctx.rescaling_uniform_constant)};
|
||||
const Id mask_f32{ctx.OpCompositeExtract(ctx.F32[1], composite, 0u)};
|
||||
const Id mask{ctx.OpBitcast(ctx.U32[1], mask_f32)};
|
||||
return BitTest(ctx, mask, ctx.Def(index));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitIsImageScaled(EmitContext& ctx, const IR::Value& index) {
|
||||
if (ctx.profile.unified_descriptor_binding) {
|
||||
const Id member_index{ctx.Const(ctx.rescaling_images_member_index)};
|
||||
return IsScaled(ctx, index, member_index, ctx.image_rescaling_index);
|
||||
} else {
|
||||
const Id composite{ctx.OpLoad(ctx.F32[4], ctx.rescaling_uniform_constant)};
|
||||
const Id mask_f32{ctx.OpCompositeExtract(ctx.F32[1], composite, 1u)};
|
||||
const Id mask{ctx.OpBitcast(ctx.U32[1], mask_f32)};
|
||||
return BitTest(ctx, mask, ctx.Def(index));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,183 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
#include "shader_recompiler/frontend/ir/modifiers.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id Image(EmitContext& ctx, IR::TextureInstInfo info) {
|
||||
if (info.type == TextureType::Buffer) {
|
||||
const ImageBufferDefinition def{ctx.image_buffers.at(info.descriptor_index)};
|
||||
return def.id;
|
||||
} else {
|
||||
const ImageDefinition def{ctx.images.at(info.descriptor_index)};
|
||||
return def.id;
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<Id, Id> AtomicArgs(EmitContext& ctx) {
|
||||
const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Device))};
|
||||
const Id semantics{ctx.u32_zero_value};
|
||||
return {scope, semantics};
|
||||
}
|
||||
|
||||
Id ImageAtomicU32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id value,
|
||||
Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
|
||||
if (!index.IsImmediate() || index.U32() != 0) {
|
||||
// TODO: handle layers
|
||||
throw NotImplementedException("Image indexing");
|
||||
}
|
||||
const auto info{inst->Flags<IR::TextureInstInfo>()};
|
||||
const Id image{Image(ctx, info)};
|
||||
const Id pointer{ctx.OpImageTexelPointer(ctx.image_u32, image, coords, ctx.Const(0U))};
|
||||
const auto [scope, semantics]{AtomicArgs(ctx)};
|
||||
return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicIAdd);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicSMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicSMin);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicUMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicUMin);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicSMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicSMax);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicUMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicUMax);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicInc32(EmitContext&, IR::Inst*, const IR::Value&, Id, Id) {
|
||||
// TODO: This is not yet implemented
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitImageAtomicDec32(EmitContext&, IR::Inst*, const IR::Value&, Id, Id) {
|
||||
// TODO: This is not yet implemented
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitImageAtomicAnd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicAnd);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicOr32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicOr);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicXor32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicXor);
|
||||
}
|
||||
|
||||
Id EmitImageAtomicExchange32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value) {
|
||||
return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicExchange);
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicIAdd32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicSMin32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicUMin32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicSMax32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicUMax32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicInc32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicDec32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicAnd32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicOr32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicXor32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBindlessImageAtomicExchange32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicIAdd32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicSMin32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicUMin32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicSMax32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicUMax32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicInc32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicDec32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicAnd32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicOr32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicXor32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitBoundImageAtomicExchange32(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,647 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <sirit/sirit.h>
|
||||
|
||||
#include "common/common_types.h"
|
||||
|
||||
namespace Shader::IR {
|
||||
enum class Attribute : u64;
|
||||
enum class Patch : u64;
|
||||
class Inst;
|
||||
class Value;
|
||||
} // namespace Shader::IR
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
using Sirit::Id;
|
||||
|
||||
class EmitContext;
|
||||
|
||||
// Microinstruction emitters
|
||||
Id EmitPhi(EmitContext& ctx, IR::Inst* inst);
|
||||
void EmitVoid(EmitContext& ctx);
|
||||
Id EmitIdentity(EmitContext& ctx, const IR::Value& value);
|
||||
Id EmitConditionRef(EmitContext& ctx, const IR::Value& value);
|
||||
void EmitReference(EmitContext&);
|
||||
void EmitPhiMove(EmitContext&);
|
||||
void EmitJoin(EmitContext& ctx);
|
||||
void EmitDemoteToHelperInvocation(EmitContext& ctx);
|
||||
void EmitBarrier(EmitContext& ctx);
|
||||
void EmitSubgroupBarrier(EmitContext& ctx);
|
||||
void EmitWorkgroupMemoryBarrier(EmitContext& ctx);
|
||||
void EmitDeviceMemoryBarrier(EmitContext& ctx);
|
||||
void EmitPrologue(EmitContext& ctx);
|
||||
void EmitEpilogue(EmitContext& ctx);
|
||||
void EmitEmitVertex(EmitContext& ctx, const IR::Value& stream);
|
||||
void EmitEndPrimitive(EmitContext& ctx, const IR::Value& stream);
|
||||
void EmitGetRegister(EmitContext& ctx);
|
||||
void EmitSetRegister(EmitContext& ctx);
|
||||
void EmitGetPred(EmitContext& ctx);
|
||||
void EmitSetPred(EmitContext& ctx);
|
||||
void EmitSetGotoVariable(EmitContext& ctx);
|
||||
void EmitGetGotoVariable(EmitContext& ctx);
|
||||
void EmitSetIndirectBranchVariable(EmitContext& ctx);
|
||||
void EmitGetIndirectBranchVariable(EmitContext& ctx);
|
||||
Id EmitGetCbufU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetCbufS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetCbufU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetCbufS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetCbufU32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetCbufF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetCbufU32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitGetAttribute(EmitContext& ctx, IR::Attribute attr, Id vertex);
|
||||
Id EmitGetAttributeU32(EmitContext& ctx, IR::Attribute attr, Id vertex);
|
||||
void EmitSetAttribute(EmitContext& ctx, IR::Attribute attr, Id value, Id vertex);
|
||||
Id EmitGetAttributeIndexed(EmitContext& ctx, Id offset, Id vertex);
|
||||
void EmitSetAttributeIndexed(EmitContext& ctx, Id offset, Id value, Id vertex);
|
||||
Id EmitGetPatch(EmitContext& ctx, IR::Patch patch);
|
||||
void EmitSetPatch(EmitContext& ctx, IR::Patch patch, Id value);
|
||||
Id EmitIsbeRead(EmitContext& ctx, Id address);
|
||||
void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value);
|
||||
void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value);
|
||||
void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value);
|
||||
void EmitMeshPrologue(EmitContext& ctx);
|
||||
void EmitMeshEpilogue(EmitContext& ctx);
|
||||
// (hookshot) a GPU without mesh shaders: the generated vertex stage that reads the mesh epilogue's records
|
||||
void EmitMeshFallbackVertexBody(EmitContext& ctx);
|
||||
void EmitSetFragColor(EmitContext& ctx, u32 index, u32 component, Id value);
|
||||
void EmitSetSampleMask(EmitContext& ctx, Id value);
|
||||
void EmitSetFragDepth(EmitContext& ctx, Id value);
|
||||
void EmitGetZFlag(EmitContext& ctx);
|
||||
void EmitGetSFlag(EmitContext& ctx);
|
||||
void EmitGetCFlag(EmitContext& ctx);
|
||||
void EmitGetOFlag(EmitContext& ctx);
|
||||
void EmitSetZFlag(EmitContext& ctx);
|
||||
void EmitSetSFlag(EmitContext& ctx);
|
||||
void EmitSetCFlag(EmitContext& ctx);
|
||||
void EmitSetOFlag(EmitContext& ctx);
|
||||
Id EmitWorkgroupId(EmitContext& ctx);
|
||||
Id EmitLocalInvocationId(EmitContext& ctx);
|
||||
Id EmitInvocationId(EmitContext& ctx);
|
||||
Id EmitInvocationInfo(EmitContext& ctx);
|
||||
Id EmitSampleId(EmitContext& ctx);
|
||||
Id EmitIsHelperInvocation(EmitContext& ctx);
|
||||
Id EmitSR_WScaleFactorXY(EmitContext& ctx);
|
||||
Id EmitSR_WScaleFactorZ(EmitContext& ctx);
|
||||
Id EmitYDirection(EmitContext& ctx);
|
||||
Id EmitResolutionDownFactor(EmitContext& ctx);
|
||||
Id EmitRenderArea(EmitContext& ctx);
|
||||
Id EmitLoadLocal(EmitContext& ctx, Id word_offset);
|
||||
void EmitWriteLocal(EmitContext& ctx, Id word_offset, Id value);
|
||||
Id EmitUndefU1(EmitContext& ctx);
|
||||
Id EmitUndefU8(EmitContext& ctx);
|
||||
Id EmitUndefU16(EmitContext& ctx);
|
||||
Id EmitUndefU32(EmitContext& ctx);
|
||||
Id EmitUndefU64(EmitContext& ctx);
|
||||
Id EmitLoadGlobalU8(EmitContext& ctx, Id address);
|
||||
Id EmitLoadGlobalS8(EmitContext& ctx, Id address);
|
||||
Id EmitLoadGlobalU16(EmitContext& ctx, Id address);
|
||||
Id EmitLoadGlobalS16(EmitContext& ctx, Id address);
|
||||
Id EmitLoadGlobal32(EmitContext& ctx, Id address);
|
||||
Id EmitLoadGlobal64(EmitContext& ctx, Id address);
|
||||
Id EmitLoadGlobal128(EmitContext& ctx, Id address);
|
||||
void EmitWriteGlobalU8(EmitContext& ctx, Id address, Id value);
|
||||
void EmitWriteGlobalS8(EmitContext& ctx, Id address, Id value);
|
||||
void EmitWriteGlobalU16(EmitContext& ctx, Id address, Id value);
|
||||
void EmitWriteGlobalS16(EmitContext& ctx, Id address, Id value);
|
||||
void EmitWriteGlobal32(EmitContext& ctx, Id address, Id value);
|
||||
void EmitWriteGlobal64(EmitContext& ctx, Id address, Id value);
|
||||
void EmitWriteGlobal128(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitLoadStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitLoadStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitLoadStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitLoadStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitLoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitLoadStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
Id EmitLoadStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset);
|
||||
void EmitWriteStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
void EmitWriteStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
void EmitWriteStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
void EmitWriteStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
void EmitWriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
void EmitWriteStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
void EmitWriteStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitLoadSharedU8(EmitContext& ctx, Id offset);
|
||||
Id EmitLoadSharedS8(EmitContext& ctx, Id offset);
|
||||
Id EmitLoadSharedU16(EmitContext& ctx, Id offset);
|
||||
Id EmitLoadSharedS16(EmitContext& ctx, Id offset);
|
||||
Id EmitLoadSharedU32(EmitContext& ctx, Id offset);
|
||||
Id EmitLoadSharedU64(EmitContext& ctx, Id offset);
|
||||
Id EmitLoadSharedU128(EmitContext& ctx, Id offset);
|
||||
void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value);
|
||||
void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value);
|
||||
void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value);
|
||||
void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value);
|
||||
void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value);
|
||||
Id EmitCompositeConstructU32x2(EmitContext& ctx, Id e1, Id e2);
|
||||
Id EmitCompositeConstructU32x3(EmitContext& ctx, Id e1, Id e2, Id e3);
|
||||
Id EmitCompositeConstructU32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4);
|
||||
Id EmitCompositeExtractU32x2(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeExtractU32x3(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeExtractU32x4(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeInsertU32x2(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertU32x3(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertU32x4(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeConstructF16x2(EmitContext& ctx, Id e1, Id e2);
|
||||
Id EmitCompositeConstructF16x3(EmitContext& ctx, Id e1, Id e2, Id e3);
|
||||
Id EmitCompositeConstructF16x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4);
|
||||
Id EmitCompositeExtractF16x2(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeExtractF16x3(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeExtractF16x4(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeInsertF16x2(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertF16x3(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertF16x4(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeConstructF32x2(EmitContext& ctx, Id e1, Id e2);
|
||||
Id EmitCompositeConstructF32x3(EmitContext& ctx, Id e1, Id e2, Id e3);
|
||||
Id EmitCompositeConstructF32x4(EmitContext& ctx, Id e1, Id e2, Id e3, Id e4);
|
||||
Id EmitCompositeExtractF32x2(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeExtractF32x3(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeExtractF32x4(EmitContext& ctx, Id composite, u32 index);
|
||||
Id EmitCompositeInsertF32x2(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertF32x3(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertF32x4(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
void EmitCompositeConstructF64x2(EmitContext& ctx);
|
||||
void EmitCompositeConstructF64x3(EmitContext& ctx);
|
||||
void EmitCompositeConstructF64x4(EmitContext& ctx);
|
||||
void EmitCompositeExtractF64x2(EmitContext& ctx);
|
||||
void EmitCompositeExtractF64x3(EmitContext& ctx);
|
||||
void EmitCompositeExtractF64x4(EmitContext& ctx);
|
||||
Id EmitCompositeInsertF64x2(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertF64x3(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitCompositeInsertF64x4(EmitContext& ctx, Id composite, Id object, u32 index);
|
||||
Id EmitSelectU1(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectU8(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectU16(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectU32(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectU64(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectF16(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectF32(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
Id EmitSelectF64(EmitContext& ctx, Id cond, Id true_value, Id false_value);
|
||||
void EmitBitCastU16F16(EmitContext& ctx);
|
||||
Id EmitBitCastU32F32(EmitContext& ctx, Id value);
|
||||
void EmitBitCastU64F64(EmitContext& ctx);
|
||||
void EmitBitCastF16U16(EmitContext&);
|
||||
Id EmitBitCastF32U32(EmitContext& ctx, Id value);
|
||||
void EmitBitCastF64U64(EmitContext& ctx);
|
||||
Id EmitPackUint2x32(EmitContext& ctx, Id value);
|
||||
Id EmitUnpackUint2x32(EmitContext& ctx, Id value);
|
||||
Id EmitPackFloat2x16(EmitContext& ctx, Id value);
|
||||
Id EmitUnpackFloat2x16(EmitContext& ctx, Id value);
|
||||
Id EmitPackHalf2x16(EmitContext& ctx, Id value);
|
||||
Id EmitUnpackHalf2x16(EmitContext& ctx, Id value);
|
||||
Id EmitPackDouble2x32(EmitContext& ctx, Id value);
|
||||
Id EmitUnpackDouble2x32(EmitContext& ctx, Id value);
|
||||
void EmitGetZeroFromOp(EmitContext& ctx);
|
||||
void EmitGetSignFromOp(EmitContext& ctx);
|
||||
void EmitGetCarryFromOp(EmitContext& ctx);
|
||||
void EmitGetOverflowFromOp(EmitContext& ctx);
|
||||
void EmitGetSparseFromOp(EmitContext& ctx);
|
||||
void EmitGetInBoundsFromOp(EmitContext& ctx);
|
||||
Id EmitFPAbs16(EmitContext& ctx, Id value);
|
||||
Id EmitFPAbs32(EmitContext& ctx, Id value);
|
||||
Id EmitFPAbs64(EmitContext& ctx, Id value);
|
||||
Id EmitFPAdd16(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitFPAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitFPAdd64(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitFPFma16(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c);
|
||||
Id EmitFPFma32(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c);
|
||||
Id EmitFPFma64(EmitContext& ctx, IR::Inst* inst, Id a, Id b, Id c);
|
||||
Id EmitFPMax32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitFPMax64(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitFPMin32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitFPMin64(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitFPMul16(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitFPMul32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitFPMul64(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitFPNeg16(EmitContext& ctx, Id value);
|
||||
Id EmitFPNeg32(EmitContext& ctx, Id value);
|
||||
Id EmitFPNeg64(EmitContext& ctx, Id value);
|
||||
Id EmitFPSin(EmitContext& ctx, Id value);
|
||||
Id EmitFPCos(EmitContext& ctx, Id value);
|
||||
Id EmitFPExp2(EmitContext& ctx, Id value);
|
||||
Id EmitFPLog2(EmitContext& ctx, Id value);
|
||||
Id EmitFPRecip32(EmitContext& ctx, Id value);
|
||||
Id EmitFPRecip64(EmitContext& ctx, Id value);
|
||||
Id EmitFPRecipSqrt32(EmitContext& ctx, Id value);
|
||||
Id EmitFPRecipSqrt64(EmitContext& ctx, Id value);
|
||||
Id EmitFPSqrt(EmitContext& ctx, Id value);
|
||||
Id EmitFPSaturate16(EmitContext& ctx, Id value);
|
||||
Id EmitFPSaturate32(EmitContext& ctx, Id value);
|
||||
Id EmitFPSaturate64(EmitContext& ctx, Id value);
|
||||
Id EmitFPClamp16(EmitContext& ctx, Id value, Id min_value, Id max_value);
|
||||
Id EmitFPClamp32(EmitContext& ctx, Id value, Id min_value, Id max_value);
|
||||
Id EmitFPClamp64(EmitContext& ctx, Id value, Id min_value, Id max_value);
|
||||
Id EmitFPRoundEven16(EmitContext& ctx, Id value);
|
||||
Id EmitFPRoundEven32(EmitContext& ctx, Id value);
|
||||
Id EmitFPRoundEven64(EmitContext& ctx, Id value);
|
||||
Id EmitFPFloor16(EmitContext& ctx, Id value);
|
||||
Id EmitFPFloor32(EmitContext& ctx, Id value);
|
||||
Id EmitFPFloor64(EmitContext& ctx, Id value);
|
||||
Id EmitFPCeil16(EmitContext& ctx, Id value);
|
||||
Id EmitFPCeil32(EmitContext& ctx, Id value);
|
||||
Id EmitFPCeil64(EmitContext& ctx, Id value);
|
||||
Id EmitFPTrunc16(EmitContext& ctx, Id value);
|
||||
Id EmitFPTrunc32(EmitContext& ctx, Id value);
|
||||
Id EmitFPTrunc64(EmitContext& ctx, Id value);
|
||||
Id EmitFPOrdEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdNotEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdNotEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdNotEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordNotEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordNotEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordNotEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdLessThan16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdLessThan32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdLessThan64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordLessThan16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordLessThan32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordLessThan64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdGreaterThan16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdGreaterThan32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdGreaterThan64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordGreaterThan16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordGreaterThan32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordGreaterThan64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordLessThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordLessThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordLessThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPOrdGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordGreaterThanEqual16(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordGreaterThanEqual32(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPUnordGreaterThanEqual64(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitFPIsNan16(EmitContext& ctx, Id value);
|
||||
Id EmitFPIsNan32(EmitContext& ctx, Id value);
|
||||
Id EmitFPIsNan64(EmitContext& ctx, Id value);
|
||||
Id EmitIAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitIAdd64(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitISub32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitISub64(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitIMul32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitSDiv32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitUDiv32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitINeg32(EmitContext& ctx, Id value);
|
||||
Id EmitINeg64(EmitContext& ctx, Id value);
|
||||
Id EmitIAbs32(EmitContext& ctx, Id value);
|
||||
Id EmitIAbs64(EmitContext& ctx, Id value);
|
||||
Id EmitShiftLeftLogical32(EmitContext& ctx, Id base, Id shift);
|
||||
Id EmitShiftLeftLogical64(EmitContext& ctx, Id base, Id shift);
|
||||
Id EmitShiftRightLogical32(EmitContext& ctx, Id base, Id shift);
|
||||
Id EmitShiftRightLogical64(EmitContext& ctx, Id base, Id shift);
|
||||
Id EmitShiftRightArithmetic32(EmitContext& ctx, Id base, Id shift);
|
||||
Id EmitShiftRightArithmetic64(EmitContext& ctx, Id base, Id shift);
|
||||
Id EmitBitwiseAnd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitBitwiseOr32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitBitwiseXor32(EmitContext& ctx, IR::Inst* inst, Id a, Id b);
|
||||
Id EmitBitFieldInsert(EmitContext& ctx, Id base, Id insert, Id offset, Id count);
|
||||
Id EmitBitFieldSExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count);
|
||||
Id EmitBitFieldUExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count);
|
||||
Id EmitBitReverse32(EmitContext& ctx, Id value);
|
||||
Id EmitBitCount32(EmitContext& ctx, Id value);
|
||||
Id EmitBitwiseNot32(EmitContext& ctx, Id value);
|
||||
Id EmitFindSMsb32(EmitContext& ctx, Id value);
|
||||
Id EmitFindUMsb32(EmitContext& ctx, Id value);
|
||||
Id EmitSMin32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitUMin32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitSMax32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitUMax32(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitSClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max);
|
||||
Id EmitUClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max);
|
||||
Id EmitSLessThan(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitULessThan(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitIEqual(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitSLessThanEqual(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitULessThanEqual(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitSGreaterThan(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitUGreaterThan(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitINotEqual(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitSGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitUGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs);
|
||||
Id EmitSharedAtomicIAdd32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicSMin32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicUMin32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicSMax32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicUMax32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicInc32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicDec32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicAnd32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicOr32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicXor32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicExchange32(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicExchange64(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitSharedAtomicExchange32x2(EmitContext& ctx, Id pointer_offset, Id value);
|
||||
Id EmitStorageAtomicIAdd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicSMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicUMin32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicSMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicUMax32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicInc32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicDec32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicAnd32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicOr32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicXor32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicCompareExchange32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id comparator, Id value);
|
||||
Id EmitStorageAtomicIAdd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicSMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicUMin64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicSMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicUMax64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicAnd64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicOr64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicXor64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicExchange64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicIAdd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicSMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicUMin32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicSMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicUMax32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicAnd32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicOr32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicXor32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicExchange32x2(EmitContext& ctx, const IR::Value& binding,
|
||||
const IR::Value& offset, Id value);
|
||||
Id EmitStorageAtomicAddF32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicAddF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicAddF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicMinF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicMinF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value);
|
||||
Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicInc32(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicDec32(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicCompareExchange32(EmitContext& ctx, Id address, Id comparator, Id value);
|
||||
Id EmitGlobalAtomicIAdd64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicSMin64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicUMin64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicSMax64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicUMax64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicInc64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicDec64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicAnd64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicOr64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicXor64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicExchange64(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicIAdd32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicSMin32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicUMin32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicSMax32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicUMax32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicInc32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicDec32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicAnd32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicOr32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicXor32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicExchange32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value);
|
||||
Id EmitGlobalAtomicAddF16x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicAddF32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicMinF16x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicMinF32x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicMaxF16x2(EmitContext& ctx);
|
||||
Id EmitGlobalAtomicMaxF32x2(EmitContext& ctx);
|
||||
Id EmitLogicalOr(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitLogicalAnd(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitLogicalXor(EmitContext& ctx, Id a, Id b);
|
||||
Id EmitLogicalNot(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS16F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS16F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS16F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS32F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS32F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS32F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS64F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS64F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS64F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU16F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU16F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU16F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU32F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU32F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU32F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU64F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU64F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU64F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU64U32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU32U64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32F16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32F64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64F32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16S8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16S16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16S32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16S64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16U8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16U16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16U32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF16U64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32S8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32S16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32S32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32S64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32U8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32U16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32U32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF32U64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64S8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64S16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64S32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64S64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64U8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64U16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64U32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertF64U64(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU16U32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU32U16(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU8U32(EmitContext& ctx, Id value);
|
||||
Id EmitConvertU32U8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS32S8(EmitContext& ctx, Id value);
|
||||
Id EmitConvertS32S16(EmitContext& ctx, Id value);
|
||||
Id EmitBindlessImageSampleImplicitLod(EmitContext&);
|
||||
Id EmitBindlessImageSampleExplicitLod(EmitContext&);
|
||||
Id EmitBindlessImageSampleDrefImplicitLod(EmitContext&);
|
||||
Id EmitBindlessImageSampleDrefExplicitLod(EmitContext&);
|
||||
Id EmitBindlessImageGather(EmitContext&);
|
||||
Id EmitBindlessImageGatherDref(EmitContext&);
|
||||
Id EmitBindlessImageFetch(EmitContext&);
|
||||
Id EmitBindlessImageQueryDimensions(EmitContext&);
|
||||
Id EmitBindlessImageQueryLod(EmitContext&);
|
||||
Id EmitBindlessImageGradient(EmitContext&);
|
||||
Id EmitBindlessImageRead(EmitContext&);
|
||||
Id EmitBindlessImageWrite(EmitContext&);
|
||||
Id EmitBoundImageSampleImplicitLod(EmitContext&);
|
||||
Id EmitBoundImageSampleExplicitLod(EmitContext&);
|
||||
Id EmitBoundImageSampleDrefImplicitLod(EmitContext&);
|
||||
Id EmitBoundImageSampleDrefExplicitLod(EmitContext&);
|
||||
Id EmitBoundImageGather(EmitContext&);
|
||||
Id EmitBoundImageGatherDref(EmitContext&);
|
||||
Id EmitBoundImageFetch(EmitContext&);
|
||||
Id EmitBoundImageQueryDimensions(EmitContext&);
|
||||
Id EmitBoundImageQueryLod(EmitContext&);
|
||||
Id EmitBoundImageGradient(EmitContext&);
|
||||
Id EmitBoundImageRead(EmitContext&);
|
||||
Id EmitBoundImageWrite(EmitContext&);
|
||||
Id EmitImageSampleImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id bias_lc, const IR::Value& offset);
|
||||
Id EmitImageSampleExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id lod, const IR::Value& offset);
|
||||
Id EmitImageSampleDrefImplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
|
||||
Id coords, Id dref, Id bias_lc, const IR::Value& offset);
|
||||
Id EmitImageSampleDrefExplicitLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index,
|
||||
Id coords, Id dref, Id lod, const IR::Value& offset);
|
||||
Id EmitImageGather(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
const IR::Value& offset, const IR::Value& offset2);
|
||||
Id EmitImageGatherDref(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
const IR::Value& offset, const IR::Value& offset2, Id dref);
|
||||
Id EmitImageFetch(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id offset,
|
||||
Id lod, Id ms);
|
||||
Id EmitImageQueryDimensions(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id lod,
|
||||
const IR::Value& skip_mips);
|
||||
Id EmitImageQueryLod(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
|
||||
Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id derivatives, const IR::Value& offset, Id lod_clamp);
|
||||
Id EmitImageRead(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
|
||||
void EmitImageWrite(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id color);
|
||||
Id EmitIsTextureScaled(EmitContext& ctx, const IR::Value& index);
|
||||
Id EmitIsImageScaled(EmitContext& ctx, const IR::Value& index);
|
||||
Id EmitBindlessImageAtomicIAdd32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicSMin32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicUMin32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicSMax32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicUMax32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicInc32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicDec32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicAnd32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicOr32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicXor32(EmitContext&);
|
||||
Id EmitBindlessImageAtomicExchange32(EmitContext&);
|
||||
Id EmitBoundImageAtomicIAdd32(EmitContext&);
|
||||
Id EmitBoundImageAtomicSMin32(EmitContext&);
|
||||
Id EmitBoundImageAtomicUMin32(EmitContext&);
|
||||
Id EmitBoundImageAtomicSMax32(EmitContext&);
|
||||
Id EmitBoundImageAtomicUMax32(EmitContext&);
|
||||
Id EmitBoundImageAtomicInc32(EmitContext&);
|
||||
Id EmitBoundImageAtomicDec32(EmitContext&);
|
||||
Id EmitBoundImageAtomicAnd32(EmitContext&);
|
||||
Id EmitBoundImageAtomicOr32(EmitContext&);
|
||||
Id EmitBoundImageAtomicXor32(EmitContext&);
|
||||
Id EmitBoundImageAtomicExchange32(EmitContext&);
|
||||
Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicSMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicUMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicSMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicUMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicInc32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicDec32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicAnd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicOr32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicXor32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitImageAtomicExchange32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
|
||||
Id value);
|
||||
Id EmitLaneId(EmitContext& ctx);
|
||||
Id EmitVoteAll(EmitContext& ctx, Id pred);
|
||||
Id EmitVoteAny(EmitContext& ctx, Id pred);
|
||||
Id EmitVoteEqual(EmitContext& ctx, Id pred);
|
||||
Id EmitSubgroupBallot(EmitContext& ctx, Id pred);
|
||||
Id EmitSubgroupEqMask(EmitContext& ctx);
|
||||
Id EmitSubgroupLtMask(EmitContext& ctx);
|
||||
Id EmitSubgroupLeMask(EmitContext& ctx);
|
||||
Id EmitSubgroupGtMask(EmitContext& ctx);
|
||||
Id EmitSubgroupGeMask(EmitContext& ctx);
|
||||
Id EmitShuffleIndex(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask);
|
||||
Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask);
|
||||
Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask);
|
||||
Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask);
|
||||
Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane);
|
||||
Id EmitMatrixMulAddF16(EmitContext& ctx, Id a, Id b, Id c);
|
||||
void DefineSoftwareMatrixFunction(EmitContext& ctx);
|
||||
Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction);
|
||||
Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle);
|
||||
Id EmitDPdxFine(EmitContext& ctx, Id op_a);
|
||||
Id EmitDPdyFine(EmitContext& ctx, Id op_a);
|
||||
Id EmitDPdxCoarse(EmitContext& ctx, Id op_a);
|
||||
Id EmitDPdyCoarse(EmitContext& ctx, Id op_a);
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,284 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
void SetZeroFlag(EmitContext& ctx, IR::Inst* inst, Id result) {
|
||||
IR::Inst* const zero{inst->GetAssociatedPseudoOperation(IR::Opcode::GetZeroFromOp)};
|
||||
if (!zero) {
|
||||
return;
|
||||
}
|
||||
zero->SetDefinition(ctx.OpIEqual(ctx.U1, result, ctx.u32_zero_value));
|
||||
zero->Invalidate();
|
||||
}
|
||||
|
||||
void SetSignFlag(EmitContext& ctx, IR::Inst* inst, Id result) {
|
||||
IR::Inst* const sign{inst->GetAssociatedPseudoOperation(IR::Opcode::GetSignFromOp)};
|
||||
if (!sign) {
|
||||
return;
|
||||
}
|
||||
sign->SetDefinition(ctx.OpSLessThan(ctx.U1, result, ctx.u32_zero_value));
|
||||
sign->Invalidate();
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitIAdd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
Id result{};
|
||||
if (IR::Inst* const carry{inst->GetAssociatedPseudoOperation(IR::Opcode::GetCarryFromOp)}) {
|
||||
const Id carry_type{ctx.TypeStruct(ctx.U32[1], ctx.U32[1])};
|
||||
const Id carry_result{ctx.OpIAddCarry(carry_type, a, b)};
|
||||
result = ctx.OpCompositeExtract(ctx.U32[1], carry_result, 0U);
|
||||
|
||||
const Id carry_value{ctx.OpCompositeExtract(ctx.U32[1], carry_result, 1U)};
|
||||
carry->SetDefinition(ctx.OpINotEqual(ctx.U1, carry_value, ctx.u32_zero_value));
|
||||
carry->Invalidate();
|
||||
} else {
|
||||
result = ctx.OpIAdd(ctx.U32[1], a, b);
|
||||
}
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
if (IR::Inst * overflow{inst->GetAssociatedPseudoOperation(IR::Opcode::GetOverflowFromOp)}) {
|
||||
// https://stackoverflow.com/questions/55468823/how-to-detect-integer-overflow-in-c
|
||||
constexpr u32 s32_max{static_cast<u32>((std::numeric_limits<s32>::max)())};
|
||||
const Id is_positive{ctx.OpSGreaterThanEqual(ctx.U1, a, ctx.u32_zero_value)};
|
||||
const Id sub_a{ctx.OpISub(ctx.U32[1], ctx.Const(s32_max), a)};
|
||||
|
||||
const Id positive_test{ctx.OpSGreaterThan(ctx.U1, b, sub_a)};
|
||||
const Id negative_test{ctx.OpSLessThan(ctx.U1, b, sub_a)};
|
||||
const Id carry_flag{ctx.OpSelect(ctx.U1, is_positive, positive_test, negative_test)};
|
||||
overflow->SetDefinition(carry_flag);
|
||||
overflow->Invalidate();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitIAdd64(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpIAdd(ctx.U64, a, b);
|
||||
}
|
||||
|
||||
Id EmitISub32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpISub(ctx.U32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitISub64(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpISub(ctx.U64, a, b);
|
||||
}
|
||||
|
||||
Id EmitIMul32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpIMul(ctx.U32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitSDiv32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpSDiv(ctx.U32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitUDiv32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpUDiv(ctx.U32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitINeg32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpSNegate(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitINeg64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpSNegate(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitIAbs32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpSAbs(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitIAbs64(EmitContext& ctx, Id value) {
|
||||
return ctx.OpSAbs(ctx.U64, value);
|
||||
}
|
||||
|
||||
Id EmitShiftLeftLogical32(EmitContext& ctx, Id base, Id shift) {
|
||||
return ctx.OpShiftLeftLogical(ctx.U32[1], base, shift);
|
||||
}
|
||||
|
||||
Id EmitShiftLeftLogical64(EmitContext& ctx, Id base, Id shift) {
|
||||
return ctx.OpShiftLeftLogical(ctx.U64, base, shift);
|
||||
}
|
||||
|
||||
Id EmitShiftRightLogical32(EmitContext& ctx, Id base, Id shift) {
|
||||
return ctx.OpShiftRightLogical(ctx.U32[1], base, shift);
|
||||
}
|
||||
|
||||
Id EmitShiftRightLogical64(EmitContext& ctx, Id base, Id shift) {
|
||||
return ctx.OpShiftRightLogical(ctx.U64, base, shift);
|
||||
}
|
||||
|
||||
Id EmitShiftRightArithmetic32(EmitContext& ctx, Id base, Id shift) {
|
||||
return ctx.OpShiftRightArithmetic(ctx.U32[1], base, shift);
|
||||
}
|
||||
|
||||
Id EmitShiftRightArithmetic64(EmitContext& ctx, Id base, Id shift) {
|
||||
return ctx.OpShiftRightArithmetic(ctx.U64, base, shift);
|
||||
}
|
||||
|
||||
Id EmitBitwiseAnd32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
const Id result{ctx.OpBitwiseAnd(ctx.U32[1], a, b)};
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitBitwiseOr32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
const Id result{ctx.OpBitwiseOr(ctx.U32[1], a, b)};
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitBitwiseXor32(EmitContext& ctx, IR::Inst* inst, Id a, Id b) {
|
||||
const Id result{ctx.OpBitwiseXor(ctx.U32[1], a, b)};
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitBitFieldInsert(EmitContext& ctx, Id base, Id insert, Id offset, Id count) {
|
||||
return ctx.OpBitFieldInsert(ctx.U32[1], base, insert, offset, count);
|
||||
}
|
||||
|
||||
Id EmitBitFieldSExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count) {
|
||||
const Id result{ctx.OpBitFieldSExtract(ctx.U32[1], base, offset, count)};
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitBitFieldUExtract(EmitContext& ctx, IR::Inst* inst, Id base, Id offset, Id count) {
|
||||
const Id result{ctx.OpBitFieldUExtract(ctx.U32[1], base, offset, count)};
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitBitReverse32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitReverse(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitBitCount32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpBitCount(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitBitwiseNot32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpNot(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFindSMsb32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFindSMsb(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitFindUMsb32(EmitContext& ctx, Id value) {
|
||||
return ctx.OpFindUMsb(ctx.U32[1], value);
|
||||
}
|
||||
|
||||
Id EmitSMin32(EmitContext& ctx, Id a, Id b) {
|
||||
const bool is_broken{ctx.profile.has_broken_signed_operations};
|
||||
if (is_broken) {
|
||||
a = ctx.OpBitcast(ctx.S32[1], a);
|
||||
b = ctx.OpBitcast(ctx.S32[1], b);
|
||||
}
|
||||
const Id result{ctx.OpSMin(ctx.U32[1], a, b)};
|
||||
return is_broken ? ctx.OpBitcast(ctx.U32[1], result) : result;
|
||||
}
|
||||
|
||||
Id EmitUMin32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpUMin(ctx.U32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitSMax32(EmitContext& ctx, Id a, Id b) {
|
||||
const bool is_broken{ctx.profile.has_broken_signed_operations};
|
||||
if (is_broken) {
|
||||
a = ctx.OpBitcast(ctx.S32[1], a);
|
||||
b = ctx.OpBitcast(ctx.S32[1], b);
|
||||
}
|
||||
const Id result{ctx.OpSMax(ctx.U32[1], a, b)};
|
||||
return is_broken ? ctx.OpBitcast(ctx.U32[1], result) : result;
|
||||
}
|
||||
|
||||
Id EmitUMax32(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpUMax(ctx.U32[1], a, b);
|
||||
}
|
||||
|
||||
Id EmitSClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max) {
|
||||
Id result{};
|
||||
if (ctx.profile.has_broken_signed_operations || ctx.profile.has_broken_spirv_clamp) {
|
||||
value = ctx.OpBitcast(ctx.S32[1], value);
|
||||
min = ctx.OpBitcast(ctx.S32[1], min);
|
||||
max = ctx.OpBitcast(ctx.S32[1], max);
|
||||
if (ctx.profile.has_broken_spirv_clamp) {
|
||||
result = ctx.OpSMax(ctx.S32[1], ctx.OpSMin(ctx.S32[1], value, max), min);
|
||||
} else {
|
||||
result = ctx.OpSClamp(ctx.S32[1], value, min, max);
|
||||
}
|
||||
result = ctx.OpBitcast(ctx.U32[1], result);
|
||||
} else {
|
||||
result = ctx.OpSClamp(ctx.U32[1], value, min, max);
|
||||
}
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitUClamp32(EmitContext& ctx, IR::Inst* inst, Id value, Id min, Id max) {
|
||||
Id result{};
|
||||
if (ctx.profile.has_broken_spirv_clamp) {
|
||||
result = ctx.OpUMax(ctx.U32[1], ctx.OpUMin(ctx.U32[1], value, max), min);
|
||||
} else {
|
||||
result = ctx.OpUClamp(ctx.U32[1], value, min, max);
|
||||
}
|
||||
SetZeroFlag(ctx, inst, result);
|
||||
SetSignFlag(ctx, inst, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id EmitSLessThan(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpSLessThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitULessThan(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpULessThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitIEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpIEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitSLessThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpSLessThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitULessThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpULessThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitSGreaterThan(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpSGreaterThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitUGreaterThan(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpUGreaterThan(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitINotEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpINotEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitSGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpSGreaterThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
Id EmitUGreaterThanEqual(EmitContext& ctx, Id lhs, Id rhs) {
|
||||
return ctx.OpUGreaterThanEqual(ctx.U1, lhs, rhs);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,25 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
Id EmitLogicalOr(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpLogicalOr(ctx.U1, a, b);
|
||||
}
|
||||
|
||||
Id EmitLogicalAnd(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpLogicalAnd(ctx.U1, a, b);
|
||||
}
|
||||
|
||||
Id EmitLogicalXor(EmitContext& ctx, Id a, Id b) {
|
||||
return ctx.OpLogicalNotEqual(ctx.U1, a, b);
|
||||
}
|
||||
|
||||
Id EmitLogicalNot(EmitContext& ctx, Id value) {
|
||||
return ctx.OpLogicalNot(ctx.U1, value);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,343 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <bit>
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id StorageIndex(EmitContext& ctx, const IR::Value& offset, size_t element_size,
|
||||
u32 index_offset = 0) {
|
||||
if (offset.IsImmediate()) {
|
||||
const u32 imm_offset{static_cast<u32>(offset.U32() / element_size) + index_offset};
|
||||
return ctx.Const(imm_offset);
|
||||
}
|
||||
const u32 shift{static_cast<u32>(std::countr_zero(element_size))};
|
||||
Id index{ctx.Def(offset)};
|
||||
if (shift != 0) {
|
||||
const Id shift_id{ctx.Const(shift)};
|
||||
index = ctx.OpShiftRightLogical(ctx.U32[1], index, shift_id);
|
||||
}
|
||||
if (index_offset != 0) {
|
||||
index = ctx.OpIAdd(ctx.U32[1], index, ctx.Const(index_offset));
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
Id StoragePointer(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
const StorageTypeDefinition& type_def, size_t element_size,
|
||||
Id StorageDefinitions::*member_ptr, u32 index_offset = 0) {
|
||||
if (!binding.IsImmediate()) {
|
||||
throw NotImplementedException("Dynamic storage buffer indexing");
|
||||
}
|
||||
const Id ssbo{ctx.ssbos[binding.U32()].*member_ptr};
|
||||
const Id index{StorageIndex(ctx, offset, element_size, index_offset)};
|
||||
return ctx.OpAccessChain(type_def.element, ssbo, ctx.u32_zero_value, index);
|
||||
}
|
||||
|
||||
Id LoadStorage(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id result_type,
|
||||
const StorageTypeDefinition& type_def, size_t element_size,
|
||||
Id StorageDefinitions::*member_ptr, u32 index_offset = 0) {
|
||||
const Id pointer{
|
||||
StoragePointer(ctx, binding, offset, type_def, element_size, member_ptr, index_offset)};
|
||||
return ctx.OpLoad(result_type, pointer);
|
||||
}
|
||||
|
||||
Id LoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
u32 index_offset = 0) {
|
||||
return LoadStorage(ctx, binding, offset, ctx.U32[1], ctx.storage_types.U32, sizeof(u32),
|
||||
&StorageDefinitions::U32, index_offset);
|
||||
}
|
||||
|
||||
void WriteStorage(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
|
||||
const StorageTypeDefinition& type_def, size_t element_size,
|
||||
Id StorageDefinitions::*member_ptr, u32 index_offset = 0) {
|
||||
const Id pointer{
|
||||
StoragePointer(ctx, binding, offset, type_def, element_size, member_ptr, index_offset)};
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
|
||||
void WriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value,
|
||||
u32 index_offset = 0) {
|
||||
WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32, sizeof(u32),
|
||||
&StorageDefinitions::U32, index_offset);
|
||||
}
|
||||
|
||||
void WriteStorageByCasLoop(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value, Id bit_offset, Id bit_count) {
|
||||
const Id pointer{StoragePointer(ctx, binding, offset, ctx.storage_types.U32, sizeof(u32),
|
||||
&StorageDefinitions::U32)};
|
||||
ctx.OpFunctionCall(ctx.TypeVoid(), ctx.write_storage_cas_loop_func, pointer, value, bit_offset,
|
||||
bit_count);
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
// (hookshot) 8- and 16-bit loads through a pointer no storage buffer was found for: the aligned 32-bit word by the same
|
||||
// search as LoadGlobal32, and the byte or half of it the address names (the console's loads are naturally aligned). They
|
||||
// threw "not implemented": 30 of OoT's mesh programs (LoadGlobalS16) were left untranslated on every card, issue 351
|
||||
namespace {
|
||||
Id LoadGlobalNarrow(EmitContext& ctx, Id address, u32 bits, bool is_signed) {
|
||||
if (!ctx.profile.support_int64) {
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
return ctx.Const(0u);
|
||||
}
|
||||
const Id aligned{ctx.OpBitwiseAnd(ctx.U64, address, ctx.Constant(ctx.U64, u64{~u64{3}}))};
|
||||
const Id word{ctx.OpFunctionCall(ctx.U32[1], ctx.load_global_func_u32, aligned)};
|
||||
const Id low{ctx.OpUConvert(ctx.U32[1], ctx.OpBitwiseAnd(ctx.U64, address, ctx.Constant(ctx.U64, u64{3})))};
|
||||
const Id shift{ctx.OpShiftLeftLogical(ctx.U32[1], low, ctx.Const(3u))};
|
||||
return is_signed ? ctx.OpBitFieldSExtract(ctx.U32[1], word, shift, ctx.Const(bits))
|
||||
: ctx.OpBitFieldUExtract(ctx.U32[1], word, shift, ctx.Const(bits));
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitLoadGlobalU8(EmitContext& ctx, Id address) {
|
||||
return LoadGlobalNarrow(ctx, address, 8, false);
|
||||
}
|
||||
|
||||
Id EmitLoadGlobalS8(EmitContext& ctx, Id address) {
|
||||
return LoadGlobalNarrow(ctx, address, 8, true);
|
||||
}
|
||||
|
||||
Id EmitLoadGlobalU16(EmitContext& ctx, Id address) {
|
||||
return LoadGlobalNarrow(ctx, address, 16, false);
|
||||
}
|
||||
|
||||
Id EmitLoadGlobalS16(EmitContext& ctx, Id address) {
|
||||
return LoadGlobalNarrow(ctx, address, 16, true);
|
||||
}
|
||||
|
||||
Id EmitLoadGlobal32(EmitContext& ctx, Id address) {
|
||||
if (ctx.profile.support_int64) {
|
||||
return ctx.OpFunctionCall(ctx.U32[1], ctx.load_global_func_u32, address);
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
return ctx.Const(0u);
|
||||
}
|
||||
|
||||
Id EmitLoadGlobal64(EmitContext& ctx, Id address) {
|
||||
if (ctx.profile.support_int64) {
|
||||
return ctx.OpFunctionCall(ctx.U32[2], ctx.load_global_func_u32x2, address);
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
return ctx.Const(0u, 0u);
|
||||
}
|
||||
|
||||
Id EmitLoadGlobal128(EmitContext& ctx, Id address) {
|
||||
if (ctx.profile.support_int64) {
|
||||
return ctx.OpFunctionCall(ctx.U32[4], ctx.load_global_func_u32x4, address);
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
return ctx.Const(0u, 0u, 0u, 0u);
|
||||
}
|
||||
|
||||
// (hookshot) 8- and 16-bit stores through a pointer no storage buffer was found for: the aligned 32-bit word by the same
|
||||
// search as LoadGlobal32, changed in place with a compare-exchange loop (spirv_emit_context.cpp). They threw "not
|
||||
// implemented", the same gap as the narrow loads (issue 192)
|
||||
namespace {
|
||||
void WriteGlobalNarrow(EmitContext& ctx, Id address, Id value, u32 bits) {
|
||||
if (!ctx.profile.support_int64) {
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
return;
|
||||
}
|
||||
const Id low{ctx.OpUConvert(ctx.U32[1], ctx.OpBitwiseAnd(ctx.U64, address, ctx.Constant(ctx.U64, u64{3})))};
|
||||
const Id bit_offset{ctx.OpShiftLeftLogical(ctx.U32[1], low, ctx.Const(3u))};
|
||||
ctx.OpFunctionCall(ctx.TypeVoid(), ctx.write_global_cas_loop_func, address, value, bit_offset, ctx.Const(bits));
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
void EmitWriteGlobalU8(EmitContext& ctx, Id address, Id value) {
|
||||
WriteGlobalNarrow(ctx, address, value, 8);
|
||||
}
|
||||
|
||||
void EmitWriteGlobalS8(EmitContext& ctx, Id address, Id value) {
|
||||
WriteGlobalNarrow(ctx, address, value, 8);
|
||||
}
|
||||
|
||||
void EmitWriteGlobalU16(EmitContext& ctx, Id address, Id value) {
|
||||
WriteGlobalNarrow(ctx, address, value, 16);
|
||||
}
|
||||
|
||||
void EmitWriteGlobalS16(EmitContext& ctx, Id address, Id value) {
|
||||
WriteGlobalNarrow(ctx, address, value, 16);
|
||||
}
|
||||
|
||||
void EmitWriteGlobal32(EmitContext& ctx, Id address, Id value) {
|
||||
if (ctx.profile.support_int64) {
|
||||
ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32, address, value);
|
||||
return;
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
}
|
||||
|
||||
void EmitWriteGlobal64(EmitContext& ctx, Id address, Id value) {
|
||||
if (ctx.profile.support_int64) {
|
||||
ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32x2, address, value);
|
||||
return;
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
}
|
||||
|
||||
void EmitWriteGlobal128(EmitContext& ctx, Id address, Id value) {
|
||||
if (ctx.profile.support_int64) {
|
||||
ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32x4, address, value);
|
||||
return;
|
||||
}
|
||||
LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation");
|
||||
}
|
||||
|
||||
Id EmitLoadStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
return ctx.OpUConvert(ctx.U32[1],
|
||||
LoadStorage(ctx, binding, offset, ctx.U8, ctx.storage_types.U8,
|
||||
sizeof(u8), &StorageDefinitions::U8));
|
||||
} else {
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
|
||||
ctx.BitOffset8(offset), ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
return ctx.OpSConvert(ctx.U32[1],
|
||||
LoadStorage(ctx, binding, offset, ctx.S8, ctx.storage_types.S8,
|
||||
sizeof(s8), &StorageDefinitions::S8));
|
||||
} else {
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
|
||||
ctx.BitOffset8(offset), ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
return ctx.OpUConvert(ctx.U32[1],
|
||||
LoadStorage(ctx, binding, offset, ctx.U16, ctx.storage_types.U16,
|
||||
sizeof(u16), &StorageDefinitions::U16));
|
||||
} else {
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
|
||||
ctx.BitOffset16(offset), ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
return ctx.OpSConvert(ctx.U32[1],
|
||||
LoadStorage(ctx, binding, offset, ctx.S16, ctx.storage_types.S16,
|
||||
sizeof(s16), &StorageDefinitions::S16));
|
||||
} else {
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], LoadStorage32(ctx, binding, offset),
|
||||
ctx.BitOffset16(offset), ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
return LoadStorage32(ctx, binding, offset);
|
||||
}
|
||||
|
||||
Id EmitLoadStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
return LoadStorage(ctx, binding, offset, ctx.U32[2], ctx.storage_types.U32x2,
|
||||
sizeof(u32[2]), &StorageDefinitions::U32x2);
|
||||
} else {
|
||||
return ctx.OpCompositeConstruct(ctx.U32[2], LoadStorage32(ctx, binding, offset, 0),
|
||||
LoadStorage32(ctx, binding, offset, 1));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
return LoadStorage(ctx, binding, offset, ctx.U32[4], ctx.storage_types.U32x4,
|
||||
sizeof(u32[4]), &StorageDefinitions::U32x4);
|
||||
} else {
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], LoadStorage32(ctx, binding, offset, 0),
|
||||
LoadStorage32(ctx, binding, offset, 1),
|
||||
LoadStorage32(ctx, binding, offset, 2),
|
||||
LoadStorage32(ctx, binding, offset, 3));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U8, value), ctx.storage_types.U8,
|
||||
sizeof(u8), &StorageDefinitions::U8);
|
||||
} else {
|
||||
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_int8 && ctx.profile.support_storage_buffer_8bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S8, value), ctx.storage_types.S8,
|
||||
sizeof(s8), &StorageDefinitions::S8);
|
||||
} else {
|
||||
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U16, value), ctx.storage_types.U16,
|
||||
sizeof(u16), &StorageDefinitions::U16);
|
||||
} else {
|
||||
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_int16 && ctx.profile.support_storage_buffer_16bit &&
|
||||
ctx.profile.support_descriptor_aliasing) {
|
||||
WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S16, value), ctx.storage_types.S16,
|
||||
sizeof(s16), &StorageDefinitions::S16);
|
||||
} else {
|
||||
WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u));
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
WriteStorage32(ctx, binding, offset, value);
|
||||
}
|
||||
|
||||
void EmitWriteStorage64(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32x2, sizeof(u32[2]),
|
||||
&StorageDefinitions::U32x2);
|
||||
} else {
|
||||
for (u32 index = 0; index < 2; ++index) {
|
||||
const Id element{ctx.OpCompositeExtract(ctx.U32[1], value, index)};
|
||||
WriteStorage32(ctx, binding, offset, element, index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteStorage128(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset,
|
||||
Id value) {
|
||||
if (ctx.profile.support_descriptor_aliasing) {
|
||||
WriteStorage(ctx, binding, offset, value, ctx.storage_types.U32x4, sizeof(u32[4]),
|
||||
&StorageDefinitions::U32x4);
|
||||
} else {
|
||||
for (u32 index = 0; index < 4; ++index) {
|
||||
const Id element{ctx.OpCompositeExtract(ctx.U32[1], value, index)};
|
||||
WriteStorage32(ctx, binding, offset, element, index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
656
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp
Normal file
656
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_mesh.cpp
Normal file
@@ -0,0 +1,656 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Switch 2 (NVN) task and mesh programs. They talk to the hardware through the stage buffer
|
||||
// (ISBE): a task program writes the number of mesh workgroups at byte 4 of its output region and
|
||||
// its payload after it; each mesh workgroup reads that region (its own index at byte 0) and
|
||||
// writes its vertices and primitives to an attribute region in groups of 32, one 32-word line per
|
||||
// attribute, plus the primitive count and 8-bit vertex indices to a map region. Here the task's
|
||||
// output region is the task payload, the mesh output regions live in workgroup memory, and the
|
||||
// epilogue (reached by every invocation, see structured_control_flow.cpp) hands them to Vulkan.
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <span>
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
void Barrier(EmitContext& ctx) {
|
||||
ctx.OpControlBarrier(ctx.Const(static_cast<u32>(spv::Scope::Workgroup)),
|
||||
ctx.Const(static_cast<u32>(spv::Scope::Workgroup)),
|
||||
ctx.Const(static_cast<u32>(spv::MemorySemanticsMask::AcquireRelease |
|
||||
spv::MemorySemanticsMask::WorkgroupMemory)));
|
||||
}
|
||||
|
||||
Id LocalInvocation(EmitContext& ctx) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.local_invocation_id), 0U);
|
||||
}
|
||||
|
||||
Id Clamped(EmitContext& ctx, Id index, u32 count) {
|
||||
return ctx.OpUMin(ctx.U32[1], index, ctx.Const(count - 1));
|
||||
}
|
||||
|
||||
Id PushU64(EmitContext& ctx, u32 member);
|
||||
Id PushU32(EmitContext& ctx, u32 member) {
|
||||
const Id pointer{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U32[1])};
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.OpAccessChain(pointer, ctx.rescaling_push_constants, ctx.Const(member)));
|
||||
}
|
||||
Id WorkgroupId(EmitContext& ctx);
|
||||
Id FallbackSlot(EmitContext& ctx);
|
||||
Id BufferWordAt(EmitContext& ctx, Id base, Id word);
|
||||
Id BufferWord(EmitContext& ctx, u32 member, Id word);
|
||||
|
||||
Id PayloadWord(EmitContext& ctx, Id index) {
|
||||
if (ctx.mesh_fallback && ctx.stage == Stage::Mesh && ctx.runtime_info.mesh_fallback_task) {
|
||||
// (the task wrote it out: the mesh workgroups are a later dispatch, and read their task workgroup's payload, a slot
|
||||
// being task workgroup * threads + its index in the task)
|
||||
const Id task{ctx.OpUDiv(ctx.U32[1], FallbackSlot(ctx), ctx.Const(ctx.runtime_info.mesh_fallback_task_threads))};
|
||||
const Id first{ctx.OpIAdd(
|
||||
ctx.U32[1], ctx.Const(MESH_FALLBACK_CAPACITY),
|
||||
ctx.OpIMul(ctx.U32[1], task, ctx.Const(ctx.mesh_info.payload_words)))};
|
||||
return BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index),
|
||||
ctx.OpIAdd(ctx.U32[1], first,
|
||||
Clamped(ctx, index, ctx.mesh_info.payload_words)));
|
||||
}
|
||||
return ctx.OpAccessChain(ctx.mesh_payload_u32, ctx.mesh_payload,
|
||||
Clamped(ctx, index, ctx.mesh_info.payload_words));
|
||||
}
|
||||
|
||||
Id SharedWord(EmitContext& ctx, Id array, Id index) {
|
||||
return ctx.OpAccessChain(ctx.mesh_shared_u32, array, index);
|
||||
}
|
||||
|
||||
// Runs body under `if (condition)`.
|
||||
template <typename Func>
|
||||
void If(EmitContext& ctx, Id condition, Func&& body) {
|
||||
const Id then_label{ctx.OpLabel()};
|
||||
const Id merge_label{ctx.OpLabel()};
|
||||
ctx.OpSelectionMerge(merge_label, spv::SelectionControlMask::MaskNone);
|
||||
ctx.OpBranchConditional(condition, then_label, merge_label);
|
||||
ctx.AddLabel(then_label);
|
||||
body();
|
||||
ctx.OpBranch(merge_label);
|
||||
ctx.AddLabel(merge_label);
|
||||
}
|
||||
|
||||
// Copies the lines of element `index` (a vertex or a primitive) from the attribute region.
|
||||
void CopyLines(EmitContext& ctx, Id index, u32 base, u32 num_lines,
|
||||
const std::vector<EmitContext::MeshLineTarget>& targets) {
|
||||
// Word of line 0: base + (index / 32) * num_lines * 32 + index % 32.
|
||||
const Id group{ctx.OpShiftRightLogical(ctx.U32[1], index, ctx.Const(5U))};
|
||||
const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U))};
|
||||
const Id first{ctx.OpIAdd(ctx.U32[1],
|
||||
ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)),
|
||||
ctx.OpIAdd(ctx.U32[1], slot, ctx.Const(base)))};
|
||||
for (u32 line = 0; line < num_lines; ++line) {
|
||||
const EmitContext::MeshLineTarget& target{targets[line]};
|
||||
if (!Sirit::ValidId(target.variable)) {
|
||||
continue;
|
||||
}
|
||||
const Id word{ctx.OpLoad(
|
||||
ctx.U32[1],
|
||||
SharedWord(ctx, ctx.mesh_attributes, ctx.OpIAdd(ctx.U32[1], first, ctx.Const(line * 32))))};
|
||||
const Id value{target.is_float ? ctx.OpBitcast(ctx.F32[1], word) : word};
|
||||
const Id pointer_type{target.is_float ? ctx.output_f32 : ctx.output_u32};
|
||||
const Id pointer{target.element == ~0U
|
||||
? ctx.OpAccessChain(pointer_type, target.variable, index)
|
||||
: ctx.OpAccessChain(pointer_type, target.variable, index,
|
||||
ctx.Const(target.element))};
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
}
|
||||
|
||||
// OpenGL-style clip space: z = (z + w) / 2, from the stage buffer's position lines.
|
||||
void ConvertDepth(EmitContext& ctx, Id vertex) {
|
||||
const auto& lines{ctx.mesh_info.vertex_lines};
|
||||
const auto z_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionZ))};
|
||||
const auto w_line{std::ranges::find(lines, static_cast<u32>(IR::Attribute::PositionW))};
|
||||
if (z_line == lines.end() || w_line == lines.end()) {
|
||||
return;
|
||||
}
|
||||
const u32 num_lines{static_cast<u32>(lines.size())};
|
||||
const Id group{ctx.OpShiftRightLogical(ctx.U32[1], vertex, ctx.Const(5U))};
|
||||
const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], vertex, ctx.Const(31U))};
|
||||
const Id first{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)),
|
||||
slot)};
|
||||
const auto read = [&](auto line) {
|
||||
const u32 offset{static_cast<u32>(line - lines.begin()) * 32};
|
||||
return ctx.OpBitcast(ctx.F32[1],
|
||||
ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_attributes,
|
||||
ctx.OpIAdd(ctx.U32[1], first,
|
||||
ctx.Const(offset)))));
|
||||
};
|
||||
const Id z{ctx.OpFMul(ctx.F32[1], ctx.OpFAdd(ctx.F32[1], read(z_line), read(w_line)),
|
||||
ctx.Const(0.5f))};
|
||||
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, ctx.output_position, vertex, ctx.Const(2U)), z);
|
||||
}
|
||||
|
||||
/// (hookshot) The fallback's records: one per primitive corner, its validity word and the stage buffer's vertex and
|
||||
/// primitive lines. A mesh workgroup writes its workgroup index and a fixed number of records for it; the generated
|
||||
/// vertex stage reads one record per vertex.
|
||||
u32 FallbackRecordWords(const MeshInfo& mesh) {
|
||||
return 1 + static_cast<u32>(mesh.vertex_lines.size()) +
|
||||
static_cast<u32>(mesh.primitive_lines.size());
|
||||
}
|
||||
|
||||
Id PushU64(EmitContext& ctx, u32 member) {
|
||||
const Id pointer_type{ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U64)};
|
||||
return ctx.OpLoad(
|
||||
ctx.U64, ctx.OpAccessChain(pointer_type, ctx.rescaling_push_constants, ctx.Const(member)));
|
||||
}
|
||||
|
||||
/// The word at a device address.
|
||||
Id BufferWordAt(EmitContext& ctx, Id base, Id word) {
|
||||
const Id address{
|
||||
ctx.OpIAdd(ctx.U64, base,
|
||||
ctx.OpUConvert(ctx.U64, ctx.OpIMul(ctx.U32[1], word, ctx.Const(4U))))};
|
||||
return ctx.MeshFallbackWord(address);
|
||||
}
|
||||
|
||||
/// The word at a pushed device address.
|
||||
Id BufferWord(EmitContext& ctx, u32 member, Id word) {
|
||||
return BufferWordAt(ctx, PushU64(ctx, member), word);
|
||||
}
|
||||
|
||||
/// A load and a store of such a word. (PhysicalStorageBuffer accesses must say their alignment: spirv-val rejects them
|
||||
/// without it.) Workgroup words take the same, which is valid there too.
|
||||
Id LoadWord(EmitContext& ctx, Id pointer) {
|
||||
return ctx.OpLoad(ctx.U32[1], pointer, spv::MemoryAccessMask::Aligned, 4U);
|
||||
}
|
||||
|
||||
void StoreWord(EmitContext& ctx, Id pointer, Id value) {
|
||||
ctx.OpStore(pointer, value, spv::MemoryAccessMask::Aligned, 4U);
|
||||
}
|
||||
|
||||
Id WorkgroupId(EmitContext& ctx) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.workgroup_id), 0U);
|
||||
}
|
||||
|
||||
/// A fallback mesh workgroup's slot: the one its task's map gave it (EmitMeshPrologue), else its workgroup id
|
||||
Id FallbackSlot(EmitContext& ctx) {
|
||||
return Sirit::ValidId(ctx.mesh_fallback_slot) ? ctx.mesh_fallback_slot : WorkgroupId(ctx);
|
||||
}
|
||||
|
||||
/// Runs body once for each index invocation, invocation + threads, ... below count. (Unrolled: count and threads are
|
||||
/// known here. A loop's index variable declared at the call, mid-function, was invalid SPIR-V: a function's OpVariables
|
||||
/// must open its first block)
|
||||
template <typename Func>
|
||||
void Strided(EmitContext& ctx, u32 count, u32 threads, Id invocation, Func&& body) {
|
||||
for (u32 first = 0; first < count; first += std::max(threads, 1U)) {
|
||||
const Id index{first ? ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first)) : invocation};
|
||||
If(ctx, ctx.OpULessThan(ctx.U1, index, ctx.Const(count)), [&] { body(index); });
|
||||
}
|
||||
}
|
||||
|
||||
/// The raw stage buffer word of line `line` for element `element` (a vertex or a primitive).
|
||||
Id LineWord(EmitContext& ctx, Id array, Id element, u32 base, u32 num_lines, u32 line) {
|
||||
const Id group{ctx.OpShiftRightLogical(ctx.U32[1], element, ctx.Const(5U))};
|
||||
const Id slot{ctx.OpBitwiseAnd(ctx.U32[1], element, ctx.Const(31U))};
|
||||
const Id first{ctx.OpIAdd(
|
||||
ctx.U32[1], ctx.OpIMul(ctx.U32[1], group, ctx.Const(num_lines * 32)),
|
||||
ctx.OpIAdd(ctx.U32[1], slot, ctx.Const(base)))};
|
||||
return ctx.OpLoad(
|
||||
ctx.U32[1],
|
||||
SharedWord(ctx, array, ctx.OpIAdd(ctx.U32[1], first, ctx.Const(line * 32))));
|
||||
}
|
||||
|
||||
/// A mesh workgroup takes records for its primitives' corners from the records buffer's allocator and writes them; the vertex
|
||||
/// stage's indirect draw covers the records taken (MESH_FALLBACK_DRAW_WORD). A task's flag first bounds its count.
|
||||
void MeshFallbackEpilogue(EmitContext& ctx) {
|
||||
const MeshInfo& mesh{ctx.mesh_info};
|
||||
const u32 corners{mesh.Corners()};
|
||||
const u32 words{FallbackRecordWords(mesh)};
|
||||
const u32 fit{MESH_FALLBACK_RECORD_WORDS / words}; // (this program's records the buffer holds)
|
||||
const Id none{ctx.Const(0xFFFFFFFFU)};
|
||||
Barrier(ctx);
|
||||
// (with a task program every workgroup dispatched was started by it: the dispatch is the map's, TaskEpilogue)
|
||||
const Id slot{FallbackSlot(ctx)};
|
||||
const Id in_range{ctx.OpULessThan(ctx.U1, slot, ctx.Const(MESH_FALLBACK_CAPACITY))};
|
||||
Id count{ctx.OpUMin(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count),
|
||||
ctx.Const(mesh.max_primitives))};
|
||||
count = ctx.OpSelect(ctx.U32[1], in_range, count, ctx.u32_zero_value);
|
||||
const Id taken{ctx.OpIMul(ctx.U32[1], count, ctx.Const(corners))};
|
||||
const auto atomic_add = [&](u32 word, Id value) {
|
||||
return ctx.OpAtomicIAdd(ctx.U32[1], BufferWord(ctx, ctx.mesh_fallback_records_member_index, ctx.Const(word)),
|
||||
ctx.Const(static_cast<u32>(spv::Scope::Device)), ctx.u32_zero_value, value);
|
||||
};
|
||||
// (every thread has read the primitive count above: thread 0 hands the others its first record in that word. Records
|
||||
// that do not fit are not taken; taking only grows, so the ones that fit are the first, and the draw's count is theirs)
|
||||
Barrier(ctx);
|
||||
If(ctx, ctx.OpIEqual(ctx.U1, LocalInvocation(ctx), ctx.u32_zero_value), [&] {
|
||||
ctx.OpStore(ctx.mesh_primitive_count, none);
|
||||
If(ctx, ctx.OpINotEqual(ctx.U1, taken, ctx.u32_zero_value), [&] {
|
||||
const Id base{atomic_add(MESH_FALLBACK_ALLOC_WORD, taken)};
|
||||
const Id fits{ctx.OpLogicalAnd(ctx.U1, ctx.OpULessThanEqual(ctx.U1, base, ctx.Const(fit)),
|
||||
ctx.OpULessThanEqual(ctx.U1, taken,
|
||||
ctx.OpISub(ctx.U32[1], ctx.Const(fit), base)))};
|
||||
If(ctx, fits, [&] {
|
||||
atomic_add(MESH_FALLBACK_DRAW_WORD, taken);
|
||||
ctx.OpStore(ctx.mesh_primitive_count, base);
|
||||
});
|
||||
});
|
||||
});
|
||||
Barrier(ctx);
|
||||
const Id base{ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count)};
|
||||
If(ctx, ctx.OpINotEqual(ctx.U1, base, none), [&] {
|
||||
const Id invocation{LocalInvocation(ctx)};
|
||||
Strided(ctx, mesh.max_primitives, ctx.mesh_threads, invocation, [&](Id primitive) {
|
||||
If(ctx, ctx.OpULessThan(ctx.U1, primitive, count), [&] {
|
||||
for (u32 corner = 0; corner < corners; ++corner) {
|
||||
const Id byte{ctx.OpIAdd(
|
||||
ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(corners)),
|
||||
ctx.Const(4U + corner))};
|
||||
// (an index byte the program left unwritten is held to the vertex count)
|
||||
const Id vertex{Clamped(
|
||||
ctx, ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_map, byte)),
|
||||
mesh.max_vertices)};
|
||||
const Id record{ctx.OpIAdd(
|
||||
ctx.U32[1], base,
|
||||
ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(corners)),
|
||||
ctx.Const(corner)))};
|
||||
const Id word{ctx.OpIAdd(ctx.U32[1], ctx.Const(MESH_FALLBACK_HEADER_WORDS),
|
||||
ctx.OpIMul(ctx.U32[1], record, ctx.Const(words)))};
|
||||
StoreWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index, word),
|
||||
ctx.Const(1U));
|
||||
for (u32 line = 0; line < mesh.vertex_lines.size(); ++line) {
|
||||
const Id value{LineWord(ctx, ctx.mesh_attributes, vertex, 0,
|
||||
static_cast<u32>(mesh.vertex_lines.size()), line)};
|
||||
StoreWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index,
|
||||
ctx.OpIAdd(ctx.U32[1], word, ctx.Const(1U + line))),
|
||||
value);
|
||||
}
|
||||
for (u32 line = 0; line < mesh.primitive_lines.size(); ++line) {
|
||||
const Id value{LineWord(ctx, ctx.mesh_attributes, primitive,
|
||||
ctx.mesh_primitive_base,
|
||||
static_cast<u32>(mesh.primitive_lines.size()), line)};
|
||||
StoreWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index,
|
||||
ctx.OpIAdd(ctx.U32[1], word,
|
||||
ctx.Const(static_cast<u32>(1U + mesh.vertex_lines.size()) +
|
||||
line))),
|
||||
value);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void StorePosition(EmitContext& ctx, u32 element, Id value) {
|
||||
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, ctx.output_position, ctx.Const(element)), value);
|
||||
}
|
||||
|
||||
void StoreFallbackOutput(EmitContext& ctx, IR::Attribute attribute, Id value) {
|
||||
if (IR::IsGeneric(attribute)) {
|
||||
const u32 index{IR::GenericAttributeIndex(attribute)};
|
||||
const u32 element{IR::GenericAttributeElement(attribute)};
|
||||
const GenericElementInfo& info{ctx.output_generics.at(index).at(element)};
|
||||
if (!Sirit::ValidId(info.id)) {
|
||||
return;
|
||||
}
|
||||
if (info.num_components == 1) {
|
||||
ctx.OpStore(info.id, value);
|
||||
} else {
|
||||
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, info.id,
|
||||
ctx.Const(element - info.first_element)),
|
||||
value);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (attribute == IR::Attribute::PointSize && Sirit::ValidId(ctx.output_point_size)) {
|
||||
ctx.OpStore(ctx.output_point_size, value);
|
||||
}
|
||||
}
|
||||
|
||||
u32 task_cap(u32 threads) { // (RX_VK_TASK_CAP=<n>: mesh workgroups a task may start; 0: one a thread)
|
||||
static const char* e = getenv("RX_VK_TASK_CAP");
|
||||
const u32 n = e ? static_cast<u32>(atoi(e)) : 0U;
|
||||
return n ? n : threads;
|
||||
}
|
||||
|
||||
void TaskEpilogue(EmitContext& ctx) {
|
||||
Barrier(ctx);
|
||||
// (hookshot) at most one mesh workgroup a task thread: each thread keeps or culls one meshlet (the programs compact them by subgroup
|
||||
// ballot). It was 65535, and a payload word a task left unwritten is anything: when the task culled everything, tens of thousands of
|
||||
// mesh workgroups for a draw held the GPU past Windows' two seconds, and the device was lost, on about half the first starts
|
||||
const Id count{ctx.OpUMin(ctx.U32[1], LoadWord(ctx, PayloadWord(ctx, ctx.Const(1U))),
|
||||
ctx.Const(task_cap(ctx.mesh_threads)))};
|
||||
if (ctx.mesh_fallback) {
|
||||
// (no OpEmitMeshTasksEXT: each thread that started a mesh workgroup takes an index of the mesh dispatch
|
||||
// (MESH_FALLBACK_DISPATCH_WORD) and writes its slot, task workgroup * threads + its index, at that index of the
|
||||
// payload buffer's map. The mesh dispatch is those indices: no more, whatever the CPU knew of the task count)
|
||||
const Id slot{ctx.OpIAdd(ctx.U32[1],
|
||||
ctx.OpIMul(ctx.U32[1], WorkgroupId(ctx), ctx.Const(ctx.mesh_threads)),
|
||||
LocalInvocation(ctx))};
|
||||
// (the count is how many meshlets this task workgroup started: the first that many threads, not all of them)
|
||||
const Id started{ctx.OpLogicalAnd(ctx.U1, ctx.OpULessThan(ctx.U1, LocalInvocation(ctx), count),
|
||||
ctx.OpULessThan(ctx.U1, slot, ctx.Const(MESH_FALLBACK_CAPACITY)))};
|
||||
If(ctx, started, [&] {
|
||||
const Id index{ctx.OpAtomicIAdd(
|
||||
ctx.U32[1],
|
||||
BufferWord(ctx, ctx.mesh_fallback_records_member_index, ctx.Const(MESH_FALLBACK_DISPATCH_WORD)),
|
||||
ctx.Const(static_cast<u32>(spv::Scope::Device)), ctx.u32_zero_value, ctx.Const(1U))};
|
||||
// (fewer than MESH_FALLBACK_CAPACITY: a started slot is below it)
|
||||
StoreWord(ctx, BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index), index), slot);
|
||||
});
|
||||
// (the payload once a task workgroup, its threads sharing the words, where its mesh workgroups read it (PayloadWord).
|
||||
// A copy a slot was 32 times the writes and the memory: 52 MB a draw at 16384 slots)
|
||||
const Id task{WorkgroupId(ctx)};
|
||||
const Id first_slot{ctx.OpIMul(ctx.U32[1], task, ctx.Const(ctx.mesh_threads))};
|
||||
If(ctx, ctx.OpULessThan(ctx.U1, first_slot, ctx.Const(MESH_FALLBACK_CAPACITY)), [&] {
|
||||
const Id first{ctx.OpIAdd(
|
||||
ctx.U32[1], ctx.Const(MESH_FALLBACK_CAPACITY),
|
||||
ctx.OpIMul(ctx.U32[1], task, ctx.Const(ctx.mesh_info.payload_words)))};
|
||||
Strided(ctx, ctx.mesh_info.payload_words, ctx.mesh_threads, LocalInvocation(ctx), [&](Id index) {
|
||||
StoreWord(ctx, BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index),
|
||||
ctx.OpIAdd(ctx.U32[1], first, index)),
|
||||
LoadWord(ctx, PayloadWord(ctx, index)));
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
const Id one{ctx.Const(1U)};
|
||||
// Marker for OpEmitMeshTasksEXT count, 1, 1, payload (PatchMeshInstructions).
|
||||
ctx.OpStore(count, one);
|
||||
ctx.OpStore(one, ctx.mesh_payload);
|
||||
ctx.mesh_tasks_marker = {count, one, one, ctx.mesh_payload};
|
||||
}
|
||||
|
||||
void MeshEpilogue(EmitContext& ctx) {
|
||||
const MeshInfo& mesh{ctx.mesh_info};
|
||||
if (ctx.mesh_fallback) {
|
||||
MeshFallbackEpilogue(ctx);
|
||||
return;
|
||||
}
|
||||
Barrier(ctx);
|
||||
const Id count{ctx.OpUMin(ctx.U32[1], ctx.OpLoad(ctx.U32[1], ctx.mesh_primitive_count),
|
||||
ctx.Const(mesh.max_primitives))};
|
||||
const Id max_vertices{ctx.Const(mesh.max_vertices)};
|
||||
// Marker for OpSetMeshOutputsEXT max_vertices, count (PatchMeshInstructions).
|
||||
ctx.OpStore(max_vertices, count);
|
||||
ctx.mesh_outputs_marker = {max_vertices, count};
|
||||
|
||||
const Id invocation{LocalInvocation(ctx)};
|
||||
const u32 threads{ctx.mesh_threads};
|
||||
const bool convert_depth{ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc &&
|
||||
Sirit::ValidId(ctx.output_position)};
|
||||
const u32 vertex_lines{static_cast<u32>(mesh.vertex_lines.size())};
|
||||
for (u32 first = 0; first < mesh.max_vertices; first += threads) {
|
||||
const Id vertex{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))};
|
||||
const auto copy = [&] {
|
||||
for (const auto& d : ctx.mesh_vertex_defaults) {
|
||||
for (u32 c = 0; c < 4; ++c) {
|
||||
if (d.missing & (1U << c)) {
|
||||
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, d.variable, vertex, ctx.Const(c)),
|
||||
ctx.Const(c == 3 ? 1.0f : 0.0f));
|
||||
}
|
||||
}
|
||||
}
|
||||
CopyLines(ctx, vertex, 0, vertex_lines, ctx.mesh_vertex_targets);
|
||||
if (convert_depth) {
|
||||
ConvertDepth(ctx, vertex);
|
||||
}
|
||||
};
|
||||
if (first + threads > mesh.max_vertices) {
|
||||
If(ctx, ctx.OpULessThan(ctx.U1, vertex, max_vertices), copy);
|
||||
} else {
|
||||
copy();
|
||||
}
|
||||
}
|
||||
const u32 primitive_lines{static_cast<u32>(mesh.primitive_lines.size())};
|
||||
for (u32 first = 0; first < mesh.max_primitives; first += threads) {
|
||||
const Id primitive{ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first))};
|
||||
If(ctx, ctx.OpULessThan(ctx.U1, primitive, count), [&] {
|
||||
// Vertex indices: map bytes 4 + 3 * primitive (lines: 4 + 2 * primitive, points: 4 + primitive).
|
||||
const u32 corners{mesh.Corners()};
|
||||
const Id byte{ctx.OpIAdd(ctx.U32[1], ctx.OpIMul(ctx.U32[1], primitive, ctx.Const(corners)),
|
||||
ctx.Const(4U))};
|
||||
std::array<Id, 3> indices;
|
||||
for (u32 element = 0; element < corners; ++element) {
|
||||
const Id address{ctx.OpIAdd(ctx.U32[1], byte, ctx.Const(element))};
|
||||
// (hookshot) within the vertices: an index byte the program left unwritten is anything, and an index
|
||||
// past the vertex count is undefined (a device fault on NVIDIA)
|
||||
indices[element] = Clamped(ctx, ctx.OpLoad(ctx.U32[1], SharedWord(ctx, ctx.mesh_map, address)),
|
||||
mesh.max_vertices);
|
||||
}
|
||||
const Id output_indices{ctx.TypePointer(spv::StorageClass::Output, ctx.U32[corners])};
|
||||
ctx.OpStore(ctx.OpAccessChain(output_indices, ctx.mesh_primitive_indices, primitive),
|
||||
corners == 1 ? indices[0] // (a point's index is a scalar)
|
||||
: ctx.OpCompositeConstruct(ctx.U32[corners],
|
||||
std::span<const Id>(indices.data(), corners)));
|
||||
CopyLines(ctx, primitive, ctx.mesh_primitive_base, primitive_lines,
|
||||
ctx.mesh_primitive_targets);
|
||||
});
|
||||
}
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
void EmitMeshFallbackVertexBody(EmitContext& ctx) {
|
||||
// (the fragment stage's inputs for the mesh program's per-primitive generics are Flat: the generated vertex
|
||||
// stage's outputs must match, and DefineOutputs only knows the stores, not which of them are per-primitive)
|
||||
for (u32 index = 0; index < IR::NUM_GENERICS; ++index) {
|
||||
if (!ctx.runtime_info.per_primitive_generics.test(index)) {
|
||||
continue;
|
||||
}
|
||||
for (u32 element = 0; element < 4; ++element) {
|
||||
const Id id{ctx.output_generics[index][element].id};
|
||||
if (!Sirit::ValidId(id)) {
|
||||
continue;
|
||||
}
|
||||
bool seen{};
|
||||
for (u32 before = 0; before < element; ++before) {
|
||||
seen |= ctx.output_generics[index][before].id.value == id.value;
|
||||
}
|
||||
if (!seen) {
|
||||
ctx.Decorate(id, spv::Decoration::Flat);
|
||||
}
|
||||
}
|
||||
}
|
||||
// (every output component starts at (0,0,0,1), as the native mesh stage's do (6dcf6f1) and a vertex stage's; the record's
|
||||
// lines below go over the ones the mesh program wrote. Left undefined, AMD's driver gave the fragment stage whatever its
|
||||
// registers held: specks of colour in Kokiri Forest's bushes on an RX 5500M)
|
||||
for (u32 index = 0; index < IR::NUM_GENERICS; ++index) {
|
||||
for (u32 element = 0; element < 4; ++element) {
|
||||
const GenericElementInfo& info{ctx.output_generics[index][element]};
|
||||
if (!Sirit::ValidId(info.id)) {
|
||||
continue;
|
||||
}
|
||||
const Id value{element == 3 ? ctx.Const(1.0f) : ctx.f32_zero_value};
|
||||
if (info.num_components == 1) {
|
||||
ctx.OpStore(info.id, value);
|
||||
} else {
|
||||
ctx.OpStore(ctx.OpAccessChain(ctx.output_f32, info.id, ctx.Const(element - info.first_element)), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Sirit::ValidId(ctx.output_point_size)) {
|
||||
ctx.OpStore(ctx.output_point_size, ctx.Const(1.0f));
|
||||
}
|
||||
const MeshInfo& mesh{ctx.mesh_info};
|
||||
const u32 vertex_lines{static_cast<u32>(mesh.vertex_lines.size())};
|
||||
const u32 primitive_lines{static_cast<u32>(mesh.primitive_lines.size())};
|
||||
const u32 words{FallbackRecordWords(mesh)};
|
||||
// (a record a vertex, in the order the mesh workgroups took them (MeshFallbackEpilogue): the indirect draw's count is the
|
||||
// records taken, so every one it reaches was written)
|
||||
const Id vertex{ctx.OpLoad(ctx.U32[1], ctx.vertex_index)};
|
||||
const Id first{ctx.OpIAdd(ctx.U32[1], ctx.Const(MESH_FALLBACK_HEADER_WORDS),
|
||||
ctx.OpIMul(ctx.U32[1], vertex, ctx.Const(words)))};
|
||||
const Id valid{LoadWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index, first))};
|
||||
const auto read = [&](u32 word) {
|
||||
return ctx.OpBitcast(
|
||||
ctx.F32[1],
|
||||
LoadWord(ctx, BufferWord(ctx, ctx.mesh_fallback_records_member_index,
|
||||
ctx.OpIAdd(ctx.U32[1], first, ctx.Const(word)))));
|
||||
};
|
||||
std::array<Id, 4> position{};
|
||||
for (u32 line = 0; line < vertex_lines; ++line) {
|
||||
const IR::Attribute attribute{mesh.vertex_lines[line]};
|
||||
const Id value{read(1U + line)};
|
||||
const u32 raw{static_cast<u32>(attribute)};
|
||||
if (raw >= static_cast<u32>(IR::Attribute::PositionX) &&
|
||||
raw <= static_cast<u32>(IR::Attribute::PositionW)) {
|
||||
position[raw - static_cast<u32>(IR::Attribute::PositionX)] = value;
|
||||
} else {
|
||||
StoreFallbackOutput(ctx, attribute, value);
|
||||
}
|
||||
}
|
||||
for (u32 line = 0; line < primitive_lines; ++line) {
|
||||
StoreFallbackOutput(ctx, IR::Attribute{mesh.primitive_lines[line]},
|
||||
read(1U + vertex_lines + line));
|
||||
}
|
||||
// (OpenGL-style clip space, as the mesh epilogue's ConvertDepth)
|
||||
if (ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc &&
|
||||
Sirit::ValidId(position[2]) && Sirit::ValidId(position[3])) {
|
||||
position[2] = ctx.OpFMul(ctx.F32[1], ctx.OpFAdd(ctx.F32[1], position[2], position[3]),
|
||||
ctx.Const(0.5f));
|
||||
}
|
||||
for (u32 element = 0; element < 4; ++element) {
|
||||
if (Sirit::ValidId(position[element])) {
|
||||
StorePosition(ctx, element, position[element]);
|
||||
}
|
||||
}
|
||||
// (a record no mesh workgroup wrote, which the draw's count should never reach: outside the clip volume)
|
||||
If(ctx, ctx.OpIEqual(ctx.U1, valid, ctx.u32_zero_value),
|
||||
[&] {
|
||||
StorePosition(ctx, 0, ctx.f32_zero_value);
|
||||
StorePosition(ctx, 1, ctx.f32_zero_value);
|
||||
StorePosition(ctx, 2, ctx.f32_zero_value);
|
||||
StorePosition(ctx, 3, ctx.Const(-1.0f));
|
||||
});
|
||||
}
|
||||
|
||||
Id EmitIsbeRead(EmitContext& ctx, Id address) {
|
||||
const Id native_workgroup{
|
||||
ctx.OpCompositeExtract(ctx.U32[1], ctx.OpLoad(ctx.U32[3], ctx.workgroup_id), 0U)};
|
||||
if (ctx.stage == Stage::Task) {
|
||||
const Id workgroup{native_workgroup};
|
||||
if (ctx.mesh_fallback) {
|
||||
const Id input{PushU64(ctx, ctx.mesh_fallback_indirect_input_member_index)};
|
||||
const Id indirect{ctx.OpINotEqual(ctx.U1, input, ctx.Constant(ctx.U64, u64{0}))};
|
||||
// On a direct draw the header's instance count is a valid, read-only word for the unused load.
|
||||
const Id base{ctx.OpSelect(ctx.U64, indirect, input, PushU64(ctx, ctx.mesh_fallback_records_member_index))};
|
||||
const Id gpu_first{LoadWord(ctx, BufferWordAt(ctx, base, ctx.Const(1U)))};
|
||||
const Id first{ctx.OpSelect(ctx.U32[1], indirect, gpu_first, PushU32(ctx, ctx.mesh_first_member_index))};
|
||||
return ctx.OpIAdd(ctx.U32[1], workgroup, first);
|
||||
}
|
||||
if (ctx.wants_mesh_first && Sirit::ValidId(ctx.rescaling_push_constants)) {
|
||||
// (hookshot) + the draw's first workgroup: NVN draws [first, first + count). Without it every draw began at 0: the
|
||||
// foliage's draws (first up to ~1900) drew the first records again and again, and the forest had no leaves
|
||||
const Id table{ctx.OpLoad(ctx.U64, ctx.OpAccessChain(ctx.TypePointer(spv::StorageClass::PushConstant, ctx.U64),
|
||||
ctx.rescaling_push_constants, ctx.Const(ctx.mesh_first_member_index)))};
|
||||
const Id draw{ctx.OpLoad(ctx.U32[1], ctx.draw_index)};
|
||||
const Id at{ctx.OpIAdd(ctx.U64, table, ctx.OpUConvert(ctx.U64, ctx.OpIMul(ctx.U32[1], draw, ctx.Const(16U))))};
|
||||
const Id member{ctx.OpAccessChain(ctx.mesh_first_member_pointer, ctx.OpConvertUToPtr(ctx.mesh_first_block_pointer, at),
|
||||
ctx.Const(0U))};
|
||||
const Id first{ctx.OpLoad(ctx.U32[1], member, spv::MemoryAccessMask::Aligned, 4U)};
|
||||
return ctx.OpIAdd(ctx.U32[1], workgroup, first);
|
||||
}
|
||||
return workgroup; // the input region holds the workgroup index (VERTEX_ID)
|
||||
}
|
||||
const Id workgroup{native_workgroup};
|
||||
const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))};
|
||||
const Id pointer{PayloadWord(ctx, index)};
|
||||
const Id word{ctx.mesh_fallback ? LoadWord(ctx, pointer) : ctx.OpLoad(ctx.U32[1], pointer)}; // (the fallback's: a device address)
|
||||
// (hookshot) a fallback mesh workgroup is a slot, task workgroup * threads + its index: address 0 is that index in its task,
|
||||
// from 0 as OpEmitMeshTasksEXT numbers them. It was the slot, which picked a meshlet past the end of every task's list
|
||||
const Id within{ctx.mesh_fallback && ctx.runtime_info.mesh_fallback_task
|
||||
? ctx.OpUMod(ctx.U32[1], FallbackSlot(ctx), ctx.Const(ctx.runtime_info.mesh_fallback_task_threads))
|
||||
: workgroup};
|
||||
return ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, address, ctx.u32_zero_value), within,
|
||||
word);
|
||||
}
|
||||
|
||||
void EmitIsbeWriteAttribute(EmitContext& ctx, Id address, Id value) {
|
||||
const Id index{ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.Const(2U))};
|
||||
if (ctx.stage == Stage::Task) {
|
||||
StoreWord(ctx, PayloadWord(ctx, index), value);
|
||||
return;
|
||||
}
|
||||
ctx.OpStore(SharedWord(ctx, ctx.mesh_attributes, Clamped(ctx, index, ctx.mesh_attribute_words)),
|
||||
value);
|
||||
}
|
||||
|
||||
void EmitIsbeWriteMapU8(EmitContext& ctx, Id address, Id value) {
|
||||
if (ctx.stage != Stage::Mesh) {
|
||||
throw NotImplementedException("Stage buffer map write outside mesh programs");
|
||||
}
|
||||
ctx.OpStore(SharedWord(ctx, ctx.mesh_map, Clamped(ctx, address, ctx.mesh_map_bytes)),
|
||||
ctx.OpBitwiseAnd(ctx.U32[1], value, ctx.Const(0xffU)));
|
||||
}
|
||||
|
||||
void EmitIsbeWriteMapU32(EmitContext& ctx, const IR::Value& address, Id value) {
|
||||
// The primitive count: the map is written from its end, so its first word is at byte 3.
|
||||
if (ctx.stage != Stage::Mesh || !address.IsImmediate() || address.U32() != 3) {
|
||||
throw NotImplementedException("32-bit stage buffer map write");
|
||||
}
|
||||
ctx.OpStore(ctx.mesh_primitive_count, value);
|
||||
}
|
||||
|
||||
void EmitMeshPrologue(EmitContext& ctx) {
|
||||
if (ctx.stage == Stage::Task) {
|
||||
// (hookshot) the count starts at nothing: task payload memory holds whatever the last task left, and a task that keeps no meshlet
|
||||
// never writes it. The barrier: every thread has done this before any of them goes on to write the real count
|
||||
StoreWord(ctx, PayloadWord(ctx, ctx.Const(1U)), ctx.u32_zero_value);
|
||||
Barrier(ctx);
|
||||
}
|
||||
if (ctx.stage != Stage::Mesh) {
|
||||
return;
|
||||
}
|
||||
if (ctx.mesh_fallback && ctx.runtime_info.mesh_fallback_task) {
|
||||
// (hookshot) the slot its task gave this workgroup, from the map at its index of the dispatch (TaskEpilogue). Read here,
|
||||
// in the entry block, it serves every use; a load, no branch (see below)
|
||||
ctx.mesh_fallback_slot =
|
||||
LoadWord(ctx, BufferWordAt(ctx, PushU64(ctx, ctx.mesh_fallback_payload_member_index), WorkgroupId(ctx)));
|
||||
}
|
||||
// (local) The stage buffer's map and attribute regions start at zero, as they come out on NVIDIA: workgroup memory holds
|
||||
// whatever the last workgroup on that unit left, and a vertex or an index byte the program left unwritten was that. On an
|
||||
// RX 7700S the foliage drawn into the 2400 x 2400 shadow map around Link came out with hundreds of long thin triangles
|
||||
// across the whole map (vertices from other meshlets' leftovers), dark lines on the ground, the water and Link that
|
||||
// changed every frame. Zeroed, such a primitive is (0,0,0): no area. Straight-line stores, each clamped into its array
|
||||
// (no branch: see below). RX_VK_MESH_NO_ZERO=1: as before
|
||||
static const bool no_zero{std::getenv("RX_VK_MESH_NO_ZERO") != nullptr};
|
||||
if (!no_zero && ctx.mesh_threads) {
|
||||
const Id invocation{LocalInvocation(ctx)};
|
||||
const auto zero_all = [&](Id array, u32 count) {
|
||||
if (!Sirit::ValidId(array) || count == 0) {
|
||||
return;
|
||||
}
|
||||
for (u32 first = 0; first < count; first += ctx.mesh_threads) {
|
||||
const Id at{Clamped(ctx, ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first)), count)};
|
||||
ctx.OpStore(SharedWord(ctx, array, at), ctx.u32_zero_value);
|
||||
}
|
||||
};
|
||||
zero_all(ctx.mesh_map, ctx.mesh_map_bytes);
|
||||
zero_all(ctx.mesh_attributes, ctx.mesh_attribute_words);
|
||||
}
|
||||
if (ctx.mesh_fallback && !ctx.runtime_info.mesh_fallback_task && ctx.mesh_threads) {
|
||||
// (a mesh program with no task reads its workgroup index from the payload: NVN's hardware puts it at byte 0)
|
||||
const Id invocation{LocalInvocation(ctx)};
|
||||
for (u32 first = 0; first < ctx.mesh_info.payload_words; first += ctx.mesh_threads) {
|
||||
const Id at{Clamped(ctx, ctx.OpIAdd(ctx.U32[1], invocation, ctx.Const(first)),
|
||||
ctx.mesh_info.payload_words)};
|
||||
ctx.OpStore(SharedWord(ctx, ctx.mesh_payload, at), ctx.u32_zero_value);
|
||||
}
|
||||
if (ctx.mesh_info.payload_words) {
|
||||
ctx.OpStore(SharedWord(ctx, ctx.mesh_payload, ctx.u32_zero_value), WorkgroupId(ctx));
|
||||
}
|
||||
Barrier(ctx);
|
||||
}
|
||||
// Every invocation stores the zero: no branch here, the entry block's label is a phi
|
||||
// predecessor for later blocks.
|
||||
ctx.OpStore(ctx.mesh_primitive_count, ctx.u32_zero_value);
|
||||
Barrier(ctx);
|
||||
}
|
||||
|
||||
void EmitMeshEpilogue(EmitContext& ctx) {
|
||||
if (ctx.stage == Stage::Task) {
|
||||
TaskEpilogue(ctx);
|
||||
} else if (ctx.stage == Stage::Mesh) {
|
||||
MeshEpilogue(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,41 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
Id EmitSelectU1(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.U1, cond, true_value, false_value);
|
||||
}
|
||||
|
||||
Id EmitSelectU8(EmitContext&, Id, Id, Id) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitSelectU16(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.U16, cond, true_value, false_value);
|
||||
}
|
||||
|
||||
Id EmitSelectU32(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.U32[1], cond, true_value, false_value);
|
||||
}
|
||||
|
||||
Id EmitSelectU64(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.U64, cond, true_value, false_value);
|
||||
}
|
||||
|
||||
Id EmitSelectF16(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.F16[1], cond, true_value, false_value);
|
||||
}
|
||||
|
||||
Id EmitSelectF32(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.F32[1], cond, true_value, false_value);
|
||||
}
|
||||
|
||||
Id EmitSelectF64(EmitContext& ctx, Id cond, Id true_value, Id false_value) {
|
||||
return ctx.OpSelect(ctx.F64[1], cond, true_value, false_value);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,176 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id Pointer(EmitContext& ctx, Id pointer_type, Id array, Id offset, u32 shift) {
|
||||
const Id shift_id{ctx.Const(shift)};
|
||||
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
return ctx.OpAccessChain(pointer_type, array, ctx.u32_zero_value, index);
|
||||
}
|
||||
|
||||
Id Word(EmitContext& ctx, Id offset) {
|
||||
const Id shift_id{ctx.Const(2U)};
|
||||
const Id index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)};
|
||||
return ctx.OpLoad(ctx.U32[1], pointer);
|
||||
}
|
||||
|
||||
std::pair<Id, Id> ExtractArgs(EmitContext& ctx, Id offset, u32 mask, u32 count) {
|
||||
const Id shift{ctx.OpShiftLeftLogical(ctx.U32[1], offset, ctx.Const(3U))};
|
||||
const Id bit{ctx.OpBitwiseAnd(ctx.U32[1], shift, ctx.Const(mask))};
|
||||
const Id count_id{ctx.Const(count)};
|
||||
return {bit, count_id};
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitLoadSharedU8(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{
|
||||
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
|
||||
return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U8, pointer));
|
||||
} else {
|
||||
const auto [bit, count]{ExtractArgs(ctx, offset, 24, 8)};
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], Word(ctx, offset), bit, count);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadSharedS8(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{
|
||||
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
|
||||
return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U8, pointer));
|
||||
} else {
|
||||
const auto [bit, count]{ExtractArgs(ctx, offset, 24, 8)};
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], Word(ctx, offset), bit, count);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
|
||||
return ctx.OpUConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
|
||||
} else {
|
||||
const auto [bit, count]{ExtractArgs(ctx, offset, 16, 16)};
|
||||
return ctx.OpBitFieldUExtract(ctx.U32[1], Word(ctx, offset), bit, count);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadSharedS16(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
|
||||
return ctx.OpSConvert(ctx.U32[1], ctx.OpLoad(ctx.U16, pointer));
|
||||
} else {
|
||||
const auto [bit, count]{ExtractArgs(ctx, offset, 16, 16)};
|
||||
return ctx.OpBitFieldSExtract(ctx.U32[1], Word(ctx, offset), bit, count);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2)};
|
||||
return ctx.OpLoad(ctx.U32[1], pointer);
|
||||
} else {
|
||||
return Word(ctx, offset);
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
|
||||
return ctx.OpLoad(ctx.U32[2], pointer);
|
||||
} else {
|
||||
const Id shift_id{ctx.Const(2U)};
|
||||
const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
const Id next_index{ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(1U))};
|
||||
const Id lhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, base_index)};
|
||||
const Id rhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, next_index)};
|
||||
return ctx.OpCompositeConstruct(ctx.U32[2], ctx.OpLoad(ctx.U32[1], lhs_pointer),
|
||||
ctx.OpLoad(ctx.U32[1], rhs_pointer));
|
||||
}
|
||||
}
|
||||
|
||||
Id EmitLoadSharedU128(EmitContext& ctx, Id offset) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
|
||||
return ctx.OpLoad(ctx.U32[4], pointer);
|
||||
}
|
||||
const Id shift_id{ctx.Const(2U)};
|
||||
const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift_id)};
|
||||
std::array<Id, 4> values{};
|
||||
for (u32 i = 0; i < 4; ++i) {
|
||||
const Id index{i == 0 ? base_index : ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(i))};
|
||||
const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)};
|
||||
values[i] = ctx.OpLoad(ctx.U32[1], pointer);
|
||||
}
|
||||
return ctx.OpCompositeConstruct(ctx.U32[4], values);
|
||||
}
|
||||
|
||||
void EmitWriteSharedU8(EmitContext& ctx, Id offset, Id value) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{
|
||||
ctx.OpAccessChain(ctx.shared_u8, ctx.shared_memory_u8, ctx.u32_zero_value, offset)};
|
||||
ctx.OpStore(pointer, ctx.OpUConvert(ctx.U8, value));
|
||||
} else {
|
||||
ctx.OpFunctionCall(ctx.void_id, ctx.shared_store_u8_func, offset, value);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u16, ctx.shared_memory_u16, offset, 1)};
|
||||
ctx.OpStore(pointer, ctx.OpUConvert(ctx.U16, value));
|
||||
} else {
|
||||
ctx.OpFunctionCall(ctx.void_id, ctx.shared_store_u16_func, offset, value);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
|
||||
Id pointer{};
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
pointer = Pointer(ctx, ctx.shared_u32, ctx.shared_memory_u32, offset, 2);
|
||||
} else {
|
||||
const Id shift{ctx.Const(2U)};
|
||||
const Id word_offset{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)};
|
||||
pointer = ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, word_offset);
|
||||
}
|
||||
ctx.OpStore(pointer, value);
|
||||
}
|
||||
|
||||
void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u32x2, ctx.shared_memory_u32x2, offset, 3)};
|
||||
ctx.OpStore(pointer, value);
|
||||
return;
|
||||
}
|
||||
const Id shift{ctx.Const(2U)};
|
||||
const Id word_offset{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)};
|
||||
const Id next_offset{ctx.OpIAdd(ctx.U32[1], word_offset, ctx.Const(1U))};
|
||||
const Id lhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, word_offset)};
|
||||
const Id rhs_pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, next_offset)};
|
||||
ctx.OpStore(lhs_pointer, ctx.OpCompositeExtract(ctx.U32[1], value, 0U));
|
||||
ctx.OpStore(rhs_pointer, ctx.OpCompositeExtract(ctx.U32[1], value, 1U));
|
||||
}
|
||||
|
||||
void EmitWriteSharedU128(EmitContext& ctx, Id offset, Id value) {
|
||||
if (ctx.uses_explicit_workgroup_layout) {
|
||||
const Id pointer{Pointer(ctx, ctx.shared_u32x4, ctx.shared_memory_u32x4, offset, 4)};
|
||||
ctx.OpStore(pointer, value);
|
||||
return;
|
||||
}
|
||||
const Id shift{ctx.Const(2U)};
|
||||
const Id base_index{ctx.OpShiftRightArithmetic(ctx.U32[1], offset, shift)};
|
||||
for (u32 i = 0; i < 4; ++i) {
|
||||
const Id index{i == 0 ? base_index : ctx.OpIAdd(ctx.U32[1], base_index, ctx.Const(i))};
|
||||
const Id pointer{ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, index)};
|
||||
ctx.OpStore(pointer, ctx.OpCompositeExtract(ctx.U32[1], value, i));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,186 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv.h"
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
void ConvertDepthMode(EmitContext& ctx) {
|
||||
const Id type{ctx.F32[1]};
|
||||
const Id position{ctx.OpLoad(ctx.F32[4], ctx.output_position)};
|
||||
const Id z{ctx.OpCompositeExtract(type, position, 2u)};
|
||||
const Id w{ctx.OpCompositeExtract(type, position, 3u)};
|
||||
const Id screen_depth{ctx.OpFMul(type, ctx.OpFAdd(type, z, w), ctx.Constant(type, 0.5f))};
|
||||
const Id vector{ctx.OpCompositeInsert(ctx.F32[4], screen_depth, position, 2u)};
|
||||
ctx.OpStore(ctx.output_position, vector);
|
||||
}
|
||||
|
||||
void SetFixedPipelinePointSize(EmitContext& ctx) {
|
||||
if (ctx.runtime_info.fixed_state_point_size) {
|
||||
const float point_size{*ctx.runtime_info.fixed_state_point_size};
|
||||
ctx.OpStore(ctx.output_point_size, ctx.Const(point_size));
|
||||
}
|
||||
}
|
||||
|
||||
Id DefaultVarying(EmitContext& ctx, u32 num_components, u32 element, Id zero, Id one,
|
||||
Id default_vector) {
|
||||
switch (num_components) {
|
||||
case 1:
|
||||
return element == 3 ? one : zero;
|
||||
case 2:
|
||||
return ctx.ConstantComposite(ctx.F32[2], zero, element + 1 == 3 ? one : zero);
|
||||
case 3:
|
||||
return ctx.ConstantComposite(ctx.F32[3], zero, zero, element + 2 == 3 ? one : zero);
|
||||
case 4:
|
||||
return default_vector;
|
||||
}
|
||||
throw InvalidArgument("Bad element");
|
||||
}
|
||||
|
||||
Id ComparisonFunction(EmitContext& ctx, CompareFunction comparison, Id operand_1, Id operand_2) {
|
||||
switch (comparison) {
|
||||
case CompareFunction::Never:
|
||||
return ctx.false_value;
|
||||
case CompareFunction::Less:
|
||||
return ctx.OpFOrdLessThan(ctx.U1, operand_1, operand_2);
|
||||
case CompareFunction::Equal:
|
||||
return ctx.OpFOrdEqual(ctx.U1, operand_1, operand_2);
|
||||
case CompareFunction::LessThanEqual:
|
||||
return ctx.OpFOrdLessThanEqual(ctx.U1, operand_1, operand_2);
|
||||
case CompareFunction::Greater:
|
||||
return ctx.OpFOrdGreaterThan(ctx.U1, operand_1, operand_2);
|
||||
case CompareFunction::NotEqual:
|
||||
return ctx.OpFOrdNotEqual(ctx.U1, operand_1, operand_2);
|
||||
case CompareFunction::GreaterThanEqual:
|
||||
return ctx.OpFOrdGreaterThanEqual(ctx.U1, operand_1, operand_2);
|
||||
case CompareFunction::Always:
|
||||
return ctx.true_value;
|
||||
}
|
||||
throw InvalidArgument("Comparison function {}", comparison);
|
||||
}
|
||||
|
||||
void AlphaTest(EmitContext& ctx) {
|
||||
if (!ctx.runtime_info.alpha_test_func) {
|
||||
return;
|
||||
}
|
||||
const auto comparison{*ctx.runtime_info.alpha_test_func};
|
||||
if (comparison == CompareFunction::Always) {
|
||||
return;
|
||||
}
|
||||
if (!Sirit::ValidId(ctx.frag_color[0])) {
|
||||
return;
|
||||
}
|
||||
|
||||
const Id type{ctx.F32[1]};
|
||||
const Id rt0_color{ctx.OpLoad(ctx.F32[4], ctx.frag_color[0])};
|
||||
const Id alpha{ctx.OpCompositeExtract(type, rt0_color, 3u)};
|
||||
|
||||
const Id true_label{ctx.OpLabel()};
|
||||
const Id discard_label{ctx.OpLabel()};
|
||||
const Id alpha_reference{ctx.Const(ctx.runtime_info.alpha_test_reference)};
|
||||
const Id condition{ComparisonFunction(ctx, comparison, alpha, alpha_reference)};
|
||||
|
||||
ctx.OpSelectionMerge(true_label, spv::SelectionControlMask::MaskNone);
|
||||
ctx.OpBranchConditional(condition, true_label, discard_label);
|
||||
ctx.AddLabel(discard_label);
|
||||
ctx.OpKill();
|
||||
ctx.AddLabel(true_label);
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
void EmitPrologue(EmitContext& ctx) {
|
||||
if (ctx.stage == Stage::Fragment && ctx.runtime_info.dual_source_blend) {
|
||||
// Initialize dual-source blending outputs - prevents MoltenVK crash.
|
||||
const Id zero{ctx.Const(0.0f)};
|
||||
const Id one{ctx.Const(1.0f)};
|
||||
const Id default_color{ctx.ConstantComposite(ctx.F32[4], zero, zero, zero, one)};
|
||||
for (u32 i = 0; i < 2; ++i) {
|
||||
if (Sirit::ValidId(ctx.frag_color[i])) {
|
||||
ctx.OpStore(ctx.frag_color[i], default_color);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ctx.stage == Stage::VertexB) {
|
||||
const Id zero{ctx.Const(0.0f)};
|
||||
const Id one{ctx.Const(1.0f)};
|
||||
const Id default_vector{ctx.ConstantComposite(ctx.F32[4], zero, zero, zero, one)};
|
||||
ctx.OpStore(ctx.output_position, default_vector);
|
||||
for (const auto& info : ctx.output_generics) {
|
||||
if (info[0].num_components == 0) {
|
||||
continue;
|
||||
}
|
||||
u32 element{0};
|
||||
while (element < 4) {
|
||||
const auto& element_info{info[element]};
|
||||
const u32 num{element_info.num_components};
|
||||
const Id value{DefaultVarying(ctx, num, element, zero, one, default_vector)};
|
||||
ctx.OpStore(element_info.id, value);
|
||||
element += num;
|
||||
}
|
||||
}
|
||||
|
||||
if (Sirit::ValidId(ctx.clip_distances)) {
|
||||
for (u32 i = 0; i < ctx.profile.max_user_clip_distances; ++i) {
|
||||
if (!clip_distance_written.test(i)) {
|
||||
const Id idx = ctx.Const(i);
|
||||
const Id element = ctx.OpAccessChain(ctx.output_f32, ctx.clip_distances, idx);
|
||||
ctx.OpStore(element, ctx.Const(0.0f));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ctx.stage == Stage::VertexB || ctx.stage == Stage::Geometry) {
|
||||
SetFixedPipelinePointSize(ctx);
|
||||
}
|
||||
if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) {
|
||||
EmitMeshPrologue(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitEpilogue(EmitContext& ctx) {
|
||||
if (ctx.stage == Stage::Task || ctx.stage == Stage::Mesh) {
|
||||
EmitMeshEpilogue(ctx);
|
||||
return;
|
||||
}
|
||||
if (ctx.stage == Stage::VertexB && ctx.runtime_info.convert_depth_mode &&
|
||||
!ctx.profile.support_native_ndc) {
|
||||
ConvertDepthMode(ctx);
|
||||
}
|
||||
if (ctx.stage == Stage::Fragment) {
|
||||
AlphaTest(ctx);
|
||||
}
|
||||
}
|
||||
|
||||
void EmitEmitVertex(EmitContext& ctx, const IR::Value& stream) {
|
||||
if (ctx.runtime_info.convert_depth_mode && !ctx.profile.support_native_ndc) {
|
||||
ConvertDepthMode(ctx);
|
||||
}
|
||||
if (!ctx.profile.support_geometry_streams) {
|
||||
throw NotImplementedException("Geometry streams");
|
||||
} else if (stream.IsImmediate()) {
|
||||
ctx.OpEmitStreamVertex(ctx.Def(stream));
|
||||
} else {
|
||||
LOG_WARNING(Shader_SPIRV, "Stream is not immediate");
|
||||
ctx.OpEmitStreamVertex(ctx.u32_zero_value);
|
||||
}
|
||||
// Restore fixed pipeline point size after emitting the vertex
|
||||
SetFixedPipelinePointSize(ctx);
|
||||
}
|
||||
|
||||
void EmitEndPrimitive(EmitContext& ctx, const IR::Value& stream) {
|
||||
if (!ctx.profile.support_geometry_streams) {
|
||||
throw NotImplementedException("Geometry streams");
|
||||
} else if (stream.IsImmediate()) {
|
||||
ctx.OpEndStreamPrimitive(ctx.Def(stream));
|
||||
} else {
|
||||
LOG_WARNING(Shader_SPIRV, "Stream is not immediate");
|
||||
ctx.OpEndStreamPrimitive(ctx.u32_zero_value);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
@@ -0,0 +1,47 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include <cstdlib>
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
// Registers read before any write are zero rather than OpUndef, as NVIDIA hardware and drivers
|
||||
// give them; AMD's OpUndef is whatever the register held. Title 1's glow pass exits before
|
||||
// writing its colour on some pixels, which AMD blended in as green patches on the road.
|
||||
// EDEN_NX2_UNDEF_ZERO=0 emits OpUndef again.
|
||||
bool UndefIsZero() {
|
||||
static const bool zero = [] {
|
||||
const char* const value = std::getenv("EDEN_NX2_UNDEF_ZERO");
|
||||
return value == nullptr || value[0] != '0';
|
||||
}();
|
||||
return zero;
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitUndefU1(EmitContext& ctx) {
|
||||
return UndefIsZero() ? ctx.false_value : ctx.OpUndef(ctx.U1);
|
||||
}
|
||||
|
||||
Id EmitUndefU8(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitUndefU16(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
Id EmitUndefU32(EmitContext& ctx) {
|
||||
return UndefIsZero() ? ctx.u32_zero_value : ctx.OpUndef(ctx.U32[1]);
|
||||
}
|
||||
|
||||
Id EmitUndefU64(EmitContext&) {
|
||||
throw NotImplementedException("SPIR-V Instruction");
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
540
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_warp.cpp
Normal file
540
gpu/vk/eden/shader_recompiler/backend/spirv/emit_spirv_warp.cpp
Normal file
@@ -0,0 +1,540 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#include "shader_recompiler/backend/spirv/emit_spirv_instructions.h"
|
||||
#include "shader_recompiler/backend/spirv/spirv_emit_context.h"
|
||||
#include "shader_recompiler/backend/spirv/mma_layout.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
namespace {
|
||||
Id SubgroupScope(EmitContext& ctx) {
|
||||
return ctx.Const(static_cast<u32>(spv::Scope::Subgroup));
|
||||
}
|
||||
|
||||
bool StageSupportsSubgroups(EmitContext& ctx) {
|
||||
return ctx.profile.SupportsSubgroupStage(ctx.stage);
|
||||
}
|
||||
|
||||
Id GetThreadId(EmitContext& ctx) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.u32_zero_value;
|
||||
}
|
||||
return ctx.OpLoad(ctx.U32[1], ctx.subgroup_local_invocation_id);
|
||||
}
|
||||
|
||||
Id WarpExtract(EmitContext& ctx, Id value) {
|
||||
const Id thread_id{GetThreadId(ctx)};
|
||||
const Id local_index{ctx.OpShiftRightArithmetic(ctx.U32[1], thread_id, ctx.Const(5U))};
|
||||
if (ctx.profile.has_broken_spirv_subgroup_mask_vector_extract_dynamic) {
|
||||
const Id c0_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(0U)),
|
||||
ctx.OpCompositeExtract(ctx.U32[1], value, 0U), ctx.Const(0U))};
|
||||
const Id c1_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(1U)),
|
||||
ctx.OpCompositeExtract(ctx.U32[1], value, 1U), ctx.Const(0U))};
|
||||
const Id c2_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(2U)),
|
||||
ctx.OpCompositeExtract(ctx.U32[1], value, 2U), ctx.Const(0U))};
|
||||
const Id c3_sel{ctx.OpSelect(ctx.U32[1], ctx.OpIEqual(ctx.U1, local_index, ctx.Const(3U)),
|
||||
ctx.OpCompositeExtract(ctx.U32[1], value, 3U), ctx.Const(0U))};
|
||||
const Id c0_or_c1{ctx.OpBitwiseOr(ctx.U32[1], c0_sel, c1_sel)};
|
||||
const Id c2_or_c3{ctx.OpBitwiseOr(ctx.U32[1], c2_sel, c3_sel)};
|
||||
const Id c0_or_c1_or_c2_or_c3{ctx.OpBitwiseOr(ctx.U32[1], c0_or_c1, c2_or_c3)};
|
||||
return c0_or_c1_or_c2_or_c3;
|
||||
} else {
|
||||
return ctx.OpVectorExtractDynamic(ctx.U32[1], value, local_index);
|
||||
}
|
||||
}
|
||||
|
||||
Id LoadMask(EmitContext& ctx, Id mask) {
|
||||
const Id value{ctx.OpLoad(ctx.U32[4], mask)};
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], value, 0U);
|
||||
}
|
||||
return WarpExtract(ctx, value);
|
||||
}
|
||||
|
||||
void SetInBoundsFlag(IR::Inst* inst, Id result) {
|
||||
IR::Inst* const in_bounds{inst->GetAssociatedPseudoOperation(IR::Opcode::GetInBoundsFromOp)};
|
||||
if (!in_bounds) {
|
||||
return;
|
||||
}
|
||||
in_bounds->SetDefinition(result);
|
||||
in_bounds->Invalidate();
|
||||
}
|
||||
|
||||
Id ComputeMinThreadId(EmitContext& ctx, Id thread_id, Id segmentation_mask) {
|
||||
return ctx.OpBitwiseAnd(ctx.U32[1], thread_id, segmentation_mask);
|
||||
}
|
||||
|
||||
Id ComputeMaxThreadId(EmitContext& ctx, Id min_thread_id, Id clamp, Id not_seg_mask) {
|
||||
return ctx.OpBitwiseOr(ctx.U32[1], min_thread_id,
|
||||
ctx.OpBitwiseAnd(ctx.U32[1], clamp, not_seg_mask));
|
||||
}
|
||||
|
||||
Id GetMaxThreadId(EmitContext& ctx, Id thread_id, Id clamp, Id segmentation_mask) {
|
||||
const Id not_seg_mask{ctx.OpNot(ctx.U32[1], segmentation_mask)};
|
||||
const Id min_thread_id{ComputeMinThreadId(ctx, thread_id, segmentation_mask)};
|
||||
return ComputeMaxThreadId(ctx, min_thread_id, clamp, not_seg_mask);
|
||||
}
|
||||
|
||||
Id HostThreadId(EmitContext& ctx, Id thread_id) {
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return thread_id;
|
||||
}
|
||||
const Id partition_idx{ctx.OpShiftRightLogical(ctx.U32[1], GetThreadId(ctx), ctx.Const(5u))};
|
||||
const Id partition_base{ctx.OpShiftLeftLogical(ctx.U32[1], partition_idx, ctx.Const(5u))};
|
||||
return ctx.OpIAdd(ctx.U32[1], thread_id, partition_base);
|
||||
}
|
||||
|
||||
Id GuestLane(EmitContext& ctx, Id index) {
|
||||
return ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U));
|
||||
}
|
||||
|
||||
bool UseShuffleFallback(EmitContext& ctx) {
|
||||
return ctx.profile.has_broken_spirv_subgroup_shuffle ||
|
||||
(ctx.profile.has_broken_spirv_fragment_shuffle && ctx.stage == Stage::Fragment);
|
||||
}
|
||||
|
||||
Id ShuffleAbsolute(EmitContext& ctx, Id value, Id src_thread_id) {
|
||||
if (!UseShuffleFallback(ctx)) {
|
||||
return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id);
|
||||
}
|
||||
Id result{ctx.u32_zero_value};
|
||||
for (u32 lane = 0; lane < ctx.profile.max_subgroup_size; ++lane) {
|
||||
const Id read{
|
||||
ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, ctx.Const(lane))};
|
||||
const Id matches{ctx.OpIEqual(ctx.U1, src_thread_id, ctx.Const(lane))};
|
||||
result = ctx.OpSelect(ctx.U32[1], matches, read, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Id ShuffleRelative(EmitContext& ctx, Id value, Id delta, Id src_thread_id, spv::Op op) {
|
||||
if (!ctx.profile.support_shuffle_relative || UseShuffleFallback(ctx)) {
|
||||
return ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id));
|
||||
}
|
||||
const Id result{ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, delta)};
|
||||
ctx.opcode_rewrites.emplace_back(result.value, op);
|
||||
return result;
|
||||
}
|
||||
|
||||
Id BroadcastLane(EmitContext& ctx, Id value, u32 lane) {
|
||||
Id result{
|
||||
ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, ctx.Const(lane))};
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return result;
|
||||
}
|
||||
const Id partition_idx{ctx.OpShiftRightLogical(ctx.U32[1], GetThreadId(ctx), ctx.Const(5u))};
|
||||
for (u32 base = 32; base < ctx.profile.max_subgroup_size; base += 32) {
|
||||
const Id read{ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value,
|
||||
ctx.Const(base + lane))};
|
||||
const Id matches{ctx.OpIEqual(ctx.U1, partition_idx, ctx.Const(base >> 5))};
|
||||
result = ctx.OpSelect(ctx.U32[1], matches, read, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} // Anonymous namespace
|
||||
|
||||
Id EmitLaneId(EmitContext& ctx) {
|
||||
const Id id{GetThreadId(ctx)};
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return id;
|
||||
}
|
||||
return ctx.OpBitwiseAnd(ctx.U32[1], id, ctx.Const(31U));
|
||||
}
|
||||
|
||||
Id EmitVoteAll(EmitContext& ctx, Id pred) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return pred;
|
||||
}
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return ctx.OpGroupNonUniformAll(ctx.U1, SubgroupScope(ctx), pred);
|
||||
}
|
||||
const Id mask_ballot{
|
||||
ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), ctx.true_value)};
|
||||
const Id active_mask{WarpExtract(ctx, mask_ballot)};
|
||||
const Id ballot{
|
||||
WarpExtract(ctx, ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred))};
|
||||
const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], ballot, active_mask)};
|
||||
return ctx.OpIEqual(ctx.U1, lhs, active_mask);
|
||||
}
|
||||
|
||||
Id EmitVoteAny(EmitContext& ctx, Id pred) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return pred;
|
||||
}
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return ctx.OpGroupNonUniformAny(ctx.U1, SubgroupScope(ctx), pred);
|
||||
}
|
||||
const Id mask_ballot{
|
||||
ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), ctx.true_value)};
|
||||
const Id active_mask{WarpExtract(ctx, mask_ballot)};
|
||||
const Id ballot{
|
||||
WarpExtract(ctx, ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred))};
|
||||
const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], ballot, active_mask)};
|
||||
return ctx.OpINotEqual(ctx.U1, lhs, ctx.u32_zero_value);
|
||||
}
|
||||
|
||||
Id EmitVoteEqual(EmitContext& ctx, Id pred) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.true_value;
|
||||
}
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return ctx.OpGroupNonUniformAllEqual(ctx.U1, SubgroupScope(ctx), pred);
|
||||
}
|
||||
const Id mask_ballot{
|
||||
ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), ctx.true_value)};
|
||||
const Id active_mask{WarpExtract(ctx, mask_ballot)};
|
||||
const Id ballot{
|
||||
WarpExtract(ctx, ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred))};
|
||||
const Id lhs{ctx.OpBitwiseXor(ctx.U32[1], ballot, active_mask)};
|
||||
return ctx.OpLogicalOr(ctx.U1, ctx.OpIEqual(ctx.U1, lhs, ctx.u32_zero_value),
|
||||
ctx.OpIEqual(ctx.U1, lhs, active_mask));
|
||||
}
|
||||
|
||||
Id EmitSubgroupBallot(EmitContext& ctx, Id pred) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.OpSelect(ctx.U32[1], pred, ctx.Const(1u), ctx.u32_zero_value);
|
||||
}
|
||||
const Id ballot{ctx.OpGroupNonUniformBallot(ctx.U32[4], SubgroupScope(ctx), pred)};
|
||||
if (!ctx.profile.warp_size_potentially_larger_than_guest) {
|
||||
return ctx.OpCompositeExtract(ctx.U32[1], ballot, 0U);
|
||||
}
|
||||
return WarpExtract(ctx, ballot);
|
||||
}
|
||||
|
||||
Id EmitSubgroupEqMask(EmitContext& ctx) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.Const(1u);
|
||||
}
|
||||
return LoadMask(ctx, ctx.subgroup_mask_eq);
|
||||
}
|
||||
|
||||
Id EmitSubgroupLtMask(EmitContext& ctx) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.u32_zero_value;
|
||||
}
|
||||
return LoadMask(ctx, ctx.subgroup_mask_lt);
|
||||
}
|
||||
|
||||
Id EmitSubgroupLeMask(EmitContext& ctx) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.Const(1u);
|
||||
}
|
||||
return LoadMask(ctx, ctx.subgroup_mask_le);
|
||||
}
|
||||
|
||||
Id EmitSubgroupGtMask(EmitContext& ctx) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.u32_zero_value;
|
||||
}
|
||||
return LoadMask(ctx, ctx.subgroup_mask_gt);
|
||||
}
|
||||
|
||||
Id EmitSubgroupGeMask(EmitContext& ctx) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return ctx.Const(1u);
|
||||
}
|
||||
return LoadMask(ctx, ctx.subgroup_mask_ge);
|
||||
}
|
||||
|
||||
Id EmitShuffleIndex(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask) {
|
||||
const Id not_seg_mask{ctx.OpNot(ctx.U32[1], segmentation_mask)};
|
||||
const Id thread_id{EmitLaneId(ctx)};
|
||||
const Id min_thread_id{ComputeMinThreadId(ctx, thread_id, segmentation_mask)};
|
||||
const Id max_thread_id{ComputeMaxThreadId(ctx, min_thread_id, clamp, not_seg_mask)};
|
||||
|
||||
const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], GuestLane(ctx, index), not_seg_mask)};
|
||||
const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], lhs, min_thread_id)};
|
||||
const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)};
|
||||
|
||||
SetInBoundsFlag(inst, in_range);
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
return value;
|
||||
}
|
||||
const IR::Value lane{inst->Arg(1).Resolve()};
|
||||
const IR::Value segment{inst->Arg(3).Resolve()};
|
||||
if (lane.IsImmediate() && segment.IsImmediate() && segment.U32() == 0) {
|
||||
return ctx.OpSelect(ctx.U32[1], in_range, BroadcastLane(ctx, value, lane.U32() & 31),
|
||||
value);
|
||||
}
|
||||
const Id shuffled{ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id))};
|
||||
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
|
||||
}
|
||||
|
||||
Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
SetInBoundsFlag(inst, ctx.false_value);
|
||||
return value;
|
||||
}
|
||||
const Id delta{GuestLane(ctx, index)};
|
||||
const Id thread_id{EmitLaneId(ctx)};
|
||||
const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
|
||||
const Id src_thread_id{ctx.OpISub(ctx.U32[1], thread_id, delta)};
|
||||
const Id in_range{ctx.OpSGreaterThanEqual(ctx.U1, src_thread_id, max_thread_id)};
|
||||
|
||||
SetInBoundsFlag(inst, in_range);
|
||||
const Id shuffled{ShuffleRelative(ctx, value, delta, src_thread_id,
|
||||
spv::Op::OpGroupNonUniformShuffleUp)};
|
||||
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
|
||||
}
|
||||
|
||||
Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
SetInBoundsFlag(inst, ctx.false_value);
|
||||
return value;
|
||||
}
|
||||
const Id delta{GuestLane(ctx, index)};
|
||||
const Id thread_id{EmitLaneId(ctx)};
|
||||
const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
|
||||
const Id src_thread_id{ctx.OpIAdd(ctx.U32[1], thread_id, delta)};
|
||||
const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)};
|
||||
|
||||
SetInBoundsFlag(inst, in_range);
|
||||
const Id shuffled{ShuffleRelative(ctx, value, delta, src_thread_id,
|
||||
spv::Op::OpGroupNonUniformShuffleDown)};
|
||||
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
|
||||
}
|
||||
|
||||
Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp,
|
||||
Id segmentation_mask) {
|
||||
if (!StageSupportsSubgroups(ctx)) {
|
||||
SetInBoundsFlag(inst, ctx.false_value);
|
||||
return value;
|
||||
}
|
||||
const Id mask{GuestLane(ctx, index)};
|
||||
const Id thread_id{EmitLaneId(ctx)};
|
||||
const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)};
|
||||
const Id src_thread_id{ctx.OpBitwiseXor(ctx.U32[1], thread_id, mask)};
|
||||
const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)};
|
||||
|
||||
SetInBoundsFlag(inst, in_range);
|
||||
// (hookshot) through the broadcasts where shuffles are broken (UseShuffleFallback: AMD's fragment stages, Lichii #273),
|
||||
// as the absolute and relative shuffles are; #273 left SHFL.BFLY native, and some of Kokiri Forest's plants use it
|
||||
const Id shuffled{UseShuffleFallback(ctx)
|
||||
? ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id))
|
||||
: ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, mask)};
|
||||
return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value);
|
||||
}
|
||||
|
||||
Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) {
|
||||
if (ctx.profile.support_quad_shuffles) {
|
||||
return ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, lane);
|
||||
}
|
||||
const Id base{ctx.OpBitwiseAnd(ctx.U32[1], GetThreadId(ctx), ctx.Const(~3u))};
|
||||
const Id local_lane{ctx.OpBitwiseAnd(ctx.U32[1], lane, ctx.Const(3u))};
|
||||
const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], base, local_lane)};
|
||||
return ShuffleAbsolute(ctx, value, src_thread_id);
|
||||
}
|
||||
|
||||
Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction) {
|
||||
if (ctx.profile.support_quad_shuffles) {
|
||||
const Id result{
|
||||
ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, direction)};
|
||||
ctx.opcode_rewrites.emplace_back(result.value, spv::Op::OpGroupNonUniformQuadSwap);
|
||||
return result;
|
||||
}
|
||||
const Id xor_mask{ctx.OpIAdd(ctx.U32[1], direction, ctx.Const(1u))};
|
||||
// (hookshot) not through UseShuffleFallback's broadcasts: a program with quad shuffles alone (uses_quad_shuffles) has
|
||||
// neither the lane id input nor the ballot capability they need, and the SPIR-V made with them crashed the start
|
||||
return ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, xor_mask);
|
||||
}
|
||||
|
||||
Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle) {
|
||||
const Id three{ctx.Const(3U)};
|
||||
Id mask{GetThreadId(ctx)};
|
||||
mask = ctx.OpBitwiseAnd(ctx.U32[1], mask, three);
|
||||
mask = ctx.OpShiftLeftLogical(ctx.U32[1], mask, ctx.Const(1U));
|
||||
mask = ctx.OpShiftRightLogical(ctx.U32[1], swizzle, mask);
|
||||
mask = ctx.OpBitwiseAnd(ctx.U32[1], mask, three);
|
||||
|
||||
const Id modifier_a{ctx.OpVectorExtractDynamic(ctx.F32[1], ctx.fswzadd_lut_a, mask)};
|
||||
const Id modifier_b{ctx.OpVectorExtractDynamic(ctx.F32[1], ctx.fswzadd_lut_b, mask)};
|
||||
|
||||
const Id result_a{ctx.OpFMul(ctx.F32[1], op_a, modifier_a)};
|
||||
const Id result_b{ctx.OpFMul(ctx.F32[1], op_b, modifier_b)};
|
||||
return ctx.OpFAdd(ctx.F32[1], result_a, result_b);
|
||||
}
|
||||
|
||||
Id EmitDPdxFine(EmitContext& ctx, Id op_a) {
|
||||
return ctx.OpDPdxFine(ctx.F32[1], op_a);
|
||||
}
|
||||
|
||||
Id EmitDPdyFine(EmitContext& ctx, Id op_a) {
|
||||
return ctx.OpDPdyFine(ctx.F32[1], op_a);
|
||||
}
|
||||
|
||||
Id EmitDPdxCoarse(EmitContext& ctx, Id op_a) {
|
||||
return ctx.OpDPdxCoarse(ctx.F32[1], op_a);
|
||||
}
|
||||
|
||||
Id EmitDPdyCoarse(EmitContext& ctx, Id op_a) {
|
||||
return ctx.OpDPdyCoarse(ctx.F32[1], op_a);
|
||||
}
|
||||
|
||||
// (hookshot) HMMA in software, from Lichii (zortos293/Lichii, emit_spirv_tensor.cpp, GPL-3.0): the 32 lanes' fragments
|
||||
// through subgroup shuffles in one helper function, accumulated with FMA in f32 and rounded to f16 after every 8
|
||||
// columns of K, as HMMA.F16 does. It needs no cooperative matrices (RX_VK_MMA_SOFTWARE=1, vk_coopmat.cpp).
|
||||
void DefineSoftwareMatrixFunction(EmitContext& ctx) {
|
||||
{
|
||||
constexpr bool variant = true;
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniform);
|
||||
ctx.AddCapability(spv::Capability::GroupNonUniformShuffle);
|
||||
if (!ctx.profile.SupportsSubgroupStage(ctx.stage) ||
|
||||
ctx.profile.has_broken_spirv_subgroup_shuffle)
|
||||
throw NotImplementedException("Software HMMA requires subgroup shuffle support");
|
||||
const Id u32ty = ctx.U32[1], f32ty = ctx.F32[1];
|
||||
const auto imm = [&](u32 v) { return ctx.Const(v); };
|
||||
const auto add = [&](Id a, Id b) { return ctx.OpIAdd(u32ty, a, b); };
|
||||
const auto and_ = [&](Id a, u32 b) { return ctx.OpBitwiseAnd(u32ty, a, imm(b)); };
|
||||
const auto unpack = [&](Id v) { return ctx.OpUnpackHalf2x16(ctx.F32[2], v); };
|
||||
const auto round_half = [&](Id value) {
|
||||
// Do not express the intermediate rounding as unpack(pack(value)): drivers may
|
||||
// eliminate that round trip. Round normal significands explicitly, ties to even;
|
||||
// half subnormals lie on the exactly representable 2^-24 grid.
|
||||
const Id bits = ctx.OpBitcast(u32ty, value);
|
||||
const Id magnitude = and_(bits, 0x7fffffffU);
|
||||
const Id tie = and_(ctx.OpShiftRightLogical(u32ty, magnitude, imm(13)), 1);
|
||||
const Id rounded = and_(add(add(magnitude, imm(0xfff)), tie), ~0x1fffU);
|
||||
const Id normal = ctx.OpSelect(u32ty,
|
||||
ctx.OpUGreaterThanEqual(ctx.U1, rounded, imm(0x47800000)), imm(0x7f800000), rounded);
|
||||
const Id scaled = ctx.OpFMul(f32ty, ctx.OpBitcast(f32ty, magnitude), ctx.Const(16777216.0f));
|
||||
const Id subnormal = ctx.OpBitcast(u32ty,
|
||||
ctx.OpFMul(f32ty, ctx.OpRoundEven(f32ty, scaled), ctx.Const(0x1p-24f)));
|
||||
const Id finite = ctx.OpSelect(u32ty,
|
||||
ctx.OpUGreaterThanEqual(ctx.U1, magnitude, imm(0x38800000)), normal, subnormal);
|
||||
const Id result = ctx.OpSelect(u32ty,
|
||||
ctx.OpUGreaterThanEqual(ctx.U1, magnitude, imm(0x7f800000)), magnitude, finite);
|
||||
return ctx.OpBitcast(f32ty, ctx.OpBitwiseOr(u32ty, result, and_(bits, 0x80000000)));
|
||||
};
|
||||
const Id type = ctx.TypeFunction(ctx.U32[2], ctx.U32[4], ctx.U32[2], ctx.U32[2]);
|
||||
const Id fn = ctx.OpFunction(ctx.U32[2], spv::FunctionControlMask::DontInline, type);
|
||||
ctx.software_matrix_function = fn;
|
||||
const Id matrix_a = ctx.OpFunctionParameter(ctx.U32[4]);
|
||||
const Id matrix_b = ctx.OpFunctionParameter(ctx.U32[2]);
|
||||
const Id matrix_c = ctx.OpFunctionParameter(ctx.U32[2]);
|
||||
ctx.AddLabel();
|
||||
const Id counter = ctx.AddLocalVariable(ctx.TypePointer(spv::StorageClass::Function, u32ty),
|
||||
spv::StorageClass::Function);
|
||||
std::array<Id, 4> accumulators;
|
||||
for (auto& acc : accumulators)
|
||||
acc = ctx.AddLocalVariable(ctx.TypePointer(spv::StorageClass::Function, f32ty),
|
||||
spv::StorageClass::Function);
|
||||
std::array<Id, 8> inputs;
|
||||
for (u32 i = 0; i < 4; ++i)
|
||||
inputs[i] = ctx.OpCompositeExtract(u32ty, matrix_a, i);
|
||||
for (u32 i = 0; i < 2; ++i) {
|
||||
inputs[4 + i] = ctx.OpCompositeExtract(u32ty, matrix_b, i);
|
||||
inputs[6 + i] = ctx.OpCompositeExtract(u32ty, matrix_c, i);
|
||||
}
|
||||
for (u32 row = 0; row < 2; ++row) {
|
||||
const Id c = unpack(inputs[6 + row]);
|
||||
for (u32 col = 0; col < 2; ++col)
|
||||
ctx.OpStore(accumulators[row * 2 + col], ctx.OpCompositeExtract(f32ty, c, col));
|
||||
}
|
||||
ctx.OpStore(counter, imm(0));
|
||||
const Id lane = ctx.OpLoad(u32ty, ctx.subgroup_local_invocation_id);
|
||||
const Id warp_base = and_(lane, ~31U);
|
||||
const Id row_base = and_(lane, 28);
|
||||
const Id col_base = ctx.OpShiftLeftLogical(u32ty, and_(lane, 3), imm(3));
|
||||
const Id header = ctx.OpLabel(), body = ctx.OpLabel(), next = ctx.OpLabel(), end = ctx.OpLabel();
|
||||
ctx.OpBranch(header);
|
||||
ctx.AddLabel(header);
|
||||
const Id pair = ctx.OpLoad(u32ty, counter);
|
||||
const Id more = ctx.OpULessThan(ctx.U1, pair, imm(variant ? 8 : 4));
|
||||
ctx.OpLoopMerge(end, next, spv::LoopControlMask::DontUnroll);
|
||||
ctx.OpBranchConditional(more, body, end);
|
||||
ctx.AddLabel(body);
|
||||
const Id upper = ctx.OpUGreaterThanEqual(ctx.U1, pair, imm(4));
|
||||
const Id component = and_(pair, 3);
|
||||
const Id a_lane = add(warp_base, add(row_base, component));
|
||||
const Id b_lane = add(warp_base, add(col_base, component));
|
||||
const Id b = ctx.OpSelect(u32ty, upper, inputs[5], inputs[4]);
|
||||
std::array<Id, 2> bs;
|
||||
for (u32 col = 0; col < 2; ++col)
|
||||
bs[col] = unpack(ctx.OpGroupNonUniformShuffle(u32ty, imm(3), b,
|
||||
add(b_lane, imm(col * 4))));
|
||||
const Id round_now = ctx.OpIEqual(ctx.U1, component, imm(3));
|
||||
for (u32 row = 0; row < 2; ++row) {
|
||||
const Id a = ctx.OpSelect(u32ty, upper, inputs[row + 2], inputs[row]);
|
||||
const Id as = unpack(ctx.OpGroupNonUniformShuffle(u32ty, imm(3), a, a_lane));
|
||||
for (u32 col = 0; col < 2; ++col) {
|
||||
Id value = ctx.OpLoad(f32ty, accumulators[row * 2 + col]);
|
||||
for (u32 half = 0; half < 2; ++half) {
|
||||
value = ctx.OpFma(f32ty, ctx.OpCompositeExtract(f32ty, as, half),
|
||||
ctx.OpCompositeExtract(f32ty, bs[col], half), value);
|
||||
ctx.Decorate(value, spv::Decoration::NoContraction);
|
||||
}
|
||||
ctx.OpStore(accumulators[row * 2 + col], value);
|
||||
}
|
||||
}
|
||||
// HMMA.F16 rounds after each K=8 group, including the midpoint of K=16.
|
||||
// OpSelect evaluates both operands: keep the expensive half conversion in
|
||||
// a branch so the other three pairs do no rounding work. The counter is
|
||||
// identical in every participating lane and all shuffles precede this branch.
|
||||
const Id round_block = ctx.OpLabel(), round_merge = ctx.OpLabel();
|
||||
ctx.OpSelectionMerge(round_merge, spv::SelectionControlMask::DontFlatten);
|
||||
ctx.OpBranchConditional(round_now, round_block, round_merge);
|
||||
ctx.AddLabel(round_block);
|
||||
for (const Id accumulator : accumulators) {
|
||||
ctx.OpStore(accumulator, round_half(ctx.OpLoad(f32ty, accumulator)));
|
||||
}
|
||||
ctx.OpBranch(round_merge);
|
||||
ctx.AddLabel(round_merge);
|
||||
ctx.OpBranch(next);
|
||||
ctx.AddLabel(next);
|
||||
ctx.OpStore(counter, add(pair, imm(1)));
|
||||
ctx.OpBranch(header);
|
||||
ctx.AddLabel(end);
|
||||
std::array<Id, 2> packed;
|
||||
for (u32 row = 0; row < 2; ++row)
|
||||
packed[row] = ctx.OpPackHalf2x16(u32ty, ctx.OpCompositeConstruct(ctx.F32[2],
|
||||
ctx.OpLoad(f32ty, accumulators[row * 2]), ctx.OpLoad(f32ty, accumulators[row * 2 + 1])));
|
||||
ctx.OpReturnValue(ctx.OpCompositeConstruct(ctx.U32[2], packed[0], packed[1]));
|
||||
ctx.OpFunctionEnd();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// (hookshot) HMMA_nx2: D(16x8) = A(16x16) * B(16x8) + C on the subgroup's cooperative matrices. Each
|
||||
// lane's fragment registers hold its elements in the console's order (two f16 a word); they go to and come
|
||||
// from the device's elements in its own order (mma_layout.h, measured by vk_coopmat.cpp).
|
||||
Id EmitMatrixMulAddF16(EmitContext& ctx, Id a, Id b, Id c) {
|
||||
if (mma_software) return ctx.OpFunctionCall(ctx.U32[2], ctx.software_matrix_function, a, b, c);
|
||||
const Id f16{ctx.F16[1]};
|
||||
const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Subgroup))};
|
||||
const auto type = [&](u32 rows, u32 columns, spv::CooperativeMatrixUse use) {
|
||||
return ctx.TypeCooperativeMatrixKHR(f16, scope, ctx.Const(rows), ctx.Const(columns),
|
||||
ctx.Const(static_cast<u32>(use)));
|
||||
};
|
||||
const Id a_type{type(16, 16, spv::CooperativeMatrixUse::MatrixAKHR)};
|
||||
const Id b_type{type(16, 8, spv::CooperativeMatrixUse::MatrixBKHR)};
|
||||
const Id c_type{type(16, 8, spv::CooperativeMatrixUse::MatrixAccumulatorKHR)};
|
||||
const Id zero{ctx.Constant(f16, 0U)};
|
||||
const auto load = [&](Id matrix_type, Id words, u32 count, const auto& order) {
|
||||
Id m{ctx.OpCompositeConstruct(matrix_type, zero)};
|
||||
for (u32 w = 0; w < count; ++w) {
|
||||
const Id word{count == 1 ? words : ctx.OpCompositeExtract(ctx.U32[1], words, w)};
|
||||
const Id pair{ctx.OpBitcast(ctx.F16[2], word)};
|
||||
for (u32 h = 0; h < 2; ++h) {
|
||||
m = ctx.OpCompositeInsert(matrix_type, ctx.OpCompositeExtract(f16, pair, h), m, order[w * 2 + h]);
|
||||
}
|
||||
}
|
||||
return m;
|
||||
};
|
||||
const Id d{ctx.OpCooperativeMatrixMulAddKHR(c_type, load(a_type, a, 4, mma_order_a), load(b_type, b, 2, mma_order_b),
|
||||
load(c_type, c, 2, mma_order_c))};
|
||||
Id out[2];
|
||||
for (u32 w = 0; w < 2; ++w) {
|
||||
const Id pair{ctx.OpCompositeConstruct(ctx.F16[2], ctx.OpCompositeExtract(f16, d, mma_order_c[w * 2]),
|
||||
ctx.OpCompositeExtract(f16, d, mma_order_c[w * 2 + 1]))};
|
||||
out[w] = ctx.OpBitcast(ctx.U32[1], pair);
|
||||
}
|
||||
return ctx.OpCompositeConstruct(ctx.U32[2], out[0], out[1]);
|
||||
}
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
29
gpu/vk/eden/shader_recompiler/backend/spirv/mma_layout.h
Normal file
29
gpu/vk/eden/shader_recompiler/backend/spirv/mma_layout.h
Normal file
@@ -0,0 +1,29 @@
|
||||
// (hookshot) The device's element order of the 16x8x16 f16 cooperative matrices HMMA_nx2 runs on.
|
||||
//
|
||||
// A lane's HMMA fragment registers hold its matrix elements in the console's order (PTX mma.m16n8k16's: two f16
|
||||
// a register, low half first). The host's cooperative matrices hold each lane's share in an order of the
|
||||
// implementation's own; vk_coopmat.cpp measures it at start, and only when every lane holds the same positions in
|
||||
// both does the back end turn HMMA_nx2 on. Element e of the console's fragment is then the device's element
|
||||
// mma_order_x[e].
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
inline std::array<uint32_t, 8> mma_order_a{0, 1, 2, 3, 4, 5, 6, 7};
|
||||
inline std::array<uint32_t, 4> mma_order_b{0, 1, 2, 3};
|
||||
inline std::array<uint32_t, 4> mma_order_c{0, 1, 2, 3};
|
||||
inline bool mma_software = false; // (HMMA_nx2 in software: emit_spirv_warp.cpp, RX_VK_MMA_SOFTWARE=1)
|
||||
// (amd-fix) float colour outputs carry no NaN or infinity to their target (EmitSetFragColor): one NaN texel spreads
|
||||
// through bloom/blur/downsample averages into glowing blocks (OoT's flowers on AMD, Hookshot issue 606). On for every
|
||||
// card but NVIDIA (vk_coopmat.cpp coopmat_init); RX_VK_NO_NAN_GUARD=1 turns it off. Idea from NX2navi patch 0027.
|
||||
inline bool frag_nan_guard = false;
|
||||
// (amd-fix) gl_Position decorated Invariant (vertex and mesh outputs): the game draws a depth pre-pass, then the colour
|
||||
// pass again with depth EQUAL; Maxwell computes the same position in both, AMD's compiler may not without Invariant, and
|
||||
// the colour pass is then rejected in spots: holes showing the bright sky, blown up by bloom (the glowing flowers).
|
||||
// On for every card but NVIDIA (vk_coopmat.cpp); RX_VK_NO_INVARIANT=1 off. From NX2navi patch 0007.
|
||||
inline bool position_invariant = false;
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
2481
gpu/vk/eden/shader_recompiler/backend/spirv/spirv_emit_context.cpp
Normal file
2481
gpu/vk/eden/shader_recompiler/backend/spirv/spirv_emit_context.cpp
Normal file
File diff suppressed because it is too large
Load Diff
477
gpu/vk/eden/shader_recompiler/backend/spirv/spirv_emit_context.h
Normal file
477
gpu/vk/eden/shader_recompiler/backend/spirv/spirv_emit_context.h
Normal file
@@ -0,0 +1,477 @@
|
||||
// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
|
||||
#include <sirit/sirit.h>
|
||||
#include "common/container/unordered_set.h"
|
||||
|
||||
#include "shader_recompiler/backend/bindings.h"
|
||||
#include "shader_recompiler/frontend/ir/program.h"
|
||||
#include "shader_recompiler/profile.h"
|
||||
#include "shader_recompiler/runtime_info.h"
|
||||
#include "shader_recompiler/shader_info.h"
|
||||
|
||||
namespace Shader::Backend::SPIRV {
|
||||
|
||||
static std::bitset<8> clip_distance_written;
|
||||
|
||||
using Sirit::Id;
|
||||
|
||||
class VectorTypes {
|
||||
public:
|
||||
void Define(Sirit::Module& sirit_ctx, Id base_type, std::string_view name);
|
||||
|
||||
[[nodiscard]] Id operator[](size_t size) const noexcept {
|
||||
return defs[size - 1];
|
||||
}
|
||||
|
||||
private:
|
||||
std::array<Id, 4> defs{};
|
||||
};
|
||||
|
||||
struct TextureDefinition {
|
||||
Id id;
|
||||
Id sampled_type;
|
||||
Id pointer_type;
|
||||
Id image_type;
|
||||
u32 count;
|
||||
bool is_multisample;
|
||||
bool is_integer;
|
||||
};
|
||||
|
||||
struct TextureBufferDefinition {
|
||||
Id id;
|
||||
u32 count;
|
||||
};
|
||||
|
||||
struct ImageBufferDefinition {
|
||||
Id id;
|
||||
Id image_type;
|
||||
Id pointer_type;
|
||||
u32 count;
|
||||
bool is_integer;
|
||||
};
|
||||
|
||||
struct ImageDefinition {
|
||||
Id id;
|
||||
Id image_type;
|
||||
Id pointer_type;
|
||||
u32 count;
|
||||
bool is_integer;
|
||||
};
|
||||
|
||||
struct UniformDefinitions {
|
||||
Id U8{};
|
||||
Id S8{};
|
||||
Id U16{};
|
||||
Id S16{};
|
||||
Id U32{};
|
||||
Id F32{};
|
||||
Id U32x2{};
|
||||
Id U32x4{};
|
||||
|
||||
constexpr static size_t NumElements(Id UniformDefinitions::*member_ptr) {
|
||||
if (member_ptr == &UniformDefinitions::U8) {
|
||||
return 1;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::S8) {
|
||||
return 1;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::U16) {
|
||||
return 1;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::S16) {
|
||||
return 1;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::U32) {
|
||||
return 1;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::F32) {
|
||||
return 1;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::U32x2) {
|
||||
return 2;
|
||||
}
|
||||
if (member_ptr == &UniformDefinitions::U32x4) {
|
||||
return 4;
|
||||
}
|
||||
ASSERT(false);
|
||||
return 1;
|
||||
}
|
||||
|
||||
constexpr static bool IsFloat(Id UniformDefinitions::*member_ptr) {
|
||||
if (member_ptr == &UniformDefinitions::F32) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct StorageTypeDefinition {
|
||||
Id array{};
|
||||
Id element{};
|
||||
};
|
||||
|
||||
struct StorageTypeDefinitions {
|
||||
StorageTypeDefinition U8{};
|
||||
StorageTypeDefinition S8{};
|
||||
StorageTypeDefinition U16{};
|
||||
StorageTypeDefinition S16{};
|
||||
StorageTypeDefinition U32{};
|
||||
StorageTypeDefinition U64{};
|
||||
StorageTypeDefinition F32{};
|
||||
StorageTypeDefinition U32x2{};
|
||||
StorageTypeDefinition U32x4{};
|
||||
};
|
||||
|
||||
struct StorageDefinitions {
|
||||
Id U8{};
|
||||
Id S8{};
|
||||
Id U16{};
|
||||
Id S16{};
|
||||
Id U32{};
|
||||
Id F32{};
|
||||
Id U64{};
|
||||
Id U32x2{};
|
||||
Id U32x4{};
|
||||
};
|
||||
|
||||
enum class InputGenericLoadOp {
|
||||
None,
|
||||
Bitcast,
|
||||
SToF,
|
||||
UToF,
|
||||
};
|
||||
|
||||
struct InputGenericInfo {
|
||||
Id id;
|
||||
Id pointer_type;
|
||||
Id component_type;
|
||||
InputGenericLoadOp load_op;
|
||||
};
|
||||
|
||||
struct GenericElementInfo {
|
||||
Id id{};
|
||||
u32 first_element{};
|
||||
u32 num_components{};
|
||||
};
|
||||
|
||||
class EmitContext final : public Sirit::Module {
|
||||
public:
|
||||
explicit EmitContext(const Profile& profile, const RuntimeInfo& runtime_info,
|
||||
IR::Program& program, Bindings& binding);
|
||||
~EmitContext();
|
||||
|
||||
[[nodiscard]] Id Def(const IR::Value& value);
|
||||
|
||||
[[nodiscard]] Id BitOffset8(const IR::Value& offset);
|
||||
[[nodiscard]] Id BitOffset16(const IR::Value& offset);
|
||||
|
||||
Id Const(u32 value) {
|
||||
return Constant(U32[1], value);
|
||||
}
|
||||
|
||||
Id Const(u32 element_1, u32 element_2) {
|
||||
return ConstantComposite(U32[2], Const(element_1), Const(element_2));
|
||||
}
|
||||
|
||||
Id Const(u32 element_1, u32 element_2, u32 element_3) {
|
||||
return ConstantComposite(U32[3], Const(element_1), Const(element_2), Const(element_3));
|
||||
}
|
||||
|
||||
Id Const(u32 element_1, u32 element_2, u32 element_3, u32 element_4) {
|
||||
return ConstantComposite(U32[4], Const(element_1), Const(element_2), Const(element_3),
|
||||
Const(element_4));
|
||||
}
|
||||
|
||||
Id SConst(s32 value) {
|
||||
return Constant(S32[1], value);
|
||||
}
|
||||
|
||||
Id SConst(s32 element_1, s32 element_2) {
|
||||
return ConstantComposite(S32[2], SConst(element_1), SConst(element_2));
|
||||
}
|
||||
|
||||
Id SConst(s32 element_1, s32 element_2, s32 element_3) {
|
||||
return ConstantComposite(S32[3], SConst(element_1), SConst(element_2), SConst(element_3));
|
||||
}
|
||||
|
||||
Id SConst(s32 element_1, s32 element_2, s32 element_3, s32 element_4) {
|
||||
return ConstantComposite(S32[4], SConst(element_1), SConst(element_2), SConst(element_3),
|
||||
SConst(element_4));
|
||||
}
|
||||
|
||||
Id Const(f32 value) {
|
||||
return Constant(F32[1], value);
|
||||
}
|
||||
|
||||
const Profile& profile;
|
||||
const RuntimeInfo& runtime_info;
|
||||
Stage stage{};
|
||||
|
||||
Id void_id{};
|
||||
Id U1{};
|
||||
Id U8{};
|
||||
Id S8{};
|
||||
Id U16{};
|
||||
Id S16{};
|
||||
Id U64{};
|
||||
VectorTypes F32;
|
||||
VectorTypes U32;
|
||||
VectorTypes S32;
|
||||
VectorTypes F16;
|
||||
VectorTypes F64;
|
||||
|
||||
Id true_value{};
|
||||
Id false_value{};
|
||||
Id u32_zero_value{};
|
||||
Id f32_zero_value{};
|
||||
|
||||
UniformDefinitions uniform_types;
|
||||
StorageTypeDefinitions storage_types;
|
||||
|
||||
Id private_u32{};
|
||||
|
||||
Id shared_u8{};
|
||||
Id shared_u16{};
|
||||
Id shared_u32{};
|
||||
Id shared_u64{};
|
||||
Id shared_u32x2{};
|
||||
Id shared_u32x4{};
|
||||
|
||||
Id input_f32{};
|
||||
Id input_u32{};
|
||||
Id input_s32{};
|
||||
|
||||
Id output_f32{};
|
||||
Id output_u32{};
|
||||
|
||||
Id image_buffer_type{};
|
||||
Id image_u32{};
|
||||
|
||||
std::array<UniformDefinitions, Info::MAX_CBUFS> cbufs{};
|
||||
std::array<StorageDefinitions, Info::MAX_SSBOS> ssbos{};
|
||||
std::vector<TextureBufferDefinition> texture_buffers;
|
||||
std::vector<ImageBufferDefinition> image_buffers;
|
||||
std::vector<TextureDefinition> textures;
|
||||
std::vector<ImageDefinition> images;
|
||||
|
||||
Id workgroup_id{};
|
||||
Id local_invocation_id{};
|
||||
Id invocation_id{};
|
||||
Id patch_vertices_in{};
|
||||
Id sample_id{};
|
||||
Id is_helper_invocation{};
|
||||
Id subgroup_local_invocation_id{};
|
||||
Id software_matrix_function{}; // (hookshot: emit_spirv_warp.cpp)
|
||||
Id subgroup_mask_eq{};
|
||||
Id subgroup_mask_lt{};
|
||||
Id subgroup_mask_le{};
|
||||
Id subgroup_mask_gt{};
|
||||
Id subgroup_mask_ge{};
|
||||
Id instance_id{};
|
||||
Id instance_index{};
|
||||
Id base_instance{};
|
||||
Id vertex_id{};
|
||||
Id vertex_index{};
|
||||
Id draw_index{};
|
||||
Id base_vertex{};
|
||||
Id front_face{};
|
||||
Id point_coord{};
|
||||
Id tess_coord{};
|
||||
Id clip_distances{};
|
||||
Id layer{};
|
||||
Id viewport_index{};
|
||||
Id viewport_mask{};
|
||||
Id primitive_id{};
|
||||
|
||||
Id fswzadd_lut_a{};
|
||||
Id fswzadd_lut_b{};
|
||||
|
||||
Id indexed_load_func{};
|
||||
Id indexed_store_func{};
|
||||
|
||||
Id rescaling_uniform_constant{};
|
||||
Id rescaling_push_constants{};
|
||||
Id rescaling_textures_type{};
|
||||
Id rescaling_images_type{};
|
||||
u32 rescaling_textures_member_index{};
|
||||
u32 rescaling_images_member_index{};
|
||||
u32 rescaling_downfactor_member_index{};
|
||||
u32 guest_pages_member_index{}; // (hookshot: GUEST_PAGES_PUSH_OFFSET's member, compute only)
|
||||
bool wants_guest_pages{};
|
||||
Id probe_log{}; // (EvilOOT, RX_VK_BLOCK_PROBE: the miss log's U32 storage buffer, per-block entry counters at word 128+)
|
||||
u32 mesh_first_member_index{}; // (hookshot: MESH_FIRST_PUSH_OFFSET's member, task programs)
|
||||
bool wants_mesh_first{};
|
||||
Id mesh_first_block_pointer{}; // (a PhysicalStorageBuffer pointer to a block of one u32, and to its member)
|
||||
Id mesh_first_member_pointer{};
|
||||
u32 texture_rescaling_index{};
|
||||
u32 image_rescaling_index{};
|
||||
|
||||
Id render_area_push_constant{};
|
||||
u32 render_are_member_index{};
|
||||
|
||||
Id local_memory{};
|
||||
|
||||
bool uses_explicit_workgroup_layout{};
|
||||
Id shared_memory_u8{};
|
||||
Id shared_memory_u16{};
|
||||
Id shared_memory_u32{};
|
||||
Id shared_memory_u64{};
|
||||
Id shared_memory_u32x2{};
|
||||
Id shared_memory_u32x4{};
|
||||
|
||||
Id shared_memory_u32_type{};
|
||||
|
||||
Id shared_store_u8_func{};
|
||||
Id shared_store_u16_func{};
|
||||
Id increment_cas_shared{};
|
||||
Id increment_cas_ssbo{};
|
||||
Id decrement_cas_shared{};
|
||||
Id decrement_cas_ssbo{};
|
||||
Id f32_add_cas{};
|
||||
Id f16x2_add_cas{};
|
||||
Id f16x2_min_cas{};
|
||||
Id f16x2_max_cas{};
|
||||
Id f32x2_add_cas{};
|
||||
Id f32x2_min_cas{};
|
||||
Id f32x2_max_cas{};
|
||||
|
||||
Id write_storage_cas_loop_func{};
|
||||
|
||||
// (hookshot) storage buffer tables (Info::global_tables): one array variable a type, which only the global
|
||||
// memory fallbacks use; and those fallbacks for the 32-bit atomics (IAdd SMin UMin SMax UMax And Or Xor Exchange)
|
||||
struct GlobalTableVars {
|
||||
u32 cbuf_index{};
|
||||
u32 cbuf_offset{};
|
||||
u32 count{};
|
||||
StorageDefinitions vars{};
|
||||
};
|
||||
std::vector<GlobalTableVars> ssbo_tables;
|
||||
std::array<Id, 9> atomic_global_func_u32{};
|
||||
Id atomic_global_cas_func_u32{}; // (hookshot) (address, comparator, value) -> the old value
|
||||
|
||||
Id load_global_func_u32{};
|
||||
Id load_global_func_u32x2{};
|
||||
Id load_global_func_u32x4{};
|
||||
Id write_global_func_u32{};
|
||||
Id write_global_func_u32x2{};
|
||||
Id write_global_func_u32x4{};
|
||||
Id write_global_cas_loop_func{};
|
||||
Id atomic_global_add_f32_func{}; // (hookshot) (address, value) -> the old value
|
||||
|
||||
bool need_input_position_indirect{};
|
||||
Id input_position{};
|
||||
std::array<InputGenericInfo, 32> input_generics{};
|
||||
|
||||
Id output_point_size{};
|
||||
Id output_position{};
|
||||
std::array<std::array<GenericElementInfo, 4>, 32> output_generics{};
|
||||
|
||||
Id output_tess_level_outer{};
|
||||
Id output_tess_level_inner{};
|
||||
std::array<Id, 30> patches{};
|
||||
|
||||
std::array<Id, 8> frag_color{};
|
||||
Id sample_mask{};
|
||||
Id frag_depth{};
|
||||
|
||||
std::vector<Id> interfaces;
|
||||
std::vector<std::pair<u32, spv::Op>> opcode_rewrites;
|
||||
|
||||
// Task and mesh programs. Their stage buffer: the task's output region is the task payload
|
||||
// (the mesh stage's input region); the mesh output regions live in workgroup memory until
|
||||
// the epilogue copies them to the outputs. The two mesh instructions Sirit lacks are emitted
|
||||
// as OpStore markers with these operands and replaced after assembly (PatchMeshInstructions).
|
||||
struct MeshLineTarget {
|
||||
Id variable; ///< arrayed output
|
||||
u32 element; ///< vector element, or ~0 for a scalar output
|
||||
bool is_float; ///< bitcast the stage buffer word to F32
|
||||
};
|
||||
Id mesh_payload{};
|
||||
Id mesh_payload_u32{}; ///< pointer type to one payload word
|
||||
Id mesh_attributes{};
|
||||
Id mesh_map{};
|
||||
Id mesh_primitive_count{};
|
||||
/// (hookshot) a fallback mesh workgroup's slot (task workgroup * threads + its index), read from the task's map in the
|
||||
/// entry block (EmitMeshPrologue); invalid without a task program, where the slot is the workgroup id
|
||||
Id mesh_fallback_slot{};
|
||||
Id mesh_shared_u32{}; ///< pointer type to one workgroup word
|
||||
Id mesh_primitive_indices{};
|
||||
MeshInfo mesh_info{};
|
||||
u32 mesh_threads{};
|
||||
u32 mesh_attribute_words{};
|
||||
u32 mesh_map_bytes{};
|
||||
u32 mesh_primitive_base{}; ///< first word of the per-primitive groups
|
||||
std::vector<MeshLineTarget> mesh_vertex_targets;
|
||||
// (hookshot) a vertex generic output and the components no vertex line writes: they get (0,0,0,1) as in the vertex stage
|
||||
struct MeshDefault {
|
||||
Id variable;
|
||||
u32 missing;
|
||||
};
|
||||
std::vector<MeshDefault> mesh_vertex_defaults;
|
||||
std::vector<MeshLineTarget> mesh_primitive_targets;
|
||||
std::array<Id, 2> mesh_outputs_marker{};
|
||||
bool mesh_zero_outputs_marker{}; ///< (hookshot) OpStore(zero, zero): OpSetMeshOutputsEXT 0 0 where a branch has no target
|
||||
std::array<Id, 4> mesh_tasks_marker{};
|
||||
|
||||
// (hookshot) a GPU without VK_EXT_mesh_shader: Task and Mesh are compute shaders. Their stage buffer outputs go to
|
||||
// the payload, map and record buffers, pushed as device addresses; a generated vertex stage reads the records.
|
||||
bool mesh_fallback{};
|
||||
bool wants_mesh_fallback{};
|
||||
Id mesh_fallback_word_ptr{}; ///< PhysicalStorageBuffer pointer to a block of one u32
|
||||
u32 mesh_fallback_payload_member_index{};
|
||||
u32 mesh_fallback_records_member_index{};
|
||||
u32 mesh_fallback_indirect_input_member_index{};
|
||||
/// A pointer to the u32 at a PhysicalStorageBuffer device address
|
||||
Id MeshFallbackWord(Id address);
|
||||
|
||||
Id load_const_func_u8{};
|
||||
Id load_const_func_u16{};
|
||||
Id load_const_func_u32{};
|
||||
Id load_const_func_f32{};
|
||||
Id load_const_func_u32x2{};
|
||||
Id load_const_func_u32x4{};
|
||||
|
||||
// Sirit::Id doesn't play nice with *::set<>
|
||||
::Common::unordered_set<u32> non_uniform_ids;
|
||||
|
||||
bool uses_nonuniform_sampled_image{};
|
||||
bool uses_nonuniform_storage_image{};
|
||||
bool uses_nonuniform_uniform_texel_buffer{};
|
||||
bool uses_nonuniform_storage_texel_buffer{};
|
||||
|
||||
private:
|
||||
void DefineCommonTypes(const Info& info);
|
||||
void DefineCommonConstants();
|
||||
void DefineInterfaces(const IR::Program& program);
|
||||
void DefineLocalMemory(const IR::Program& program);
|
||||
void DefineSharedMemory(const IR::Program& program);
|
||||
void DefineSharedMemoryFunctions(const IR::Program& program);
|
||||
void DefineConstantBuffers(const Info& info, u32& binding);
|
||||
void DefineConstantBufferIndirectFunctions(const Info& info);
|
||||
void DefineStorageBuffers(const Info& info, u32& binding);
|
||||
void DefineTextureBuffers(const Info& info, u32& binding);
|
||||
void DefineImageBuffers(const Info& info, u32& binding);
|
||||
void DefineTextures(const Info& info, u32& binding, u32& scaling_index);
|
||||
void DefineImages(const Info& info, u32& binding, u32& scaling_index);
|
||||
void DefineAttributeMemAccess(const Info& info);
|
||||
void DefineWriteStorageCasLoopFunction(const Info& info);
|
||||
void DefineGlobalMemoryFunctions(const Info& info);
|
||||
void DefineRescalingInput(const Info& info);
|
||||
void DefineRescalingInputPushConstant();
|
||||
void DefineRescalingInputUniformConstant();
|
||||
void DefineRenderArea(const Info& info);
|
||||
|
||||
void DefineInputs(const IR::Program& program);
|
||||
void DefineOutputs(const IR::Program& program);
|
||||
void DefineMesh(const IR::Program& program);
|
||||
};
|
||||
|
||||
} // namespace Shader::Backend::SPIRV
|
||||
Reference in New Issue
Block a user