shader: Implement Int32 SUATOM/SURED

src/shader_recompiler/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(shader_recompiler STATIC
     backend/spirv/emit_spirv_convert.cpp
     backend/spirv/emit_spirv_floating_point.cpp
     backend/spirv/emit_spirv_image.cpp
+    backend/spirv/emit_spirv_image_atomic.cpp
     backend/spirv/emit_spirv_integer.cpp
     backend/spirv/emit_spirv_logical.cpp
     backend/spirv/emit_spirv_memory.cpp
@@ -138,6 +139,7 @@ add_library(shader_recompiler STATIC
     frontend/maxwell/translate/impl/predicate_set_predicate.cpp
     frontend/maxwell/translate/impl/predicate_set_register.cpp
     frontend/maxwell/translate/impl/select_source_with_predicate.cpp
+    frontend/maxwell/translate/impl/surface_atomic_operations.cpp
     frontend/maxwell/translate/impl/surface_load_store.cpp
     frontend/maxwell/translate/impl/texture_fetch.cpp
     frontend/maxwell/translate/impl/texture_fetch_swizzled.cpp

src/shader_recompiler/backend/spirv/emit_context.cpp

@@ -1107,6 +1107,9 @@ void EmitContext::DefineTextures(const Info& info, u32& binding) {
         }
         ++binding;
     }
+    if (info.uses_atomic_image_u32) {
+        image_u32 = TypePointer(spv::StorageClass::Image, U32[1]);
+    }
 }
 
 void EmitContext::DefineImages(const Info& info, u32& binding) {

src/shader_recompiler/backend/spirv/emit_context.h

@@ -198,6 +198,7 @@ public:
 
     Id image_buffer_type{};
     Id sampled_texture_buffer_type{};
+    Id image_u32{};
 
     std::array<UniformDefinitions, Info::MAX_CBUFS> cbufs{};
     std::array<StorageDefinitions, Info::MAX_SSBOS> ssbos{};

src/shader_recompiler/backend/spirv/emit_spirv.cpp

@@ -335,6 +335,9 @@ void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ct
     if (info.uses_typeless_image_writes) {
         ctx.AddCapability(spv::Capability::StorageImageWriteWithoutFormat);
     }
+    if (info.uses_image_buffers) {
+        ctx.AddCapability(spv::Capability::ImageBuffer);
+    }
     if (info.uses_sample_id) {
         ctx.AddCapability(spv::Capability::SampleRateShading);
     }

src/shader_recompiler/backend/spirv/emit_spirv.h

@@ -509,6 +509,50 @@ Id EmitImageGradient(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, I
                      Id derivates, Id offset, Id lod_clamp);
 Id EmitImageRead(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords);
 void EmitImageWrite(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id color);
+Id EmitBindlessImageAtomicIAdd32(EmitContext&);
+Id EmitBindlessImageAtomicSMin32(EmitContext&);
+Id EmitBindlessImageAtomicUMin32(EmitContext&);
+Id EmitBindlessImageAtomicSMax32(EmitContext&);
+Id EmitBindlessImageAtomicUMax32(EmitContext&);
+Id EmitBindlessImageAtomicInc32(EmitContext&);
+Id EmitBindlessImageAtomicDec32(EmitContext&);
+Id EmitBindlessImageAtomicAnd32(EmitContext&);
+Id EmitBindlessImageAtomicOr32(EmitContext&);
+Id EmitBindlessImageAtomicXor32(EmitContext&);
+Id EmitBindlessImageAtomicExchange32(EmitContext&);
+Id EmitBoundImageAtomicIAdd32(EmitContext&);
+Id EmitBoundImageAtomicSMin32(EmitContext&);
+Id EmitBoundImageAtomicUMin32(EmitContext&);
+Id EmitBoundImageAtomicSMax32(EmitContext&);
+Id EmitBoundImageAtomicUMax32(EmitContext&);
+Id EmitBoundImageAtomicInc32(EmitContext&);
+Id EmitBoundImageAtomicDec32(EmitContext&);
+Id EmitBoundImageAtomicAnd32(EmitContext&);
+Id EmitBoundImageAtomicOr32(EmitContext&);
+Id EmitBoundImageAtomicXor32(EmitContext&);
+Id EmitBoundImageAtomicExchange32(EmitContext&);
+Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value);
+Id EmitImageAtomicSMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value);
+Id EmitImageAtomicUMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value);
+Id EmitImageAtomicSMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value);
+Id EmitImageAtomicUMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value);
+Id EmitImageAtomicInc32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                        Id value);
+Id EmitImageAtomicDec32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                        Id value);
+Id EmitImageAtomicAnd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                        Id value);
+Id EmitImageAtomicOr32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                       Id value);
+Id EmitImageAtomicXor32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                        Id value);
+Id EmitImageAtomicExchange32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                             Id value);
 Id EmitLaneId(EmitContext& ctx);
 Id EmitVoteAll(EmitContext& ctx, Id pred);
 Id EmitVoteAny(EmitContext& ctx, Id pred);

src/shader_recompiler/backend/spirv/emit_spirv_image_atomic.cpp

@@ -0,0 +1,182 @@
+// Copyright 2021 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "shader_recompiler/backend/spirv/emit_spirv.h"
+#include "shader_recompiler/frontend/ir/modifiers.h"
+
+namespace Shader::Backend::SPIRV {
+namespace {
+Id Image(EmitContext& ctx, const IR::Value& index, IR::TextureInstInfo info) {
+    if (!index.IsImmediate()) {
+        throw NotImplementedException("Indirect image indexing");
+    }
+    if (info.type == TextureType::Buffer) {
+        const ImageBufferDefinition def{ctx.image_buffers.at(index.U32())};
+        return def.id;
+    } else {
+        const ImageDefinition def{ctx.images.at(index.U32())};
+        return def.id;
+    }
+}
+
+std::pair<Id, Id> AtomicArgs(EmitContext& ctx) {
+    const Id scope{ctx.Const(static_cast<u32>(spv::Scope::Device))};
+    const Id semantics{ctx.u32_zero_value};
+    return {scope, semantics};
+}
+
+Id ImageAtomicU32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords, Id value,
+                  Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) {
+    const auto info{inst->Flags<IR::TextureInstInfo>()};
+    const Id image{Image(ctx, index, info)};
+    const Id pointer{ctx.OpImageTexelPointer(ctx.image_u32, image, coords, ctx.Const(0U))};
+    const auto [scope, semantics]{AtomicArgs(ctx)};
+    return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
+}
+} // Anonymous namespace
+
+Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicIAdd);
+}
+
+Id EmitImageAtomicSMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicSMin);
+}
+
+Id EmitImageAtomicUMin32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicUMin);
+}
+
+Id EmitImageAtomicSMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicSMax);
+}
+
+Id EmitImageAtomicUMax32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                         Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicUMax);
+}
+
+Id EmitImageAtomicInc32(EmitContext&, IR::Inst*, const IR::Value&, Id, Id) {
+    // TODO: This is not yet implemented
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitImageAtomicDec32(EmitContext&, IR::Inst*, const IR::Value&, Id, Id) {
+    // TODO: This is not yet implemented
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitImageAtomicAnd32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                        Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicAnd);
+}
+
+Id EmitImageAtomicOr32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                       Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicOr);
+}
+
+Id EmitImageAtomicXor32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                        Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicXor);
+}
+
+Id EmitImageAtomicExchange32(EmitContext& ctx, IR::Inst* inst, const IR::Value& index, Id coords,
+                             Id value) {
+    return ImageAtomicU32(ctx, inst, index, coords, value, &Sirit::Module::OpAtomicExchange);
+}
+
+Id EmitBindlessImageAtomicIAdd32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicSMin32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicUMin32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicSMax32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicUMax32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicInc32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicDec32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicAnd32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicOr32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicXor32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBindlessImageAtomicExchange32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicIAdd32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicSMin32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicUMin32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicSMax32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicUMax32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicInc32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicDec32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicAnd32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicOr32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicXor32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+Id EmitBoundImageAtomicExchange32(EmitContext&) {
+    throw NotImplementedException("SPIR-V Instruction");
+}
+
+} // namespace Shader::Backend::SPIRV

src/shader_recompiler/frontend/ir/ir_emitter.cpp

@@ -1869,6 +1869,95 @@ void IREmitter::ImageWrite(const Value& handle, const Value& coords, const Value
     Inst(op, Flags{info}, handle, coords, color);
 }
 
+Value IREmitter::ImageAtomicIAdd(const Value& handle, const Value& coords, const Value& value,
+                                 TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicIAdd32
+                                         : Opcode::BindlessImageAtomicIAdd32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicSMin(const Value& handle, const Value& coords, const Value& value,
+                                 TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicSMin32
+                                         : Opcode::BindlessImageAtomicSMin32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicUMin(const Value& handle, const Value& coords, const Value& value,
+                                 TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicUMin32
+                                         : Opcode::BindlessImageAtomicUMin32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicIMin(const Value& handle, const Value& coords, const Value& value,
+                                 bool is_signed, TextureInstInfo info) {
+    return is_signed ? ImageAtomicSMin(handle, coords, value, info)
+                     : ImageAtomicUMin(handle, coords, value, info);
+}
+
+Value IREmitter::ImageAtomicSMax(const Value& handle, const Value& coords, const Value& value,
+                                 TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicSMax32
+                                         : Opcode::BindlessImageAtomicSMax32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicUMax(const Value& handle, const Value& coords, const Value& value,
+                                 TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicUMax32
+                                         : Opcode::BindlessImageAtomicUMax32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicIMax(const Value& handle, const Value& coords, const Value& value,
+                                 bool is_signed, TextureInstInfo info) {
+    return is_signed ? ImageAtomicSMax(handle, coords, value, info)
+                     : ImageAtomicUMax(handle, coords, value, info);
+}
+
+Value IREmitter::ImageAtomicInc(const Value& handle, const Value& coords, const Value& value,
+                                TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicInc32
+                                         : Opcode::BindlessImageAtomicInc32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicDec(const Value& handle, const Value& coords, const Value& value,
+                                TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicDec32
+                                         : Opcode::BindlessImageAtomicDec32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicAnd(const Value& handle, const Value& coords, const Value& value,
+                                TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicAnd32
+                                         : Opcode::BindlessImageAtomicAnd32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicOr(const Value& handle, const Value& coords, const Value& value,
+                               TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicOr32
+                                         : Opcode::BindlessImageAtomicOr32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicXor(const Value& handle, const Value& coords, const Value& value,
+                                TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicXor32
+                                         : Opcode::BindlessImageAtomicXor32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
+Value IREmitter::ImageAtomicExchange(const Value& handle, const Value& coords, const Value& value,
+                                     TextureInstInfo info) {
+    const Opcode op{handle.IsImmediate() ? Opcode::BoundImageAtomicExchange32
+                                         : Opcode::BindlessImageAtomicExchange32};
+    return Inst(op, Flags{info}, handle, coords, value);
+}
+
 U1 IREmitter::VoteAll(const U1& value) {
     return Inst<U1>(Opcode::VoteAll, value);
 }

src/shader_recompiler/frontend/ir/ir_emitter.h

@@ -334,6 +334,32 @@ public:
     [[nodiscard]] void ImageWrite(const Value& handle, const Value& coords, const Value& color,
                                   TextureInstInfo info);
 
+    [[nodiscard]] Value ImageAtomicIAdd(const Value& handle, const Value& coords,
+                                        const Value& value, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicSMin(const Value& handle, const Value& coords,
+                                        const Value& value, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicUMin(const Value& handle, const Value& coords,
+                                        const Value& value, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicIMin(const Value& handle, const Value& coords,
+                                        const Value& value, bool is_signed, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicSMax(const Value& handle, const Value& coords,
+                                        const Value& value, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicUMax(const Value& handle, const Value& coords,
+                                        const Value& value, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicIMax(const Value& handle, const Value& coords,
+                                        const Value& value, bool is_signed, TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicInc(const Value& handle, const Value& coords, const Value& value,
+                                       TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicDec(const Value& handle, const Value& coords, const Value& value,
+                                       TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicAnd(const Value& handle, const Value& coords, const Value& value,
+                                       TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicOr(const Value& handle, const Value& coords, const Value& value,
+                                      TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicXor(const Value& handle, const Value& coords, const Value& value,
+                                       TextureInstInfo info);
+    [[nodiscard]] Value ImageAtomicExchange(const Value& handle, const Value& coords,
+                                            const Value& value, TextureInstInfo info);
     [[nodiscard]] U1 VoteAll(const U1& value);
     [[nodiscard]] U1 VoteAny(const U1& value);
     [[nodiscard]] U1 VoteEqual(const U1& value);

src/shader_recompiler/frontend/ir/microinstruction.cpp

@@ -166,6 +166,39 @@ bool Inst::MayHaveSideEffects() const noexcept {
     case Opcode::BindlessImageWrite:
     case Opcode::BoundImageWrite:
     case Opcode::ImageWrite:
+    case IR::Opcode::BindlessImageAtomicIAdd32:
+    case IR::Opcode::BindlessImageAtomicSMin32:
+    case IR::Opcode::BindlessImageAtomicUMin32:
+    case IR::Opcode::BindlessImageAtomicSMax32:
+    case IR::Opcode::BindlessImageAtomicUMax32:
+    case IR::Opcode::BindlessImageAtomicInc32:
+    case IR::Opcode::BindlessImageAtomicDec32:
+    case IR::Opcode::BindlessImageAtomicAnd32:
+    case IR::Opcode::BindlessImageAtomicOr32:
+    case IR::Opcode::BindlessImageAtomicXor32:
+    case IR::Opcode::BindlessImageAtomicExchange32:
+    case IR::Opcode::BoundImageAtomicIAdd32:
+    case IR::Opcode::BoundImageAtomicSMin32:
+    case IR::Opcode::BoundImageAtomicUMin32:
+    case IR::Opcode::BoundImageAtomicSMax32:
+    case IR::Opcode::BoundImageAtomicUMax32:
+    case IR::Opcode::BoundImageAtomicInc32:
+    case IR::Opcode::BoundImageAtomicDec32:
+    case IR::Opcode::BoundImageAtomicAnd32:
+    case IR::Opcode::BoundImageAtomicOr32:
+    case IR::Opcode::BoundImageAtomicXor32:
+    case IR::Opcode::BoundImageAtomicExchange32:
+    case IR::Opcode::ImageAtomicIAdd32:
+    case IR::Opcode::ImageAtomicSMin32:
+    case IR::Opcode::ImageAtomicUMin32:
+    case IR::Opcode::ImageAtomicSMax32:
+    case IR::Opcode::ImageAtomicUMax32:
+    case IR::Opcode::ImageAtomicInc32:
+    case IR::Opcode::ImageAtomicDec32:
+    case IR::Opcode::ImageAtomicAnd32:
+    case IR::Opcode::ImageAtomicOr32:
+    case IR::Opcode::ImageAtomicXor32:
+    case IR::Opcode::ImageAtomicExchange32:
         return true;
     default:
         return false;

src/shader_recompiler/frontend/ir/opcodes.inc

@@ -496,6 +496,44 @@ OPCODE(ImageGradient,                                       F32x4,          Opaq
 OPCODE(ImageRead,                                           U32x4,          Opaque,         Opaque,                                                         )
 OPCODE(ImageWrite,                                          Void,           Opaque,         Opaque,         U32x4,                                          )
 
+// Atomic Image operations
+
+OPCODE(BindlessImageAtomicIAdd32,                           U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicSMin32,                           U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicUMin32,                           U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicSMax32,                           U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicUMax32,                           U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicInc32,                            U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicDec32,                            U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicAnd32,                            U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicOr32,                             U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicXor32,                            U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BindlessImageAtomicExchange32,                       U32,            U32,            Opaque,            U32,                                         )
+
+OPCODE(BoundImageAtomicIAdd32,                              U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicSMin32,                              U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicUMin32,                              U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicSMax32,                              U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicUMax32,                              U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicInc32,                               U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicDec32,                               U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicAnd32,                               U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicOr32,                                U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicXor32,                               U32,            U32,            Opaque,            U32,                                         )
+OPCODE(BoundImageAtomicExchange32,                          U32,            U32,            Opaque,            U32,                                         )
+
+OPCODE(ImageAtomicIAdd32,                                   U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicSMin32,                                   U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicUMin32,                                   U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicSMax32,                                   U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicUMax32,                                   U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicInc32,                                    U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicDec32,                                    U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicAnd32,                                    U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicOr32,                                     U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicXor32,                                    U32,            Opaque,            Opaque,            U32,                                      )
+OPCODE(ImageAtomicExchange32,                               U32,            Opaque,            Opaque,            U32,                                      )
+
 // Warp operations
 OPCODE(LaneId,                                              U32,                                                                                            )
 OPCODE(VoteAll,                                             U1,             U1,                                                                             )

src/shader_recompiler/frontend/maxwell/maxwell.inc

@@ -244,7 +244,8 @@ INST(STG,          "STG",            "1110 1110 1101 1---")
 INST(STL,          "STL",            "1110 1111 0101 0---")
 INST(STP,          "STP",            "1110 1110 1010 0---")
 INST(STS,          "STS",            "1110 1111 0101 1---")
-INST(SUATOM_cas,   "SUATOM",         "1110 1010 ---- ----")
+INST(SUATOM,       "SUATOM",         "1110 1010 0--- ----")
+INST(SUATOM_cas,   "SUATOM_cas",     "1110 1010 1--- ----")
 INST(SULD,         "SULD",           "1110 1011 000- ----")
 INST(SURED,        "SURED",          "1110 1011 010- ----")
 INST(SUST,         "SUST",           "1110 1011 001- ----")

src/shader_recompiler/frontend/maxwell/translate/impl/impl.h

@@ -303,6 +303,7 @@ public:
     void STL(u64 insn);
     void STP(u64 insn);
     void STS(u64 insn);
+    void SUATOM(u64 insn);
     void SUATOM_cas(u64 insn);
     void SULD(u64 insn);
     void SURED(u64 insn);

src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp

@@ -249,10 +249,6 @@ void TranslatorVisitor::SUATOM_cas(u64) {
     ThrowNotImplemented(Opcode::SUATOM_cas);
 }
 
-void TranslatorVisitor::SURED(u64) {
-    ThrowNotImplemented(Opcode::SURED);
-}
-
 void TranslatorVisitor::SYNC(u64) {
     ThrowNotImplemented(Opcode::SYNC);
 }

src/shader_recompiler/frontend/maxwell/translate/impl/surface_atomic_operations.cpp

@@ -0,0 +1,204 @@
+// Copyright 2021 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <array>
+#include <bit>
+
+#include "common/bit_field.h"
+#include "common/common_types.h"
+#include "shader_recompiler/frontend/ir/modifiers.h"
+#include "shader_recompiler/frontend/maxwell/translate/impl/impl.h"
+
+namespace Shader::Maxwell {
+namespace {
+enum class Type : u64 {
+    _1D,
+    BUFFER_1D,
+    ARRAY_1D,
+    _2D,
+    ARRAY_2D,
+    _3D,
+};
+
+enum class Size : u64 {
+    U32,
+    S32,
+    U64,
+    S64,
+    F32FTZRN,
+    F16x2FTZRN,
+    SD32,
+    SD64,
+};
+
+enum class AtomicOp : u64 {
+    ADD,
+    MIN,
+    MAX,
+    INC,
+    DEC,
+    AND,
+    OR,
+    XOR,
+    EXCH,
+};
+
+enum class Clamp : u64 {
+    IGN,
+    Default,
+    TRAP,
+};
+
+TextureType GetType(Type type) {
+    switch (type) {
+    case Type::_1D:
+        return TextureType::Color1D;
+    case Type::BUFFER_1D:
+        return TextureType::Buffer;
+    case Type::ARRAY_1D:
+        return TextureType::ColorArray1D;
+    case Type::_2D:
+        return TextureType::Color2D;
+    case Type::ARRAY_2D:
+        return TextureType::ColorArray2D;
+    case Type::_3D:
+        return TextureType::Color3D;
+    }
+    throw NotImplementedException("Invalid type {}", type);
+}
+
+IR::Value MakeCoords(TranslatorVisitor& v, IR::Reg reg, Type type) {
+    const auto array{[&](int index) {
+        return v.ir.BitFieldExtract(v.X(reg + index), v.ir.Imm32(0), v.ir.Imm32(16));
+    }};
+    switch (type) {
+    case Type::_1D:
+    case Type::BUFFER_1D:
+        return v.X(reg);
+    default:
+        break;
+    }
+    throw NotImplementedException("Invalid type {}", type);
+}
+
+IR::Value ApplyAtomicOp(IR::IREmitter& ir, const IR::U32& handle, const IR::Value& coords,
+                        const IR::Value& op_b, IR::TextureInstInfo info, AtomicOp op,
+                        bool is_signed) {
+    switch (op) {
+    case AtomicOp::ADD:
+        return ir.ImageAtomicIAdd(handle, coords, op_b, info);
+    case AtomicOp::MIN:
+        return ir.ImageAtomicIMin(handle, coords, op_b, is_signed, info);
+    case AtomicOp::MAX:
+        return ir.ImageAtomicIMax(handle, coords, op_b, is_signed, info);
+    case AtomicOp::INC:
+        return ir.ImageAtomicInc(handle, coords, op_b, info);
+    case AtomicOp::DEC:
+        return ir.ImageAtomicDec(handle, coords, op_b, info);
+    case AtomicOp::AND:
+        return ir.ImageAtomicAnd(handle, coords, op_b, info);
+    case AtomicOp::OR:
+        return ir.ImageAtomicOr(handle, coords, op_b, info);
+    case AtomicOp::XOR:
+        return ir.ImageAtomicXor(handle, coords, op_b, info);
+    case AtomicOp::EXCH:
+        return ir.ImageAtomicExchange(handle, coords, op_b, info);
+    default:
+        throw NotImplementedException("Atomic Operation {}", op);
+    }
+}
+
+ImageFormat Format(Size size) {
+    switch (size) {
+    case Size::U32:
+    case Size::S32:
+    case Size::SD32:
+        return ImageFormat::R32_UINT;
+    default:
+        break;
+    }
+    throw NotImplementedException("Invalid size {}", size);
+}
+
+bool IsSizeInt32(Size size) {
+    switch (size) {
+    case Size::U32:
+    case Size::S32:
+    case Size::SD32:
+        return true;
+    default:
+        return false;
+    }
+}
+
+void ImageAtomOp(TranslatorVisitor& v, IR::Reg dest_reg, IR::Reg operand_reg, IR::Reg coord_reg,
+                 IR::Reg bindless_reg, AtomicOp op, Clamp clamp, Size size, Type type,
+                 u64 bound_offset, bool is_bindless, bool write_result) {
+    if (clamp != Clamp::IGN) {
+        throw NotImplementedException("Clamp {}", clamp);
+    }
+    if (!IsSizeInt32(size)) {
+        throw NotImplementedException("Size {}", size);
+    }
+    const bool is_signed{size == Size::S32};
+    const ImageFormat format{Format(size)};
+    const TextureType tex_type{GetType(type)};
+    const IR::Value coords{MakeCoords(v, coord_reg, type)};
+
+    const IR::U32 handle{is_bindless != 0 ? v.X(bindless_reg)
+                                          : v.ir.Imm32(static_cast<u32>(bound_offset * 4))};
+    IR::TextureInstInfo info{};
+    info.type.Assign(tex_type);
+    info.image_format.Assign(format);
+
+    // TODO: float/64-bit operand
+    const IR::Value op_b{v.X(operand_reg)};
+    const IR::Value color{ApplyAtomicOp(v.ir, handle, coords, op_b, info, op, is_signed)};
+
+    if (write_result) {
+        v.X(dest_reg, IR::U32{color});
+    }
+}
+} // Anonymous namespace
+
+void TranslatorVisitor::SUATOM(u64 insn) {
+    union {
+        u64 raw;
+        BitField<54, 1, u64> is_bindless;
+        BitField<29, 4, AtomicOp> op;
+        BitField<33, 3, Type> type;
+        BitField<51, 3, Size> size;
+        BitField<49, 2, Clamp> clamp;
+        BitField<0, 8, IR::Reg> dest_reg;
+        BitField<8, 8, IR::Reg> coord_reg;
+        BitField<20, 8, IR::Reg> operand_reg;
+        BitField<36, 13, u64> bound_offset;    // !is_bindless
+        BitField<39, 8, IR::Reg> bindless_reg; // is_bindless
+    } const suatom{insn};
+
+    ImageAtomOp(*this, suatom.dest_reg, suatom.operand_reg, suatom.coord_reg, suatom.bindless_reg,
+                suatom.op, suatom.clamp, suatom.size, suatom.type, suatom.bound_offset,
+                suatom.is_bindless != 0, true);
+}
+
+void TranslatorVisitor::SURED(u64 insn) {
+    // TODO: confirm offsets
+    union {
+        u64 raw;
+        BitField<51, 1, u64> is_bound;
+        BitField<21, 3, AtomicOp> op;
+        BitField<33, 3, Type> type;
+        BitField<20, 3, Size> size;
+        BitField<49, 2, Clamp> clamp;
+        BitField<0, 8, IR::Reg> operand_reg;
+        BitField<8, 8, IR::Reg> coord_reg;
+        BitField<36, 13, u64> bound_offset;    // is_bound
+        BitField<39, 8, IR::Reg> bindless_reg; // !is_bound
+    } const sured{insn};
+    ImageAtomOp(*this, IR::Reg::RZ, sured.operand_reg, sured.coord_reg, sured.bindless_reg,
+                sured.op, sured.clamp, sured.size, sured.type, sured.bound_offset,
+                sured.is_bound == 0, false);
+}
+
+} // namespace Shader::Maxwell

src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp

@@ -565,6 +565,7 @@ void VisitUsages(Info& info, IR::Inst& inst) {
     case IR::Opcode::ImageWrite: {
         const auto flags{inst.Flags<IR::TextureInstInfo>()};
         info.uses_typeless_image_writes |= flags.image_format == ImageFormat::Typeless;
+        info.uses_image_buffers |= flags.type == TextureType::Buffer;
         break;
     }
     case IR::Opcode::SubgroupEqMask:
@@ -696,6 +697,41 @@ void VisitUsages(Info& info, IR::Inst& inst) {
         info.used_storage_buffer_types |= IR::Type::U64;
         info.uses_int64_bit_atomics = true;
         break;
+    case IR::Opcode::BindlessImageAtomicIAdd32:
+    case IR::Opcode::BindlessImageAtomicSMin32:
+    case IR::Opcode::BindlessImageAtomicUMin32:
+    case IR::Opcode::BindlessImageAtomicSMax32:
+    case IR::Opcode::BindlessImageAtomicUMax32:
+    case IR::Opcode::BindlessImageAtomicInc32:
+    case IR::Opcode::BindlessImageAtomicDec32:
+    case IR::Opcode::BindlessImageAtomicAnd32:
+    case IR::Opcode::BindlessImageAtomicOr32:
+    case IR::Opcode::BindlessImageAtomicXor32:
+    case IR::Opcode::BindlessImageAtomicExchange32:
+    case IR::Opcode::BoundImageAtomicIAdd32:
+    case IR::Opcode::BoundImageAtomicSMin32:
+    case IR::Opcode::BoundImageAtomicUMin32:
+    case IR::Opcode::BoundImageAtomicSMax32:
+    case IR::Opcode::BoundImageAtomicUMax32:
+    case IR::Opcode::BoundImageAtomicInc32:
+    case IR::Opcode::BoundImageAtomicDec32:
+    case IR::Opcode::BoundImageAtomicAnd32:
+    case IR::Opcode::BoundImageAtomicOr32:
+    case IR::Opcode::BoundImageAtomicXor32:
+    case IR::Opcode::BoundImageAtomicExchange32:
+    case IR::Opcode::ImageAtomicIAdd32:
+    case IR::Opcode::ImageAtomicSMin32:
+    case IR::Opcode::ImageAtomicUMin32:
+    case IR::Opcode::ImageAtomicSMax32:
+    case IR::Opcode::ImageAtomicUMax32:
+    case IR::Opcode::ImageAtomicInc32:
+    case IR::Opcode::ImageAtomicDec32:
+    case IR::Opcode::ImageAtomicAnd32:
+    case IR::Opcode::ImageAtomicOr32:
+    case IR::Opcode::ImageAtomicXor32:
+    case IR::Opcode::ImageAtomicExchange32:
+        info.uses_atomic_image_u32 = true;
+        break;
     default:
         break;
     }

src/shader_recompiler/ir_opt/texture_pass.cpp

@@ -76,6 +76,39 @@ IR::Opcode IndexedInstruction(const IR::Inst& inst) {
     case IR::Opcode::BoundImageWrite:
     case IR::Opcode::BindlessImageWrite:
         return IR::Opcode::ImageWrite;
+    case IR::Opcode::BoundImageAtomicIAdd32:
+    case IR::Opcode::BindlessImageAtomicIAdd32:
+        return IR::Opcode::ImageAtomicIAdd32;
+    case IR::Opcode::BoundImageAtomicSMin32:
+    case IR::Opcode::BindlessImageAtomicSMin32:
+        return IR::Opcode::ImageAtomicSMin32;
+    case IR::Opcode::BoundImageAtomicUMin32:
+    case IR::Opcode::BindlessImageAtomicUMin32:
+        return IR::Opcode::ImageAtomicUMin32;
+    case IR::Opcode::BoundImageAtomicSMax32:
+    case IR::Opcode::BindlessImageAtomicSMax32:
+        return IR::Opcode::ImageAtomicSMax32;
+    case IR::Opcode::BoundImageAtomicUMax32:
+    case IR::Opcode::BindlessImageAtomicUMax32:
+        return IR::Opcode::ImageAtomicUMax32;
+    case IR::Opcode::BoundImageAtomicInc32:
+    case IR::Opcode::BindlessImageAtomicInc32:
+        return IR::Opcode::ImageAtomicInc32;
+    case IR::Opcode::BoundImageAtomicDec32:
+    case IR::Opcode::BindlessImageAtomicDec32:
+        return IR::Opcode::ImageAtomicDec32;
+    case IR::Opcode::BoundImageAtomicAnd32:
+    case IR::Opcode::BindlessImageAtomicAnd32:
+        return IR::Opcode::ImageAtomicAnd32;
+    case IR::Opcode::BoundImageAtomicOr32:
+    case IR::Opcode::BindlessImageAtomicOr32:
+        return IR::Opcode::ImageAtomicOr32;
+    case IR::Opcode::BoundImageAtomicXor32:
+    case IR::Opcode::BindlessImageAtomicXor32:
+        return IR::Opcode::ImageAtomicXor32;
+    case IR::Opcode::BoundImageAtomicExchange32:
+    case IR::Opcode::BindlessImageAtomicExchange32:
+        return IR::Opcode::ImageAtomicExchange32;
     default:
         return IR::Opcode::Void;
     }
@@ -95,6 +128,17 @@ bool IsBindless(const IR::Inst& inst) {
     case IR::Opcode::BindlessImageGradient:
     case IR::Opcode::BindlessImageRead:
     case IR::Opcode::BindlessImageWrite:
+    case IR::Opcode::BindlessImageAtomicIAdd32:
+    case IR::Opcode::BindlessImageAtomicSMin32:
+    case IR::Opcode::BindlessImageAtomicUMin32:
+    case IR::Opcode::BindlessImageAtomicSMax32:
+    case IR::Opcode::BindlessImageAtomicUMax32:
+    case IR::Opcode::BindlessImageAtomicInc32:
+    case IR::Opcode::BindlessImageAtomicDec32:
+    case IR::Opcode::BindlessImageAtomicAnd32:
+    case IR::Opcode::BindlessImageAtomicOr32:
+    case IR::Opcode::BindlessImageAtomicXor32:
+    case IR::Opcode::BindlessImageAtomicExchange32:
         return true;
     case IR::Opcode::BoundImageSampleImplicitLod:
     case IR::Opcode::BoundImageSampleExplicitLod:
@@ -108,6 +152,17 @@ bool IsBindless(const IR::Inst& inst) {
     case IR::Opcode::BoundImageGradient:
     case IR::Opcode::BoundImageRead:
     case IR::Opcode::BoundImageWrite:
+    case IR::Opcode::BoundImageAtomicIAdd32:
+    case IR::Opcode::BoundImageAtomicSMin32:
+    case IR::Opcode::BoundImageAtomicUMin32:
+    case IR::Opcode::BoundImageAtomicSMax32:
+    case IR::Opcode::BoundImageAtomicUMax32:
+    case IR::Opcode::BoundImageAtomicInc32:
+    case IR::Opcode::BoundImageAtomicDec32:
+    case IR::Opcode::BoundImageAtomicAnd32:
+    case IR::Opcode::BoundImageAtomicOr32:
+    case IR::Opcode::BoundImageAtomicXor32:
+    case IR::Opcode::BoundImageAtomicExchange32:
         return false;
     default:
         throw InvalidArgument("Invalid opcode {}", inst.GetOpcode());
@@ -359,11 +414,22 @@ void TexturePass(Environment& env, IR::Program& program) {
         u32 index;
         switch (inst->GetOpcode()) {
         case IR::Opcode::ImageRead:
+        case IR::Opcode::ImageAtomicIAdd32:
+        case IR::Opcode::ImageAtomicSMin32:
+        case IR::Opcode::ImageAtomicUMin32:
+        case IR::Opcode::ImageAtomicSMax32:
+        case IR::Opcode::ImageAtomicUMax32:
+        case IR::Opcode::ImageAtomicInc32:
+        case IR::Opcode::ImageAtomicDec32:
+        case IR::Opcode::ImageAtomicAnd32:
+        case IR::Opcode::ImageAtomicOr32:
+        case IR::Opcode::ImageAtomicXor32:
+        case IR::Opcode::ImageAtomicExchange32:
         case IR::Opcode::ImageWrite: {
             if (cbuf.has_secondary) {
                 throw NotImplementedException("Unexpected separate sampler");
             }
-            const bool is_written{inst->GetOpcode() == IR::Opcode::ImageWrite};
+            const bool is_written{inst->GetOpcode() != IR::Opcode::ImageRead};
             if (flags.type == TextureType::Buffer) {
                 index = descriptors.Add(ImageBufferDescriptor{
                     .format = flags.image_format,

src/shader_recompiler/shader_info.h

@@ -160,6 +160,7 @@ struct Info {
     bool uses_derivatives{};
     bool uses_typeless_image_reads{};
     bool uses_typeless_image_writes{};
+    bool uses_image_buffers{};
     bool uses_shared_increment{};
     bool uses_shared_decrement{};
     bool uses_global_increment{};
@@ -173,6 +174,7 @@ struct Info {
     bool uses_atomic_f32x2_max{};
     bool uses_int64_bit_atomics{};
     bool uses_global_memory{};
+    bool uses_atomic_image_u32{};
 
     IR::Type used_constant_buffer_types{};
     IR::Type used_storage_buffer_types{};