Keep track of transform feedback varyings pipeline state

app/src/main/cpp/skyline/gpu/interconnect/maxwell_3d/packed_pipeline_state.cpp

@@ -300,6 +300,37 @@ namespace skyline::gpu::interconnect::maxwell3d {
             .blendEnable = state.blendEnable
         };
     }
+
+    void PackedPipelineState::SetTransformFeedbackVaryings(const engine::StreamOutControl &control, const std::array<u8, engine::StreamOutLayoutSelectAttributeCount> &layoutSelect, size_t buffer) {
+        for (size_t i{}; i < control.componentCount; i++) {
+            // TODO: We could merge multiple component accesses from the same attribute into one varying as yuzu does
+            u8 attributeIndex{layoutSelect[i]};
+
+            if (control.strideBytes > std::numeric_limits<u16>::max())
+                throw exception("Stride too large: {}", control.strideBytes);
+
+            transformFeedbackVaryings[attributeIndex] = {
+                .buffer = static_cast<u8>(buffer),
+                .offsetWords = static_cast<u8>(i),
+                .stride = static_cast<u16>(control.strideBytes)
+            };
+        }
+    }
+
+    std::vector<Shader::TransformFeedbackVarying> PackedPipelineState::GetTransformFeedbackVaryings() const {
+        std::vector<Shader::TransformFeedbackVarying> convertedVaryings;
+        convertedVaryings.reserve(0x100);
+        for (const auto &varying : transformFeedbackVaryings)
+            convertedVaryings.push_back(
+                {
+                    .buffer = varying.buffer,
+                    .stride = varying.stride,
+                    .offset = varying.offsetWords * 4U,
+                    .components = 1,
+                });
+
+        return convertedVaryings;
+    }
 }
 
 #pragma clang diagnostic pop

app/src/main/cpp/skyline/gpu/interconnect/maxwell_3d/packed_pipeline_state.h

@@ -3,8 +3,9 @@
 
 #pragma once
 
-#include "common.h"
 #include <tuple>
+#include <shader_compiler/runtime_info.h>
+#include "common.h"
 
 namespace skyline::gpu::interconnect::maxwell3d {
     #pragma clang diagnostic push
@@ -53,6 +54,7 @@ namespace skyline::gpu::interconnect::maxwell3d {
             u8 bindlessTextureConstantBufferSlotSelect : 5;
             bool apiMandatedEarlyZ : 1;
             bool openGlNdc : 1;
+            bool transformFeedbackEnable : 1;
         };
 
         u32 patchSize;
@@ -89,6 +91,13 @@ namespace skyline::gpu::interconnect::maxwell3d {
 
         std::array<AttachmentBlendState, engine::ColorTargetCount> attachmentBlendStates;
 
+        struct TransformFeedbackVarying {
+            u16 stride;
+            u8 offsetWords;
+            u8 buffer;
+        };
+        std::array<TransformFeedbackVarying, 0x100> transformFeedbackVaryings{};
+
         void SetColorRenderTargetFormat(size_t index, engine::ColorTarget::Format format);
 
         void SetDepthRenderTargetFormat(engine::ZtFormat format);
@@ -121,8 +130,26 @@ namespace skyline::gpu::interconnect::maxwell3d {
 
         vk::PipelineColorBlendAttachmentState GetAttachmentBlendState(u32 index) const;
 
+        void SetTransformFeedbackVaryings(const engine::StreamOutControl &control, const std::array<u8, engine::StreamOutLayoutSelectAttributeCount> &layoutSelect, size_t buffer);
+
+        std::vector<Shader::TransformFeedbackVarying> GetTransformFeedbackVaryings() const;
+
         bool operator==(const PackedPipelineState &other) const {
+            // Only hash transform feedback state if it's enabled
+            if (other.transformFeedbackEnable && transformFeedbackEnable)
                 return std::memcmp(this, &other, sizeof(PackedPipelineState)) == 0;
+            else
+                return std::memcmp(this, &other, offsetof(PackedPipelineState, transformFeedbackVaryings)) == 0;
+        }
+    };
+
+    struct PackedPipelineStateHash {
+        size_t operator()(const PackedPipelineState &state) const noexcept {
+            // Only hash transform feedback state if it's enabled
+            if (state.transformFeedbackEnable)
+                return XXH64(&state, sizeof(PackedPipelineState), 0);
+
+            return XXH64(&state, offsetof(PackedPipelineState, transformFeedbackVaryings), 0);
         }
     };
 

app/src/main/cpp/skyline/gpu/interconnect/maxwell_3d/pipeline_manager.cpp

@@ -107,15 +107,14 @@ namespace skyline::gpu::interconnect::maxwell3d {
                     if (packedState.topology == engine::DrawTopology::Points)
                         info.fixed_state_point_size = packedState.pointSize;
 
-                    //if (key.state.xfb_enabled)
+                    if (packedState.transformFeedbackEnable)
-                    //info.xfb_varyings = VideoCommon::MakeTransformFeedbackVaryings(key.state.xfb_state);
+                        info.xfb_varyings = packedState.GetTransformFeedbackVaryings();
-                    //}
+
                     info.convert_depth_mode = packedState.openGlNdc;
                 }
                 ranges::transform(packedState.vertexAttributes, info.generic_input_types.begin(), &ConvertShaderAttributeType);
                 break;
             case Shader::Stage::TessellationEval:
-                // Double check this!
                 info.tess_clockwise = packedState.outputPrimitives != engine::TessellationParameters::OutputPrimitives::TrianglesCCW;
                 info.tess_primitive = ConvertShaderTessPrimitive(packedState.domainType);
                 info.tess_spacing = ConvertShaderTessSpacing(packedState.spacing);
@@ -124,9 +123,9 @@ namespace skyline::gpu::interconnect::maxwell3d {
                 if (program.output_topology == Shader::OutputTopology::PointList)
                     info.fixed_state_point_size = packedState.pointSize;
 
-             //   if (key.state.xfb_enabled != 0) {
+                if (packedState.transformFeedbackEnable)
-              //      info.xfb_varyings = VideoCommon::MakeTransformFeedbackVaryings(key.state.xfb_state);
+                    info.xfb_varyings = packedState.GetTransformFeedbackVaryings();
-               // }
+
                 info.convert_depth_mode = packedState.openGlNdc;
                 break;
             case Shader::Stage::Fragment:

app/src/main/cpp/skyline/gpu/interconnect/maxwell_3d/pipeline_state.cpp

@@ -458,6 +458,22 @@ namespace skyline::gpu::interconnect::maxwell3d {
         }
     }
 
+    /* Transform Feedback State */
+    void TransformFeedbackState::EngineRegisters::DirtyBind(DirtyManager &manager, dirty::Handle handle) const {
+        manager.Bind(handle, streamOutputEnable, streamOutControls, streamOutLayoutSelect);
+    }
+
+    TransformFeedbackState::TransformFeedbackState(dirty::Handle dirtyHandle, DirtyManager &manager, const EngineRegisters &engine) : engine{manager, dirtyHandle, engine} {}
+
+    void TransformFeedbackState::Flush(PackedPipelineState &packedState) {
+        packedState.transformFeedbackEnable = engine->streamOutputEnable;
+        packedState.transformFeedbackVaryings = {};
+
+        if (engine->streamOutputEnable)
+            for (size_t i{}; i < engine::StreamOutBufferCount; i++)
+                packedState.SetTransformFeedbackVaryings(engine->streamOutControls[i], engine->streamOutLayoutSelect[i], i);
+    }
+
     /* Global Shader Config State */
     void GlobalShaderConfigState::EngineRegisters::DirtyBind(DirtyManager &manager, dirty::Handle handle) const {
         manager.Bind(handle, postVtgShaderAttributeSkipMask, bindlessTexture, apiMandatedEarlyZ);
@@ -485,6 +501,7 @@ namespace skyline::gpu::interconnect::maxwell3d {
         bindFunc(depthStencilRegisters);
         bindFunc(colorBlendRegisters);
         bindFunc(globalShaderConfigRegisters);
+        bindFunc(transformFeedbackRegisters);
         manager.Bind(handle, ctSelect);
     }
 
@@ -498,6 +515,7 @@ namespace skyline::gpu::interconnect::maxwell3d {
           rasterization{manager, engine.rasterizationRegisters},
           depthStencil{manager, engine.depthStencilRegisters},
           colorBlend{manager, engine.colorBlendRegisters},
+          transformFeedback{manager, engine.transformFeedbackRegisters},
           directState{engine.inputAssemblyRegisters},
           globalShaderConfig{engine.globalShaderConfigRegisters},
           ctSelect{engine.ctSelect} {}
@@ -531,6 +549,7 @@ namespace skyline::gpu::interconnect::maxwell3d {
         rasterization.Update(packedState);
         depthStencil.Update(packedState);
         colorBlend.Update(packedState);
+        transformFeedback.Update(packedState);
         globalShaderConfig.Update(packedState);
 
         if (pipeline) {

app/src/main/cpp/skyline/gpu/interconnect/maxwell_3d/pipeline_state.h

@@ -249,6 +249,25 @@ namespace skyline::gpu::interconnect::maxwell3d {
         void Flush(PackedPipelineState &packedState);
     };
 
+    class TransformFeedbackState : dirty::ManualDirty {
+      public:
+        struct EngineRegisters {
+            const u32 &streamOutputEnable;
+            const std::array<engine::StreamOutControl, engine::StreamOutBufferCount> &streamOutControls;
+            const std::array<std::array<u8, engine::StreamOutLayoutSelectAttributeCount>, engine::StreamOutBufferCount> &streamOutLayoutSelect;
+
+            void DirtyBind(DirtyManager &manager, dirty::Handle handle) const;
+        };
+
+      private:
+        dirty::BoundSubresource<EngineRegisters> engine;
+
+      public:
+        TransformFeedbackState(dirty::Handle dirtyHandle, DirtyManager &manager, const EngineRegisters &engine);
+
+        void Flush(PackedPipelineState &packedState);
+    };
+
     class GlobalShaderConfigState {
       public:
         struct EngineRegisters {
@@ -283,6 +302,7 @@ namespace skyline::gpu::interconnect::maxwell3d {
             RasterizationState::EngineRegisters rasterizationRegisters;
             DepthStencilState::EngineRegisters depthStencilRegisters;
             ColorBlendState::EngineRegisters colorBlendRegisters;
+            TransformFeedbackState::EngineRegisters transformFeedbackRegisters;
             GlobalShaderConfigState::EngineRegisters globalShaderConfigRegisters;
             const engine::CtSelect &ctSelect;
 
@@ -304,6 +324,7 @@ namespace skyline::gpu::interconnect::maxwell3d {
         dirty::ManualDirtyState<RasterizationState> rasterization;
         dirty::ManualDirtyState<DepthStencilState> depthStencil;
         dirty::ManualDirtyState<ColorBlendState> colorBlend;
+        dirty::ManualDirtyState<TransformFeedbackState> transformFeedback;
         GlobalShaderConfigState globalShaderConfig;
         const engine::CtSelect &ctSelect;