renderer_vulkan: fix cropping for presentation

src/core/hle/service/nvnflinger/buffer_transform_flags.h

@@ -3,6 +3,7 @@
 
 #pragma once
 
+#include "common/common_funcs.h"
 #include "common/common_types.h"
 
 namespace Service::android {
@@ -21,5 +22,6 @@ enum class BufferTransformFlags : u32 {
     /// Rotate source image 270 degrees clockwise
     Rotate270 = 0x07,
 };
+DECLARE_ENUM_FLAG_OPERATORS(BufferTransformFlags);
 
 } // namespace Service::android

src/video_core/renderer_vulkan/vk_blit_screen.cpp

@@ -1395,63 +1395,72 @@ void BlitScreen::SetUniformData(BufferData& data, const Layout::FramebufferLayou
         MakeOrthographicMatrix(static_cast<f32>(layout.width), static_cast<f32>(layout.height));
 }
 
-void BlitScreen::SetVertexData(BufferData& data, const Tegra::FramebufferConfig& framebuffer,
-                               const Layout::FramebufferLayout layout) const {
-    const auto& framebuffer_transform_flags = framebuffer.transform_flags;
-    const auto& framebuffer_crop_rect = framebuffer.crop_rect;
-
-    static constexpr Common::Rectangle<f32> texcoords{0.f, 0.f, 1.f, 1.f};
-    auto left = texcoords.left;
-    auto right = texcoords.right;
-
-    switch (framebuffer_transform_flags) {
-    case Service::android::BufferTransformFlags::Unset:
-        break;
-    case Service::android::BufferTransformFlags::FlipV:
-        // Flip the framebuffer vertically
-        left = texcoords.right;
-        right = texcoords.left;
-        break;
-    default:
-        UNIMPLEMENTED_MSG("Unsupported framebuffer_transform_flags={}",
-                          static_cast<u32>(framebuffer_transform_flags));
-        break;
+static Common::Rectangle<f32> NormalizeCrop(Common::Rectangle<int> crop,
+                                            const Tegra::FramebufferConfig& framebuffer) {
+    f32 left, top, right, bottom;
+
+    if (!crop.IsEmpty()) {
+        // If crop rectangle is not empty, apply properties from rectangle.
+        left = static_cast<f32>(crop.left);
+        top = static_cast<f32>(crop.top);
+        right = static_cast<f32>(crop.right);
+        bottom = static_cast<f32>(crop.bottom);
+    } else {
+        // Otherwise, fall back to framebuffer dimensions.
+        left = 0;
+        top = 0;
+        right = static_cast<f32>(framebuffer.width);
+        bottom = static_cast<f32>(framebuffer.height);
     }
 
-    UNIMPLEMENTED_IF(framebuffer_crop_rect.left != 0);
+    // Apply transformation flags.
+    auto framebuffer_transform_flags = framebuffer.transform_flags;
 
-    f32 left_start{};
-    if (framebuffer_crop_rect.Top() > 0) {
-        left_start = static_cast<f32>(framebuffer_crop_rect.Top()) /
-                     static_cast<f32>(framebuffer_crop_rect.Bottom());
+    if (True(framebuffer_transform_flags & Service::android::BufferTransformFlags::FlipH)) {
+        // Switch left and right.
+        std::swap(left, right);
     }
-    f32 scale_u = static_cast<f32>(framebuffer.width) / static_cast<f32>(screen_info.width);
-    f32 scale_v = static_cast<f32>(framebuffer.height) / static_cast<f32>(screen_info.height);
-    // Scale the output by the crop width/height. This is commonly used with 1280x720 rendering
-    // (e.g. handheld mode) on a 1920x1080 framebuffer.
-    if (!fsr) {
-        if (framebuffer_crop_rect.GetWidth() > 0) {
-            scale_u = static_cast<f32>(framebuffer_crop_rect.GetWidth()) /
-                      static_cast<f32>(screen_info.width);
-        }
-        if (framebuffer_crop_rect.GetHeight() > 0) {
-            scale_v = static_cast<f32>(framebuffer_crop_rect.GetHeight()) /
-                      static_cast<f32>(screen_info.height);
-        }
+    if (True(framebuffer_transform_flags & Service::android::BufferTransformFlags::FlipV)) {
+        // Switch top and bottom.
+        std::swap(top, bottom);
+    }
+
+    framebuffer_transform_flags &= ~Service::android::BufferTransformFlags::FlipH;
+    framebuffer_transform_flags &= ~Service::android::BufferTransformFlags::FlipV;
+    if (True(framebuffer_transform_flags)) {
+        UNIMPLEMENTED_MSG("Unsupported framebuffer_transform_flags={}",
+                          static_cast<u32>(framebuffer_transform_flags));
     }
 
+    return Common::Rectangle<f32>(left, top, right, bottom);
+}
+
+void BlitScreen::SetVertexData(BufferData& data, const Tegra::FramebufferConfig& framebuffer,
+                               const Layout::FramebufferLayout layout) const {
+    // Get the normalized crop rectangle.
+    const auto crop = NormalizeCrop(framebuffer.crop_rect, framebuffer);
+
+    // Get the screen properties.
+    const f32 screen_width = static_cast<f32>(screen_info.width);
+    const f32 screen_height = static_cast<f32>(screen_info.height);
+
+    // Apply the crop.
+    const f32 left = crop.left / screen_width;
+    const f32 top = crop.top / screen_height;
+    const f32 right = crop.right / screen_width;
+    const f32 bottom = crop.bottom / screen_height;
+
+    // Map the coordinates to the screen.
     const auto& screen = layout.screen;
     const auto x = static_cast<f32>(screen.left);
     const auto y = static_cast<f32>(screen.top);
     const auto w = static_cast<f32>(screen.GetWidth());
     const auto h = static_cast<f32>(screen.GetHeight());
-    data.vertices[0] = ScreenRectVertex(x, y, texcoords.top * scale_u, left_start + left * scale_v);
-    data.vertices[1] =
-        ScreenRectVertex(x + w, y, texcoords.bottom * scale_u, left_start + left * scale_v);
-    data.vertices[2] =
-        ScreenRectVertex(x, y + h, texcoords.top * scale_u, left_start + right * scale_v);
-    data.vertices[3] =
-        ScreenRectVertex(x + w, y + h, texcoords.bottom * scale_u, left_start + right * scale_v);
+
+    data.vertices[0] = ScreenRectVertex(x, y, left, top);
+    data.vertices[1] = ScreenRectVertex(x + w, y, right, top);
+    data.vertices[2] = ScreenRectVertex(x, y + h, left, bottom);
+    data.vertices[3] = ScreenRectVertex(x + w, y + h, right, bottom);
 }
 
 void BlitScreen::CreateSMAA(VkExtent2D smaa_size) {