From b62ffb612dbd672371d163e3b511e81f0c2282e6 Mon Sep 17 00:00:00 2001
From: Fernando Sahmkow <fsahmkow27@gmail.com>
Date: Tue, 27 Dec 2022 16:40:28 -0500
Subject: Vulkan: rework stencil tracking.

---
 src/video_core/renderer_vulkan/vk_rasterizer.cpp | 106 ++++++++++++++++++-----
 1 file changed, 83 insertions(+), 23 deletions(-)

(limited to 'src/video_core/renderer_vulkan/vk_rasterizer.cpp')

diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index da76b9a22..fc746fe2c 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -886,32 +886,92 @@ void RasterizerVulkan::UpdateStencilFaces(Tegra::Engines::Maxwell3D::Regs& regs)
     if (!state_tracker.TouchStencilProperties()) {
         return;
     }
-    if (regs.stencil_two_side_enable) {
-        // Separate values per face
-        scheduler.Record(
-            [front_ref = regs.stencil_front_ref, front_write_mask = regs.stencil_front_mask,
-             front_test_mask = regs.stencil_front_func_mask, back_ref = regs.stencil_back_ref,
-             back_write_mask = regs.stencil_back_mask,
-             back_test_mask = regs.stencil_back_func_mask](vk::CommandBuffer cmdbuf) {
+    bool update_references = state_tracker.TouchStencilReference();
+    bool update_write_mask = state_tracker.TouchStencilWriteMask();
+    bool update_compare_masks = state_tracker.TouchStencilCompare();
+    if (state_tracker.TouchStencilSide(regs.stencil_two_side_enable != 0)) {
+        update_references = true;
+        update_write_mask = true;
+        update_compare_masks = true;
+    }
+    if (update_references) {
+        [&]() {
+            if (regs.stencil_two_side_enable) {
+                if (!state_tracker.CheckStencilReferenceFront(regs.stencil_front_ref) &&
+                    !state_tracker.CheckStencilReferenceBack(regs.stencil_back_ref)) {
+                    return;
+                }
+            } else {
+                if (!state_tracker.CheckStencilReferenceFront(regs.stencil_front_ref)) {
+                    return;
+                }
+            }
+            scheduler.Record([front_ref = regs.stencil_front_ref, back_ref = regs.stencil_back_ref,
+                              two_sided = regs.stencil_two_side_enable](vk::CommandBuffer cmdbuf) {
+                const bool set_back = two_sided && front_ref != back_ref;
                 // Front face
-                cmdbuf.SetStencilReference(VK_STENCIL_FACE_FRONT_BIT, front_ref);
-                cmdbuf.SetStencilWriteMask(VK_STENCIL_FACE_FRONT_BIT, front_write_mask);
-                cmdbuf.SetStencilCompareMask(VK_STENCIL_FACE_FRONT_BIT, front_test_mask);
-
-                // Back face
-                cmdbuf.SetStencilReference(VK_STENCIL_FACE_BACK_BIT, back_ref);
-                cmdbuf.SetStencilWriteMask(VK_STENCIL_FACE_BACK_BIT, back_write_mask);
-                cmdbuf.SetStencilCompareMask(VK_STENCIL_FACE_BACK_BIT, back_test_mask);
+                cmdbuf.SetStencilReference(set_back ? VK_STENCIL_FACE_FRONT_BIT
+                                                    : VK_STENCIL_FACE_FRONT_AND_BACK,
+                                           front_ref);
+                if (set_back) {
+                    cmdbuf.SetStencilReference(VK_STENCIL_FACE_BACK_BIT, back_ref);
+                }
             });
-    } else {
-        // Front face defines both faces
-        scheduler.Record([ref = regs.stencil_front_ref, write_mask = regs.stencil_front_mask,
-                          test_mask = regs.stencil_front_func_mask](vk::CommandBuffer cmdbuf) {
-            cmdbuf.SetStencilReference(VK_STENCIL_FACE_FRONT_AND_BACK, ref);
-            cmdbuf.SetStencilWriteMask(VK_STENCIL_FACE_FRONT_AND_BACK, write_mask);
-            cmdbuf.SetStencilCompareMask(VK_STENCIL_FACE_FRONT_AND_BACK, test_mask);
-        });
+        }();
+    }
+    if (update_write_mask) {
+        [&]() {
+            if (regs.stencil_two_side_enable) {
+                if (!state_tracker.CheckStencilWriteMaskFront(regs.stencil_front_mask) &&
+                    !state_tracker.CheckStencilWriteMaskBack(regs.stencil_back_mask)) {
+                    return;
+                }
+            } else {
+                if (!state_tracker.CheckStencilWriteMaskFront(regs.stencil_front_mask)) {
+                    return;
+                }
+            }
+            scheduler.Record([front_write_mask = regs.stencil_front_mask,
+                              back_write_mask = regs.stencil_back_mask,
+                              two_sided = regs.stencil_two_side_enable](vk::CommandBuffer cmdbuf) {
+                const bool set_back = two_sided && front_write_mask != back_write_mask;
+                // Front face
+                cmdbuf.SetStencilWriteMask(set_back ? VK_STENCIL_FACE_FRONT_BIT
+                                                    : VK_STENCIL_FACE_FRONT_AND_BACK,
+                                           front_write_mask);
+                if (set_back) {
+                    cmdbuf.SetStencilWriteMask(VK_STENCIL_FACE_BACK_BIT, back_write_mask);
+                }
+            });
+        }();
+    }
+    if (update_compare_masks) {
+        [&]() {
+            if (regs.stencil_two_side_enable) {
+                if (!state_tracker.CheckStencilCompareMaskFront(regs.stencil_front_func_mask) &&
+                    !state_tracker.CheckStencilCompareMaskBack(regs.stencil_back_func_mask)) {
+                    return;
+                }
+            } else {
+                if (!state_tracker.CheckStencilCompareMaskFront(regs.stencil_front_func_mask)) {
+                    return;
+                }
+            }
+            scheduler.Record([front_test_mask = regs.stencil_front_func_mask,
+                              back_test_mask = regs.stencil_back_func_mask,
+                              two_sided = regs.stencil_two_side_enable](vk::CommandBuffer cmdbuf) {
+                const bool set_back = two_sided && front_test_mask != back_test_mask;
+                // Front face
+                cmdbuf.SetStencilCompareMask(set_back ? VK_STENCIL_FACE_FRONT_BIT
+                                                      : VK_STENCIL_FACE_FRONT_AND_BACK,
+                                             front_test_mask);
+                if (set_back) {
+                    cmdbuf.SetStencilCompareMask(VK_STENCIL_FACE_BACK_BIT, back_test_mask);
+                }
+            });
+        }();
     }
+    state_tracker.ClearStencilReset();
 }
 
 void RasterizerVulkan::UpdateLineWidth(Tegra::Engines::Maxwell3D::Regs& regs) {
-- 
cgit v1.2.3