GLRasterizer: Implemented stencil testing in the hw renderer.

src/video_core/renderer_opengl/gl_rasterizer.cpp

@@ -268,7 +268,8 @@ void RasterizerOpenGL::NotifyPicaRegisterChanged(u32 id) {
         break;
 
     // Stencil test
-    case PICA_REG_INDEX(output_merger.stencil_test):
+    case PICA_REG_INDEX(output_merger.stencil_test.raw_func):
+    case PICA_REG_INDEX(output_merger.stencil_test.raw_op):
         SyncStencilTest();
         break;
 
@@ -675,7 +676,15 @@ void RasterizerOpenGL::SyncLogicOp() {
 }
 
 void RasterizerOpenGL::SyncStencilTest() {
-    // TODO: Implement stencil test, mask, and op
+    const auto& regs = Pica::g_state.regs;
+    state.stencil.test_enabled = regs.output_merger.stencil_test.enable && regs.framebuffer.depth_format == Pica::Regs::DepthFormat::D24S8;
+    state.stencil.test_func = PicaToGL::CompareFunc(regs.output_merger.stencil_test.func);
+    state.stencil.test_ref = regs.output_merger.stencil_test.reference_value;
+    state.stencil.test_mask = regs.output_merger.stencil_test.input_mask;
+    state.stencil.write_mask = regs.output_merger.stencil_test.write_mask;
+    state.stencil.action_stencil_fail = PicaToGL::StencilOp(regs.output_merger.stencil_test.action_stencil_fail);
+    state.stencil.action_depth_fail = PicaToGL::StencilOp(regs.output_merger.stencil_test.action_depth_fail);
+    state.stencil.action_depth_pass = PicaToGL::StencilOp(regs.output_merger.stencil_test.action_depth_pass);
 }
 
 void RasterizerOpenGL::SyncDepthTest() {

src/video_core/renderer_opengl/gl_state.cpp

@@ -26,6 +26,9 @@ OpenGLState::OpenGLState() {
     stencil.test_ref = 0;
     stencil.test_mask = -1;
     stencil.write_mask = -1;
+    stencil.action_depth_fail = GL_KEEP;
+    stencil.action_depth_pass = GL_KEEP;
+    stencil.action_stencil_fail = GL_KEEP;
 
     blend.enabled = false;
     blend.src_rgb_func = GL_ONE;
@@ -105,6 +108,12 @@ void OpenGLState::Apply() {
         glStencilFunc(stencil.test_func, stencil.test_ref, stencil.test_mask);
     }
 
+    if (stencil.action_depth_fail != cur_state.stencil.action_depth_fail ||
+            stencil.action_depth_pass != cur_state.stencil.action_depth_pass ||
+            stencil.action_stencil_fail != cur_state.stencil.action_stencil_fail) {
+        glStencilOp(stencil.action_stencil_fail, stencil.action_depth_fail, stencil.action_depth_pass);
+    }
+
     // Stencil mask
     if (stencil.write_mask != cur_state.stencil.write_mask) {
         glStencilMask(stencil.write_mask);

src/video_core/renderer_opengl/gl_state.h

@@ -32,6 +32,9 @@ public:
         GLint test_ref; // GL_STENCIL_REF
         GLuint test_mask; // GL_STENCIL_VALUE_MASK
         GLuint write_mask; // GL_STENCIL_WRITEMASK
+        GLenum action_stencil_fail; // GL_STENCIL_FAIL
+        GLenum action_depth_fail; // GL_STENCIL_PASS_DEPTH_FAIL
+        GLenum action_depth_pass; // GL_STENCIL_PASS_DEPTH_PASS
     } stencil;
 
     struct {

src/video_core/renderer_opengl/pica_to_gl.h

@@ -152,6 +152,27 @@ inline GLenum CompareFunc(Pica::Regs::CompareFunc func) {
     return compare_func_table[(unsigned)func];
 }
 
+inline GLenum StencilOp(Pica::Regs::StencilAction action) {
+    static const GLenum stencil_op_table[] = {
+        GL_KEEP,        // StencilAction::Keep
+        GL_KEEP,
+        GL_REPLACE,     // StencilAction::Replace
+        GL_INCR,        // StencilAction::Increment
+        GL_DECR,        // StencilAction::Decrement
+        GL_INVERT       // StencilAction::Invert
+    };
+
+    // Range check table for input
+    if ((unsigned)action >= ARRAY_SIZE(stencil_op_table)) {
+        LOG_CRITICAL(Render_OpenGL, "Unknown stencil op %d", action);
+        UNREACHABLE();
+
+        return GL_KEEP;
+    }
+
+    return stencil_op_table[(unsigned)action];
+}
+
 inline std::array<GLfloat, 4> ColorRGBA8(const u8* bytes) {
     return { { bytes[0] / 255.0f,
                bytes[1] / 255.0f,