gl_shader_disk_cache: Invalidate shader cache changes with CMake hash

CMakeLists.txt

@@ -419,19 +419,6 @@ function(create_target_directory_groups target_name)
     endforeach()
 endfunction()
 
-# Gets a UTC timstamp and sets the provided variable to it
-function(get_timestamp _var)
-    string(TIMESTAMP timestamp UTC)
-    set(${_var} "${timestamp}" PARENT_SCOPE)
-endfunction()
-
-# generate git/build information
-include(GetGitRevisionDescription)
-get_git_head_revision(GIT_REF_SPEC GIT_REV)
-git_describe(GIT_DESC --always --long --dirty)
-git_branch_name(GIT_BRANCH)
-get_timestamp(BUILD_DATE)
-
 enable_testing()
 add_subdirectory(externals)
 add_subdirectory(src)

CMakeModules/GenerateSCMRev.cmake

@@ -0,0 +1,101 @@
+# Gets a UTC timstamp and sets the provided variable to it
+function(get_timestamp _var)
+    string(TIMESTAMP timestamp UTC)
+    set(${_var} "${timestamp}" PARENT_SCOPE)
+endfunction()
+
+list(APPEND CMAKE_MODULE_PATH "${SRC_DIR}/externals/cmake-modules")
+# generate git/build information
+include(GetGitRevisionDescription)
+get_git_head_revision(GIT_REF_SPEC GIT_REV)
+git_describe(GIT_DESC --always --long --dirty)
+git_branch_name(GIT_BRANCH)
+get_timestamp(BUILD_DATE)
+
+# Generate cpp with Git revision from template
+# Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well
+set(REPO_NAME "")
+set(BUILD_VERSION "0")
+if ($ENV{CI})
+  if ($ENV{TRAVIS})
+    set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG})
+    set(BUILD_TAG $ENV{TRAVIS_TAG})
+  elseif($ENV{APPVEYOR})
+    set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME})
+    set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME})
+  endif()
+  # regex capture the string nightly or canary into CMAKE_MATCH_1
+  string(REGEX MATCH "citra-emu/citra-?(.*)" OUTVAR ${BUILD_REPOSITORY})
+  if (${CMAKE_MATCH_COUNT} GREATER 0)
+    # capitalize the first letter of each word in the repo name.
+    string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1})
+    foreach(WORD ${REPO_NAME_LIST})
+      string(SUBSTRING ${WORD} 0 1 FIRST_LETTER)
+      string(SUBSTRING ${WORD} 1 -1 REMAINDER)
+      string(TOUPPER ${FIRST_LETTER} FIRST_LETTER)
+      set(REPO_NAME "${REPO_NAME}${FIRST_LETTER}${REMAINDER}")
+    endforeach()
+    if (BUILD_TAG)
+      string(REGEX MATCH "${CMAKE_MATCH_1}-([0-9]+)" OUTVAR ${BUILD_TAG})
+      if (${CMAKE_MATCH_COUNT} GREATER 0)
+        set(BUILD_VERSION ${CMAKE_MATCH_1})
+      endif()
+      if (BUILD_VERSION)
+        # This leaves a trailing space on the last word, but we actually want that
+        # because of how it's styled in the title bar.
+        set(BUILD_FULLNAME "${REPO_NAME} ${BUILD_VERSION} ")
+      else()
+        set(BUILD_FULLNAME "")
+      endif()
+    endif()
+  endif()
+endif()
+
+# The variable SRC_DIR must be passed into the script (since it uses the current build directory for all values of CMAKE_*_DIR)
+set(VIDEO_CORE "${SRC_DIR}/src/video_core")
+set(HASH_FILES
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.h"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.h"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.h"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.cpp"
+    "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.h"
+    "${VIDEO_CORE}/shader/decode/arithmetic.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_half.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_half_immediate.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_immediate.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_integer.cpp"
+    "${VIDEO_CORE}/shader/decode/arithmetic_integer_immediate.cpp"
+    "${VIDEO_CORE}/shader/decode/bfe.cpp"
+    "${VIDEO_CORE}/shader/decode/bfi.cpp"
+    "${VIDEO_CORE}/shader/decode/conversion.cpp"
+    "${VIDEO_CORE}/shader/decode/ffma.cpp"
+    "${VIDEO_CORE}/shader/decode/float_set.cpp"
+    "${VIDEO_CORE}/shader/decode/float_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/half_set.cpp"
+    "${VIDEO_CORE}/shader/decode/half_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/hfma2.cpp"
+    "${VIDEO_CORE}/shader/decode/integer_set.cpp"
+    "${VIDEO_CORE}/shader/decode/integer_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/memory.cpp"
+    "${VIDEO_CORE}/shader/decode/other.cpp"
+    "${VIDEO_CORE}/shader/decode/predicate_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/predicate_set_register.cpp"
+    "${VIDEO_CORE}/shader/decode/register_set_predicate.cpp"
+    "${VIDEO_CORE}/shader/decode/shift.cpp"
+    "${VIDEO_CORE}/shader/decode/video.cpp"
+    "${VIDEO_CORE}/shader/decode/xmad.cpp"
+    "${VIDEO_CORE}/shader/decode.cpp"
+    "${VIDEO_CORE}/shader/shader_ir.cpp"
+    "${VIDEO_CORE}/shader/shader_ir.h"
+    "${VIDEO_CORE}/shader/track.cpp"
+)
+set(COMBINED "")
+foreach (F IN LISTS HASH_FILES)
+    file(READ ${F} TMP)
+    set(COMBINED "${COMBINED}${TMP}")
+endforeach()
+string(MD5 SHADER_CACHE_VERSION "${COMBINED}")
+configure_file("${SRC_DIR}/src/common/scm_rev.cpp.in" "scm_rev.cpp" @ONLY)

src/common/CMakeLists.txt

@@ -1,42 +1,56 @@
-# Generate cpp with Git revision from template
-# Also if this is a CI build, add the build name (ie: Nightly, Canary) to the scm_rev file as well
-set(REPO_NAME "")
-set(BUILD_VERSION "0")
-if ($ENV{CI})
-  if ($ENV{TRAVIS})
-    set(BUILD_REPOSITORY $ENV{TRAVIS_REPO_SLUG})
-    set(BUILD_TAG $ENV{TRAVIS_TAG})
-  elseif($ENV{APPVEYOR})
-    set(BUILD_REPOSITORY $ENV{APPVEYOR_REPO_NAME})
-    set(BUILD_TAG $ENV{APPVEYOR_REPO_TAG_NAME})
-  endif()
-  # regex capture the string nightly or canary into CMAKE_MATCH_1
-  string(REGEX MATCH "yuzu-emu/yuzu-?(.*)" OUTVAR ${BUILD_REPOSITORY})
-  if (${CMAKE_MATCH_COUNT} GREATER 0)
-    # capitalize the first letter of each word in the repo name.
-    string(REPLACE "-" ";" REPO_NAME_LIST ${CMAKE_MATCH_1})
-    foreach(WORD ${REPO_NAME_LIST})
-      string(SUBSTRING ${WORD} 0 1 FIRST_LETTER)
-      string(SUBSTRING ${WORD} 1 -1 REMAINDER)
-      string(TOUPPER ${FIRST_LETTER} FIRST_LETTER)
-      set(REPO_NAME "${REPO_NAME}${FIRST_LETTER}${REMAINDER}")
-    endforeach()
-    if (BUILD_TAG)
-      string(REGEX MATCH "${CMAKE_MATCH_1}-([0-9]+)" OUTVAR ${BUILD_TAG})
-      if (${CMAKE_MATCH_COUNT} GREATER 0)
-        set(BUILD_VERSION ${CMAKE_MATCH_1})
-      endif()
-      if (BUILD_VERSION)
-        # This leaves a trailing space on the last word, but we actually want that
-        # because of how it's styled in the title bar.
-        set(BUILD_FULLNAME "${REPO_NAME} ${BUILD_VERSION} ")
-      else()
-        set(BUILD_FULLNAME "")
-      endif()
-    endif()
-  endif()
-endif()
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp.in" "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp" @ONLY)
+# Add a custom command to generate a new shader_cache_version hash when any of the following files change
+# NOTE: This is an approximation of what files affect shader generation, its possible something else
+# could affect the result, but much more unlikely than the following files. Keeping a list of files
+# like this allows for much better caching since it doesn't force the user to recompile binary shaders every update
+set(VIDEO_CORE "${CMAKE_SOURCE_DIR}/src/video_core")
+add_custom_command(OUTPUT scm_rev.cpp
+    COMMAND cmake -DSRC_DIR="${CMAKE_SOURCE_DIR}" -P "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake"
+    DEPENDS
+      # WARNING! It was too much work to try and make a common location for this list,
+      # so if you need to change it, please update CMakeModules/GenerateSCMRev.cmake as well
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_cache.h"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_decompiler.h"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_disk_cache.h"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.cpp"
+      "${VIDEO_CORE}/renderer_opengl/gl_shader_gen.h"
+      "${VIDEO_CORE}/shader/decode/arithmetic.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_half.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_half_immediate.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_immediate.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_integer.cpp"
+      "${VIDEO_CORE}/shader/decode/arithmetic_integer_immediate.cpp"
+      "${VIDEO_CORE}/shader/decode/bfe.cpp"
+      "${VIDEO_CORE}/shader/decode/bfi.cpp"
+      "${VIDEO_CORE}/shader/decode/conversion.cpp"
+      "${VIDEO_CORE}/shader/decode/ffma.cpp"
+      "${VIDEO_CORE}/shader/decode/float_set.cpp"
+      "${VIDEO_CORE}/shader/decode/float_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/half_set.cpp"
+      "${VIDEO_CORE}/shader/decode/half_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/hfma2.cpp"
+      "${VIDEO_CORE}/shader/decode/integer_set.cpp"
+      "${VIDEO_CORE}/shader/decode/integer_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/memory.cpp"
+      "${VIDEO_CORE}/shader/decode/other.cpp"
+      "${VIDEO_CORE}/shader/decode/predicate_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/predicate_set_register.cpp"
+      "${VIDEO_CORE}/shader/decode/register_set_predicate.cpp"
+      "${VIDEO_CORE}/shader/decode/shift.cpp"
+      "${VIDEO_CORE}/shader/decode/video.cpp"
+      "${VIDEO_CORE}/shader/decode/xmad.cpp"
+      "${VIDEO_CORE}/shader/decode.cpp"
+      "${VIDEO_CORE}/shader/shader_ir.cpp"
+      "${VIDEO_CORE}/shader/shader_ir.h"
+      "${VIDEO_CORE}/shader/track.cpp"
+      # and also check that the scm_rev files haven't changed
+      "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.cpp.in"
+      "${CMAKE_CURRENT_SOURCE_DIR}/scm_rev.h"
+      # technically we should regenerate if the git version changed, but its not worth the effort imo
+      "${CMAKE_SOURCE_DIR}/CMakeModules/GenerateSCMRev.cmake"
+)
 
 add_library(common STATIC
     alignment.h

src/common/scm_rev.cpp.in

@@ -11,6 +11,7 @@
 #define BUILD_DATE   "@BUILD_DATE@"
 #define BUILD_FULLNAME "@BUILD_FULLNAME@"
 #define BUILD_VERSION "@BUILD_VERSION@"
+#define SHADER_CACHE_VERSION "@SHADER_CACHE_VERSION@"
 
 namespace Common {
 
@@ -21,6 +22,7 @@ const char g_build_name[]   = BUILD_NAME;
 const char g_build_date[]   = BUILD_DATE;
 const char g_build_fullname[] = BUILD_FULLNAME;
 const char g_build_version[]  = BUILD_VERSION;
+const char g_shader_cache_version[] = SHADER_CACHE_VERSION;
 
 } // namespace
 

src/common/scm_rev.h

@@ -13,5 +13,6 @@ extern const char g_build_name[];
 extern const char g_build_date[];
 extern const char g_build_fullname[];
 extern const char g_build_version[];
+extern const char g_shader_cache_version[];
 
 } // namespace Common

src/video_core/renderer_opengl/gl_shader_disk_cache.cpp

@@ -4,6 +4,8 @@
 
 #pragma once
 
+#include <cstring>
+
 #include <fmt/format.h>
 
 #include "common/assert.h"
@@ -11,6 +13,7 @@
 #include "common/common_types.h"
 #include "common/file_util.h"
 #include "common/logging/log.h"
+#include "common/scm_rev.h"
 
 #include "core/core.h"
 #include "core/hle/kernel/process.h"
@@ -26,9 +29,7 @@ enum class EntryKind : u32 {
 };
 
 constexpr u32 NativeVersion = 1;
-
-// TODO(Rodrigo): Hash files
-constexpr u64 PrecompiledHash = 0xdeadbeefdeadbeef;
+constexpr u32 ShaderHashSize = 64;
 
 // Making sure sizes doesn't change by accident
 static_assert(sizeof(BaseBindings) == 12);
@@ -38,6 +39,12 @@ namespace {
 std::string GetTitleID() {
     return fmt::format("{:016X}", Core::CurrentProcess()->GetTitleID());
 }
+
+std::string GetShaderHash() {
+    std::array<char, ShaderHashSize> hash{};
+    std::strncpy(hash.data(), Common::g_shader_cache_version, ShaderHashSize);
+    return std::string(hash.data());
+}
 } // namespace
 
 ShaderDiskCacheRaw::ShaderDiskCacheRaw(FileUtil::IOFile& file) {
@@ -130,9 +137,9 @@ std::vector<ShaderDiskCachePrecompiledEntry> ShaderDiskCacheOpenGL::LoadPrecompi
     }
     const u64 file_size = file.GetSize();
 
-    u64 precompiled_hash{};
-    file.ReadBytes(&precompiled_hash, sizeof(precompiled_hash));
-    if (precompiled_hash != PrecompiledHash) {
+    char precompiled_hash[ShaderHashSize];
+    file.ReadBytes(&precompiled_hash, ShaderHashSize);
+    if (std::string(precompiled_hash) != GetShaderHash()) {
         LOG_INFO(Render_OpenGL, "Precompiled cache is from another version of yuzu - removing");
         file.Close();
         InvalidatePrecompiled();
@@ -255,7 +262,9 @@ FileUtil::IOFile ShaderDiskCacheOpenGL::AppendPrecompiledFile() const {
     }
 
     if (!existed || file.GetSize() == 0) {
-        file.WriteObject(PrecompiledHash);
+        std::array<char, ShaderHashSize> hash{};
+        std::strcpy(hash.data(), GetShaderHash().c_str());
+        file.WriteArray(hash.data(), hash.size());
     }
     return file;
 }