Buffer Cache: Fully rework the buffer cache.

src/video_core/CMakeLists.txt

@@ -11,8 +11,11 @@ endif()
 
 add_library(video_core STATIC
     buffer_cache/buffer_base.h
+    buffer_cache/buffer_cache_base.h
     buffer_cache/buffer_cache.cpp
     buffer_cache/buffer_cache.h
+    buffer_cache/memory_tracker_base.h
+    buffer_cache/word_manager.h
     cache_types.h
     cdma_pusher.cpp
     cdma_pusher.h
@@ -104,6 +107,7 @@ add_library(video_core STATIC
     renderer_null/renderer_null.h
     renderer_opengl/blit_image.cpp
     renderer_opengl/blit_image.h
+    renderer_opengl/gl_buffer_cache_base.cpp
     renderer_opengl/gl_buffer_cache.cpp
     renderer_opengl/gl_buffer_cache.h
     renderer_opengl/gl_compute_pipeline.cpp
@@ -154,6 +158,7 @@ add_library(video_core STATIC
     renderer_vulkan/renderer_vulkan.cpp
     renderer_vulkan/vk_blit_screen.cpp
     renderer_vulkan/vk_blit_screen.h
+    renderer_vulkan/vk_buffer_cache_base.cpp
     renderer_vulkan/vk_buffer_cache.cpp
     renderer_vulkan/vk_buffer_cache.h
     renderer_vulkan/vk_command_pool.cpp

src/video_core/buffer_cache/buffer_base.h

@@ -1,5 +1,5 @@
-// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
 
 #pragma once
 
@@ -11,9 +11,7 @@
 #include "common/alignment.h"
 #include "common/common_funcs.h"
 #include "common/common_types.h"
-#include "common/div_ceil.h"
-#include "common/settings.h"
-#include "core/memory.h"
+#include "video_core/buffer_cache/word_manager.h"
 
 namespace VideoCommon {
 
@@ -36,116 +34,14 @@ struct NullBufferParams {};
  */
 template <class RasterizerInterface>
 class BufferBase {
-    static constexpr u64 PAGES_PER_WORD = 64;
-    static constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE;
-    static constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
-
-    /// Vector tracking modified pages tightly packed with small vector optimization
-    union WordsArray {
-        /// Returns the pointer to the words state
-        [[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
-            return is_short ? &stack : heap;
-        }
-
-        /// Returns the pointer to the words state
-        [[nodiscard]] u64* Pointer(bool is_short) noexcept {
-            return is_short ? &stack : heap;
-        }
-
-        u64 stack = 0; ///< Small buffers storage
-        u64* heap;     ///< Not-small buffers pointer to the storage
-    };
-
-    struct Words {
-        explicit Words() = default;
-        explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
-            if (IsShort()) {
-                cpu.stack = ~u64{0};
-                gpu.stack = 0;
-                cached_cpu.stack = 0;
-                untracked.stack = ~u64{0};
-            } else {
-                // Share allocation between CPU and GPU pages and set their default values
-                const size_t num_words = NumWords();
-                u64* const alloc = new u64[num_words * 4];
-                cpu.heap = alloc;
-                gpu.heap = alloc + num_words;
-                cached_cpu.heap = alloc + num_words * 2;
-                untracked.heap = alloc + num_words * 3;
-                std::fill_n(cpu.heap, num_words, ~u64{0});
-                std::fill_n(gpu.heap, num_words, 0);
-                std::fill_n(cached_cpu.heap, num_words, 0);
-                std::fill_n(untracked.heap, num_words, ~u64{0});
-            }
-            // Clean up tailing bits
-            const u64 last_word_size = size_bytes % BYTES_PER_WORD;
-            const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
-            const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
-            const u64 last_word = (~u64{0} << shift) >> shift;
-            cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
-            untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
-        }
-
-        ~Words() {
-            Release();
-        }
-
-        Words& operator=(Words&& rhs) noexcept {
-            Release();
-            size_bytes = rhs.size_bytes;
-            cpu = rhs.cpu;
-            gpu = rhs.gpu;
-            cached_cpu = rhs.cached_cpu;
-            untracked = rhs.untracked;
-            rhs.cpu.heap = nullptr;
-            return *this;
-        }
-
-        Words(Words&& rhs) noexcept
-            : size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu},
-              cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} {
-            rhs.cpu.heap = nullptr;
-        }
-
-        Words& operator=(const Words&) = delete;
-        Words(const Words&) = delete;
-
-        /// Returns true when the buffer fits in the small vector optimization
-        [[nodiscard]] bool IsShort() const noexcept {
-            return size_bytes <= BYTES_PER_WORD;
-        }
-
-        /// Returns the number of words of the buffer
-        [[nodiscard]] size_t NumWords() const noexcept {
-            return Common::DivCeil(size_bytes, BYTES_PER_WORD);
-        }
-
-        /// Release buffer resources
-        void Release() {
-            if (!IsShort()) {
-                // CPU written words is the base for the heap allocation
-                delete[] cpu.heap;
-            }
-        }
-
-        u64 size_bytes = 0;
-        WordsArray cpu;
-        WordsArray gpu;
-        WordsArray cached_cpu;
-        WordsArray untracked;
-    };
-
-    enum class Type {
-        CPU,
-        GPU,
-        CachedCPU,
-        Untracked,
-    };
-
 public:
+    static constexpr u64 BASE_PAGE_BITS = 16;
+    static constexpr u64 BASE_PAGE_SIZE = 1ULL << BASE_PAGE_BITS;
+
     explicit BufferBase(RasterizerInterface& rasterizer_, VAddr cpu_addr_, u64 size_bytes)
-        : rasterizer{&rasterizer_}, cpu_addr{Common::AlignDown(cpu_addr_, BYTES_PER_PAGE)},
-          words(Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BYTES_PER_PAGE)) {}
+        : cpu_addr{Common::AlignDown(cpu_addr_, BASE_PAGE_SIZE)},
+          word_manager(cpu_addr, rasterizer_,
+                       Common::AlignUp(size_bytes + (cpu_addr_ - cpu_addr), BASE_PAGE_SIZE)) {}
 
     explicit BufferBase(NullBufferParams) {}
 
@@ -159,94 +55,82 @@ public:
     [[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr,
                                                         u64 query_size) const noexcept {
         const u64 offset = query_cpu_addr - cpu_addr;
-        return ModifiedRegion<Type::CPU>(offset, query_size);
+        return word_manager.ModifiedRegion<Type::CPU>(offset, query_size);
     }
 
     /// Returns the inclusive GPU modified range in a begin end pair
     [[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr,
                                                         u64 query_size) const noexcept {
         const u64 offset = query_cpu_addr - cpu_addr;
-        return ModifiedRegion<Type::GPU>(offset, query_size);
+        return word_manager.ModifiedRegion<Type::GPU>(offset, query_size);
     }
 
     /// Returns true if a region has been modified from the CPU
     [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept {
         const u64 offset = query_cpu_addr - cpu_addr;
-        return IsRegionModified<Type::CPU>(offset, query_size);
+        return word_manager.IsRegionModified<Type::CPU>(offset, query_size);
     }
 
     /// Returns true if a region has been modified from the GPU
     [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) const noexcept {
         const u64 offset = query_cpu_addr - cpu_addr;
-        return IsRegionModified<Type::GPU>(offset, query_size);
+        return word_manager.IsRegionModified<Type::GPU>(offset, query_size);
     }
 
     /// Mark region as CPU modified, notifying the rasterizer about this change
     void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) {
-        ChangeRegionState<Type::CPU, true>(dirty_cpu_addr, size);
+        word_manager.ChangeRegionState<Type::CPU, true>(dirty_cpu_addr, size);
     }
 
     /// Unmark region as CPU modified, notifying the rasterizer about this change
     void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 size) {
-        ChangeRegionState<Type::CPU, false>(dirty_cpu_addr, size);
+        word_manager.ChangeRegionState<Type::CPU, false>(dirty_cpu_addr, size);
     }
 
     /// Mark region as modified from the host GPU
     void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept {
-        ChangeRegionState<Type::GPU, true>(dirty_cpu_addr, size);
+        word_manager.ChangeRegionState<Type::GPU, true>(dirty_cpu_addr, size);
     }
 
     /// Unmark region as modified from the host GPU
     void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 size) noexcept {
-        ChangeRegionState<Type::GPU, false>(dirty_cpu_addr, size);
+        word_manager.ChangeRegionState<Type::GPU, false>(dirty_cpu_addr, size);
     }
 
     /// Mark region as modified from the CPU
     /// but don't mark it as modified until FlusHCachedWrites is called.
     void CachedCpuWrite(VAddr dirty_cpu_addr, u64 size) {
         flags |= BufferFlagBits::CachedWrites;
-        ChangeRegionState<Type::CachedCPU, true>(dirty_cpu_addr, size);
+        word_manager.ChangeRegionState<Type::CachedCPU, true>(dirty_cpu_addr, size);
     }
 
     /// Flushes cached CPU writes, and notify the rasterizer about the deltas
     void FlushCachedWrites() noexcept {
         flags &= ~BufferFlagBits::CachedWrites;
-        const u64 num_words = NumWords();
-        u64* const cached_words = Array<Type::CachedCPU>();
-        u64* const untracked_words = Array<Type::Untracked>();
-        u64* const cpu_words = Array<Type::CPU>();
-        for (u64 word_index = 0; word_index < num_words; ++word_index) {
-            const u64 cached_bits = cached_words[word_index];
-            NotifyRasterizer<false>(word_index, untracked_words[word_index], cached_bits);
-            untracked_words[word_index] |= cached_bits;
-            cpu_words[word_index] |= cached_bits;
-            if (!Settings::values.use_pessimistic_flushes) {
-                cached_words[word_index] = 0;
-            }
-        }
+        word_manager.FlushCachedWrites();
     }
 
     /// Call 'func' for each CPU modified range and unmark those pages as CPU modified
     template <typename Func>
     void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) {
-        ForEachModifiedRange<Type::CPU>(query_cpu_range, size, true, func);
+        word_manager.ForEachModifiedRange<Type::CPU>(query_cpu_range, size, true, func);
     }
 
     /// Call 'func' for each GPU modified range and unmark those pages as GPU modified
     template <typename Func>
     void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) {
-        ForEachModifiedRange<Type::GPU>(query_cpu_range, size, clear, func);
+        word_manager.ForEachModifiedRange<Type::GPU>(query_cpu_range, size, clear, func);
     }
 
     template <typename Func>
     void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) {
-        ForEachModifiedRange<Type::GPU>(query_cpu_range, size, true, func);
+        word_manager.ForEachModifiedRange<Type::GPU>(query_cpu_range, size, true, func);
     }
 
     /// Call 'func' for each GPU modified range and unmark those pages as GPU modified
     template <typename Func>
     void ForEachDownloadRange(Func&& func) {
-        ForEachModifiedRange<Type::GPU>(cpu_addr, SizeBytes(), true, func);
+        word_manager.ForEachModifiedRange<Type::GPU>(cpu_addr, SizeBytes(), true, func);
     }
 
     /// Mark buffer as picked
@@ -297,7 +181,7 @@ public:
 
     /// Returns the size in bytes of the buffer
     [[nodiscard]] u64 SizeBytes() const noexcept {
-        return words.size_bytes;
+        return word_manager.SizeBytes();
     }
 
     size_t getLRUID() const noexcept {
@@ -309,301 +193,8 @@ public:
     }
 
 private:
-    template <Type type>
-    u64* Array() noexcept {
-        if constexpr (type == Type::CPU) {
-            return words.cpu.Pointer(IsShort());
-        } else if constexpr (type == Type::GPU) {
-            return words.gpu.Pointer(IsShort());
-        } else if constexpr (type == Type::CachedCPU) {
-            return words.cached_cpu.Pointer(IsShort());
-        } else if constexpr (type == Type::Untracked) {
-            return words.untracked.Pointer(IsShort());
-        }
-    }
-
-    template <Type type>
-    const u64* Array() const noexcept {
-        if constexpr (type == Type::CPU) {
-            return words.cpu.Pointer(IsShort());
-        } else if constexpr (type == Type::GPU) {
-            return words.gpu.Pointer(IsShort());
-        } else if constexpr (type == Type::CachedCPU) {
-            return words.cached_cpu.Pointer(IsShort());
-        } else if constexpr (type == Type::Untracked) {
-            return words.untracked.Pointer(IsShort());
-        }
-    }
-
-    /**
-     * Change the state of a range of pages
-     *
-     * @param dirty_addr    Base address to mark or unmark as modified
-     * @param size          Size in bytes to mark or unmark as modified
-     */
-    template <Type type, bool enable>
-    void ChangeRegionState(u64 dirty_addr, s64 size) noexcept(type == Type::GPU) {
-        const s64 difference = dirty_addr - cpu_addr;
-        const u64 offset = std::max<s64>(difference, 0);
-        size += std::min<s64>(difference, 0);
-        if (offset >= SizeBytes() || size < 0) {
-            return;
-        }
-        u64* const untracked_words = Array<Type::Untracked>();
-        u64* const state_words = Array<type>();
-        const u64 offset_end = std::min(offset + size, SizeBytes());
-        const u64 begin_page_index = offset / BYTES_PER_PAGE;
-        const u64 begin_word_index = begin_page_index / PAGES_PER_WORD;
-        const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE);
-        const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD);
-        u64 page_index = begin_page_index % PAGES_PER_WORD;
-        u64 word_index = begin_word_index;
-        while (word_index < end_word_index) {
-            const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD;
-            const u64 left_offset =
-                std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD;
-            const u64 right_offset = page_index;
-            u64 bits = ~u64{0};
-            bits = (bits >> right_offset) << right_offset;
-            bits = (bits << left_offset) >> left_offset;
-            if constexpr (type == Type::CPU || type == Type::CachedCPU) {
-                NotifyRasterizer<!enable>(word_index, untracked_words[word_index], bits);
-            }
-            if constexpr (enable) {
-                state_words[word_index] |= bits;
-                if constexpr (type == Type::CPU || type == Type::CachedCPU) {
-                    untracked_words[word_index] |= bits;
-                }
-            } else {
-                state_words[word_index] &= ~bits;
-                if constexpr (type == Type::CPU || type == Type::CachedCPU) {
-                    untracked_words[word_index] &= ~bits;
-                }
-            }
-            page_index = 0;
-            ++word_index;
-        }
-    }
-
-    /**
-     * Notify rasterizer about changes in the CPU tracking state of a word in the buffer
-     *
-     * @param word_index   Index to the word to notify to the rasterizer
-     * @param current_bits Current state of the word
-     * @param new_bits     New state of the word
-     *
-     * @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages
-     */
-    template <bool add_to_rasterizer>
-    void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const {
-        u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits;
-        VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
-        while (changed_bits != 0) {
-            const int empty_bits = std::countr_zero(changed_bits);
-            addr += empty_bits * BYTES_PER_PAGE;
-            changed_bits >>= empty_bits;
-
-            const u32 continuous_bits = std::countr_one(changed_bits);
-            const u64 size = continuous_bits * BYTES_PER_PAGE;
-            const VAddr begin_addr = addr;
-            addr += size;
-            changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0;
-            rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1);
-        }
-    }
-
-    /**
-     * Loop over each page in the given range, turn off those bits and notify the rasterizer if
-     * needed. Call the given function on each turned off range.
-     *
-     * @param query_cpu_range Base CPU address to loop over
-     * @param size            Size in bytes of the CPU range to loop over
-     * @param func            Function to call for each turned off region
-     */
-    template <Type type, typename Func>
-    void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) {
-        static_assert(type != Type::Untracked);
-
-        const s64 difference = query_cpu_range - cpu_addr;
-        const u64 query_begin = std::max<s64>(difference, 0);
-        size += std::min<s64>(difference, 0);
-        if (query_begin >= SizeBytes() || size < 0) {
-            return;
-        }
-        u64* const untracked_words = Array<Type::Untracked>();
-        u64* const state_words = Array<type>();
-        const u64 query_end = query_begin + std::min(static_cast<u64>(size), SizeBytes());
-        u64* const words_begin = state_words + query_begin / BYTES_PER_WORD;
-        u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD);
-
-        const auto modified = [](u64 word) { return word != 0; };
-        const auto first_modified_word = std::find_if(words_begin, words_end, modified);
-        if (first_modified_word == words_end) {
-            // Exit early when the buffer is not modified
-            return;
-        }
-        const auto last_modified_word = std::find_if_not(first_modified_word, words_end, modified);
-
-        const u64 word_index_begin = std::distance(state_words, first_modified_word);
-        const u64 word_index_end = std::distance(state_words, last_modified_word);
-
-        const unsigned local_page_begin = std::countr_zero(*first_modified_word);
-        const unsigned local_page_end =
-            static_cast<unsigned>(PAGES_PER_WORD) - std::countl_zero(last_modified_word[-1]);
-        const u64 word_page_begin = word_index_begin * PAGES_PER_WORD;
-        const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD;
-        const u64 query_page_begin = query_begin / BYTES_PER_PAGE;
-        const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE);
-        const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin);
-        const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end);
-        const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD;
-        const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1;
-
-        u64 page_begin = first_word_page_begin;
-        u64 current_base = 0;
-        u64 current_size = 0;
-        bool on_going = false;
-        for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) {
-            const bool is_last_word = word_index + 1 == word_index_end;
-            const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD;
-            const u64 right_offset = page_begin;
-            const u64 left_offset = PAGES_PER_WORD - page_end;
-            u64 bits = ~u64{0};
-            bits = (bits >> right_offset) << right_offset;
-            bits = (bits << left_offset) >> left_offset;
-
-            const u64 current_word = state_words[word_index] & bits;
-            if (clear) {
-                state_words[word_index] &= ~bits;
-            }
-
-            if constexpr (type == Type::CPU) {
-                const u64 current_bits = untracked_words[word_index] & bits;
-                untracked_words[word_index] &= ~bits;
-                NotifyRasterizer<true>(word_index, current_bits, ~u64{0});
-            }
-            // Exclude CPU modified pages when visiting GPU pages
-            const u64 word = current_word & ~(type == Type::GPU ? untracked_words[word_index] : 0);
-            u64 page = page_begin;
-            page_begin = 0;
-
-            while (page < page_end) {
-                const int empty_bits = std::countr_zero(word >> page);
-                if (on_going && empty_bits != 0) {
-                    InvokeModifiedRange(func, current_size, current_base);
-                    current_size = 0;
-                    on_going = false;
-                }
-                if (empty_bits == PAGES_PER_WORD) {
-                    break;
-                }
-                page += empty_bits;
-
-                const int continuous_bits = std::countr_one(word >> page);
-                if (!on_going && continuous_bits != 0) {
-                    current_base = word_index * PAGES_PER_WORD + page;
-                    on_going = true;
-                }
-                current_size += continuous_bits;
-                page += continuous_bits;
-            }
-        }
-        if (on_going && current_size > 0) {
-            InvokeModifiedRange(func, current_size, current_base);
-        }
-    }
-
-    template <typename Func>
-    void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) {
-        const u64 current_size_bytes = current_size * BYTES_PER_PAGE;
-        const u64 offset_begin = current_base * BYTES_PER_PAGE;
-        const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes());
-        func(offset_begin, offset_end - offset_begin);
-    }
-
-    /**
-     * Returns true when a region has been modified
-     *
-     * @param offset Offset in bytes from the start of the buffer
-     * @param size   Size in bytes of the region to query for modifications
-     */
-    template <Type type>
-    [[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
-        static_assert(type != Type::Untracked);
-
-        const u64* const untracked_words = Array<Type::Untracked>();
-        const u64* const state_words = Array<type>();
-        const u64 num_query_words = size / BYTES_PER_WORD + 1;
-        const u64 word_begin = offset / BYTES_PER_WORD;
-        const u64 word_end = std::min<u64>(word_begin + num_query_words, NumWords());
-        const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
-        u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD;
-        for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) {
-            const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0;
-            const u64 word = state_words[word_index] & ~off_word;
-            if (word == 0) {
-                continue;
-            }
-            const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit);
-            const u64 local_page_end = page_end % PAGES_PER_WORD;
-            const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD;
-            if (((word >> page_index) << page_index) << page_end_shift != 0) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Returns a begin end pair with the inclusive modified region
-     *
-     * @param offset Offset in bytes from the start of the buffer
-     * @param size   Size in bytes of the region to query for modifications
-     */
-    template <Type type>
-    [[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept {
-        static_assert(type != Type::Untracked);
-
-        const u64* const untracked_words = Array<Type::Untracked>();
-        const u64* const state_words = Array<type>();
-        const u64 num_query_words = size / BYTES_PER_WORD + 1;
-        const u64 word_begin = offset / BYTES_PER_WORD;
-        const u64 word_end = std::min<u64>(word_begin + num_query_words, NumWords());
-        const u64 page_base = offset / BYTES_PER_PAGE;
-        const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
-        u64 begin = std::numeric_limits<u64>::max();
-        u64 end = 0;
-        for (u64 word_index = word_begin; word_index < word_end; ++word_index) {
-            const u64 off_word = type == Type::GPU ? untracked_words[word_index] : 0;
-            const u64 word = state_words[word_index] & ~off_word;
-            if (word == 0) {
-                continue;
-            }
-            const u64 local_page_begin = std::countr_zero(word);
-            const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word);
-            const u64 page_index = word_index * PAGES_PER_WORD;
-            const u64 page_begin = std::max(page_index + local_page_begin, page_base);
-            const u64 page_end = std::min(page_index + local_page_end, page_limit);
-            begin = std::min(begin, page_begin);
-            end = std::max(end, page_end);
-        }
-        static constexpr std::pair<u64, u64> EMPTY{0, 0};
-        return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY;
-    }
-
-    /// Returns the number of words of the buffer
-    [[nodiscard]] size_t NumWords() const noexcept {
-        return words.NumWords();
-    }
-
-    /// Returns true when the buffer fits in the small vector optimization
-    [[nodiscard]] bool IsShort() const noexcept {
-        return words.IsShort();
-    }
-
-    RasterizerInterface* rasterizer = nullptr;
     VAddr cpu_addr = 0;
-    Words words;
+    WordManager<RasterizerInterface> word_manager;
     BufferFlagBits flags{};
     int stream_score = 0;
     size_t lru_id = SIZE_MAX;

src/video_core/buffer_cache/buffer_cache.cpp

@@ -1,5 +1,5 @@
-// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "common/microprofile.h"
 

src/video_core/buffer_cache/buffer_cache_base.h

@@ -0,0 +1,507 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <span>
+#include <unordered_map>
+#include <vector>
+
+#include <boost/container/small_vector.hpp>
+#define BOOST_NO_MT
+#include <boost/pool/detail/mutex.hpp>
+#undef BOOST_NO_MT
+#include <boost/icl/interval_set.hpp>
+#include <boost/pool/pool.hpp>
+#include <boost/pool/pool_alloc.hpp>
+
+#include "common/common_types.h"
+#include "common/div_ceil.h"
+#include "common/literals.h"
+#include "common/lru_cache.h"
+#include "common/microprofile.h"
+#include "common/scope_exit.h"
+#include "common/settings.h"
+#include "core/memory.h"
+#include "video_core/buffer_cache/buffer_base.h"
+#include "video_core/control/channel_state_cache.h"
+#include "video_core/delayed_destruction_ring.h"
+#include "video_core/dirty_flags.h"
+#include "video_core/engines/draw_manager.h"
+#include "video_core/engines/kepler_compute.h"
+#include "video_core/engines/maxwell_3d.h"
+#include "video_core/memory_manager.h"
+#include "video_core/rasterizer_interface.h"
+#include "video_core/surface.h"
+#include "video_core/texture_cache/slot_vector.h"
+#include "video_core/texture_cache/types.h"
+
+
+namespace boost {
+template <typename T>
+class fast_pool_allocator<T, default_user_allocator_new_delete, details::pool::default_mutex, 4096,
+                          0>;
+}
+
+namespace VideoCommon {
+
+MICROPROFILE_DECLARE(GPU_PrepareBuffers);
+MICROPROFILE_DECLARE(GPU_BindUploadBuffers);
+MICROPROFILE_DECLARE(GPU_DownloadMemory);
+
+using BufferId = SlotId;
+
+using VideoCore::Surface::PixelFormat;
+using namespace Common::Literals;
+
+constexpr u32 NUM_VERTEX_BUFFERS = 32;
+constexpr u32 NUM_TRANSFORM_FEEDBACK_BUFFERS = 4;
+constexpr u32 NUM_GRAPHICS_UNIFORM_BUFFERS = 18;
+constexpr u32 NUM_COMPUTE_UNIFORM_BUFFERS = 8;
+constexpr u32 NUM_STORAGE_BUFFERS = 16;
+constexpr u32 NUM_TEXTURE_BUFFERS = 16;
+constexpr u32 NUM_STAGES = 5;
+
+using UniformBufferSizes = std::array<std::array<u32, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES>;
+using ComputeUniformBufferSizes = std::array<u32, NUM_COMPUTE_UNIFORM_BUFFERS>;
+
+enum class ObtainBufferSynchronize : u32 {
+    NoSynchronize = 0,
+    FullSynchronize = 1,
+    SynchronizeNoDirty = 2,
+};
+
+enum class ObtainBufferOperation : u32 {
+    DoNothing = 0,
+    MarkAsWritten = 1,
+    DiscardWrite = 2,
+    MarkQuery = 3,
+};
+
+template <typename P>
+class BufferCache : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> {
+    // Page size for caching purposes.
+    // This is unrelated to the CPU page size and it can be changed as it seems optimal.
+    static constexpr u32 PAGE_BITS = 16;
+    static constexpr u64 PAGE_SIZE = u64{1} << PAGE_BITS;
+    static constexpr u32 CPU_PAGE_BITS = 12;
+    static constexpr u64 CPU_PAGE_SIZE = u64{1} << CPU_PAGE_BITS;
+
+    static constexpr bool IS_OPENGL = P::IS_OPENGL;
+    static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS =
+        P::HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS;
+    static constexpr bool HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT =
+        P::HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT;
+    static constexpr bool NEEDS_BIND_UNIFORM_INDEX = P::NEEDS_BIND_UNIFORM_INDEX;
+    static constexpr bool NEEDS_BIND_STORAGE_INDEX = P::NEEDS_BIND_STORAGE_INDEX;
+    static constexpr bool USE_MEMORY_MAPS = P::USE_MEMORY_MAPS;
+    static constexpr bool SEPARATE_IMAGE_BUFFERS_BINDINGS = P::SEPARATE_IMAGE_BUFFER_BINDINGS;
+    static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = P::IMPLEMENTS_ASYNC_DOWNLOADS;
+
+    static constexpr BufferId NULL_BUFFER_ID{0};
+
+    static constexpr s64 DEFAULT_EXPECTED_MEMORY = 512_MiB;
+    static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB;
+    static constexpr s64 TARGET_THRESHOLD = 4_GiB;
+
+    using Maxwell = Tegra::Engines::Maxwell3D::Regs;
+
+    using Runtime = typename P::Runtime;
+    using Buffer = typename P::Buffer;
+    using Async_Buffer = typename P::Async_Buffer;
+    using MemoryTracker = typename P::MemoryTracker;
+
+    using IntervalCompare = ICL_COMPARE_INSTANCE(ICL_COMPARE_DEFAULT, VAddr);
+    using IntervalInstance = ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, IntervalCompare);
+    using IntervalAllocator = boost::fast_pool_allocator;
+    using IntervalSet =
+        boost::icl::interval_set<VAddr, IntervalCompare, IntervalInstance, IntervalAllocator>;
+    using IntervalType = typename IntervalSet::interval_type;
+
+    struct Empty {};
+
+    struct OverlapResult {
+        std::vector<BufferId> ids;
+        VAddr begin;
+        VAddr end;
+        bool has_stream_leap = false;
+    };
+
+    struct Binding {
+        VAddr cpu_addr{};
+        u32 size{};
+        BufferId buffer_id;
+    };
+
+    struct TextureBufferBinding : Binding {
+        PixelFormat format;
+    };
+
+    static constexpr Binding NULL_BINDING{
+        .cpu_addr = 0,
+        .size = 0,
+        .buffer_id = NULL_BUFFER_ID,
+    };
+
+public:
+    static constexpr u32 DEFAULT_SKIP_CACHE_SIZE = static_cast<u32>(4_KiB);
+
+    explicit BufferCache(VideoCore::RasterizerInterface& rasterizer_,
+                         Core::Memory::Memory& cpu_memory_, Runtime& runtime_);
+
+    void TickFrame();
+
+    void WriteMemory(VAddr cpu_addr, u64 size);
+
+    void CachedWriteMemory(VAddr cpu_addr, u64 size);
+
+    void DownloadMemory(VAddr cpu_addr, u64 size);
+
+    bool InlineMemory(VAddr dest_address, size_t copy_size, std::span<const u8> inlined_buffer);
+
+    void BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size);
+
+    void DisableGraphicsUniformBuffer(size_t stage, u32 index);
+
+    void UpdateGraphicsBuffers(bool is_indexed);
+
+    void UpdateComputeBuffers();
+
+    void BindHostGeometryBuffers(bool is_indexed);
+
+    void BindHostStageBuffers(size_t stage);
+
+    void BindHostComputeBuffers();
+
+    void SetUniformBuffersState(const std::array<u32, NUM_STAGES>& mask,
+                                const UniformBufferSizes* sizes);
+
+    void SetComputeUniformBufferState(u32 mask, const ComputeUniformBufferSizes* sizes);
+
+    void UnbindGraphicsStorageBuffers(size_t stage);
+
+    void BindGraphicsStorageBuffer(size_t stage, size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset,
+                                   bool is_written);
+
+    void UnbindGraphicsTextureBuffers(size_t stage);
+
+    void BindGraphicsTextureBuffer(size_t stage, size_t tbo_index, GPUVAddr gpu_addr, u32 size,
+                                   PixelFormat format, bool is_written, bool is_image);
+
+    void UnbindComputeStorageBuffers();
+
+    void BindComputeStorageBuffer(size_t ssbo_index, u32 cbuf_index, u32 cbuf_offset,
+                                  bool is_written);
+
+    void UnbindComputeTextureBuffers();
+
+    void BindComputeTextureBuffer(size_t tbo_index, GPUVAddr gpu_addr, u32 size, PixelFormat format,
+                                  bool is_written, bool is_image);
+
+    [[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(GPUVAddr gpu_addr, u32 size,
+                                                       ObtainBufferSynchronize sync_info,
+                                                       ObtainBufferOperation post_op);
+    void FlushCachedWrites();
+
+    /// Return true when there are uncommitted buffers to be downloaded
+    [[nodiscard]] bool HasUncommittedFlushes() const noexcept;
+
+    void AccumulateFlushes();
+
+    /// Return true when the caller should wait for async downloads
+    [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
+
+    /// Commit asynchronous downloads
+    void CommitAsyncFlushes();
+    void CommitAsyncFlushesHigh();
+    void CommitAsyncQueries();
+
+    /// Pop asynchronous downloads
+    void PopAsyncFlushes();
+
+    void PopAsyncQueries();
+    void PopAsyncBuffers();
+
+    bool DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount);
+
+    bool DMAClear(GPUVAddr src_address, u64 amount, u32 value);
+
+    /// Return true when a CPU region is modified from the GPU
+    [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
+
+    /// Return true when a region is registered on the cache
+    [[nodiscard]] bool IsRegionRegistered(VAddr addr, size_t size);
+
+    /// Return true when a CPU region is modified from the CPU
+    [[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
+
+    void SetDrawIndirect(const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect_) {
+        current_draw_indirect = current_draw_indirect_;
+    }
+
+    [[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectCount();
+
+    [[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectBuffer();
+
+    std::recursive_mutex mutex;
+    Runtime& runtime;
+
+private:
+    template <typename Func>
+    static void ForEachEnabledBit(u32 enabled_mask, Func&& func) {
+        for (u32 index = 0; enabled_mask != 0; ++index, enabled_mask >>= 1) {
+            const int disabled_bits = std::countr_zero(enabled_mask);
+            index += disabled_bits;
+            enabled_mask >>= disabled_bits;
+            func(index);
+        }
+    }
+
+    template <typename Func>
+    void ForEachBufferInRange(VAddr cpu_addr, u64 size, Func&& func) {
+        const u64 page_end = Common::DivCeil(cpu_addr + size, PAGE_SIZE);
+        for (u64 page = cpu_addr >> PAGE_BITS; page < page_end;) {
+            const BufferId buffer_id = page_table[page];
+            if (!buffer_id) {
+                ++page;
+                continue;
+            }
+            Buffer& buffer = slot_buffers[buffer_id];
+            func(buffer_id, buffer);
+
+            const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
+            page = Common::DivCeil(end_addr, PAGE_SIZE);
+        }
+    }
+
+    template <typename Func>
+    void ForEachInRangeSet(IntervalSet& current_range, VAddr cpu_addr, u64 size, Func&& func) {
+        const VAddr start_address = cpu_addr;
+        const VAddr end_address = start_address + size;
+        const IntervalType search_interval{start_address, end_address};
+        auto it = current_range.lower_bound(search_interval);
+        if (it == current_range.end()) {
+            return;
+        }
+        auto end_it = current_range.upper_bound(search_interval);
+        for (; it != end_it; it++) {
+            VAddr inter_addr_end = it->upper();
+            VAddr inter_addr = it->lower();
+            if (inter_addr_end > end_address) {
+                inter_addr_end = end_address;
+            }
+            if (inter_addr < start_address) {
+                inter_addr = start_address;
+            }
+            func(inter_addr, inter_addr_end);
+        }
+    }
+
+    static bool IsRangeGranular(VAddr cpu_addr, size_t size) {
+        return (cpu_addr & ~Core::Memory::YUZU_PAGEMASK) ==
+               ((cpu_addr + size) & ~Core::Memory::YUZU_PAGEMASK);
+    }
+
+    void RunGarbageCollector();
+
+    void BindHostIndexBuffer();
+
+    void BindHostVertexBuffers();
+
+    void BindHostDrawIndirectBuffers();
+
+    void BindHostGraphicsUniformBuffers(size_t stage);
+
+    void BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind);
+
+    void BindHostGraphicsStorageBuffers(size_t stage);
+
+    void BindHostGraphicsTextureBuffers(size_t stage);
+
+    void BindHostTransformFeedbackBuffers();
+
+    void BindHostComputeUniformBuffers();
+
+    void BindHostComputeStorageBuffers();
+
+    void BindHostComputeTextureBuffers();
+
+    void DoUpdateGraphicsBuffers(bool is_indexed);
+
+    void DoUpdateComputeBuffers();
+
+    void UpdateIndexBuffer();
+
+    void UpdateVertexBuffers();
+
+    void UpdateVertexBuffer(u32 index);
+
+    void UpdateDrawIndirect();
+
+    void UpdateUniformBuffers(size_t stage);
+
+    void UpdateStorageBuffers(size_t stage);
+
+    void UpdateTextureBuffers(size_t stage);
+
+    void UpdateTransformFeedbackBuffers();
+
+    void UpdateTransformFeedbackBuffer(u32 index);
+
+    void UpdateComputeUniformBuffers();
+
+    void UpdateComputeStorageBuffers();
+
+    void UpdateComputeTextureBuffers();
+
+    void MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 size);
+
+    [[nodiscard]] BufferId FindBuffer(VAddr cpu_addr, u32 size);
+
+    [[nodiscard]] OverlapResult ResolveOverlaps(VAddr cpu_addr, u32 wanted_size);
+
+    void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
+
+    [[nodiscard]] BufferId CreateBuffer(VAddr cpu_addr, u32 wanted_size);
+
+    void Register(BufferId buffer_id);
+
+    void Unregister(BufferId buffer_id);
+
+    template <bool insert>
+    void ChangeRegister(BufferId buffer_id);
+
+    void TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept;
+
+    bool SynchronizeBuffer(Buffer& buffer, VAddr cpu_addr, u32 size);
+
+    bool SynchronizeBufferImpl(Buffer& buffer, VAddr cpu_addr, u32 size);
+
+    bool SynchronizeBufferNoModified(Buffer& buffer, VAddr cpu_addr, u32 size);
+
+    void UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy,
+                      std::span<BufferCopy> copies);
+
+    void ImmediateUploadMemory(Buffer& buffer, u64 largest_copy,
+                               std::span<const BufferCopy> copies);
+
+    void MappedUploadMemory(Buffer& buffer, u64 total_size_bytes, std::span<BufferCopy> copies);
+
+    void DownloadBufferMemory(Buffer& buffer_id);
+
+    void DownloadBufferMemory(Buffer& buffer_id, VAddr cpu_addr, u64 size);
+
+    void DeleteBuffer(BufferId buffer_id, bool do_not_mark = false);
+
+    void NotifyBufferDeletion();
+
+    [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr, u32 cbuf_index, bool is_written) const;
+
+    [[nodiscard]] TextureBufferBinding GetTextureBufferBinding(GPUVAddr gpu_addr, u32 size,
+                                                               PixelFormat format);
+
+    [[nodiscard]] std::span<const u8> ImmediateBufferWithData(VAddr cpu_addr, size_t size);
+
+    [[nodiscard]] std::span<u8> ImmediateBuffer(size_t wanted_capacity);
+
+    [[nodiscard]] bool HasFastUniformBufferBound(size_t stage, u32 binding_index) const noexcept;
+
+    void ClearDownload(IntervalType subtract_interval);
+
+    VideoCore::RasterizerInterface& rasterizer;
+    Core::Memory::Memory& cpu_memory;
+
+    SlotVector<Buffer> slot_buffers;
+    DelayedDestructionRing<Buffer, 8> delayed_destruction_ring;
+
+    const Tegra::Engines::DrawManager::IndirectParams* current_draw_indirect{};
+
+    u32 last_index_count = 0;
+
+    Binding index_buffer;
+    std::array<Binding, NUM_VERTEX_BUFFERS> vertex_buffers;
+    std::array<std::array<Binding, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES> uniform_buffers;
+    std::array<std::array<Binding, NUM_STORAGE_BUFFERS>, NUM_STAGES> storage_buffers;
+    std::array<std::array<TextureBufferBinding, NUM_TEXTURE_BUFFERS>, NUM_STAGES> texture_buffers;
+    std::array<Binding, NUM_TRANSFORM_FEEDBACK_BUFFERS> transform_feedback_buffers;
+    Binding count_buffer_binding;
+    Binding indirect_buffer_binding;
+
+    std::array<Binding, NUM_COMPUTE_UNIFORM_BUFFERS> compute_uniform_buffers;
+    std::array<Binding, NUM_STORAGE_BUFFERS> compute_storage_buffers;
+    std::array<TextureBufferBinding, NUM_TEXTURE_BUFFERS> compute_texture_buffers;
+
+    std::array<u32, NUM_STAGES> enabled_uniform_buffer_masks{};
+    u32 enabled_compute_uniform_buffer_mask = 0;
+
+    const UniformBufferSizes* uniform_buffer_sizes{};
+    const ComputeUniformBufferSizes* compute_uniform_buffer_sizes{};
+
+    std::array<u32, NUM_STAGES> enabled_storage_buffers{};
+    std::array<u32, NUM_STAGES> written_storage_buffers{};
+    u32 enabled_compute_storage_buffers = 0;
+    u32 written_compute_storage_buffers = 0;
+
+    std::array<u32, NUM_STAGES> enabled_texture_buffers{};
+    std::array<u32, NUM_STAGES> written_texture_buffers{};
+    std::array<u32, NUM_STAGES> image_texture_buffers{};
+    u32 enabled_compute_texture_buffers = 0;
+    u32 written_compute_texture_buffers = 0;
+    u32 image_compute_texture_buffers = 0;
+
+    std::array<u32, 16> uniform_cache_hits{};
+    std::array<u32, 16> uniform_cache_shots{};
+
+    u32 uniform_buffer_skip_cache_size = DEFAULT_SKIP_CACHE_SIZE;
+
+    bool has_deleted_buffers = false;
+
+    std::conditional_t<HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS, std::array<u32, NUM_STAGES>, Empty>
+        dirty_uniform_buffers{};
+    std::conditional_t<IS_OPENGL, std::array<u32, NUM_STAGES>, Empty> fast_bound_uniform_buffers{};
+    std::conditional_t<HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS,
+                       std::array<std::array<u32, NUM_GRAPHICS_UNIFORM_BUFFERS>, NUM_STAGES>, Empty>
+        uniform_buffer_binding_sizes{};
+
+    std::vector<BufferId> cached_write_buffer_ids;
+
+    MemoryTracker memory_tracker;
+    IntervalSet uncommitted_ranges;
+    IntervalSet common_ranges;
+    IntervalSet cached_ranges;
+    std::deque<IntervalSet> committed_ranges;
+
+    // Async Buffers
+    std::deque<IntervalSet> async_downloads;
+    std::deque<std::optional<Async_Buffer>> async_buffers;
+    std::deque<boost::container::small_vector<BufferCopy, 4>> pending_downloads;
+    std::optional<Async_Buffer> current_buffer;
+
+    // queries
+    boost::container::small_vector<std::pair<VAddr, size_t>, 8> pending_queries;
+    std::deque<boost::container::small_vector<BufferCopy, 8>> committed_queries;
+    boost::container::small_vector<u64, 8> flushed_queries;
+    std::deque<std::optional<Async_Buffer>> query_async_buffers;
+
+    size_t immediate_buffer_capacity = 0;
+    Common::ScratchBuffer<u8> immediate_buffer_alloc;
+
+    struct LRUItemParams {
+        using ObjectType = BufferId;
+        using TickType = u64;
+    };
+    Common::LeastRecentlyUsedCache<LRUItemParams> lru_cache;
+    u64 frame_tick = 0;
+    u64 total_used_memory = 0;
+    u64 minimum_memory = 0;
+    u64 critical_memory = 0;
+
+    std::array<BufferId, ((1ULL << 39) >> PAGE_BITS)> page_table;
+};
+
+} // namespace VideoCommon

src/video_core/buffer_cache/memory_tracker_base.h

@@ -0,0 +1,258 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <deque>
+#include <limits>
+#include <type_traits>
+#include <unordered_set>
+#include <utility>
+
+#include "common/alignment.h"
+#include "common/common_types.h"
+#include "video_core/buffer_cache/word_manager.h"
+
+namespace VideoCommon {
+
+template <class RasterizerInterface>
+class MemoryTrackerBase {
+    static constexpr size_t MAX_CPU_PAGE_BITS = 39;
+    static constexpr size_t HIGHER_PAGE_BITS = 22;
+    static constexpr size_t HIGHER_PAGE_SIZE = 1ULL << HIGHER_PAGE_BITS;
+    static constexpr size_t HIGHER_PAGE_MASK = HIGHER_PAGE_SIZE - 1ULL;
+    static constexpr size_t NUM_HIGH_PAGES = 1ULL << (MAX_CPU_PAGE_BITS - HIGHER_PAGE_BITS);
+    static constexpr size_t MANAGER_POOL_SIZE = 32;
+    static constexpr size_t WORDS_STACK_NEEDED = HIGHER_PAGE_SIZE / BYTES_PER_WORD;
+    using Manager = WordManager<RasterizerInterface, WORDS_STACK_NEEDED>;
+
+public:
+    MemoryTrackerBase(RasterizerInterface& rasterizer_) : rasterizer{&rasterizer_} {}
+    ~MemoryTrackerBase() = default;
+
+    /// Returns the inclusive CPU modified range in a begin end pair
+    [[nodiscard]] std::pair<u64, u64> ModifiedCpuRegion(VAddr query_cpu_addr,
+                                                        u64 query_size) noexcept {
+        return IteratePairs<true>(query_cpu_addr, query_size,
+                                  [](Manager* manager, u64 offset, size_t size) {
+                                      return manager->ModifiedRegion<Type::CPU>(offset, size);
+                                  });
+    }
+
+    /// Returns the inclusive GPU modified range in a begin end pair
+    [[nodiscard]] std::pair<u64, u64> ModifiedGpuRegion(VAddr query_cpu_addr,
+                                                        u64 query_size) noexcept {
+        return IteratePairs<false>(query_cpu_addr, query_size,
+                                   [](Manager* manager, u64 offset, size_t size) {
+                                       return manager->ModifiedRegion<Type::GPU>(offset, size);
+                                   });
+    }
+
+    /// Returns true if a region has been modified from the CPU
+    [[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
+        return IteratePages<true>(query_cpu_addr, query_size,
+                                  [](Manager* manager, u64 offset, size_t size) {
+                                      return manager->IsRegionModified<Type::CPU>(offset, size);
+                                  });
+    }
+
+    /// Returns true if a region has been modified from the GPU
+    [[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
+        return IteratePages<false>(query_cpu_addr, query_size,
+                                   [](Manager* manager, u64 offset, size_t size) {
+                                       return manager->IsRegionModified<Type::GPU>(offset, size);
+                                   });
+    }
+
+    /// Mark region as CPU modified, notifying the rasterizer about this change
+    void MarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
+        IteratePages<true>(
+            dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
+                manager->ChangeRegionState<Type::CPU, true>(manager->GetCpuAddr() + offset, size);
+            });
+    }
+
+    /// Unmark region as CPU modified, notifying the rasterizer about this change
+    void UnmarkRegionAsCpuModified(VAddr dirty_cpu_addr, u64 query_size) {
+        IteratePages<true>(
+            dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
+                manager->ChangeRegionState<Type::CPU, false>(manager->GetCpuAddr() + offset, size);
+            });
+    }
+
+    /// Mark region as modified from the host GPU
+    void MarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
+        IteratePages<true>(
+            dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
+                manager->ChangeRegionState<Type::GPU, true>(manager->GetCpuAddr() + offset, size);
+            });
+    }
+
+    /// Unmark region as modified from the host GPU
+    void UnmarkRegionAsGpuModified(VAddr dirty_cpu_addr, u64 query_size) noexcept {
+        IteratePages<true>(
+            dirty_cpu_addr, query_size, [](Manager* manager, u64 offset, size_t size) {
+                manager->ChangeRegionState<Type::GPU, false>(manager->GetCpuAddr() + offset, size);
+            });
+    }
+
+    /// Mark region as modified from the CPU
+    /// but don't mark it as modified until FlusHCachedWrites is called.
+    void CachedCpuWrite(VAddr dirty_cpu_addr, u64 query_size) {
+        IteratePages<true>(
+            dirty_cpu_addr, query_size, [this](Manager* manager, u64 offset, size_t size) {
+                const VAddr cpu_address = manager->GetCpuAddr() + offset;
+                manager->ChangeRegionState<Type::CachedCPU, true>(cpu_address, size);
+                cached_pages.insert(static_cast<u32>(cpu_address >> HIGHER_PAGE_BITS));
+            });
+    }
+
+    /// Flushes cached CPU writes, and notify the rasterizer about the deltas
+    void FlushCachedWrites(VAddr query_cpu_addr, u64 query_size) noexcept {
+        IteratePages<false>(query_cpu_addr, query_size,
+                            [](Manager* manager, [[maybe_unused]] u64 offset,
+                               [[maybe_unused]] size_t size) { manager->FlushCachedWrites(); });
+    }
+
+    void FlushCachedWrites() noexcept {
+        for (auto id : cached_pages) {
+            top_tier[id]->FlushCachedWrites();
+        }
+        cached_pages.clear();
+    }
+
+    /// Call 'func' for each CPU modified range and unmark those pages as CPU modified
+    template <typename Func>
+    void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
+        IteratePages<true>(query_cpu_range, query_size,
+                           [&func](Manager* manager, u64 offset, size_t size) {
+                               manager->ForEachModifiedRange<Type::CPU>(
+                                   manager->GetCpuAddr() + offset, size, true, func);
+                           });
+    }
+
+    /// Call 'func' for each GPU modified range and unmark those pages as GPU modified
+    template <typename Func>
+    void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, bool clear, Func&& func) {
+        IteratePages<false>(query_cpu_range, query_size,
+                            [&func, clear](Manager* manager, u64 offset, size_t size) {
+                                manager->ForEachModifiedRange<Type::GPU>(
+                                    manager->GetCpuAddr() + offset, size, clear, func);
+                            });
+    }
+
+    template <typename Func>
+    void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 query_size, Func&& func) {
+        IteratePages<false>(query_cpu_range, query_size,
+                            [&func](Manager* manager, u64 offset, size_t size) {
+                                manager->ForEachModifiedRange<Type::GPU>(
+                                    manager->GetCpuAddr() + offset, size, true, func);
+                            });
+    }
+
+private:
+    template <bool create_region_on_fail, typename Func>
+    bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
+        using FuncReturn = typename std::invoke_result<Func, Manager*, u64, size_t>::type;
+        static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
+        std::size_t remaining_size{size};
+        std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
+        u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
+        while (remaining_size > 0) {
+            const std::size_t copy_amount{std::min(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
+            auto* manager{top_tier[page_index]};
+            if (manager) {
+                if constexpr (BOOL_BREAK) {
+                    if (func(manager, page_offset, copy_amount)) {
+                        return true;
+                    }
+                } else {
+                    func(manager, page_offset, copy_amount);
+                }
+            } else if constexpr (create_region_on_fail) {
+                CreateRegion(page_index);
+                manager = top_tier[page_index];
+                if constexpr (BOOL_BREAK) {
+                    if (func(manager, page_offset, copy_amount)) {
+                        return true;
+                    }
+                } else {
+                    func(manager, page_offset, copy_amount);
+                }
+            }
+            page_index++;
+            page_offset = 0;
+            remaining_size -= copy_amount;
+        }
+        return false;
+    }
+
+    template <bool create_region_on_fail, typename Func>
+    std::pair<u64, u64> IteratePairs(VAddr cpu_address, size_t size, Func&& func) {
+        std::size_t remaining_size{size};
+        std::size_t page_index{cpu_address >> HIGHER_PAGE_BITS};
+        u64 page_offset{cpu_address & HIGHER_PAGE_MASK};
+        u64 begin = std::numeric_limits<u64>::max();
+        u64 end = 0;
+        while (remaining_size > 0) {
+            const std::size_t copy_amount{std::min(HIGHER_PAGE_SIZE - page_offset, remaining_size)};
+            auto* manager{top_tier[page_index]};
+            const auto execute = [&] {
+                auto [new_begin, new_end] = func(manager, page_offset, copy_amount);
+                if (new_begin != 0 || new_end != 0) {
+                    const u64 base_address = page_index << HIGHER_PAGE_BITS;
+                    begin = std::min(new_begin + base_address, begin);
+                    end = std::max(new_end + base_address, end);
+                }
+            };
+            if (manager) {
+                execute();
+            } else if constexpr (create_region_on_fail) {
+                CreateRegion(page_index);
+                manager = top_tier[page_index];
+                execute();
+            }
+            page_index++;
+            page_offset = 0;
+            remaining_size -= copy_amount;
+        }
+        return begin < end ? std::make_pair(begin, end) : std::make_pair(0ULL, 0ULL);
+    }
+
+    void CreateRegion(std::size_t page_index) {
+        const VAddr base_cpu_addr = page_index << HIGHER_PAGE_BITS;
+        top_tier[page_index] = GetNewManager(base_cpu_addr);
+    }
+
+    Manager* GetNewManager(VAddr base_cpu_addess) {
+        const auto on_return = [&] {
+            auto* new_manager = free_managers.front();
+            new_manager->SetCpuAddress(base_cpu_addess);
+            free_managers.pop_front();
+            return new_manager;
+        };
+        if (!free_managers.empty()) {
+            return on_return();
+        }
+        manager_pool.emplace_back();
+        auto& last_pool = manager_pool.back();
+        for (size_t i = 0; i < MANAGER_POOL_SIZE; i++) {
+            new (&last_pool[i]) Manager(0, *rasterizer, HIGHER_PAGE_SIZE);
+            free_managers.push_back(&last_pool[i]);
+        }
+        return on_return();
+    }
+
+    std::deque<std::array<Manager, MANAGER_POOL_SIZE>> manager_pool;
+    std::deque<Manager*> free_managers;
+
+    std::array<Manager*, NUM_HIGH_PAGES> top_tier{};
+
+    std::unordered_set<u32> cached_pages;
+
+    RasterizerInterface* rasterizer = nullptr;
+};
+
+} // namespace VideoCommon

src/video_core/buffer_cache/word_manager.h

@@ -0,0 +1,474 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include <algorithm>
+#include <bit>
+#include <limits>
+#include <utility>
+
+#include "common/alignment.h"
+#include "common/common_funcs.h"
+#include "common/common_types.h"
+#include "common/div_ceil.h"
+#include "core/memory.h"
+
+namespace VideoCommon {
+
+constexpr u64 PAGES_PER_WORD = 64;
+constexpr u64 BYTES_PER_PAGE = Core::Memory::YUZU_PAGESIZE;
+constexpr u64 BYTES_PER_WORD = PAGES_PER_WORD * BYTES_PER_PAGE;
+
+/// Vector tracking modified pages tightly packed with small vector optimization
+template <size_t stack_words = 1>
+union WordsArray {
+    /// Returns the pointer to the words state
+    [[nodiscard]] const u64* Pointer(bool is_short) const noexcept {
+        return is_short ? stack.data() : heap;
+    }
+
+    /// Returns the pointer to the words state
+    [[nodiscard]] u64* Pointer(bool is_short) noexcept {
+        return is_short ? stack.data() : heap;
+    }
+
+    std::array<u64, stack_words> stack{}; ///< Small buffers storage
+    u64* heap;                            ///< Not-small buffers pointer to the storage
+};
+
+template <size_t stack_words = 1>
+struct Words {
+    explicit Words() = default;
+    explicit Words(u64 size_bytes_) : size_bytes{size_bytes_} {
+        if (IsShort()) {
+            cpu.stack.fill(~u64{0});
+            gpu.stack.fill(0);
+            cached_cpu.stack.fill(0);
+            untracked.stack.fill(~u64{0});
+        } else {
+            const size_t num_words = NumWords();
+            // Share allocation between CPU and GPU pages and set their default values
+            u64* const alloc = new u64[num_words * 4];
+            cpu.heap = alloc;
+            gpu.heap = alloc + num_words;
+            cached_cpu.heap = alloc + num_words * 2;
+            untracked.heap = alloc + num_words * 3;
+            std::fill_n(cpu.heap, num_words, ~u64{0});
+            std::fill_n(gpu.heap, num_words, 0);
+            std::fill_n(cached_cpu.heap, num_words, 0);
+            std::fill_n(untracked.heap, num_words, ~u64{0});
+        }
+        // Clean up tailing bits
+        const u64 last_word_size = size_bytes % BYTES_PER_WORD;
+        const u64 last_local_page = Common::DivCeil(last_word_size, BYTES_PER_PAGE);
+        const u64 shift = (PAGES_PER_WORD - last_local_page) % PAGES_PER_WORD;
+        const u64 last_word = (~u64{0} << shift) >> shift;
+        cpu.Pointer(IsShort())[NumWords() - 1] = last_word;
+        untracked.Pointer(IsShort())[NumWords() - 1] = last_word;
+    }
+
+    ~Words() {
+        Release();
+    }
+
+    Words& operator=(Words&& rhs) noexcept {
+        Release();
+        size_bytes = rhs.size_bytes;
+        cpu = rhs.cpu;
+        gpu = rhs.gpu;
+        cached_cpu = rhs.cached_cpu;
+        untracked = rhs.untracked;
+        rhs.cpu.heap = nullptr;
+        return *this;
+    }
+
+    Words(Words&& rhs) noexcept
+        : size_bytes{rhs.size_bytes}, cpu{rhs.cpu}, gpu{rhs.gpu},
+          cached_cpu{rhs.cached_cpu}, untracked{rhs.untracked} {
+        rhs.cpu.heap = nullptr;
+    }
+
+    Words& operator=(const Words&) = delete;
+    Words(const Words&) = delete;
+
+    /// Returns true when the buffer fits in the small vector optimization
+    [[nodiscard]] bool IsShort() const noexcept {
+        return size_bytes <= stack_words * BYTES_PER_WORD;
+    }
+
+    /// Returns the number of words of the buffer
+    [[nodiscard]] size_t NumWords() const noexcept {
+        return Common::DivCeil(size_bytes, BYTES_PER_WORD);
+    }
+
+    /// Release buffer resources
+    void Release() {
+        if (!IsShort()) {
+            // CPU written words is the base for the heap allocation
+            delete[] cpu.heap;
+        }
+    }
+
+    u64 size_bytes = 0;
+    WordsArray<stack_words> cpu;
+    WordsArray<stack_words> gpu;
+    WordsArray<stack_words> cached_cpu;
+    WordsArray<stack_words> untracked;
+};
+
+enum class Type {
+    CPU,
+    GPU,
+    CachedCPU,
+    Untracked,
+};
+
+template <class RasterizerInterface, size_t stack_words = 1>
+class WordManager {
+public:
+    explicit WordManager(VAddr cpu_addr_, RasterizerInterface& rasterizer_, u64 size_bytes)
+        : cpu_addr{cpu_addr_}, rasterizer{&rasterizer_}, words{size_bytes} {}
+
+    explicit WordManager() = default;
+
+    void SetCpuAddress(VAddr new_cpu_addr) {
+        cpu_addr = new_cpu_addr;
+    }
+
+    VAddr GetCpuAddr() const {
+        return cpu_addr;
+    }
+
+    /**
+     * Change the state of a range of pages
+     *
+     * @param dirty_addr    Base address to mark or unmark as modified
+     * @param size          Size in bytes to mark or unmark as modified
+     */
+    template <Type type, bool enable>
+    void ChangeRegionState(u64 dirty_addr, s64 size) noexcept(type == Type::GPU) {
+        const s64 difference = dirty_addr - cpu_addr;
+        const u64 offset = std::max<s64>(difference, 0);
+        size += std::min<s64>(difference, 0);
+        if (offset >= SizeBytes() || size < 0) {
+            return;
+        }
+        u64* const untracked_words = Array<Type::Untracked>();
+        u64* const state_words = Array<type>();
+        const u64 offset_end = std::min(offset + size, SizeBytes());
+        const u64 begin_page_index = offset / BYTES_PER_PAGE;
+        const u64 begin_word_index = begin_page_index / PAGES_PER_WORD;
+        const u64 end_page_index = Common::DivCeil(offset_end, BYTES_PER_PAGE);
+        const u64 end_word_index = Common::DivCeil(end_page_index, PAGES_PER_WORD);
+        u64 page_index = begin_page_index % PAGES_PER_WORD;
+        u64 word_index = begin_word_index;
+        while (word_index < end_word_index) {
+            const u64 next_word_first_page = (word_index + 1) * PAGES_PER_WORD;
+            const u64 left_offset =
+                std::min(next_word_first_page - end_page_index, PAGES_PER_WORD) % PAGES_PER_WORD;
+            const u64 right_offset = page_index;
+            u64 bits = ~u64{0};
+            bits = (bits >> right_offset) << right_offset;
+            bits = (bits << left_offset) >> left_offset;
+            if constexpr (type == Type::CPU || type == Type::CachedCPU) {
+                NotifyRasterizer<!enable>(word_index, untracked_words[word_index], bits);
+            }
+            if constexpr (enable) {
+                state_words[word_index] |= bits;
+                if constexpr (type == Type::CPU || type == Type::CachedCPU) {
+                    untracked_words[word_index] |= bits;
+                }
+            } else {
+                state_words[word_index] &= ~bits;
+                if constexpr (type == Type::CPU || type == Type::CachedCPU) {
+                    untracked_words[word_index] &= ~bits;
+                }
+            }
+            page_index = 0;
+            ++word_index;
+        }
+    }
+
+    /**
+     * Loop over each page in the given range, turn off those bits and notify the rasterizer if
+     * needed. Call the given function on each turned off range.
+     *
+     * @param query_cpu_range Base CPU address to loop over
+     * @param size            Size in bytes of the CPU range to loop over
+     * @param func            Function to call for each turned off region
+     */
+    template <Type type, typename Func>
+    void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) {
+        static_assert(type != Type::Untracked);
+
+        const s64 difference = query_cpu_range - cpu_addr;
+        const u64 query_begin = std::max<s64>(difference, 0);
+        size += std::min<s64>(difference, 0);
+        if (query_begin >= SizeBytes() || size < 0) {
+            return;
+        }
+        [[maybe_unused]] u64* const untracked_words = Array<Type::Untracked>();
+        [[maybe_unused]] u64* const cpu_words = Array<Type::CPU>();
+        u64* const state_words = Array<type>();
+        const u64 query_end = query_begin + std::min(static_cast<u64>(size), SizeBytes());
+        u64* const words_begin = state_words + query_begin / BYTES_PER_WORD;
+        u64* const words_end = state_words + Common::DivCeil(query_end, BYTES_PER_WORD);
+        u64 first_page = (query_begin / BYTES_PER_PAGE) % PAGES_PER_WORD;
+
+        const auto modified = [](u64 word) { return word != 0; };
+        const auto first_modified_word = std::find_if(words_begin, words_end, modified);
+        if (first_modified_word == words_end) {
+            // Exit early when the buffer is not modified
+            return;
+        }
+        if (first_modified_word != words_begin) {
+            first_page = 0;
+        }
+        std::reverse_iterator<u64*> first_word_reverse(first_modified_word);
+        std::reverse_iterator<u64*> last_word_iterator(words_end);
+        auto last_word_result = std::find_if(last_word_iterator, first_word_reverse, modified);
+        u64* const last_modified_word = &(*last_word_result) + 1;
+
+        const u64 word_index_begin = std::distance(state_words, first_modified_word);
+        const u64 word_index_end = std::distance(state_words, last_modified_word);
+        const unsigned local_page_begin = std::countr_zero(*first_modified_word);
+        const unsigned local_page_end =
+            static_cast<unsigned>(PAGES_PER_WORD) - std::countl_zero(last_modified_word[-1]);
+        const u64 word_page_begin = word_index_begin * PAGES_PER_WORD;
+        const u64 word_page_end = (word_index_end - 1) * PAGES_PER_WORD;
+        const u64 query_page_begin = query_begin / BYTES_PER_PAGE;
+        const u64 query_page_end = Common::DivCeil(query_end, BYTES_PER_PAGE);
+        const u64 page_index_begin = std::max(word_page_begin + local_page_begin, query_page_begin);
+        const u64 page_index_end = std::min(word_page_end + local_page_end, query_page_end);
+        const u64 first_word_page_begin = page_index_begin % PAGES_PER_WORD;
+        const u64 last_word_page_end = (page_index_end - 1) % PAGES_PER_WORD + 1;
+
+        u64 page_begin = std::max(first_word_page_begin, first_page);
+        u64 current_base = 0;
+        u64 current_size = 0;
+        bool on_going = false;
+        for (u64 word_index = word_index_begin; word_index < word_index_end; ++word_index) {
+            const bool is_last_word = word_index + 1 == word_index_end;
+            const u64 page_end = is_last_word ? last_word_page_end : PAGES_PER_WORD;
+            const u64 right_offset = page_begin;
+            const u64 left_offset = PAGES_PER_WORD - page_end;
+            u64 bits = ~u64{0};
+            bits = (bits >> right_offset) << right_offset;
+            bits = (bits << left_offset) >> left_offset;
+
+            const u64 current_word = state_words[word_index] & bits;
+            if (clear) {
+                state_words[word_index] &= ~bits;
+            }
+
+            if constexpr (type == Type::CachedCPU) {
+                NotifyRasterizer<false>(word_index, untracked_words[word_index], current_word);
+                untracked_words[word_index] |= current_word;
+                cpu_words[word_index] |= current_word;
+            }
+
+            if constexpr (type == Type::CPU) {
+                const u64 current_bits = untracked_words[word_index] & bits;
+                untracked_words[word_index] &= ~bits;
+                NotifyRasterizer<true>(word_index, current_bits, ~u64{0});
+            }
+            const u64 word = current_word;
+            u64 page = page_begin;
+            page_begin = 0;
+
+            while (page < page_end) {
+                const int empty_bits = std::countr_zero(word >> page);
+                if (on_going && empty_bits != 0) {
+                    InvokeModifiedRange(func, current_size, current_base);
+                    current_size = 0;
+                    on_going = false;
+                }
+                if (empty_bits == PAGES_PER_WORD) {
+                    break;
+                }
+                page += empty_bits;
+
+                const int continuous_bits = std::countr_one(word >> page);
+                if (!on_going && continuous_bits != 0) {
+                    current_base = word_index * PAGES_PER_WORD + page;
+                    on_going = true;
+                }
+                current_size += continuous_bits;
+                page += continuous_bits;
+            }
+        }
+        if (on_going && current_size > 0) {
+            InvokeModifiedRange(func, current_size, current_base);
+        }
+    }
+
+    template <typename Func>
+    void InvokeModifiedRange(Func&& func, u64 current_size, u64 current_base) {
+        const u64 current_size_bytes = current_size * BYTES_PER_PAGE;
+        const u64 offset_begin = current_base * BYTES_PER_PAGE;
+        const u64 offset_end = std::min(offset_begin + current_size_bytes, SizeBytes());
+        func(cpu_addr + offset_begin, offset_end - offset_begin);
+    }
+
+    /**
+     * Returns true when a region has been modified
+     *
+     * @param offset Offset in bytes from the start of the buffer
+     * @param size   Size in bytes of the region to query for modifications
+     */
+    template <Type type>
+    [[nodiscard]] bool IsRegionModified(u64 offset, u64 size) const noexcept {
+        static_assert(type != Type::Untracked);
+
+        const u64* const state_words = Array<type>();
+        const u64 num_query_words = size / BYTES_PER_WORD + 1;
+        const u64 word_begin = offset / BYTES_PER_WORD;
+        const u64 word_end = std::min(word_begin + num_query_words, NumWords());
+        const u64 page_limit = Common::DivCeil(offset + size, BYTES_PER_PAGE);
+        u64 page_index = (offset / BYTES_PER_PAGE) % PAGES_PER_WORD;
+        for (u64 word_index = word_begin; word_index < word_end; ++word_index, page_index = 0) {
+            const u64 word = state_words[word_index];
+            if (word == 0) {
+                continue;
+            }
+            const u64 page_end = std::min((word_index + 1) * PAGES_PER_WORD, page_limit);
+            const u64 local_page_end = page_end % PAGES_PER_WORD;
+            const u64 page_end_shift = (PAGES_PER_WORD - local_page_end) % PAGES_PER_WORD;
+            if (((word >> page_index) << page_index) << page_end_shift != 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns a begin end pair with the inclusive modified region
+     *
+     * @param offset Offset in bytes from the start of the buffer
+     * @param size   Size in bytes of the region to query for modifications
+     */
+    template <Type type>
+    [[nodiscard]] std::pair<u64, u64> ModifiedRegion(u64 offset, u64 size) const noexcept {
+        static_assert(type != Type::Untracked);
+        const u64* const state_words = Array<type>();
+        const u64 num_query_words = size / BYTES_PER_WORD + 1;
+        const u64 word_begin = offset / BYTES_PER_WORD;
+        const u64 word_end = std::min(word_begin + num_query_words, NumWords());
+        const u64 page_base = offset / BYTES_PER_PAGE;
+        u64 page_begin = page_base & (PAGES_PER_WORD - 1);
+        u64 page_end =
+            Common::DivCeil(offset + size, BYTES_PER_PAGE) - (page_base & ~(PAGES_PER_WORD - 1));
+        u64 begin = std::numeric_limits<u64>::max();
+        u64 end = 0;
+        for (u64 word_index = word_begin; word_index < word_end; ++word_index) {
+            const u64 base_mask = (1ULL << page_begin) - 1ULL;
+            const u64 end_mask = page_end >= PAGES_PER_WORD ? 0ULL : ~((1ULL << page_end) - 1ULL);
+            const u64 off_word = end_mask | base_mask;
+            const u64 word = state_words[word_index] & ~off_word;
+            if (word == 0) {
+                page_begin = 0;
+                page_end -= PAGES_PER_WORD;
+                continue;
+            }
+            const u64 local_page_begin = std::countr_zero(word);
+            const u64 local_page_end = PAGES_PER_WORD - std::countl_zero(word);
+            const u64 page_index = word_index * PAGES_PER_WORD;
+            begin = std::min(begin, page_index + local_page_begin);
+            end = page_index + local_page_end;
+            page_begin = 0;
+            page_end -= PAGES_PER_WORD;
+        }
+        static constexpr std::pair<u64, u64> EMPTY{0, 0};
+        return begin < end ? std::make_pair(begin * BYTES_PER_PAGE, end * BYTES_PER_PAGE) : EMPTY;
+    }
+
+    /// Returns the number of words of the manager
+    [[nodiscard]] size_t NumWords() const noexcept {
+        return words.NumWords();
+    }
+
+    /// Returns the size in bytes of the manager
+    [[nodiscard]] u64 SizeBytes() const noexcept {
+        return words.size_bytes;
+    }
+
+    /// Returns true when the buffer fits in the small vector optimization
+    [[nodiscard]] bool IsShort() const noexcept {
+        return words.IsShort();
+    }
+
+    void FlushCachedWrites() noexcept {
+        const u64 num_words = NumWords();
+        u64* const cached_words = Array<Type::CachedCPU>();
+        u64* const untracked_words = Array<Type::Untracked>();
+        u64* const cpu_words = Array<Type::CPU>();
+        for (u64 word_index = 0; word_index < num_words; ++word_index) {
+            const u64 cached_bits = cached_words[word_index];
+            NotifyRasterizer<false>(word_index, untracked_words[word_index], cached_bits);
+            untracked_words[word_index] |= cached_bits;
+            cpu_words[word_index] |= cached_bits;
+            cached_words[word_index] = 0;
+        }
+    }
+
+private:
+    template <Type type>
+    u64* Array() noexcept {
+        if constexpr (type == Type::CPU) {
+            return words.cpu.Pointer(IsShort());
+        } else if constexpr (type == Type::GPU) {
+            return words.gpu.Pointer(IsShort());
+        } else if constexpr (type == Type::CachedCPU) {
+            return words.cached_cpu.Pointer(IsShort());
+        } else if constexpr (type == Type::Untracked) {
+            return words.untracked.Pointer(IsShort());
+        }
+    }
+
+    template <Type type>
+    const u64* Array() const noexcept {
+        if constexpr (type == Type::CPU) {
+            return words.cpu.Pointer(IsShort());
+        } else if constexpr (type == Type::GPU) {
+            return words.gpu.Pointer(IsShort());
+        } else if constexpr (type == Type::CachedCPU) {
+            return words.cached_cpu.Pointer(IsShort());
+        } else if constexpr (type == Type::Untracked) {
+            return words.untracked.Pointer(IsShort());
+        }
+    }
+
+    /**
+     * Notify rasterizer about changes in the CPU tracking state of a word in the buffer
+     *
+     * @param word_index   Index to the word to notify to the rasterizer
+     * @param current_bits Current state of the word
+     * @param new_bits     New state of the word
+     *
+     * @tparam add_to_rasterizer True when the rasterizer should start tracking the new pages
+     */
+    template <bool add_to_rasterizer>
+    void NotifyRasterizer(u64 word_index, u64 current_bits, u64 new_bits) const {
+        u64 changed_bits = (add_to_rasterizer ? current_bits : ~current_bits) & new_bits;
+        VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
+        while (changed_bits != 0) {
+            const int empty_bits = std::countr_zero(changed_bits);
+            addr += empty_bits * BYTES_PER_PAGE;
+            changed_bits >>= empty_bits;
+
+            const u32 continuous_bits = std::countr_one(changed_bits);
+            const u64 size = continuous_bits * BYTES_PER_PAGE;
+            const VAddr begin_addr = addr;
+            addr += size;
+            changed_bits = continuous_bits < PAGES_PER_WORD ? (changed_bits >> continuous_bits) : 0;
+            rasterizer->UpdatePagesCachedCount(begin_addr, size, add_to_rasterizer ? 1 : -1);
+        }
+    }
+
+    VAddr cpu_addr = 0;
+    RasterizerInterface* rasterizer = nullptr;
+    Words<stack_words> words;
+};
+
+} // namespace VideoCommon

src/video_core/renderer_opengl/gl_buffer_cache.h

@@ -8,6 +8,7 @@
 
 #include "common/common_types.h"
 #include "video_core/buffer_cache/buffer_cache.h"
+#include "video_core/buffer_cache/memory_tracker_base.h"
 #include "video_core/rasterizer_interface.h"
 #include "video_core/renderer_opengl/gl_device.h"
 #include "video_core/renderer_opengl/gl_resource_manager.h"
@@ -200,6 +201,8 @@ private:
 struct BufferCacheParams {
     using Runtime = OpenGL::BufferCacheRuntime;
     using Buffer = OpenGL::Buffer;
+    using Async_Buffer = u32;
+    using MemoryTracker = VideoCommon::MemoryTrackerBase<VideoCore::RasterizerInterface>;
 
     static constexpr bool IS_OPENGL = true;
     static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = true;
@@ -208,6 +211,7 @@ struct BufferCacheParams {
     static constexpr bool NEEDS_BIND_STORAGE_INDEX = true;
     static constexpr bool USE_MEMORY_MAPS = false;
     static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = true;
+    static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = false;
 };
 
 using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;

src/video_core/renderer_opengl/gl_buffer_cache_base.cpp

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "video_core/buffer_cache/buffer_cache.h"
+#include "video_core/renderer_opengl/gl_buffer_cache.h"
+
+namespace VideoCommon {
+template class VideoCommon::BufferCache<OpenGL::BufferCacheParams>;
+}

src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@@ -314,8 +314,12 @@ StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) {
     return staging_pool.Request(size, MemoryUsage::Upload);
 }
 
-StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size) {
-    return staging_pool.Request(size, MemoryUsage::Download);
+StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size, bool deferred) {
+    return staging_pool.Request(size, MemoryUsage::Download, deferred);
+}
+
+void BufferCacheRuntime::FreeDeferredStagingBuffer(StagingBufferRef& ref) {
+    staging_pool.FreeDeferred(ref);
 }
 
 u64 BufferCacheRuntime::GetDeviceLocalMemory() const {

src/video_core/renderer_vulkan/vk_buffer_cache.h

@@ -4,6 +4,7 @@
 #pragma once
 
 #include "video_core/buffer_cache/buffer_cache.h"
+#include "video_core/buffer_cache/memory_tracker_base.h"
 #include "video_core/engines/maxwell_3d.h"
 #include "video_core/renderer_vulkan/vk_compute_pass.h"
 #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
@@ -75,7 +76,9 @@ public:
 
     [[nodiscard]] StagingBufferRef UploadStagingBuffer(size_t size);
 
-    [[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size);
+    [[nodiscard]] StagingBufferRef DownloadStagingBuffer(size_t size, bool deferred = false);
+
+    void FreeDeferredStagingBuffer(StagingBufferRef& ref);
 
     void PreCopyBarrier();
 
@@ -142,6 +145,8 @@ private:
 struct BufferCacheParams {
     using Runtime = Vulkan::BufferCacheRuntime;
     using Buffer = Vulkan::Buffer;
+    using Async_Buffer = Vulkan::StagingBufferRef;
+    using MemoryTracker = VideoCommon::MemoryTrackerBase<VideoCore::RasterizerInterface>;
 
     static constexpr bool IS_OPENGL = false;
     static constexpr bool HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS = false;
@@ -150,6 +155,7 @@ struct BufferCacheParams {
     static constexpr bool NEEDS_BIND_STORAGE_INDEX = false;
     static constexpr bool USE_MEMORY_MAPS = true;
     static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false;
+    static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true;
 };
 
 using BufferCache = VideoCommon::BufferCache<BufferCacheParams>;

src/video_core/renderer_vulkan/vk_buffer_cache_base.cpp

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "video_core/buffer_cache/buffer_cache.h"
+#include "video_core/renderer_vulkan/vk_buffer_cache.h"
+
+namespace VideoCommon {
+template class VideoCommon::BufferCache<Vulkan::BufferCacheParams>;
+}