Merge pull request #6557 from FernandoS27/staceys-mom-has-got-it-goin-on

Buffer Cache: Fix High downloads / Fence manager: Improve fence checking.

src/tests/video_core/buffer_base.cpp

@@ -536,7 +536,7 @@ TEST_CASE("BufferBase: Cached write downloads") {
     REQUIRE(rasterizer.Count() == 63);
     buffer.MarkRegionAsGpuModified(c + PAGE, PAGE);
     int num = 0;
-    buffer.ForEachDownloadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
+    buffer.ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; });
     buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; });
     REQUIRE(num == 0);
     REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE));

src/video_core/buffer_cache/buffer_base.h

@@ -226,19 +226,24 @@ public:
     /// Call 'func' for each CPU modified range and unmark those pages as CPU modified
     template <typename Func>
     void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) {
-        ForEachModifiedRange<Type::CPU>(query_cpu_range, size, func);
+        ForEachModifiedRange<Type::CPU>(query_cpu_range, size, true, func);
     }
 
     /// Call 'func' for each GPU modified range and unmark those pages as GPU modified
     template <typename Func>
-    void ForEachDownloadRange(VAddr query_cpu_range, u64 size, Func&& func) {
-        ForEachModifiedRange<Type::GPU>(query_cpu_range, size, func);
+    void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) {
+        ForEachModifiedRange<Type::GPU>(query_cpu_range, size, clear, func);
+    }
+
+    template <typename Func>
+    void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) {
+        ForEachModifiedRange<Type::GPU>(query_cpu_range, size, true, func);
     }
 
     /// Call 'func' for each GPU modified range and unmark those pages as GPU modified
     template <typename Func>
     void ForEachDownloadRange(Func&& func) {
-        ForEachModifiedRange<Type::GPU>(cpu_addr, SizeBytes(), func);
+        ForEachModifiedRange<Type::GPU>(cpu_addr, SizeBytes(), true, func);
     }
 
     /// Mark buffer as picked
@@ -415,7 +420,7 @@ private:
      * @param func            Function to call for each turned off region
      */
     template <Type type, typename Func>
-    void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
+    void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) {
         static_assert(type != Type::Untracked);
 
         const s64 difference = query_cpu_range - cpu_addr;
@@ -467,7 +472,9 @@ private:
             bits = (bits << left_offset) >> left_offset;
 
             const u64 current_word = state_words[word_index] & bits;
-            state_words[word_index] &= ~bits;
+            if (clear) {
+                state_words[word_index] &= ~bits;
+            }
 
             if constexpr (type == Type::CPU) {
                 const u64 current_bits = untracked_words[word_index] & bits;

src/video_core/buffer_cache/buffer_cache.h

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include <boost/container/small_vector.hpp>
+#include <boost/icl/interval_set.hpp>
 
 #include "common/common_types.h"
 #include "common/div_ceil.h"
@@ -77,6 +78,9 @@ class BufferCache {
     using Runtime = typename P::Runtime;
     using Buffer = typename P::Buffer;
 
+    using IntervalSet = boost::icl::interval_set<VAddr>;
+    using IntervalType = typename IntervalSet::interval_type;
+
     struct Empty {};
 
     struct OverlapResult {
@@ -148,11 +152,14 @@ public:
     /// Return true when there are uncommitted buffers to be downloaded
     [[nodiscard]] bool HasUncommittedFlushes() const noexcept;
 
+    void AccumulateFlushes();
+
     /// Return true when the caller should wait for async downloads
     [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
 
     /// Commit asynchronous downloads
     void CommitAsyncFlushes();
+    void CommitAsyncFlushesHigh();
 
     /// Pop asynchronous downloads
     void PopAsyncFlushes();
@@ -160,6 +167,9 @@ public:
     /// Return true when a CPU region is modified from the GPU
     [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
 
+    /// Return true when a CPU region is modified from the CPU
+    [[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size);
+
     std::mutex mutex;
 
 private:
@@ -272,8 +282,6 @@ private:
 
     void DeleteBuffer(BufferId buffer_id);
 
-    void ReplaceBufferDownloads(BufferId old_buffer_id, BufferId new_buffer_id);
-
     void NotifyBufferDeletion();
 
     [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr) const;
@@ -327,9 +335,9 @@ private:
 
     std::vector<BufferId> cached_write_buffer_ids;
 
-    // TODO: This data structure is not optimal and it should be reworked
-    std::vector<BufferId> uncommitted_downloads;
-    std::deque<std::vector<BufferId>> committed_downloads;
+    IntervalSet uncommitted_ranges;
+    IntervalSet common_ranges;
+    std::deque<IntervalSet> committed_ranges;
 
     size_t immediate_buffer_capacity = 0;
     std::unique_ptr<u8[]> immediate_buffer_alloc;
@@ -352,6 +360,7 @@ BufferCache<P>::BufferCache(VideoCore::RasterizerInterface& rasterizer_,
     // Ensure the first slot is used for the null buffer
     void(slot_buffers.insert(runtime, NullBufferParams{}));
     deletion_iterator = slot_buffers.end();
+    common_ranges.clear();
 }
 
 template <class P>
@@ -547,29 +556,30 @@ void BufferCache<P>::FlushCachedWrites() {
 
 template <class P>
 bool BufferCache<P>::HasUncommittedFlushes() const noexcept {
-    return !uncommitted_downloads.empty();
+    return !uncommitted_ranges.empty() || !committed_ranges.empty();
 }
 
 template <class P>
-bool BufferCache<P>::ShouldWaitAsyncFlushes() const noexcept {
-    return !committed_downloads.empty() && !committed_downloads.front().empty();
+void BufferCache<P>::AccumulateFlushes() {
+    if (Settings::values.gpu_accuracy.GetValue() != Settings::GPUAccuracy::High) {
+        uncommitted_ranges.clear();
+        return;
+    }
+    if (uncommitted_ranges.empty()) {
+        return;
+    }
+    committed_ranges.emplace_back(std::move(uncommitted_ranges));
 }
 
 template <class P>
-void BufferCache<P>::CommitAsyncFlushes() {
-    // This is intentionally passing the value by copy
-    committed_downloads.push_front(uncommitted_downloads);
-    uncommitted_downloads.clear();
+bool BufferCache<P>::ShouldWaitAsyncFlushes() const noexcept {
+    return false;
 }
 
 template <class P>
-void BufferCache<P>::PopAsyncFlushes() {
-    if (committed_downloads.empty()) {
-        return;
-    }
-    auto scope_exit_pop_download = detail::ScopeExit([this] { committed_downloads.pop_back(); });
-    const std::span<const BufferId> download_ids = committed_downloads.back();
-    if (download_ids.empty()) {
+void BufferCache<P>::CommitAsyncFlushesHigh() {
+    AccumulateFlushes();
+    if (committed_ranges.empty()) {
         return;
     }
     MICROPROFILE_SCOPE(GPU_DownloadMemory);
@@ -577,20 +587,66 @@ void BufferCache<P>::PopAsyncFlushes() {
     boost::container::small_vector<std::pair<BufferCopy, BufferId>, 1> downloads;
     u64 total_size_bytes = 0;
     u64 largest_copy = 0;
-    for (const BufferId buffer_id : download_ids) {
-        slot_buffers[buffer_id].ForEachDownloadRange([&](u64 range_offset, u64 range_size) {
-            downloads.push_back({
-                BufferCopy{
-                    .src_offset = range_offset,
-                    .dst_offset = total_size_bytes,
-                    .size = range_size,
-                },
-                buffer_id,
+    for (const IntervalSet& intervals : committed_ranges) {
+        for (auto& interval : intervals) {
+            const std::size_t size = interval.upper() - interval.lower();
+            const VAddr cpu_addr = interval.lower();
+            ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) {
+                boost::container::small_vector<BufferCopy, 1> copies;
+                buffer.ForEachDownloadRangeAndClear(
+                    cpu_addr, size, [&](u64 range_offset, u64 range_size) {
+                        const VAddr buffer_addr = buffer.CpuAddr();
+                        const auto add_download = [&](VAddr start, VAddr end) {
+                            const u64 new_offset = start - buffer_addr;
+                            const u64 new_size = end - start;
+                            downloads.push_back({
+                                BufferCopy{
+                                    .src_offset = new_offset,
+                                    .dst_offset = total_size_bytes,
+                                    .size = new_size,
+                                },
+                                buffer_id,
+                            });
+                            // Align up to avoid cache conflicts
+                            constexpr u64 align = 256ULL;
+                            constexpr u64 mask = ~(align - 1ULL);
+                            total_size_bytes += (new_size + align - 1) & mask;
+                            largest_copy = std::max(largest_copy, new_size);
+                        };
+
+                        const VAddr start_address = buffer_addr + range_offset;
+                        const VAddr end_address = start_address + range_size;
+                        const IntervalType search_interval{cpu_addr, 1};
+                        auto it = common_ranges.lower_bound(search_interval);
+                        if (it == common_ranges.end()) {
+                            it = common_ranges.begin();
+                        }
+                        while (it != common_ranges.end()) {
+                            VAddr inter_addr_end = it->upper();
+                            VAddr inter_addr = it->lower();
+                            if (inter_addr >= end_address) {
+                                break;
+                            }
+                            if (inter_addr_end <= start_address) {
+                                it++;
+                                continue;
+                            }
+                            if (inter_addr_end > end_address) {
+                                inter_addr_end = end_address;
+                            }
+                            if (inter_addr < start_address) {
+                                inter_addr = start_address;
+                            }
+                            add_download(inter_addr, inter_addr_end);
+                            it++;
+                        }
+                        const IntervalType subtract_interval{start_address, end_address};
+                        common_ranges.subtract(subtract_interval);
+                    });
             });
-            total_size_bytes += range_size;
-            largest_copy = std::max(largest_copy, range_size);
-        });
+        }
     }
+    committed_ranges.clear();
     if (downloads.empty()) {
         return;
     }
@@ -622,6 +678,19 @@ void BufferCache<P>::PopAsyncFlushes() {
     }
 }
 
+template <class P>
+void BufferCache<P>::CommitAsyncFlushes() {
+    if (Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High) {
+        CommitAsyncFlushesHigh();
+    } else {
+        uncommitted_ranges.clear();
+        committed_ranges.clear();
+    }
+}
+
+template <class P>
+void BufferCache<P>::PopAsyncFlushes() {}
+
 template <class P>
 bool BufferCache<P>::IsRegionGpuModified(VAddr addr, size_t size) {
     const u64 page_end = Common::DivCeil(addr + size, PAGE_SIZE);
@@ -641,6 +710,25 @@ bool BufferCache<P>::IsRegionGpuModified(VAddr addr, size_t size) {
     return false;
 }
 
+template <class P>
+bool BufferCache<P>::IsRegionCpuModified(VAddr addr, size_t size) {
+    const u64 page_end = Common::DivCeil(addr + size, PAGE_SIZE);
+    for (u64 page = addr >> PAGE_BITS; page < page_end;) {
+        const BufferId image_id = page_table[page];
+        if (!image_id) {
+            ++page;
+            continue;
+        }
+        Buffer& buffer = slot_buffers[image_id];
+        if (buffer.IsRegionCpuModified(addr, size)) {
+            return true;
+        }
+        const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
+        page = Common::DivCeil(end_addr, PAGE_SIZE);
+    }
+    return false;
+}
+
 template <class P>
 void BufferCache<P>::BindHostIndexBuffer() {
     Buffer& buffer = slot_buffers[index_buffer.buffer_id];
@@ -1010,16 +1098,16 @@ void BufferCache<P>::MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 s
     Buffer& buffer = slot_buffers[buffer_id];
     buffer.MarkRegionAsGpuModified(cpu_addr, size);
 
-    const bool is_accuracy_high = Settings::IsGPULevelHigh();
+    const IntervalType base_interval{cpu_addr, cpu_addr + size};
+    common_ranges.add(base_interval);
+
+    const bool is_accuracy_high =
+        Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High;
     const bool is_async = Settings::values.use_asynchronous_gpu_emulation.GetValue();
-    if (!is_accuracy_high || !is_async) {
-        return;
-    }
-    if (std::ranges::find(uncommitted_downloads, buffer_id) != uncommitted_downloads.end()) {
-        // Already inserted
+    if (!is_async && !is_accuracy_high) {
         return;
     }
-    uncommitted_downloads.push_back(buffer_id);
+    uncommitted_ranges.add(base_interval);
 }
 
 template <class P>
@@ -1103,7 +1191,6 @@ void BufferCache<P>::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id,
     if (!copies.empty()) {
         runtime.CopyBuffer(slot_buffers[new_buffer_id], overlap, copies);
     }
-    ReplaceBufferDownloads(overlap_id, new_buffer_id);
     DeleteBuffer(overlap_id);
 }
 
@@ -1244,14 +1331,51 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, VAddr cpu_addr, u64 si
     boost::container::small_vector<BufferCopy, 1> copies;
     u64 total_size_bytes = 0;
     u64 largest_copy = 0;
-    buffer.ForEachDownloadRange(cpu_addr, size, [&](u64 range_offset, u64 range_size) {
-        copies.push_back(BufferCopy{
-            .src_offset = range_offset,
-            .dst_offset = total_size_bytes,
-            .size = range_size,
-        });
-        total_size_bytes += range_size;
-        largest_copy = std::max(largest_copy, range_size);
+    buffer.ForEachDownloadRangeAndClear(cpu_addr, size, [&](u64 range_offset, u64 range_size) {
+        const VAddr buffer_addr = buffer.CpuAddr();
+        const auto add_download = [&](VAddr start, VAddr end) {
+            const u64 new_offset = start - buffer_addr;
+            const u64 new_size = end - start;
+            copies.push_back(BufferCopy{
+                .src_offset = new_offset,
+                .dst_offset = total_size_bytes,
+                .size = new_size,
+            });
+            // Align up to avoid cache conflicts
+            constexpr u64 align = 256ULL;
+            constexpr u64 mask = ~(align - 1ULL);
+            total_size_bytes += (new_size + align - 1) & mask;
+            largest_copy = std::max(largest_copy, new_size);
+        };
+
+        const VAddr start_address = buffer_addr + range_offset;
+        const VAddr end_address = start_address + range_size;
+        const IntervalType search_interval{start_address - range_size, 1};
+        auto it = common_ranges.lower_bound(search_interval);
+        if (it == common_ranges.end()) {
+            it = common_ranges.begin();
+        }
+        while (it != common_ranges.end()) {
+            VAddr inter_addr_end = it->upper();
+            VAddr inter_addr = it->lower();
+            if (inter_addr >= end_address) {
+                break;
+            }
+            if (inter_addr_end <= start_address) {
+                it++;
+                continue;
+            }
+            if (inter_addr_end > end_address) {
+                inter_addr_end = end_address;
+            }
+            if (inter_addr < start_address) {
+                inter_addr = start_address;
+            }
+            add_download(inter_addr, inter_addr_end);
+            it++;
+        }
+        const IntervalType subtract_interval{start_address, end_address};
+        common_ranges.subtract(subtract_interval);
     });
     if (total_size_bytes == 0) {
         return;
@@ -1315,18 +1439,6 @@ void BufferCache<P>::DeleteBuffer(BufferId buffer_id) {
     NotifyBufferDeletion();
 }
 
-template <class P>
-void BufferCache<P>::ReplaceBufferDownloads(BufferId old_buffer_id, BufferId new_buffer_id) {
-    const auto replace = [old_buffer_id, new_buffer_id](std::vector<BufferId>& buffers) {
-        std::ranges::replace(buffers, old_buffer_id, new_buffer_id);
-        if (auto it = std::ranges::find(buffers, new_buffer_id); it != buffers.end()) {
-            buffers.erase(std::remove(it + 1, buffers.end(), new_buffer_id), buffers.end());
-        }
-    };
-    replace(uncommitted_downloads);
-    std::ranges::for_each(committed_downloads, replace);
-}
-
 template <class P>
 void BufferCache<P>::NotifyBufferDeletion() {
     if constexpr (HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS) {
@@ -1349,15 +1461,9 @@ typename BufferCache<P>::Binding BufferCache<P>::StorageBufferBinding(GPUVAddr s
     if (!cpu_addr || size == 0) {
         return NULL_BINDING;
     }
-    // HACK(Rodrigo): This is the number of bytes bound in host beyond the guest API's range.
-    // It exists due to some games like Astral Chain operate out of bounds.
-    // Binding the whole map range would be technically correct, but games have large maps that make
-    // this approach unaffordable for now.
-    static constexpr u32 arbitrary_extra_bytes = 0xc000;
-    const u32 bytes_to_map_end = static_cast<u32>(gpu_memory.BytesToMapEnd(gpu_addr));
     const Binding binding{
         .cpu_addr = *cpu_addr,
-        .size = std::min(size + arbitrary_extra_bytes, bytes_to_map_end),
+        .size = size,
         .buffer_id = BufferId{},
     };
     return binding;

src/video_core/dma_pusher.cpp

@@ -4,6 +4,7 @@
 
 #include "common/cityhash.h"
 #include "common/microprofile.h"
+#include "common/settings.h"
 #include "core/core.h"
 #include "core/memory.h"
 #include "video_core/dma_pusher.h"
@@ -76,8 +77,13 @@ bool DmaPusher::Step() {
 
         // Push buffer non-empty, read a word
         command_headers.resize(command_list_header.size);
-        gpu.MemoryManager().ReadBlockUnsafe(dma_get, command_headers.data(),
-                                            command_list_header.size * sizeof(u32));
+        if (Settings::IsGPULevelHigh()) {
+            gpu.MemoryManager().ReadBlock(dma_get, command_headers.data(),
+                                          command_list_header.size * sizeof(u32));
+        } else {
+            gpu.MemoryManager().ReadBlockUnsafe(dma_get, command_headers.data(),
+                                                command_list_header.size * sizeof(u32));
+        }
     }
     for (std::size_t index = 0; index < command_headers.size();) {
         const CommandHeader& command_header = command_headers[index];

src/video_core/fence_manager.h

@@ -8,6 +8,7 @@
 #include <queue>
 
 #include "common/common_types.h"
+#include "common/settings.h"
 #include "core/core.h"
 #include "video_core/delayed_destruction_ring.h"
 #include "video_core/gpu.h"
@@ -53,6 +54,12 @@ public:
         delayed_destruction_ring.Tick();
     }
 
+    // Unlike other fences, this one doesn't
+    void SignalOrdering() {
+        std::scoped_lock lock{buffer_cache.mutex};
+        buffer_cache.AccumulateFlushes();
+    }
+
     void SignalSemaphore(GPUVAddr addr, u32 value) {
         TryReleasePendingFences();
         const bool should_flush = ShouldFlush();

src/video_core/gpu.cpp

@@ -268,11 +268,13 @@ void GPU::CallPullerMethod(const MethodCall& method_call) {
     case BufferMethods::SemaphoreAddressHigh:
     case BufferMethods::SemaphoreAddressLow:
     case BufferMethods::SemaphoreSequence:
-    case BufferMethods::RefCnt:
     case BufferMethods::UnkCacheFlush:
     case BufferMethods::WrcacheFlush:
     case BufferMethods::FenceValue:
         break;
+    case BufferMethods::RefCnt:
+        rasterizer->SignalReference();
+        break;
     case BufferMethods::FenceAction:
         ProcessFenceActionMethod();
         break;

src/video_core/rasterizer_interface.h

@@ -63,6 +63,9 @@ public:
     /// Signal a GPU based syncpoint as a fence
     virtual void SignalSyncPoint(u32 value) = 0;
 
+    /// Signal a GPU based reference as point
+    virtual void SignalReference() = 0;
+
     /// Release all pending fences.
     virtual void ReleaseFences() = 0;
 

src/video_core/renderer_opengl/gl_rasterizer.cpp

@@ -634,6 +634,13 @@ void RasterizerOpenGL::SignalSyncPoint(u32 value) {
     fence_manager.SignalSyncPoint(value);
 }
 
+void RasterizerOpenGL::SignalReference() {
+    if (!gpu.IsAsync()) {
+        return;
+    }
+    fence_manager.SignalOrdering();
+}
+
 void RasterizerOpenGL::ReleaseFences() {
     if (!gpu.IsAsync()) {
         return;
@@ -650,6 +657,7 @@ void RasterizerOpenGL::FlushAndInvalidateRegion(VAddr addr, u64 size) {
 
 void RasterizerOpenGL::WaitForIdle() {
     glMemoryBarrier(GL_ALL_BARRIER_BITS);
+    SignalReference();
 }
 
 void RasterizerOpenGL::FragmentBarrier() {

src/video_core/renderer_opengl/gl_rasterizer.h

@@ -83,6 +83,7 @@ public:
     void ModifyGPUMemory(GPUVAddr addr, u64 size) override;
     void SignalSemaphore(GPUVAddr addr, u32 value) override;
     void SignalSyncPoint(u32 value) override;
+    void SignalReference() override;
     void ReleaseFences() override;
     void FlushAndInvalidateRegion(VAddr addr, u64 size) override;
     void WaitForIdle() override;

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -580,6 +580,13 @@ void RasterizerVulkan::SignalSyncPoint(u32 value) {
     fence_manager.SignalSyncPoint(value);
 }
 
+void RasterizerVulkan::SignalReference() {
+    if (!gpu.IsAsync()) {
+        return;
+    }
+    fence_manager.SignalOrdering();
+}
+
 void RasterizerVulkan::ReleaseFences() {
     if (!gpu.IsAsync()) {
         return;
@@ -612,6 +619,7 @@ void RasterizerVulkan::WaitForIdle() {
         cmdbuf.SetEvent(event, flags);
         cmdbuf.WaitEvents(event, flags, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, {}, {}, {});
     });
+    SignalReference();
 }
 
 void RasterizerVulkan::FragmentBarrier() {

src/video_core/renderer_vulkan/vk_rasterizer.h

@@ -75,6 +75,7 @@ public:
     void ModifyGPUMemory(GPUVAddr addr, u64 size) override;
     void SignalSemaphore(GPUVAddr addr, u32 value) override;
     void SignalSyncPoint(u32 value) override;
+    void SignalReference() override;
     void ReleaseFences() override;
     void FlushAndInvalidateRegion(VAddr addr, u64 size) override;
     void WaitForIdle() override;

src/video_core/texture_cache/types.h

@@ -133,8 +133,8 @@ struct BufferImageCopy {
 };
 
 struct BufferCopy {
-    size_t src_offset;
-    size_t dst_offset;
+    u64 src_offset;
+    u64 dst_offset;
     size_t size;
 };