texture_cache.h 78 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950
  1. // Copyright 2019 yuzu Emulator Project
  2. // Licensed under GPLv2 or any later version
  3. // Refer to the license.txt file included.
  4. #pragma once
  5. #include <algorithm>
  6. #include <array>
  7. #include <bit>
  8. #include <memory>
  9. #include <mutex>
  10. #include <optional>
  11. #include <span>
  12. #include <type_traits>
  13. #include <unordered_map>
  14. #include <unordered_set>
  15. #include <utility>
  16. #include <vector>
  17. #include <boost/container/small_vector.hpp>
  18. #include "common/alignment.h"
  19. #include "common/common_types.h"
  20. #include "common/literals.h"
  21. #include "common/logging/log.h"
  22. #include "common/settings.h"
  23. #include "video_core/compatible_formats.h"
  24. #include "video_core/delayed_destruction_ring.h"
  25. #include "video_core/dirty_flags.h"
  26. #include "video_core/engines/fermi_2d.h"
  27. #include "video_core/engines/kepler_compute.h"
  28. #include "video_core/engines/maxwell_3d.h"
  29. #include "video_core/memory_manager.h"
  30. #include "video_core/rasterizer_interface.h"
  31. #include "video_core/surface.h"
  32. #include "video_core/texture_cache/descriptor_table.h"
  33. #include "video_core/texture_cache/format_lookup_table.h"
  34. #include "video_core/texture_cache/formatter.h"
  35. #include "video_core/texture_cache/image_base.h"
  36. #include "video_core/texture_cache/image_info.h"
  37. #include "video_core/texture_cache/image_view_base.h"
  38. #include "video_core/texture_cache/image_view_info.h"
  39. #include "video_core/texture_cache/render_targets.h"
  40. #include "video_core/texture_cache/samples_helper.h"
  41. #include "video_core/texture_cache/slot_vector.h"
  42. #include "video_core/texture_cache/types.h"
  43. #include "video_core/texture_cache/util.h"
  44. #include "video_core/textures/texture.h"
  45. namespace VideoCommon {
  46. using Tegra::Texture::SwizzleSource;
  47. using Tegra::Texture::TextureType;
  48. using Tegra::Texture::TICEntry;
  49. using Tegra::Texture::TSCEntry;
  50. using VideoCore::Surface::GetFormatType;
  51. using VideoCore::Surface::IsCopyCompatible;
  52. using VideoCore::Surface::PixelFormat;
  53. using VideoCore::Surface::PixelFormatFromDepthFormat;
  54. using VideoCore::Surface::PixelFormatFromRenderTargetFormat;
  55. using VideoCore::Surface::SurfaceType;
  56. using namespace Common::Literals;
  57. template <class P>
  58. class TextureCache {
  59. /// Address shift for caching images into a hash table
  60. static constexpr u64 PAGE_BITS = 20;
  61. /// Enables debugging features to the texture cache
  62. static constexpr bool ENABLE_VALIDATION = P::ENABLE_VALIDATION;
  63. /// Implement blits as copies between framebuffers
  64. static constexpr bool FRAMEBUFFER_BLITS = P::FRAMEBUFFER_BLITS;
  65. /// True when some copies have to be emulated
  66. static constexpr bool HAS_EMULATED_COPIES = P::HAS_EMULATED_COPIES;
  67. /// True when the API can provide info about the memory of the device.
  68. static constexpr bool HAS_DEVICE_MEMORY_INFO = P::HAS_DEVICE_MEMORY_INFO;
  69. /// Image view ID for null descriptors
  70. static constexpr ImageViewId NULL_IMAGE_VIEW_ID{0};
  71. /// Sampler ID for bugged sampler ids
  72. static constexpr SamplerId NULL_SAMPLER_ID{0};
  73. static constexpr u64 DEFAULT_EXPECTED_MEMORY = 1_GiB;
  74. static constexpr u64 DEFAULT_CRITICAL_MEMORY = 2_GiB;
  75. using Runtime = typename P::Runtime;
  76. using Image = typename P::Image;
  77. using ImageAlloc = typename P::ImageAlloc;
  78. using ImageView = typename P::ImageView;
  79. using Sampler = typename P::Sampler;
  80. using Framebuffer = typename P::Framebuffer;
  81. struct BlitImages {
  82. ImageId dst_id;
  83. ImageId src_id;
  84. PixelFormat dst_format;
  85. PixelFormat src_format;
  86. };
  87. template <typename T>
  88. struct IdentityHash {
  89. [[nodiscard]] size_t operator()(T value) const noexcept {
  90. return static_cast<size_t>(value);
  91. }
  92. };
  93. public:
  94. explicit TextureCache(Runtime&, VideoCore::RasterizerInterface&, Tegra::Engines::Maxwell3D&,
  95. Tegra::Engines::KeplerCompute&, Tegra::MemoryManager&);
  96. /// Notify the cache that a new frame has been queued
  97. void TickFrame();
  98. /// Return a constant reference to the given image view id
  99. [[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept;
  100. /// Return a reference to the given image view id
  101. [[nodiscard]] ImageView& GetImageView(ImageViewId id) noexcept;
  102. /// Fill image_view_ids with the graphics images in indices
  103. void FillGraphicsImageViews(std::span<const u32> indices,
  104. std::span<ImageViewId> image_view_ids);
  105. /// Fill image_view_ids with the compute images in indices
  106. void FillComputeImageViews(std::span<const u32> indices, std::span<ImageViewId> image_view_ids);
  107. /// Get the sampler from the graphics descriptor table in the specified index
  108. Sampler* GetGraphicsSampler(u32 index);
  109. /// Get the sampler from the compute descriptor table in the specified index
  110. Sampler* GetComputeSampler(u32 index);
  111. /// Refresh the state for graphics image view and sampler descriptors
  112. void SynchronizeGraphicsDescriptors();
  113. /// Refresh the state for compute image view and sampler descriptors
  114. void SynchronizeComputeDescriptors();
  115. /// Update bound render targets and upload memory if necessary
  116. /// @param is_clear True when the render targets are being used for clears
  117. void UpdateRenderTargets(bool is_clear);
  118. /// Find a framebuffer with the currently bound render targets
  119. /// UpdateRenderTargets should be called before this
  120. Framebuffer* GetFramebuffer();
  121. /// Mark images in a range as modified from the CPU
  122. void WriteMemory(VAddr cpu_addr, size_t size);
  123. /// Download contents of host images to guest memory in a region
  124. void DownloadMemory(VAddr cpu_addr, size_t size);
  125. /// Remove images in a region
  126. void UnmapMemory(VAddr cpu_addr, size_t size);
  127. /// Remove images in a region
  128. void UnmapGPUMemory(GPUVAddr gpu_addr, size_t size);
  129. /// Blit an image with the given parameters
  130. void BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
  131. const Tegra::Engines::Fermi2D::Surface& src,
  132. const Tegra::Engines::Fermi2D::Config& copy,
  133. std::optional<Region2D> src_region_override = {},
  134. std::optional<Region2D> dst_region_override = {});
  135. /// Invalidate the contents of the color buffer index
  136. /// These contents become unspecified, the cache can assume aggressive optimizations.
  137. void InvalidateColorBuffer(size_t index);
  138. /// Invalidate the contents of the depth buffer
  139. /// These contents become unspecified, the cache can assume aggressive optimizations.
  140. void InvalidateDepthBuffer();
  141. /// Try to find a cached image view in the given CPU address
  142. [[nodiscard]] ImageView* TryFindFramebufferImageView(VAddr cpu_addr);
  143. /// Return true when there are uncommitted images to be downloaded
  144. [[nodiscard]] bool HasUncommittedFlushes() const noexcept;
  145. /// Return true when the caller should wait for async downloads
  146. [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept;
  147. /// Commit asynchronous downloads
  148. void CommitAsyncFlushes();
  149. /// Pop asynchronous downloads
  150. void PopAsyncFlushes();
  151. /// Return true when a CPU region is modified from the GPU
  152. [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
  153. std::mutex mutex;
  154. private:
  155. /// Iterate over all page indices in a range
  156. template <typename Func>
  157. static void ForEachCPUPage(VAddr addr, size_t size, Func&& func) {
  158. static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
  159. const u64 page_end = (addr + size - 1) >> PAGE_BITS;
  160. for (u64 page = addr >> PAGE_BITS; page <= page_end; ++page) {
  161. if constexpr (RETURNS_BOOL) {
  162. if (func(page)) {
  163. break;
  164. }
  165. } else {
  166. func(page);
  167. }
  168. }
  169. }
  170. template <typename Func>
  171. static void ForEachGPUPage(GPUVAddr addr, size_t size, Func&& func) {
  172. static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
  173. const u64 page_end = (addr + size - 1) >> PAGE_BITS;
  174. for (u64 page = addr >> PAGE_BITS; page <= page_end; ++page) {
  175. if constexpr (RETURNS_BOOL) {
  176. if (func(page)) {
  177. break;
  178. }
  179. } else {
  180. func(page);
  181. }
  182. }
  183. }
  184. /// Runs the Garbage Collector.
  185. void RunGarbageCollector();
  186. /// Fills image_view_ids in the image views in indices
  187. void FillImageViews(DescriptorTable<TICEntry>& table,
  188. std::span<ImageViewId> cached_image_view_ids, std::span<const u32> indices,
  189. std::span<ImageViewId> image_view_ids);
  190. /// Find or create an image view in the guest descriptor table
  191. ImageViewId VisitImageView(DescriptorTable<TICEntry>& table,
  192. std::span<ImageViewId> cached_image_view_ids, u32 index);
  193. /// Find or create a framebuffer with the given render target parameters
  194. FramebufferId GetFramebufferId(const RenderTargets& key);
  195. /// Refresh the contents (pixel data) of an image
  196. void RefreshContents(Image& image, ImageId image_id);
  197. /// Upload data from guest to an image
  198. template <typename StagingBuffer>
  199. void UploadImageContents(Image& image, StagingBuffer& staging_buffer);
  200. /// Find or create an image view from a guest descriptor
  201. [[nodiscard]] ImageViewId FindImageView(const TICEntry& config);
  202. /// Create a new image view from a guest descriptor
  203. [[nodiscard]] ImageViewId CreateImageView(const TICEntry& config);
  204. /// Find or create an image from the given parameters
  205. [[nodiscard]] ImageId FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
  206. RelaxedOptions options = RelaxedOptions{});
  207. /// Find an image from the given parameters
  208. [[nodiscard]] ImageId FindImage(const ImageInfo& info, GPUVAddr gpu_addr,
  209. RelaxedOptions options);
  210. /// Create an image from the given parameters
  211. [[nodiscard]] ImageId InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
  212. RelaxedOptions options);
  213. /// Create a new image and join perfectly matching existing images
  214. /// Remove joined images from the cache
  215. [[nodiscard]] ImageId JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VAddr cpu_addr);
  216. /// Return a blit image pair from the given guest blit parameters
  217. [[nodiscard]] BlitImages GetBlitImages(const Tegra::Engines::Fermi2D::Surface& dst,
  218. const Tegra::Engines::Fermi2D::Surface& src);
  219. /// Find or create a sampler from a guest descriptor sampler
  220. [[nodiscard]] SamplerId FindSampler(const TSCEntry& config);
  221. /// Find or create an image view for the given color buffer index
  222. [[nodiscard]] ImageViewId FindColorBuffer(size_t index, bool is_clear);
  223. /// Find or create an image view for the depth buffer
  224. [[nodiscard]] ImageViewId FindDepthBuffer(bool is_clear);
  225. /// Find or create a view for a render target with the given image parameters
  226. [[nodiscard]] ImageViewId FindRenderTargetView(const ImageInfo& info, GPUVAddr gpu_addr,
  227. bool is_clear);
  228. /// Iterates over all the images in a region calling func
  229. template <typename Func>
  230. void ForEachImageInRegion(VAddr cpu_addr, size_t size, Func&& func);
  231. template <typename Func>
  232. void ForEachImageInRegionGPU(GPUVAddr gpu_addr, size_t size, Func&& func);
  233. template <typename Func>
  234. void ForEachSparseImageInRegion(GPUVAddr gpu_addr, size_t size, Func&& func);
  235. /// Iterates over all the images in a region calling func
  236. template <typename Func>
  237. void ForEachSparseSegment(ImageBase& image, Func&& func);
  238. /// Find or create an image view in the given image with the passed parameters
  239. [[nodiscard]] ImageViewId FindOrEmplaceImageView(ImageId image_id, const ImageViewInfo& info);
  240. /// Register image in the page table
  241. void RegisterImage(ImageId image);
  242. /// Unregister image from the page table
  243. void UnregisterImage(ImageId image);
  244. /// Track CPU reads and writes for image
  245. void TrackImage(ImageBase& image, ImageId image_id);
  246. /// Stop tracking CPU reads and writes for image
  247. void UntrackImage(ImageBase& image, ImageId image_id);
  248. /// Delete image from the cache
  249. void DeleteImage(ImageId image);
  250. /// Remove image views references from the cache
  251. void RemoveImageViewReferences(std::span<const ImageViewId> removed_views);
  252. /// Remove framebuffers using the given image views from the cache
  253. void RemoveFramebuffers(std::span<const ImageViewId> removed_views);
  254. /// Mark an image as modified from the GPU
  255. void MarkModification(ImageBase& image) noexcept;
  256. /// Synchronize image aliases, copying data if needed
  257. void SynchronizeAliases(ImageId image_id);
  258. /// Prepare an image to be used
  259. void PrepareImage(ImageId image_id, bool is_modification, bool invalidate);
  260. /// Prepare an image view to be used
  261. void PrepareImageView(ImageViewId image_view_id, bool is_modification, bool invalidate);
  262. /// Execute copies from one image to the other, even if they are incompatible
  263. void CopyImage(ImageId dst_id, ImageId src_id, std::span<const ImageCopy> copies);
  264. /// Bind an image view as render target, downloading resources preemtively if needed
  265. void BindRenderTarget(ImageViewId* old_id, ImageViewId new_id);
  266. /// Create a render target from a given image and image view parameters
  267. [[nodiscard]] std::pair<FramebufferId, ImageViewId> RenderTargetFromImage(
  268. ImageId, const ImageViewInfo& view_info);
  269. /// Returns true if the current clear parameters clear the whole image of a given image view
  270. [[nodiscard]] bool IsFullClear(ImageViewId id);
  271. Runtime& runtime;
  272. VideoCore::RasterizerInterface& rasterizer;
  273. Tegra::Engines::Maxwell3D& maxwell3d;
  274. Tegra::Engines::KeplerCompute& kepler_compute;
  275. Tegra::MemoryManager& gpu_memory;
  276. DescriptorTable<TICEntry> graphics_image_table{gpu_memory};
  277. DescriptorTable<TSCEntry> graphics_sampler_table{gpu_memory};
  278. std::vector<SamplerId> graphics_sampler_ids;
  279. std::vector<ImageViewId> graphics_image_view_ids;
  280. DescriptorTable<TICEntry> compute_image_table{gpu_memory};
  281. DescriptorTable<TSCEntry> compute_sampler_table{gpu_memory};
  282. std::vector<SamplerId> compute_sampler_ids;
  283. std::vector<ImageViewId> compute_image_view_ids;
  284. RenderTargets render_targets;
  285. std::unordered_map<TICEntry, ImageViewId> image_views;
  286. std::unordered_map<TSCEntry, SamplerId> samplers;
  287. std::unordered_map<RenderTargets, FramebufferId> framebuffers;
  288. std::unordered_map<u64, std::vector<ImageMapId>, IdentityHash<u64>> page_table;
  289. std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>> gpu_page_table;
  290. std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>> sparse_page_table;
  291. std::unordered_map<ImageId, std::vector<ImageViewId>> sparse_views;
  292. VAddr virtual_invalid_space{};
  293. bool has_deleted_images = false;
  294. u64 total_used_memory = 0;
  295. u64 minimum_memory;
  296. u64 expected_memory;
  297. u64 critical_memory;
  298. SlotVector<Image> slot_images;
  299. SlotVector<ImageMapView> slot_map_views;
  300. SlotVector<ImageView> slot_image_views;
  301. SlotVector<ImageAlloc> slot_image_allocs;
  302. SlotVector<Sampler> slot_samplers;
  303. SlotVector<Framebuffer> slot_framebuffers;
  304. // TODO: This data structure is not optimal and it should be reworked
  305. std::vector<ImageId> uncommitted_downloads;
  306. std::queue<std::vector<ImageId>> committed_downloads;
  307. static constexpr size_t TICKS_TO_DESTROY = 6;
  308. DelayedDestructionRing<Image, TICKS_TO_DESTROY> sentenced_images;
  309. DelayedDestructionRing<ImageView, TICKS_TO_DESTROY> sentenced_image_view;
  310. DelayedDestructionRing<Framebuffer, TICKS_TO_DESTROY> sentenced_framebuffers;
  311. std::unordered_map<GPUVAddr, ImageAllocId> image_allocs_table;
  312. u64 modification_tick = 0;
  313. u64 frame_tick = 0;
  314. typename SlotVector<Image>::Iterator deletion_iterator;
  315. };
  316. template <class P>
  317. TextureCache<P>::TextureCache(Runtime& runtime_, VideoCore::RasterizerInterface& rasterizer_,
  318. Tegra::Engines::Maxwell3D& maxwell3d_,
  319. Tegra::Engines::KeplerCompute& kepler_compute_,
  320. Tegra::MemoryManager& gpu_memory_)
  321. : runtime{runtime_}, rasterizer{rasterizer_}, maxwell3d{maxwell3d_},
  322. kepler_compute{kepler_compute_}, gpu_memory{gpu_memory_} {
  323. // Configure null sampler
  324. TSCEntry sampler_descriptor{};
  325. sampler_descriptor.min_filter.Assign(Tegra::Texture::TextureFilter::Linear);
  326. sampler_descriptor.mag_filter.Assign(Tegra::Texture::TextureFilter::Linear);
  327. sampler_descriptor.mipmap_filter.Assign(Tegra::Texture::TextureMipmapFilter::Linear);
  328. sampler_descriptor.cubemap_anisotropy.Assign(1);
  329. // Make sure the first index is reserved for the null resources
  330. // This way the null resource becomes a compile time constant
  331. void(slot_image_views.insert(runtime, NullImageParams{}));
  332. void(slot_samplers.insert(runtime, sampler_descriptor));
  333. deletion_iterator = slot_images.begin();
  334. if constexpr (HAS_DEVICE_MEMORY_INFO) {
  335. const auto device_memory = runtime.GetDeviceLocalMemory();
  336. const u64 possible_expected_memory = (device_memory * 3) / 10;
  337. const u64 possible_critical_memory = (device_memory * 6) / 10;
  338. expected_memory = std::max(possible_expected_memory, DEFAULT_EXPECTED_MEMORY);
  339. critical_memory = std::max(possible_critical_memory, DEFAULT_CRITICAL_MEMORY);
  340. minimum_memory = 0;
  341. } else {
  342. // on OGL we can be more conservatives as the driver takes care.
  343. expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB;
  344. critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB;
  345. minimum_memory = expected_memory;
  346. }
  347. }
  348. template <class P>
  349. void TextureCache<P>::RunGarbageCollector() {
  350. const bool high_priority_mode = total_used_memory >= expected_memory;
  351. const bool aggressive_mode = total_used_memory >= critical_memory;
  352. const u64 ticks_to_destroy = high_priority_mode ? 60 : 100;
  353. int num_iterations = aggressive_mode ? 256 : (high_priority_mode ? 128 : 64);
  354. for (; num_iterations > 0; --num_iterations) {
  355. if (deletion_iterator == slot_images.end()) {
  356. deletion_iterator = slot_images.begin();
  357. if (deletion_iterator == slot_images.end()) {
  358. break;
  359. }
  360. }
  361. auto [image_id, image_tmp] = *deletion_iterator;
  362. Image* image = image_tmp; // fix clang error.
  363. const bool is_alias = True(image->flags & ImageFlagBits::Alias);
  364. const bool is_bad_overlap = True(image->flags & ImageFlagBits::BadOverlap);
  365. const bool must_download = image->IsSafeDownload();
  366. bool should_care = is_bad_overlap || is_alias || (high_priority_mode && !must_download);
  367. const u64 ticks_needed =
  368. is_bad_overlap
  369. ? ticks_to_destroy >> 4
  370. : ((should_care && aggressive_mode) ? ticks_to_destroy >> 1 : ticks_to_destroy);
  371. should_care |= aggressive_mode;
  372. if (should_care && image->frame_tick + ticks_needed < frame_tick) {
  373. if (is_bad_overlap) {
  374. const bool overlap_check = std::ranges::all_of(
  375. image->overlapping_images, [&, image](const ImageId& overlap_id) {
  376. auto& overlap = slot_images[overlap_id];
  377. return overlap.frame_tick >= image->frame_tick;
  378. });
  379. if (!overlap_check) {
  380. ++deletion_iterator;
  381. continue;
  382. }
  383. }
  384. if (!is_bad_overlap && must_download) {
  385. const bool alias_check = std::ranges::none_of(
  386. image->aliased_images, [&, image](const AliasedImage& alias) {
  387. auto& alias_image = slot_images[alias.id];
  388. return (alias_image.frame_tick < image->frame_tick) ||
  389. (alias_image.modification_tick < image->modification_tick);
  390. });
  391. if (alias_check) {
  392. auto map = runtime.DownloadStagingBuffer(image->unswizzled_size_bytes);
  393. const auto copies = FullDownloadCopies(image->info);
  394. image->DownloadMemory(map, copies);
  395. runtime.Finish();
  396. SwizzleImage(gpu_memory, image->gpu_addr, image->info, copies, map.mapped_span);
  397. }
  398. }
  399. if (True(image->flags & ImageFlagBits::Tracked)) {
  400. UntrackImage(*image, image_id);
  401. }
  402. UnregisterImage(image_id);
  403. DeleteImage(image_id);
  404. if (is_bad_overlap) {
  405. ++num_iterations;
  406. }
  407. }
  408. ++deletion_iterator;
  409. }
  410. }
  411. template <class P>
  412. void TextureCache<P>::TickFrame() {
  413. if (Settings::values.use_caches_gc.GetValue() && total_used_memory > minimum_memory) {
  414. RunGarbageCollector();
  415. }
  416. sentenced_images.Tick();
  417. sentenced_framebuffers.Tick();
  418. sentenced_image_view.Tick();
  419. ++frame_tick;
  420. }
  421. template <class P>
  422. const typename P::ImageView& TextureCache<P>::GetImageView(ImageViewId id) const noexcept {
  423. return slot_image_views[id];
  424. }
  425. template <class P>
  426. typename P::ImageView& TextureCache<P>::GetImageView(ImageViewId id) noexcept {
  427. return slot_image_views[id];
  428. }
  429. template <class P>
  430. void TextureCache<P>::FillGraphicsImageViews(std::span<const u32> indices,
  431. std::span<ImageViewId> image_view_ids) {
  432. FillImageViews(graphics_image_table, graphics_image_view_ids, indices, image_view_ids);
  433. }
  434. template <class P>
  435. void TextureCache<P>::FillComputeImageViews(std::span<const u32> indices,
  436. std::span<ImageViewId> image_view_ids) {
  437. FillImageViews(compute_image_table, compute_image_view_ids, indices, image_view_ids);
  438. }
  439. template <class P>
  440. typename P::Sampler* TextureCache<P>::GetGraphicsSampler(u32 index) {
  441. [[unlikely]] if (index > graphics_sampler_table.Limit()) {
  442. LOG_ERROR(HW_GPU, "Invalid sampler index={}", index);
  443. return &slot_samplers[NULL_SAMPLER_ID];
  444. }
  445. const auto [descriptor, is_new] = graphics_sampler_table.Read(index);
  446. SamplerId& id = graphics_sampler_ids[index];
  447. [[unlikely]] if (is_new) {
  448. id = FindSampler(descriptor);
  449. }
  450. return &slot_samplers[id];
  451. }
  452. template <class P>
  453. typename P::Sampler* TextureCache<P>::GetComputeSampler(u32 index) {
  454. [[unlikely]] if (index > compute_sampler_table.Limit()) {
  455. LOG_ERROR(HW_GPU, "Invalid sampler index={}", index);
  456. return &slot_samplers[NULL_SAMPLER_ID];
  457. }
  458. const auto [descriptor, is_new] = compute_sampler_table.Read(index);
  459. SamplerId& id = compute_sampler_ids[index];
  460. [[unlikely]] if (is_new) {
  461. id = FindSampler(descriptor);
  462. }
  463. return &slot_samplers[id];
  464. }
  465. template <class P>
  466. void TextureCache<P>::SynchronizeGraphicsDescriptors() {
  467. using SamplerIndex = Tegra::Engines::Maxwell3D::Regs::SamplerIndex;
  468. const bool linked_tsc = maxwell3d.regs.sampler_index == SamplerIndex::ViaHeaderIndex;
  469. const u32 tic_limit = maxwell3d.regs.tic.limit;
  470. const u32 tsc_limit = linked_tsc ? tic_limit : maxwell3d.regs.tsc.limit;
  471. if (graphics_sampler_table.Synchornize(maxwell3d.regs.tsc.Address(), tsc_limit)) {
  472. graphics_sampler_ids.resize(tsc_limit + 1, CORRUPT_ID);
  473. }
  474. if (graphics_image_table.Synchornize(maxwell3d.regs.tic.Address(), tic_limit)) {
  475. graphics_image_view_ids.resize(tic_limit + 1, CORRUPT_ID);
  476. }
  477. }
  478. template <class P>
  479. void TextureCache<P>::SynchronizeComputeDescriptors() {
  480. const bool linked_tsc = kepler_compute.launch_description.linked_tsc;
  481. const u32 tic_limit = kepler_compute.regs.tic.limit;
  482. const u32 tsc_limit = linked_tsc ? tic_limit : kepler_compute.regs.tsc.limit;
  483. const GPUVAddr tsc_gpu_addr = kepler_compute.regs.tsc.Address();
  484. if (compute_sampler_table.Synchornize(tsc_gpu_addr, tsc_limit)) {
  485. compute_sampler_ids.resize(tsc_limit + 1, CORRUPT_ID);
  486. }
  487. if (compute_image_table.Synchornize(kepler_compute.regs.tic.Address(), tic_limit)) {
  488. compute_image_view_ids.resize(tic_limit + 1, CORRUPT_ID);
  489. }
  490. }
  491. template <class P>
  492. void TextureCache<P>::UpdateRenderTargets(bool is_clear) {
  493. using namespace VideoCommon::Dirty;
  494. auto& flags = maxwell3d.dirty.flags;
  495. if (!flags[Dirty::RenderTargets]) {
  496. return;
  497. }
  498. flags[Dirty::RenderTargets] = false;
  499. // Render target control is used on all render targets, so force look ups when this one is up
  500. const bool force = flags[Dirty::RenderTargetControl];
  501. flags[Dirty::RenderTargetControl] = false;
  502. for (size_t index = 0; index < NUM_RT; ++index) {
  503. ImageViewId& color_buffer_id = render_targets.color_buffer_ids[index];
  504. if (flags[Dirty::ColorBuffer0 + index] || force) {
  505. flags[Dirty::ColorBuffer0 + index] = false;
  506. BindRenderTarget(&color_buffer_id, FindColorBuffer(index, is_clear));
  507. }
  508. PrepareImageView(color_buffer_id, true, is_clear && IsFullClear(color_buffer_id));
  509. }
  510. if (flags[Dirty::ZetaBuffer] || force) {
  511. flags[Dirty::ZetaBuffer] = false;
  512. BindRenderTarget(&render_targets.depth_buffer_id, FindDepthBuffer(is_clear));
  513. }
  514. const ImageViewId depth_buffer_id = render_targets.depth_buffer_id;
  515. PrepareImageView(depth_buffer_id, true, is_clear && IsFullClear(depth_buffer_id));
  516. for (size_t index = 0; index < NUM_RT; ++index) {
  517. render_targets.draw_buffers[index] = static_cast<u8>(maxwell3d.regs.rt_control.Map(index));
  518. }
  519. render_targets.size = Extent2D{
  520. maxwell3d.regs.render_area.width,
  521. maxwell3d.regs.render_area.height,
  522. };
  523. }
  524. template <class P>
  525. typename P::Framebuffer* TextureCache<P>::GetFramebuffer() {
  526. return &slot_framebuffers[GetFramebufferId(render_targets)];
  527. }
  528. template <class P>
  529. void TextureCache<P>::FillImageViews(DescriptorTable<TICEntry>& table,
  530. std::span<ImageViewId> cached_image_view_ids,
  531. std::span<const u32> indices,
  532. std::span<ImageViewId> image_view_ids) {
  533. ASSERT(indices.size() <= image_view_ids.size());
  534. do {
  535. has_deleted_images = false;
  536. std::ranges::transform(indices, image_view_ids.begin(), [&](u32 index) {
  537. return VisitImageView(table, cached_image_view_ids, index);
  538. });
  539. } while (has_deleted_images);
  540. }
  541. template <class P>
  542. ImageViewId TextureCache<P>::VisitImageView(DescriptorTable<TICEntry>& table,
  543. std::span<ImageViewId> cached_image_view_ids,
  544. u32 index) {
  545. if (index > table.Limit()) {
  546. LOG_ERROR(HW_GPU, "Invalid image view index={}", index);
  547. return NULL_IMAGE_VIEW_ID;
  548. }
  549. const auto [descriptor, is_new] = table.Read(index);
  550. ImageViewId& image_view_id = cached_image_view_ids[index];
  551. if (is_new) {
  552. image_view_id = FindImageView(descriptor);
  553. }
  554. if (image_view_id != NULL_IMAGE_VIEW_ID) {
  555. PrepareImageView(image_view_id, false, false);
  556. }
  557. return image_view_id;
  558. }
  559. template <class P>
  560. FramebufferId TextureCache<P>::GetFramebufferId(const RenderTargets& key) {
  561. const auto [pair, is_new] = framebuffers.try_emplace(key);
  562. FramebufferId& framebuffer_id = pair->second;
  563. if (!is_new) {
  564. return framebuffer_id;
  565. }
  566. std::array<ImageView*, NUM_RT> color_buffers;
  567. std::ranges::transform(key.color_buffer_ids, color_buffers.begin(),
  568. [this](ImageViewId id) { return id ? &slot_image_views[id] : nullptr; });
  569. ImageView* const depth_buffer =
  570. key.depth_buffer_id ? &slot_image_views[key.depth_buffer_id] : nullptr;
  571. framebuffer_id = slot_framebuffers.insert(runtime, color_buffers, depth_buffer, key);
  572. return framebuffer_id;
  573. }
  574. template <class P>
  575. void TextureCache<P>::WriteMemory(VAddr cpu_addr, size_t size) {
  576. ForEachImageInRegion(cpu_addr, size, [this](ImageId image_id, Image& image) {
  577. if (True(image.flags & ImageFlagBits::CpuModified)) {
  578. return;
  579. }
  580. image.flags |= ImageFlagBits::CpuModified;
  581. if (True(image.flags & ImageFlagBits::Tracked)) {
  582. UntrackImage(image, image_id);
  583. }
  584. });
  585. }
  586. template <class P>
  587. void TextureCache<P>::DownloadMemory(VAddr cpu_addr, size_t size) {
  588. std::vector<ImageId> images;
  589. ForEachImageInRegion(cpu_addr, size, [this, &images](ImageId image_id, ImageBase& image) {
  590. if (!image.IsSafeDownload()) {
  591. return;
  592. }
  593. image.flags &= ~ImageFlagBits::GpuModified;
  594. images.push_back(image_id);
  595. });
  596. if (images.empty()) {
  597. return;
  598. }
  599. std::ranges::sort(images, [this](ImageId lhs, ImageId rhs) {
  600. return slot_images[lhs].modification_tick < slot_images[rhs].modification_tick;
  601. });
  602. for (const ImageId image_id : images) {
  603. Image& image = slot_images[image_id];
  604. auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes);
  605. const auto copies = FullDownloadCopies(image.info);
  606. image.DownloadMemory(map, copies);
  607. runtime.Finish();
  608. SwizzleImage(gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span);
  609. }
  610. }
  611. template <class P>
  612. void TextureCache<P>::UnmapMemory(VAddr cpu_addr, size_t size) {
  613. std::vector<ImageId> deleted_images;
  614. ForEachImageInRegion(cpu_addr, size, [&](ImageId id, Image&) { deleted_images.push_back(id); });
  615. for (const ImageId id : deleted_images) {
  616. Image& image = slot_images[id];
  617. if (True(image.flags & ImageFlagBits::Tracked)) {
  618. UntrackImage(image, id);
  619. }
  620. UnregisterImage(id);
  621. DeleteImage(id);
  622. }
  623. }
  624. template <class P>
  625. void TextureCache<P>::UnmapGPUMemory(GPUVAddr gpu_addr, size_t size) {
  626. std::vector<ImageId> deleted_images;
  627. ForEachImageInRegionGPU(gpu_addr, size,
  628. [&](ImageId id, Image&) { deleted_images.push_back(id); });
  629. for (const ImageId id : deleted_images) {
  630. Image& image = slot_images[id];
  631. if (True(image.flags & ImageFlagBits::Remapped)) {
  632. continue;
  633. }
  634. image.flags |= ImageFlagBits::Remapped;
  635. if (True(image.flags & ImageFlagBits::Tracked)) {
  636. UntrackImage(image, id);
  637. }
  638. }
  639. }
  640. template <class P>
  641. void TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
  642. const Tegra::Engines::Fermi2D::Surface& src,
  643. const Tegra::Engines::Fermi2D::Config& copy,
  644. std::optional<Region2D> src_override,
  645. std::optional<Region2D> dst_override) {
  646. const BlitImages images = GetBlitImages(dst, src);
  647. const ImageId dst_id = images.dst_id;
  648. const ImageId src_id = images.src_id;
  649. PrepareImage(src_id, false, false);
  650. PrepareImage(dst_id, true, false);
  651. ImageBase& dst_image = slot_images[dst_id];
  652. const ImageBase& src_image = slot_images[src_id];
  653. // TODO: Deduplicate
  654. const std::optional dst_base = dst_image.TryFindBase(dst.Address());
  655. const SubresourceRange dst_range{.base = dst_base.value(), .extent = {1, 1}};
  656. const ImageViewInfo dst_view_info(ImageViewType::e2D, images.dst_format, dst_range);
  657. const auto [dst_framebuffer_id, dst_view_id] = RenderTargetFromImage(dst_id, dst_view_info);
  658. const auto [src_samples_x, src_samples_y] = SamplesLog2(src_image.info.num_samples);
  659. // out of bounds texture blit checking
  660. const bool use_override = src_override.has_value();
  661. const s32 src_x0 = copy.src_x0 >> src_samples_x;
  662. s32 src_x1 = use_override ? src_override->end.x : copy.src_x1 >> src_samples_x;
  663. const s32 src_y0 = copy.src_y0 >> src_samples_y;
  664. const s32 src_y1 = copy.src_y1 >> src_samples_y;
  665. const auto src_width = static_cast<s32>(src_image.info.size.width);
  666. const bool width_oob = src_x1 > src_width;
  667. const auto width_diff = width_oob ? src_x1 - src_width : 0;
  668. if (width_oob) {
  669. src_x1 = src_width;
  670. }
  671. const Region2D src_dimensions{
  672. Offset2D{.x = src_x0, .y = src_y0},
  673. Offset2D{.x = src_x1, .y = src_y1},
  674. };
  675. const auto src_region = use_override ? *src_override : src_dimensions;
  676. const std::optional src_base = src_image.TryFindBase(src.Address());
  677. const SubresourceRange src_range{.base = src_base.value(), .extent = {1, 1}};
  678. const ImageViewInfo src_view_info(ImageViewType::e2D, images.src_format, src_range);
  679. const auto [src_framebuffer_id, src_view_id] = RenderTargetFromImage(src_id, src_view_info);
  680. const auto [dst_samples_x, dst_samples_y] = SamplesLog2(dst_image.info.num_samples);
  681. const s32 dst_x0 = copy.dst_x0 >> dst_samples_x;
  682. const s32 dst_x1 = copy.dst_x1 >> dst_samples_x;
  683. const s32 dst_y0 = copy.dst_y0 >> dst_samples_y;
  684. const s32 dst_y1 = copy.dst_y1 >> dst_samples_y;
  685. const Region2D dst_dimensions{
  686. Offset2D{.x = dst_x0, .y = dst_y0},
  687. Offset2D{.x = dst_x1 - width_diff, .y = dst_y1},
  688. };
  689. const auto dst_region = use_override ? *dst_override : dst_dimensions;
  690. // Always call this after src_framebuffer_id was queried, as the address might be invalidated.
  691. Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_framebuffer_id];
  692. if constexpr (FRAMEBUFFER_BLITS) {
  693. // OpenGL blits from framebuffers, not images
  694. Framebuffer* const src_framebuffer = &slot_framebuffers[src_framebuffer_id];
  695. runtime.BlitFramebuffer(dst_framebuffer, src_framebuffer, dst_region, src_region,
  696. copy.filter, copy.operation);
  697. } else {
  698. // Vulkan can blit images, but it lacks format reinterpretations
  699. // Provide a framebuffer in case it's necessary
  700. ImageView& dst_view = slot_image_views[dst_view_id];
  701. ImageView& src_view = slot_image_views[src_view_id];
  702. runtime.BlitImage(dst_framebuffer, dst_view, src_view, dst_region, src_region, copy.filter,
  703. copy.operation);
  704. }
  705. if (width_oob) {
  706. // Continue copy of the oob region of the texture on the next row
  707. auto oob_src = src;
  708. oob_src.height++;
  709. const Region2D src_region_override{
  710. Offset2D{.x = 0, .y = src_y0 + 1},
  711. Offset2D{.x = width_diff, .y = src_y1 + 1},
  712. };
  713. const Region2D dst_region_override{
  714. Offset2D{.x = dst_x1 - width_diff, .y = dst_y0},
  715. Offset2D{.x = dst_x1, .y = dst_y1},
  716. };
  717. BlitImage(dst, oob_src, copy, src_region_override, dst_region_override);
  718. }
  719. }
  720. template <class P>
  721. void TextureCache<P>::InvalidateColorBuffer(size_t index) {
  722. ImageViewId& color_buffer_id = render_targets.color_buffer_ids[index];
  723. color_buffer_id = FindColorBuffer(index, false);
  724. if (!color_buffer_id) {
  725. LOG_ERROR(HW_GPU, "Invalidating invalid color buffer in index={}", index);
  726. return;
  727. }
  728. // When invalidating a color buffer, the old contents are no longer relevant
  729. ImageView& color_buffer = slot_image_views[color_buffer_id];
  730. Image& image = slot_images[color_buffer.image_id];
  731. image.flags &= ~ImageFlagBits::CpuModified;
  732. image.flags &= ~ImageFlagBits::GpuModified;
  733. runtime.InvalidateColorBuffer(color_buffer, index);
  734. }
  735. template <class P>
  736. void TextureCache<P>::InvalidateDepthBuffer() {
  737. ImageViewId& depth_buffer_id = render_targets.depth_buffer_id;
  738. depth_buffer_id = FindDepthBuffer(false);
  739. if (!depth_buffer_id) {
  740. LOG_ERROR(HW_GPU, "Invalidating invalid depth buffer");
  741. return;
  742. }
  743. // When invalidating the depth buffer, the old contents are no longer relevant
  744. ImageBase& image = slot_images[slot_image_views[depth_buffer_id].image_id];
  745. image.flags &= ~ImageFlagBits::CpuModified;
  746. image.flags &= ~ImageFlagBits::GpuModified;
  747. ImageView& depth_buffer = slot_image_views[depth_buffer_id];
  748. runtime.InvalidateDepthBuffer(depth_buffer);
  749. }
  750. template <class P>
  751. typename P::ImageView* TextureCache<P>::TryFindFramebufferImageView(VAddr cpu_addr) {
  752. // TODO: Properly implement this
  753. const auto it = page_table.find(cpu_addr >> PAGE_BITS);
  754. if (it == page_table.end()) {
  755. return nullptr;
  756. }
  757. const auto& image_map_ids = it->second;
  758. for (const ImageMapId map_id : image_map_ids) {
  759. const ImageMapView& map = slot_map_views[map_id];
  760. const ImageBase& image = slot_images[map.image_id];
  761. if (image.cpu_addr != cpu_addr) {
  762. continue;
  763. }
  764. if (image.image_view_ids.empty()) {
  765. continue;
  766. }
  767. return &slot_image_views[image.image_view_ids.at(0)];
  768. }
  769. return nullptr;
  770. }
  771. template <class P>
  772. bool TextureCache<P>::HasUncommittedFlushes() const noexcept {
  773. return !uncommitted_downloads.empty();
  774. }
  775. template <class P>
  776. bool TextureCache<P>::ShouldWaitAsyncFlushes() const noexcept {
  777. return !committed_downloads.empty() && !committed_downloads.front().empty();
  778. }
  779. template <class P>
  780. void TextureCache<P>::CommitAsyncFlushes() {
  781. // This is intentionally passing the value by copy
  782. committed_downloads.push(uncommitted_downloads);
  783. uncommitted_downloads.clear();
  784. }
  785. template <class P>
  786. void TextureCache<P>::PopAsyncFlushes() {
  787. if (committed_downloads.empty()) {
  788. return;
  789. }
  790. const std::span<const ImageId> download_ids = committed_downloads.front();
  791. if (download_ids.empty()) {
  792. committed_downloads.pop();
  793. return;
  794. }
  795. size_t total_size_bytes = 0;
  796. for (const ImageId image_id : download_ids) {
  797. total_size_bytes += slot_images[image_id].unswizzled_size_bytes;
  798. }
  799. auto download_map = runtime.DownloadStagingBuffer(total_size_bytes);
  800. const size_t original_offset = download_map.offset;
  801. for (const ImageId image_id : download_ids) {
  802. Image& image = slot_images[image_id];
  803. const auto copies = FullDownloadCopies(image.info);
  804. image.DownloadMemory(download_map, copies);
  805. download_map.offset += image.unswizzled_size_bytes;
  806. }
  807. // Wait for downloads to finish
  808. runtime.Finish();
  809. download_map.offset = original_offset;
  810. std::span<u8> download_span = download_map.mapped_span;
  811. for (const ImageId image_id : download_ids) {
  812. const ImageBase& image = slot_images[image_id];
  813. const auto copies = FullDownloadCopies(image.info);
  814. SwizzleImage(gpu_memory, image.gpu_addr, image.info, copies, download_span);
  815. download_map.offset += image.unswizzled_size_bytes;
  816. download_span = download_span.subspan(image.unswizzled_size_bytes);
  817. }
  818. committed_downloads.pop();
  819. }
  820. template <class P>
  821. bool TextureCache<P>::IsRegionGpuModified(VAddr addr, size_t size) {
  822. bool is_modified = false;
  823. ForEachImageInRegion(addr, size, [&is_modified](ImageId, ImageBase& image) {
  824. if (False(image.flags & ImageFlagBits::GpuModified)) {
  825. return false;
  826. }
  827. is_modified = true;
  828. return true;
  829. });
  830. return is_modified;
  831. }
  832. template <class P>
  833. void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) {
  834. if (False(image.flags & ImageFlagBits::CpuModified)) {
  835. // Only upload modified images
  836. return;
  837. }
  838. image.flags &= ~ImageFlagBits::CpuModified;
  839. TrackImage(image, image_id);
  840. if (image.info.num_samples > 1) {
  841. LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented");
  842. return;
  843. }
  844. auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image));
  845. UploadImageContents(image, staging);
  846. runtime.InsertUploadMemoryBarrier();
  847. }
  848. template <class P>
  849. template <typename StagingBuffer>
  850. void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging) {
  851. const std::span<u8> mapped_span = staging.mapped_span;
  852. const GPUVAddr gpu_addr = image.gpu_addr;
  853. if (True(image.flags & ImageFlagBits::AcceleratedUpload)) {
  854. gpu_memory.ReadBlockUnsafe(gpu_addr, mapped_span.data(), mapped_span.size_bytes());
  855. const auto uploads = FullUploadSwizzles(image.info);
  856. runtime.AccelerateImageUpload(image, staging, uploads);
  857. } else if (True(image.flags & ImageFlagBits::Converted)) {
  858. std::vector<u8> unswizzled_data(image.unswizzled_size_bytes);
  859. auto copies = UnswizzleImage(gpu_memory, gpu_addr, image.info, unswizzled_data);
  860. ConvertImage(unswizzled_data, image.info, mapped_span, copies);
  861. image.UploadMemory(staging, copies);
  862. } else if (image.info.type == ImageType::Buffer) {
  863. const std::array copies{UploadBufferCopy(gpu_memory, gpu_addr, image, mapped_span)};
  864. image.UploadMemory(staging, copies);
  865. } else {
  866. const auto copies = UnswizzleImage(gpu_memory, gpu_addr, image.info, mapped_span);
  867. image.UploadMemory(staging, copies);
  868. }
  869. }
  870. template <class P>
  871. ImageViewId TextureCache<P>::FindImageView(const TICEntry& config) {
  872. if (!IsValidEntry(gpu_memory, config)) {
  873. return NULL_IMAGE_VIEW_ID;
  874. }
  875. const auto [pair, is_new] = image_views.try_emplace(config);
  876. ImageViewId& image_view_id = pair->second;
  877. if (is_new) {
  878. image_view_id = CreateImageView(config);
  879. }
  880. return image_view_id;
  881. }
  882. template <class P>
  883. ImageViewId TextureCache<P>::CreateImageView(const TICEntry& config) {
  884. const ImageInfo info(config);
  885. const GPUVAddr image_gpu_addr = config.Address() - config.BaseLayer() * info.layer_stride;
  886. const ImageId image_id = FindOrInsertImage(info, image_gpu_addr);
  887. if (!image_id) {
  888. return NULL_IMAGE_VIEW_ID;
  889. }
  890. ImageBase& image = slot_images[image_id];
  891. const SubresourceBase base = image.TryFindBase(config.Address()).value();
  892. ASSERT(base.level == 0);
  893. const ImageViewInfo view_info(config, base.layer);
  894. const ImageViewId image_view_id = FindOrEmplaceImageView(image_id, view_info);
  895. ImageViewBase& image_view = slot_image_views[image_view_id];
  896. image_view.flags |= ImageViewFlagBits::Strong;
  897. image.flags |= ImageFlagBits::Strong;
  898. return image_view_id;
  899. }
  900. template <class P>
  901. ImageId TextureCache<P>::FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
  902. RelaxedOptions options) {
  903. if (const ImageId image_id = FindImage(info, gpu_addr, options); image_id) {
  904. return image_id;
  905. }
  906. return InsertImage(info, gpu_addr, options);
  907. }
  908. template <class P>
  909. ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr,
  910. RelaxedOptions options) {
  911. std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
  912. if (!cpu_addr) {
  913. cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
  914. if (!cpu_addr) {
  915. return ImageId{};
  916. }
  917. }
  918. const bool broken_views = runtime.HasBrokenTextureViewFormats();
  919. const bool native_bgr = runtime.HasNativeBgr();
  920. ImageId image_id;
  921. const auto lambda = [&](ImageId existing_image_id, ImageBase& existing_image) {
  922. if (True(existing_image.flags & ImageFlagBits::Remapped)) {
  923. return false;
  924. }
  925. if (info.type == ImageType::Linear || existing_image.info.type == ImageType::Linear) {
  926. const bool strict_size = False(options & RelaxedOptions::Size) &&
  927. True(existing_image.flags & ImageFlagBits::Strong);
  928. const ImageInfo& existing = existing_image.info;
  929. if (existing_image.gpu_addr == gpu_addr && existing.type == info.type &&
  930. existing.pitch == info.pitch &&
  931. IsPitchLinearSameSize(existing, info, strict_size) &&
  932. IsViewCompatible(existing.format, info.format, broken_views, native_bgr)) {
  933. image_id = existing_image_id;
  934. return true;
  935. }
  936. } else if (IsSubresource(info, existing_image, gpu_addr, options, broken_views,
  937. native_bgr)) {
  938. image_id = existing_image_id;
  939. return true;
  940. }
  941. return false;
  942. };
  943. ForEachImageInRegion(*cpu_addr, CalculateGuestSizeInBytes(info), lambda);
  944. return image_id;
  945. }
  946. template <class P>
  947. ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
  948. RelaxedOptions options) {
  949. std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
  950. if (!cpu_addr) {
  951. const auto size = CalculateGuestSizeInBytes(info);
  952. cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr, size);
  953. if (!cpu_addr) {
  954. const VAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space;
  955. virtual_invalid_space += Common::AlignUp(size, 32);
  956. cpu_addr = std::optional<VAddr>(fake_addr);
  957. }
  958. }
  959. ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr=0x{:x}", gpu_addr);
  960. const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr);
  961. const Image& image = slot_images[image_id];
  962. // Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different
  963. const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr);
  964. if (is_new) {
  965. it->second = slot_image_allocs.insert();
  966. }
  967. slot_image_allocs[it->second].images.push_back(image_id);
  968. return image_id;
  969. }
  970. template <class P>
  971. ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VAddr cpu_addr) {
  972. ImageInfo new_info = info;
  973. const size_t size_bytes = CalculateGuestSizeInBytes(new_info);
  974. const bool broken_views = runtime.HasBrokenTextureViewFormats();
  975. const bool native_bgr = runtime.HasNativeBgr();
  976. std::vector<ImageId> overlap_ids;
  977. std::unordered_set<ImageId> overlaps_found;
  978. std::vector<ImageId> left_aliased_ids;
  979. std::vector<ImageId> right_aliased_ids;
  980. std::unordered_set<ImageId> ignore_textures;
  981. std::vector<ImageId> bad_overlap_ids;
  982. const auto region_check = [&](ImageId overlap_id, ImageBase& overlap) {
  983. if (True(overlap.flags & ImageFlagBits::Remapped)) {
  984. ignore_textures.insert(overlap_id);
  985. return;
  986. }
  987. if (info.type == ImageType::Linear) {
  988. if (info.pitch == overlap.info.pitch && gpu_addr == overlap.gpu_addr) {
  989. // Alias linear images with the same pitch
  990. left_aliased_ids.push_back(overlap_id);
  991. }
  992. return;
  993. }
  994. overlaps_found.insert(overlap_id);
  995. static constexpr bool strict_size = true;
  996. const std::optional<OverlapResult> solution = ResolveOverlap(
  997. new_info, gpu_addr, cpu_addr, overlap, strict_size, broken_views, native_bgr);
  998. if (solution) {
  999. gpu_addr = solution->gpu_addr;
  1000. cpu_addr = solution->cpu_addr;
  1001. new_info.resources = solution->resources;
  1002. overlap_ids.push_back(overlap_id);
  1003. return;
  1004. }
  1005. static constexpr auto options = RelaxedOptions::Size | RelaxedOptions::Format;
  1006. const ImageBase new_image_base(new_info, gpu_addr, cpu_addr);
  1007. if (IsSubresource(new_info, overlap, gpu_addr, options, broken_views, native_bgr)) {
  1008. left_aliased_ids.push_back(overlap_id);
  1009. overlap.flags |= ImageFlagBits::Alias;
  1010. } else if (IsSubresource(overlap.info, new_image_base, overlap.gpu_addr, options,
  1011. broken_views, native_bgr)) {
  1012. right_aliased_ids.push_back(overlap_id);
  1013. overlap.flags |= ImageFlagBits::Alias;
  1014. } else {
  1015. bad_overlap_ids.push_back(overlap_id);
  1016. overlap.flags |= ImageFlagBits::BadOverlap;
  1017. }
  1018. };
  1019. ForEachImageInRegion(cpu_addr, size_bytes, region_check);
  1020. const auto region_check_gpu = [&](ImageId overlap_id, ImageBase& overlap) {
  1021. if (!overlaps_found.contains(overlap_id)) {
  1022. if (True(overlap.flags & ImageFlagBits::Remapped)) {
  1023. ignore_textures.insert(overlap_id);
  1024. }
  1025. if (overlap.gpu_addr == gpu_addr && overlap.guest_size_bytes == size_bytes) {
  1026. ignore_textures.insert(overlap_id);
  1027. }
  1028. }
  1029. };
  1030. ForEachSparseImageInRegion(gpu_addr, size_bytes, region_check_gpu);
  1031. const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr);
  1032. Image& new_image = slot_images[new_image_id];
  1033. if (!gpu_memory.IsContinousRange(new_image.gpu_addr, new_image.guest_size_bytes)) {
  1034. new_image.flags |= ImageFlagBits::Sparse;
  1035. }
  1036. for (const ImageId overlap_id : ignore_textures) {
  1037. Image& overlap = slot_images[overlap_id];
  1038. if (True(overlap.flags & ImageFlagBits::GpuModified)) {
  1039. UNIMPLEMENTED();
  1040. }
  1041. if (True(overlap.flags & ImageFlagBits::Tracked)) {
  1042. UntrackImage(overlap, overlap_id);
  1043. }
  1044. UnregisterImage(overlap_id);
  1045. DeleteImage(overlap_id);
  1046. }
  1047. // TODO: Only upload what we need
  1048. RefreshContents(new_image, new_image_id);
  1049. for (const ImageId overlap_id : overlap_ids) {
  1050. Image& overlap = slot_images[overlap_id];
  1051. if (overlap.info.num_samples != new_image.info.num_samples) {
  1052. LOG_WARNING(HW_GPU, "Copying between images with different samples is not implemented");
  1053. } else {
  1054. const SubresourceBase base = new_image.TryFindBase(overlap.gpu_addr).value();
  1055. const auto copies = MakeShrinkImageCopies(new_info, overlap.info, base);
  1056. runtime.CopyImage(new_image, overlap, copies);
  1057. }
  1058. if (True(overlap.flags & ImageFlagBits::Tracked)) {
  1059. UntrackImage(overlap, overlap_id);
  1060. }
  1061. UnregisterImage(overlap_id);
  1062. DeleteImage(overlap_id);
  1063. }
  1064. ImageBase& new_image_base = new_image;
  1065. for (const ImageId aliased_id : right_aliased_ids) {
  1066. ImageBase& aliased = slot_images[aliased_id];
  1067. AddImageAlias(new_image_base, aliased, new_image_id, aliased_id);
  1068. new_image.flags |= ImageFlagBits::Alias;
  1069. }
  1070. for (const ImageId aliased_id : left_aliased_ids) {
  1071. ImageBase& aliased = slot_images[aliased_id];
  1072. AddImageAlias(aliased, new_image_base, aliased_id, new_image_id);
  1073. new_image.flags |= ImageFlagBits::Alias;
  1074. }
  1075. for (const ImageId aliased_id : bad_overlap_ids) {
  1076. ImageBase& aliased = slot_images[aliased_id];
  1077. aliased.overlapping_images.push_back(new_image_id);
  1078. new_image.overlapping_images.push_back(aliased_id);
  1079. new_image.flags |= ImageFlagBits::BadOverlap;
  1080. }
  1081. RegisterImage(new_image_id);
  1082. return new_image_id;
  1083. }
  1084. template <class P>
  1085. typename TextureCache<P>::BlitImages TextureCache<P>::GetBlitImages(
  1086. const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Surface& src) {
  1087. static constexpr auto FIND_OPTIONS = RelaxedOptions::Format | RelaxedOptions::Samples;
  1088. const GPUVAddr dst_addr = dst.Address();
  1089. const GPUVAddr src_addr = src.Address();
  1090. ImageInfo dst_info(dst);
  1091. ImageInfo src_info(src);
  1092. ImageId dst_id;
  1093. ImageId src_id;
  1094. do {
  1095. has_deleted_images = false;
  1096. dst_id = FindImage(dst_info, dst_addr, FIND_OPTIONS);
  1097. src_id = FindImage(src_info, src_addr, FIND_OPTIONS);
  1098. const ImageBase* const dst_image = dst_id ? &slot_images[dst_id] : nullptr;
  1099. const ImageBase* const src_image = src_id ? &slot_images[src_id] : nullptr;
  1100. DeduceBlitImages(dst_info, src_info, dst_image, src_image);
  1101. if (GetFormatType(dst_info.format) != GetFormatType(src_info.format)) {
  1102. continue;
  1103. }
  1104. if (!dst_id) {
  1105. dst_id = InsertImage(dst_info, dst_addr, RelaxedOptions{});
  1106. }
  1107. if (!src_id) {
  1108. src_id = InsertImage(src_info, src_addr, RelaxedOptions{});
  1109. }
  1110. } while (has_deleted_images);
  1111. return BlitImages{
  1112. .dst_id = dst_id,
  1113. .src_id = src_id,
  1114. .dst_format = dst_info.format,
  1115. .src_format = src_info.format,
  1116. };
  1117. }
  1118. template <class P>
  1119. SamplerId TextureCache<P>::FindSampler(const TSCEntry& config) {
  1120. if (std::ranges::all_of(config.raw, [](u64 value) { return value == 0; })) {
  1121. return NULL_SAMPLER_ID;
  1122. }
  1123. const auto [pair, is_new] = samplers.try_emplace(config);
  1124. if (is_new) {
  1125. pair->second = slot_samplers.insert(runtime, config);
  1126. }
  1127. return pair->second;
  1128. }
  1129. template <class P>
  1130. ImageViewId TextureCache<P>::FindColorBuffer(size_t index, bool is_clear) {
  1131. const auto& regs = maxwell3d.regs;
  1132. if (index >= regs.rt_control.count) {
  1133. return ImageViewId{};
  1134. }
  1135. const auto& rt = regs.rt[index];
  1136. const GPUVAddr gpu_addr = rt.Address();
  1137. if (gpu_addr == 0) {
  1138. return ImageViewId{};
  1139. }
  1140. if (rt.format == Tegra::RenderTargetFormat::NONE) {
  1141. return ImageViewId{};
  1142. }
  1143. const ImageInfo info(regs, index);
  1144. return FindRenderTargetView(info, gpu_addr, is_clear);
  1145. }
  1146. template <class P>
  1147. ImageViewId TextureCache<P>::FindDepthBuffer(bool is_clear) {
  1148. const auto& regs = maxwell3d.regs;
  1149. if (!regs.zeta_enable) {
  1150. return ImageViewId{};
  1151. }
  1152. const GPUVAddr gpu_addr = regs.zeta.Address();
  1153. if (gpu_addr == 0) {
  1154. return ImageViewId{};
  1155. }
  1156. const ImageInfo info(regs);
  1157. return FindRenderTargetView(info, gpu_addr, is_clear);
  1158. }
  1159. template <class P>
  1160. ImageViewId TextureCache<P>::FindRenderTargetView(const ImageInfo& info, GPUVAddr gpu_addr,
  1161. bool is_clear) {
  1162. const auto options = is_clear ? RelaxedOptions::Samples : RelaxedOptions{};
  1163. const ImageId image_id = FindOrInsertImage(info, gpu_addr, options);
  1164. if (!image_id) {
  1165. return NULL_IMAGE_VIEW_ID;
  1166. }
  1167. Image& image = slot_images[image_id];
  1168. const ImageViewType view_type = RenderTargetImageViewType(info);
  1169. SubresourceBase base;
  1170. if (image.info.type == ImageType::Linear) {
  1171. base = SubresourceBase{.level = 0, .layer = 0};
  1172. } else {
  1173. base = image.TryFindBase(gpu_addr).value();
  1174. }
  1175. const s32 layers = image.info.type == ImageType::e3D ? info.size.depth : info.resources.layers;
  1176. const SubresourceRange range{
  1177. .base = base,
  1178. .extent = {.levels = 1, .layers = layers},
  1179. };
  1180. return FindOrEmplaceImageView(image_id, ImageViewInfo(view_type, info.format, range));
  1181. }
  1182. template <class P>
  1183. template <typename Func>
  1184. void TextureCache<P>::ForEachImageInRegion(VAddr cpu_addr, size_t size, Func&& func) {
  1185. using FuncReturn = typename std::invoke_result<Func, ImageId, Image&>::type;
  1186. static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
  1187. boost::container::small_vector<ImageId, 32> images;
  1188. boost::container::small_vector<ImageMapId, 32> maps;
  1189. ForEachCPUPage(cpu_addr, size, [this, &images, &maps, cpu_addr, size, func](u64 page) {
  1190. const auto it = page_table.find(page);
  1191. if (it == page_table.end()) {
  1192. if constexpr (BOOL_BREAK) {
  1193. return false;
  1194. } else {
  1195. return;
  1196. }
  1197. }
  1198. for (const ImageMapId map_id : it->second) {
  1199. ImageMapView& map = slot_map_views[map_id];
  1200. if (map.picked) {
  1201. continue;
  1202. }
  1203. if (!map.Overlaps(cpu_addr, size)) {
  1204. continue;
  1205. }
  1206. map.picked = true;
  1207. maps.push_back(map_id);
  1208. Image& image = slot_images[map.image_id];
  1209. if (True(image.flags & ImageFlagBits::Picked)) {
  1210. continue;
  1211. }
  1212. image.flags |= ImageFlagBits::Picked;
  1213. images.push_back(map.image_id);
  1214. if constexpr (BOOL_BREAK) {
  1215. if (func(map.image_id, image)) {
  1216. return true;
  1217. }
  1218. } else {
  1219. func(map.image_id, image);
  1220. }
  1221. }
  1222. if constexpr (BOOL_BREAK) {
  1223. return false;
  1224. }
  1225. });
  1226. for (const ImageId image_id : images) {
  1227. slot_images[image_id].flags &= ~ImageFlagBits::Picked;
  1228. }
  1229. for (const ImageMapId map_id : maps) {
  1230. slot_map_views[map_id].picked = false;
  1231. }
  1232. }
  1233. template <class P>
  1234. template <typename Func>
  1235. void TextureCache<P>::ForEachImageInRegionGPU(GPUVAddr gpu_addr, size_t size, Func&& func) {
  1236. using FuncReturn = typename std::invoke_result<Func, ImageId, Image&>::type;
  1237. static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
  1238. boost::container::small_vector<ImageId, 8> images;
  1239. ForEachGPUPage(gpu_addr, size, [this, &images, gpu_addr, size, func](u64 page) {
  1240. const auto it = gpu_page_table.find(page);
  1241. if (it == gpu_page_table.end()) {
  1242. if constexpr (BOOL_BREAK) {
  1243. return false;
  1244. } else {
  1245. return;
  1246. }
  1247. }
  1248. for (const ImageId image_id : it->second) {
  1249. Image& image = slot_images[image_id];
  1250. if (True(image.flags & ImageFlagBits::Picked)) {
  1251. continue;
  1252. }
  1253. if (!image.OverlapsGPU(gpu_addr, size)) {
  1254. continue;
  1255. }
  1256. image.flags |= ImageFlagBits::Picked;
  1257. images.push_back(image_id);
  1258. if constexpr (BOOL_BREAK) {
  1259. if (func(image_id, image)) {
  1260. return true;
  1261. }
  1262. } else {
  1263. func(image_id, image);
  1264. }
  1265. }
  1266. if constexpr (BOOL_BREAK) {
  1267. return false;
  1268. }
  1269. });
  1270. for (const ImageId image_id : images) {
  1271. slot_images[image_id].flags &= ~ImageFlagBits::Picked;
  1272. }
  1273. }
  1274. template <class P>
  1275. template <typename Func>
  1276. void TextureCache<P>::ForEachSparseImageInRegion(GPUVAddr gpu_addr, size_t size, Func&& func) {
  1277. using FuncReturn = typename std::invoke_result<Func, ImageId, Image&>::type;
  1278. static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
  1279. boost::container::small_vector<ImageId, 8> images;
  1280. ForEachGPUPage(gpu_addr, size, [this, &images, gpu_addr, size, func](u64 page) {
  1281. const auto it = sparse_page_table.find(page);
  1282. if (it == sparse_page_table.end()) {
  1283. if constexpr (BOOL_BREAK) {
  1284. return false;
  1285. } else {
  1286. return;
  1287. }
  1288. }
  1289. for (const ImageId image_id : it->second) {
  1290. Image& image = slot_images[image_id];
  1291. if (True(image.flags & ImageFlagBits::Picked)) {
  1292. continue;
  1293. }
  1294. if (!image.OverlapsGPU(gpu_addr, size)) {
  1295. continue;
  1296. }
  1297. image.flags |= ImageFlagBits::Picked;
  1298. images.push_back(image_id);
  1299. if constexpr (BOOL_BREAK) {
  1300. if (func(image_id, image)) {
  1301. return true;
  1302. }
  1303. } else {
  1304. func(image_id, image);
  1305. }
  1306. }
  1307. if constexpr (BOOL_BREAK) {
  1308. return false;
  1309. }
  1310. });
  1311. for (const ImageId image_id : images) {
  1312. slot_images[image_id].flags &= ~ImageFlagBits::Picked;
  1313. }
  1314. }
  1315. template <class P>
  1316. template <typename Func>
  1317. void TextureCache<P>::ForEachSparseSegment(ImageBase& image, Func&& func) {
  1318. using FuncReturn = typename std::invoke_result<Func, GPUVAddr, VAddr, size_t>::type;
  1319. static constexpr bool RETURNS_BOOL = std::is_same_v<FuncReturn, bool>;
  1320. const auto segments = gpu_memory.GetSubmappedRange(image.gpu_addr, image.guest_size_bytes);
  1321. for (auto& segment : segments) {
  1322. const auto gpu_addr = segment.first;
  1323. const auto size = segment.second;
  1324. std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
  1325. ASSERT(cpu_addr);
  1326. if constexpr (RETURNS_BOOL) {
  1327. if (func(gpu_addr, *cpu_addr, size)) {
  1328. break;
  1329. }
  1330. } else {
  1331. func(gpu_addr, *cpu_addr, size);
  1332. }
  1333. }
  1334. }
  1335. template <class P>
  1336. ImageViewId TextureCache<P>::FindOrEmplaceImageView(ImageId image_id, const ImageViewInfo& info) {
  1337. Image& image = slot_images[image_id];
  1338. if (const ImageViewId image_view_id = image.FindView(info); image_view_id) {
  1339. return image_view_id;
  1340. }
  1341. const ImageViewId image_view_id = slot_image_views.insert(runtime, info, image_id, image);
  1342. image.InsertView(info, image_view_id);
  1343. return image_view_id;
  1344. }
  1345. template <class P>
  1346. void TextureCache<P>::RegisterImage(ImageId image_id) {
  1347. ImageBase& image = slot_images[image_id];
  1348. ASSERT_MSG(False(image.flags & ImageFlagBits::Registered),
  1349. "Trying to register an already registered image");
  1350. image.flags |= ImageFlagBits::Registered;
  1351. u64 tentative_size = std::max(image.guest_size_bytes, image.unswizzled_size_bytes);
  1352. if ((IsPixelFormatASTC(image.info.format) &&
  1353. True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
  1354. True(image.flags & ImageFlagBits::Converted)) {
  1355. tentative_size = EstimatedDecompressedSize(tentative_size, image.info.format);
  1356. }
  1357. total_used_memory += Common::AlignUp(tentative_size, 1024);
  1358. ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
  1359. [this, image_id](u64 page) { gpu_page_table[page].push_back(image_id); });
  1360. if (False(image.flags & ImageFlagBits::Sparse)) {
  1361. auto map_id =
  1362. slot_map_views.insert(image.gpu_addr, image.cpu_addr, image.guest_size_bytes, image_id);
  1363. ForEachCPUPage(image.cpu_addr, image.guest_size_bytes,
  1364. [this, map_id](u64 page) { page_table[page].push_back(map_id); });
  1365. image.map_view_id = map_id;
  1366. return;
  1367. }
  1368. std::vector<ImageViewId> sparse_maps{};
  1369. ForEachSparseSegment(
  1370. image, [this, image_id, &sparse_maps](GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
  1371. auto map_id = slot_map_views.insert(gpu_addr, cpu_addr, size, image_id);
  1372. ForEachCPUPage(cpu_addr, size,
  1373. [this, map_id](u64 page) { page_table[page].push_back(map_id); });
  1374. sparse_maps.push_back(map_id);
  1375. });
  1376. sparse_views.emplace(image_id, std::move(sparse_maps));
  1377. ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
  1378. [this, image_id](u64 page) { sparse_page_table[page].push_back(image_id); });
  1379. }
  1380. template <class P>
  1381. void TextureCache<P>::UnregisterImage(ImageId image_id) {
  1382. Image& image = slot_images[image_id];
  1383. ASSERT_MSG(True(image.flags & ImageFlagBits::Registered),
  1384. "Trying to unregister an already registered image");
  1385. image.flags &= ~ImageFlagBits::Registered;
  1386. image.flags &= ~ImageFlagBits::BadOverlap;
  1387. u64 tentative_size = std::max(image.guest_size_bytes, image.unswizzled_size_bytes);
  1388. if ((IsPixelFormatASTC(image.info.format) &&
  1389. True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
  1390. True(image.flags & ImageFlagBits::Converted)) {
  1391. tentative_size = EstimatedDecompressedSize(tentative_size, image.info.format);
  1392. }
  1393. total_used_memory -= Common::AlignUp(tentative_size, 1024);
  1394. const auto& clear_page_table =
  1395. [this, image_id](
  1396. u64 page,
  1397. std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>>& selected_page_table) {
  1398. const auto page_it = selected_page_table.find(page);
  1399. if (page_it == selected_page_table.end()) {
  1400. UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
  1401. return;
  1402. }
  1403. std::vector<ImageId>& image_ids = page_it->second;
  1404. const auto vector_it = std::ranges::find(image_ids, image_id);
  1405. if (vector_it == image_ids.end()) {
  1406. UNREACHABLE_MSG("Unregistering unregistered image in page=0x{:x}",
  1407. page << PAGE_BITS);
  1408. return;
  1409. }
  1410. image_ids.erase(vector_it);
  1411. };
  1412. ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
  1413. [this, &clear_page_table](u64 page) { clear_page_table(page, gpu_page_table); });
  1414. if (False(image.flags & ImageFlagBits::Sparse)) {
  1415. const auto map_id = image.map_view_id;
  1416. ForEachCPUPage(image.cpu_addr, image.guest_size_bytes, [this, map_id](u64 page) {
  1417. const auto page_it = page_table.find(page);
  1418. if (page_it == page_table.end()) {
  1419. UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
  1420. return;
  1421. }
  1422. std::vector<ImageMapId>& image_map_ids = page_it->second;
  1423. const auto vector_it = std::ranges::find(image_map_ids, map_id);
  1424. if (vector_it == image_map_ids.end()) {
  1425. UNREACHABLE_MSG("Unregistering unregistered image in page=0x{:x}",
  1426. page << PAGE_BITS);
  1427. return;
  1428. }
  1429. image_map_ids.erase(vector_it);
  1430. });
  1431. slot_map_views.erase(map_id);
  1432. return;
  1433. }
  1434. ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, &clear_page_table](u64 page) {
  1435. clear_page_table(page, sparse_page_table);
  1436. });
  1437. auto it = sparse_views.find(image_id);
  1438. ASSERT(it != sparse_views.end());
  1439. auto& sparse_maps = it->second;
  1440. for (auto& map_view_id : sparse_maps) {
  1441. const auto& map_range = slot_map_views[map_view_id];
  1442. const VAddr cpu_addr = map_range.cpu_addr;
  1443. const std::size_t size = map_range.size;
  1444. ForEachCPUPage(cpu_addr, size, [this, image_id](u64 page) {
  1445. const auto page_it = page_table.find(page);
  1446. if (page_it == page_table.end()) {
  1447. UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
  1448. return;
  1449. }
  1450. std::vector<ImageMapId>& image_map_ids = page_it->second;
  1451. auto vector_it = image_map_ids.begin();
  1452. while (vector_it != image_map_ids.end()) {
  1453. ImageMapView& map = slot_map_views[*vector_it];
  1454. if (map.image_id != image_id) {
  1455. vector_it++;
  1456. continue;
  1457. }
  1458. if (!map.picked) {
  1459. map.picked = true;
  1460. }
  1461. vector_it = image_map_ids.erase(vector_it);
  1462. }
  1463. });
  1464. slot_map_views.erase(map_view_id);
  1465. }
  1466. sparse_views.erase(it);
  1467. }
  1468. template <class P>
  1469. void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
  1470. ASSERT(False(image.flags & ImageFlagBits::Tracked));
  1471. image.flags |= ImageFlagBits::Tracked;
  1472. if (False(image.flags & ImageFlagBits::Sparse)) {
  1473. rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
  1474. return;
  1475. }
  1476. if (True(image.flags & ImageFlagBits::Registered)) {
  1477. auto it = sparse_views.find(image_id);
  1478. ASSERT(it != sparse_views.end());
  1479. auto& sparse_maps = it->second;
  1480. for (auto& map_view_id : sparse_maps) {
  1481. const auto& map = slot_map_views[map_view_id];
  1482. const VAddr cpu_addr = map.cpu_addr;
  1483. const std::size_t size = map.size;
  1484. rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
  1485. }
  1486. return;
  1487. }
  1488. ForEachSparseSegment(image,
  1489. [this]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
  1490. rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
  1491. });
  1492. }
  1493. template <class P>
  1494. void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
  1495. ASSERT(True(image.flags & ImageFlagBits::Tracked));
  1496. image.flags &= ~ImageFlagBits::Tracked;
  1497. if (False(image.flags & ImageFlagBits::Sparse)) {
  1498. rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
  1499. return;
  1500. }
  1501. ASSERT(True(image.flags & ImageFlagBits::Registered));
  1502. auto it = sparse_views.find(image_id);
  1503. ASSERT(it != sparse_views.end());
  1504. auto& sparse_maps = it->second;
  1505. for (auto& map_view_id : sparse_maps) {
  1506. const auto& map = slot_map_views[map_view_id];
  1507. const VAddr cpu_addr = map.cpu_addr;
  1508. const std::size_t size = map.size;
  1509. rasterizer.UpdatePagesCachedCount(cpu_addr, size, -1);
  1510. }
  1511. }
  1512. template <class P>
  1513. void TextureCache<P>::DeleteImage(ImageId image_id) {
  1514. ImageBase& image = slot_images[image_id];
  1515. const GPUVAddr gpu_addr = image.gpu_addr;
  1516. const auto alloc_it = image_allocs_table.find(gpu_addr);
  1517. if (alloc_it == image_allocs_table.end()) {
  1518. UNREACHABLE_MSG("Trying to delete an image alloc that does not exist in address 0x{:x}",
  1519. gpu_addr);
  1520. return;
  1521. }
  1522. const ImageAllocId alloc_id = alloc_it->second;
  1523. std::vector<ImageId>& alloc_images = slot_image_allocs[alloc_id].images;
  1524. const auto alloc_image_it = std::ranges::find(alloc_images, image_id);
  1525. if (alloc_image_it == alloc_images.end()) {
  1526. UNREACHABLE_MSG("Trying to delete an image that does not exist");
  1527. return;
  1528. }
  1529. ASSERT_MSG(False(image.flags & ImageFlagBits::Tracked), "Image was not untracked");
  1530. ASSERT_MSG(False(image.flags & ImageFlagBits::Registered), "Image was not unregistered");
  1531. // Mark render targets as dirty
  1532. auto& dirty = maxwell3d.dirty.flags;
  1533. dirty[Dirty::RenderTargets] = true;
  1534. dirty[Dirty::ZetaBuffer] = true;
  1535. for (size_t rt = 0; rt < NUM_RT; ++rt) {
  1536. dirty[Dirty::ColorBuffer0 + rt] = true;
  1537. }
  1538. const std::span<const ImageViewId> image_view_ids = image.image_view_ids;
  1539. for (const ImageViewId image_view_id : image_view_ids) {
  1540. std::ranges::replace(render_targets.color_buffer_ids, image_view_id, ImageViewId{});
  1541. if (render_targets.depth_buffer_id == image_view_id) {
  1542. render_targets.depth_buffer_id = ImageViewId{};
  1543. }
  1544. }
  1545. RemoveImageViewReferences(image_view_ids);
  1546. RemoveFramebuffers(image_view_ids);
  1547. for (const AliasedImage& alias : image.aliased_images) {
  1548. ImageBase& other_image = slot_images[alias.id];
  1549. [[maybe_unused]] const size_t num_removed_aliases =
  1550. std::erase_if(other_image.aliased_images, [image_id](const AliasedImage& other_alias) {
  1551. return other_alias.id == image_id;
  1552. });
  1553. other_image.CheckAliasState();
  1554. ASSERT_MSG(num_removed_aliases == 1, "Invalid number of removed aliases: {}",
  1555. num_removed_aliases);
  1556. }
  1557. for (const ImageId overlap_id : image.overlapping_images) {
  1558. ImageBase& other_image = slot_images[overlap_id];
  1559. [[maybe_unused]] const size_t num_removed_overlaps = std::erase_if(
  1560. other_image.overlapping_images,
  1561. [image_id](const ImageId other_overlap_id) { return other_overlap_id == image_id; });
  1562. other_image.CheckBadOverlapState();
  1563. ASSERT_MSG(num_removed_overlaps == 1, "Invalid number of removed overlapps: {}",
  1564. num_removed_overlaps);
  1565. }
  1566. for (const ImageViewId image_view_id : image_view_ids) {
  1567. sentenced_image_view.Push(std::move(slot_image_views[image_view_id]));
  1568. slot_image_views.erase(image_view_id);
  1569. }
  1570. sentenced_images.Push(std::move(slot_images[image_id]));
  1571. slot_images.erase(image_id);
  1572. alloc_images.erase(alloc_image_it);
  1573. if (alloc_images.empty()) {
  1574. image_allocs_table.erase(alloc_it);
  1575. }
  1576. if constexpr (ENABLE_VALIDATION) {
  1577. std::ranges::fill(graphics_image_view_ids, CORRUPT_ID);
  1578. std::ranges::fill(compute_image_view_ids, CORRUPT_ID);
  1579. }
  1580. graphics_image_table.Invalidate();
  1581. compute_image_table.Invalidate();
  1582. has_deleted_images = true;
  1583. }
  1584. template <class P>
  1585. void TextureCache<P>::RemoveImageViewReferences(std::span<const ImageViewId> removed_views) {
  1586. auto it = image_views.begin();
  1587. while (it != image_views.end()) {
  1588. const auto found = std::ranges::find(removed_views, it->second);
  1589. if (found != removed_views.end()) {
  1590. it = image_views.erase(it);
  1591. } else {
  1592. ++it;
  1593. }
  1594. }
  1595. }
  1596. template <class P>
  1597. void TextureCache<P>::RemoveFramebuffers(std::span<const ImageViewId> removed_views) {
  1598. auto it = framebuffers.begin();
  1599. while (it != framebuffers.end()) {
  1600. if (it->first.Contains(removed_views)) {
  1601. it = framebuffers.erase(it);
  1602. } else {
  1603. ++it;
  1604. }
  1605. }
  1606. }
  1607. template <class P>
  1608. void TextureCache<P>::MarkModification(ImageBase& image) noexcept {
  1609. image.flags |= ImageFlagBits::GpuModified;
  1610. image.modification_tick = ++modification_tick;
  1611. }
  1612. template <class P>
  1613. void TextureCache<P>::SynchronizeAliases(ImageId image_id) {
  1614. boost::container::small_vector<const AliasedImage*, 1> aliased_images;
  1615. ImageBase& image = slot_images[image_id];
  1616. u64 most_recent_tick = image.modification_tick;
  1617. for (const AliasedImage& aliased : image.aliased_images) {
  1618. ImageBase& aliased_image = slot_images[aliased.id];
  1619. if (image.modification_tick < aliased_image.modification_tick) {
  1620. most_recent_tick = std::max(most_recent_tick, aliased_image.modification_tick);
  1621. aliased_images.push_back(&aliased);
  1622. }
  1623. }
  1624. if (aliased_images.empty()) {
  1625. return;
  1626. }
  1627. image.modification_tick = most_recent_tick;
  1628. std::ranges::sort(aliased_images, [this](const AliasedImage* lhs, const AliasedImage* rhs) {
  1629. const ImageBase& lhs_image = slot_images[lhs->id];
  1630. const ImageBase& rhs_image = slot_images[rhs->id];
  1631. return lhs_image.modification_tick < rhs_image.modification_tick;
  1632. });
  1633. for (const AliasedImage* const aliased : aliased_images) {
  1634. CopyImage(image_id, aliased->id, aliased->copies);
  1635. }
  1636. }
  1637. template <class P>
  1638. void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool invalidate) {
  1639. Image& image = slot_images[image_id];
  1640. if (invalidate) {
  1641. image.flags &= ~(ImageFlagBits::CpuModified | ImageFlagBits::GpuModified);
  1642. if (False(image.flags & ImageFlagBits::Tracked)) {
  1643. TrackImage(image, image_id);
  1644. }
  1645. } else {
  1646. RefreshContents(image, image_id);
  1647. SynchronizeAliases(image_id);
  1648. }
  1649. if (is_modification) {
  1650. MarkModification(image);
  1651. }
  1652. image.frame_tick = frame_tick;
  1653. }
  1654. template <class P>
  1655. void TextureCache<P>::PrepareImageView(ImageViewId image_view_id, bool is_modification,
  1656. bool invalidate) {
  1657. if (!image_view_id) {
  1658. return;
  1659. }
  1660. const ImageViewBase& image_view = slot_image_views[image_view_id];
  1661. PrepareImage(image_view.image_id, is_modification, invalidate);
  1662. }
  1663. template <class P>
  1664. void TextureCache<P>::CopyImage(ImageId dst_id, ImageId src_id, std::span<const ImageCopy> copies) {
  1665. Image& dst = slot_images[dst_id];
  1666. Image& src = slot_images[src_id];
  1667. const auto dst_format_type = GetFormatType(dst.info.format);
  1668. const auto src_format_type = GetFormatType(src.info.format);
  1669. if (src_format_type == dst_format_type) {
  1670. if constexpr (HAS_EMULATED_COPIES) {
  1671. if (!runtime.CanImageBeCopied(dst, src)) {
  1672. return runtime.EmulateCopyImage(dst, src, copies);
  1673. }
  1674. }
  1675. return runtime.CopyImage(dst, src, copies);
  1676. }
  1677. UNIMPLEMENTED_IF(dst.info.type != ImageType::e2D);
  1678. UNIMPLEMENTED_IF(src.info.type != ImageType::e2D);
  1679. for (const ImageCopy& copy : copies) {
  1680. UNIMPLEMENTED_IF(copy.dst_subresource.num_layers != 1);
  1681. UNIMPLEMENTED_IF(copy.src_subresource.num_layers != 1);
  1682. UNIMPLEMENTED_IF(copy.src_offset != Offset3D{});
  1683. UNIMPLEMENTED_IF(copy.dst_offset != Offset3D{});
  1684. const SubresourceBase dst_base{
  1685. .level = copy.dst_subresource.base_level,
  1686. .layer = copy.dst_subresource.base_layer,
  1687. };
  1688. const SubresourceBase src_base{
  1689. .level = copy.src_subresource.base_level,
  1690. .layer = copy.src_subresource.base_layer,
  1691. };
  1692. const SubresourceExtent dst_extent{.levels = 1, .layers = 1};
  1693. const SubresourceExtent src_extent{.levels = 1, .layers = 1};
  1694. const SubresourceRange dst_range{.base = dst_base, .extent = dst_extent};
  1695. const SubresourceRange src_range{.base = src_base, .extent = src_extent};
  1696. const ImageViewInfo dst_view_info(ImageViewType::e2D, dst.info.format, dst_range);
  1697. const ImageViewInfo src_view_info(ImageViewType::e2D, src.info.format, src_range);
  1698. const auto [dst_framebuffer_id, dst_view_id] = RenderTargetFromImage(dst_id, dst_view_info);
  1699. Framebuffer* const dst_framebuffer = &slot_framebuffers[dst_framebuffer_id];
  1700. const ImageViewId src_view_id = FindOrEmplaceImageView(src_id, src_view_info);
  1701. ImageView& dst_view = slot_image_views[dst_view_id];
  1702. ImageView& src_view = slot_image_views[src_view_id];
  1703. [[maybe_unused]] const Extent3D expected_size{
  1704. .width = std::min(dst_view.size.width, src_view.size.width),
  1705. .height = std::min(dst_view.size.height, src_view.size.height),
  1706. .depth = std::min(dst_view.size.depth, src_view.size.depth),
  1707. };
  1708. UNIMPLEMENTED_IF(copy.extent != expected_size);
  1709. runtime.ConvertImage(dst_framebuffer, dst_view, src_view);
  1710. }
  1711. }
  1712. template <class P>
  1713. void TextureCache<P>::BindRenderTarget(ImageViewId* old_id, ImageViewId new_id) {
  1714. if (*old_id == new_id) {
  1715. return;
  1716. }
  1717. if (*old_id) {
  1718. const ImageViewBase& old_view = slot_image_views[*old_id];
  1719. if (True(old_view.flags & ImageViewFlagBits::PreemtiveDownload)) {
  1720. uncommitted_downloads.push_back(old_view.image_id);
  1721. }
  1722. }
  1723. *old_id = new_id;
  1724. }
  1725. template <class P>
  1726. std::pair<FramebufferId, ImageViewId> TextureCache<P>::RenderTargetFromImage(
  1727. ImageId image_id, const ImageViewInfo& view_info) {
  1728. const ImageViewId view_id = FindOrEmplaceImageView(image_id, view_info);
  1729. const ImageBase& image = slot_images[image_id];
  1730. const bool is_color = GetFormatType(image.info.format) == SurfaceType::ColorTexture;
  1731. const ImageViewId color_view_id = is_color ? view_id : ImageViewId{};
  1732. const ImageViewId depth_view_id = is_color ? ImageViewId{} : view_id;
  1733. const Extent3D extent = MipSize(image.info.size, view_info.range.base.level);
  1734. const u32 num_samples = image.info.num_samples;
  1735. const auto [samples_x, samples_y] = SamplesLog2(num_samples);
  1736. const FramebufferId framebuffer_id = GetFramebufferId(RenderTargets{
  1737. .color_buffer_ids = {color_view_id},
  1738. .depth_buffer_id = depth_view_id,
  1739. .size = {extent.width >> samples_x, extent.height >> samples_y},
  1740. });
  1741. return {framebuffer_id, view_id};
  1742. }
  1743. template <class P>
  1744. bool TextureCache<P>::IsFullClear(ImageViewId id) {
  1745. if (!id) {
  1746. return true;
  1747. }
  1748. const ImageViewBase& image_view = slot_image_views[id];
  1749. const ImageBase& image = slot_images[image_view.image_id];
  1750. const Extent3D size = image_view.size;
  1751. const auto& regs = maxwell3d.regs;
  1752. const auto& scissor = regs.scissor_test[0];
  1753. if (image.info.resources.levels > 1 || image.info.resources.layers > 1) {
  1754. // Images with multiple resources can't be cleared in a single call
  1755. return false;
  1756. }
  1757. if (regs.clear_flags.scissor == 0) {
  1758. // If scissor testing is disabled, the clear is always full
  1759. return true;
  1760. }
  1761. // Make sure the clear covers all texels in the subresource
  1762. return scissor.min_x == 0 && scissor.min_y == 0 && scissor.max_x >= size.width &&
  1763. scissor.max_y >= size.height;
  1764. }
  1765. } // namespace VideoCommon