memory_manager.cpp 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605
  1. // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project
  2. // SPDX-License-Identifier: GPL-2.0-or-later
  3. #include <algorithm>
  4. #include "common/alignment.h"
  5. #include "common/assert.h"
  6. #include "common/logging/log.h"
  7. #include "core/core.h"
  8. #include "core/device_memory.h"
  9. #include "core/hle/kernel/k_page_table.h"
  10. #include "core/hle/kernel/k_process.h"
  11. #include "core/memory.h"
  12. #include "video_core/memory_manager.h"
  13. #include "video_core/rasterizer_interface.h"
  14. #include "video_core/renderer_base.h"
  15. namespace Tegra {
  16. std::atomic<size_t> MemoryManager::unique_identifier_generator{};
  17. MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 big_page_bits_,
  18. u64 page_bits_)
  19. : system{system_}, memory{system.Memory()}, device_memory{system.DeviceMemory()},
  20. address_space_bits{address_space_bits_}, page_bits{page_bits_}, big_page_bits{big_page_bits_},
  21. entries{}, big_entries{}, page_table{address_space_bits, address_space_bits + page_bits - 38,
  22. page_bits != big_page_bits ? page_bits : 0},
  23. unique_identifier{unique_identifier_generator.fetch_add(1, std::memory_order_acq_rel)} {
  24. address_space_size = 1ULL << address_space_bits;
  25. page_size = 1ULL << page_bits;
  26. page_mask = page_size - 1ULL;
  27. big_page_size = 1ULL << big_page_bits;
  28. big_page_mask = big_page_size - 1ULL;
  29. const u64 page_table_bits = address_space_bits - page_bits;
  30. const u64 big_page_table_bits = address_space_bits - big_page_bits;
  31. const u64 page_table_size = 1ULL << page_table_bits;
  32. const u64 big_page_table_size = 1ULL << big_page_table_bits;
  33. page_table_mask = page_table_size - 1;
  34. big_page_table_mask = big_page_table_size - 1;
  35. big_entries.resize(big_page_table_size / 32, 0);
  36. big_page_table_cpu.resize(big_page_table_size);
  37. big_page_continous.resize(big_page_table_size / continous_bits, 0);
  38. entries.resize(page_table_size / 32, 0);
  39. }
  40. MemoryManager::~MemoryManager() = default;
  41. template <bool is_big_page>
  42. MemoryManager::EntryType MemoryManager::GetEntry(size_t position) const {
  43. if constexpr (is_big_page) {
  44. position = position >> big_page_bits;
  45. const u64 entry_mask = big_entries[position / 32];
  46. const size_t sub_index = position % 32;
  47. return static_cast<EntryType>((entry_mask >> (2 * sub_index)) & 0x03ULL);
  48. } else {
  49. position = position >> page_bits;
  50. const u64 entry_mask = entries[position / 32];
  51. const size_t sub_index = position % 32;
  52. return static_cast<EntryType>((entry_mask >> (2 * sub_index)) & 0x03ULL);
  53. }
  54. }
  55. template <bool is_big_page>
  56. void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) {
  57. if constexpr (is_big_page) {
  58. position = position >> big_page_bits;
  59. const u64 entry_mask = big_entries[position / 32];
  60. const size_t sub_index = position % 32;
  61. big_entries[position / 32] =
  62. (~(3ULL << sub_index * 2) & entry_mask) | (static_cast<u64>(entry) << sub_index * 2);
  63. } else {
  64. position = position >> page_bits;
  65. const u64 entry_mask = entries[position / 32];
  66. const size_t sub_index = position % 32;
  67. entries[position / 32] =
  68. (~(3ULL << sub_index * 2) & entry_mask) | (static_cast<u64>(entry) << sub_index * 2);
  69. }
  70. }
  71. inline bool MemoryManager::IsBigPageContinous(size_t big_page_index) const {
  72. const u64 entry_mask = big_page_continous[big_page_index / continous_bits];
  73. const size_t sub_index = big_page_index % continous_bits;
  74. return ((entry_mask >> sub_index) & 0x1ULL) != 0;
  75. }
  76. inline void MemoryManager::SetBigPageContinous(size_t big_page_index, bool value) {
  77. const u64 continous_mask = big_page_continous[big_page_index / continous_bits];
  78. const size_t sub_index = big_page_index % continous_bits;
  79. big_page_continous[big_page_index / continous_bits] =
  80. (~(1ULL << sub_index) & continous_mask) | (value ? 1ULL << sub_index : 0);
  81. }
  82. template <MemoryManager::EntryType entry_type>
  83. GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr,
  84. size_t size) {
  85. u64 remaining_size{size};
  86. if constexpr (entry_type == EntryType::Mapped) {
  87. page_table.ReserveRange(gpu_addr, size);
  88. }
  89. for (u64 offset{}; offset < size; offset += page_size) {
  90. const GPUVAddr current_gpu_addr = gpu_addr + offset;
  91. [[maybe_unused]] const auto current_entry_type = GetEntry<false>(current_gpu_addr);
  92. SetEntry<false>(current_gpu_addr, entry_type);
  93. if (current_entry_type != entry_type) {
  94. rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, page_size);
  95. }
  96. if constexpr (entry_type == EntryType::Mapped) {
  97. const VAddr current_cpu_addr = cpu_addr + offset;
  98. const auto index = PageEntryIndex<false>(current_gpu_addr);
  99. const u32 sub_value = static_cast<u32>(current_cpu_addr >> cpu_page_bits);
  100. page_table[index] = sub_value;
  101. }
  102. remaining_size -= page_size;
  103. }
  104. return gpu_addr;
  105. }
  106. template <MemoryManager::EntryType entry_type>
  107. GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr,
  108. size_t size) {
  109. u64 remaining_size{size};
  110. for (u64 offset{}; offset < size; offset += big_page_size) {
  111. const GPUVAddr current_gpu_addr = gpu_addr + offset;
  112. [[maybe_unused]] const auto current_entry_type = GetEntry<true>(current_gpu_addr);
  113. SetEntry<true>(current_gpu_addr, entry_type);
  114. if (current_entry_type != entry_type) {
  115. rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, big_page_size);
  116. }
  117. if constexpr (entry_type == EntryType::Mapped) {
  118. const VAddr current_cpu_addr = cpu_addr + offset;
  119. const auto index = PageEntryIndex<true>(current_gpu_addr);
  120. const u32 sub_value = static_cast<u32>(current_cpu_addr >> cpu_page_bits);
  121. big_page_table_cpu[index] = sub_value;
  122. const bool is_continous = ([&] {
  123. uintptr_t base_ptr{
  124. reinterpret_cast<uintptr_t>(memory.GetPointerSilent(current_cpu_addr))};
  125. if (base_ptr == 0) {
  126. return false;
  127. }
  128. for (VAddr start_cpu = current_cpu_addr + page_size;
  129. start_cpu < current_cpu_addr + big_page_size; start_cpu += page_size) {
  130. base_ptr += page_size;
  131. auto next_ptr = reinterpret_cast<uintptr_t>(memory.GetPointerSilent(start_cpu));
  132. if (next_ptr == 0 || base_ptr != next_ptr) {
  133. return false;
  134. }
  135. }
  136. return true;
  137. })();
  138. SetBigPageContinous(index, is_continous);
  139. }
  140. remaining_size -= big_page_size;
  141. }
  142. return gpu_addr;
  143. }
  144. void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) {
  145. rasterizer = rasterizer_;
  146. }
  147. GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size,
  148. bool is_big_pages) {
  149. if (is_big_pages) [[likely]] {
  150. return BigPageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size);
  151. }
  152. return PageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size);
  153. }
  154. GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) {
  155. if (is_big_pages) [[likely]] {
  156. return BigPageTableOp<EntryType::Reserved>(gpu_addr, 0, size);
  157. }
  158. return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size);
  159. }
  160. void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) {
  161. if (size == 0) {
  162. return;
  163. }
  164. const auto submapped_ranges = GetSubmappedRange(gpu_addr, size);
  165. for (const auto& [map_addr, map_size] : submapped_ranges) {
  166. // Flush and invalidate through the GPU interface, to be asynchronous if possible.
  167. const std::optional<VAddr> cpu_addr = GpuToCpuAddress(map_addr);
  168. ASSERT(cpu_addr);
  169. rasterizer->UnmapMemory(*cpu_addr, map_size);
  170. }
  171. BigPageTableOp<EntryType::Free>(gpu_addr, 0, size);
  172. PageTableOp<EntryType::Free>(gpu_addr, 0, size);
  173. }
  174. std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const {
  175. if (GetEntry<true>(gpu_addr) != EntryType::Mapped) [[unlikely]] {
  176. if (GetEntry<false>(gpu_addr) != EntryType::Mapped) {
  177. return std::nullopt;
  178. }
  179. const VAddr cpu_addr_base = static_cast<VAddr>(page_table[PageEntryIndex<false>(gpu_addr)])
  180. << cpu_page_bits;
  181. return cpu_addr_base + (gpu_addr & page_mask);
  182. }
  183. const VAddr cpu_addr_base =
  184. static_cast<VAddr>(big_page_table_cpu[PageEntryIndex<true>(gpu_addr)]) << cpu_page_bits;
  185. return cpu_addr_base + (gpu_addr & big_page_mask);
  186. }
  187. std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr addr, std::size_t size) const {
  188. size_t page_index{addr >> page_bits};
  189. const size_t page_last{(addr + size + page_size - 1) >> page_bits};
  190. while (page_index < page_last) {
  191. const auto page_addr{GpuToCpuAddress(page_index << page_bits)};
  192. if (page_addr) {
  193. return page_addr;
  194. }
  195. ++page_index;
  196. }
  197. return std::nullopt;
  198. }
  199. template <typename T>
  200. T MemoryManager::Read(GPUVAddr addr) const {
  201. if (auto page_pointer{GetPointer(addr)}; page_pointer) {
  202. // NOTE: Avoid adding any extra logic to this fast-path block
  203. T value;
  204. std::memcpy(&value, page_pointer, sizeof(T));
  205. return value;
  206. }
  207. ASSERT(false);
  208. return {};
  209. }
  210. template <typename T>
  211. void MemoryManager::Write(GPUVAddr addr, T data) {
  212. if (auto page_pointer{GetPointer(addr)}; page_pointer) {
  213. // NOTE: Avoid adding any extra logic to this fast-path block
  214. std::memcpy(page_pointer, &data, sizeof(T));
  215. return;
  216. }
  217. ASSERT(false);
  218. }
  219. template u8 MemoryManager::Read<u8>(GPUVAddr addr) const;
  220. template u16 MemoryManager::Read<u16>(GPUVAddr addr) const;
  221. template u32 MemoryManager::Read<u32>(GPUVAddr addr) const;
  222. template u64 MemoryManager::Read<u64>(GPUVAddr addr) const;
  223. template void MemoryManager::Write<u8>(GPUVAddr addr, u8 data);
  224. template void MemoryManager::Write<u16>(GPUVAddr addr, u16 data);
  225. template void MemoryManager::Write<u32>(GPUVAddr addr, u32 data);
  226. template void MemoryManager::Write<u64>(GPUVAddr addr, u64 data);
  227. u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) {
  228. const auto address{GpuToCpuAddress(gpu_addr)};
  229. if (!address) {
  230. return {};
  231. }
  232. return memory.GetPointer(*address);
  233. }
  234. const u8* MemoryManager::GetPointer(GPUVAddr gpu_addr) const {
  235. const auto address{GpuToCpuAddress(gpu_addr)};
  236. if (!address) {
  237. return {};
  238. }
  239. return memory.GetPointer(*address);
  240. }
  241. #ifdef _MSC_VER // no need for gcc / clang but msvc's compiler is more conservative with inlining.
  242. #pragma inline_recursion(on)
  243. #endif
  244. template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped>
  245. inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size,
  246. FuncMapped&& func_mapped, FuncReserved&& func_reserved,
  247. FuncUnmapped&& func_unmapped) const {
  248. static constexpr bool BOOL_BREAK_MAPPED = std::is_same_v<FuncMapped, bool>;
  249. static constexpr bool BOOL_BREAK_RESERVED = std::is_same_v<FuncReserved, bool>;
  250. static constexpr bool BOOL_BREAK_UNMAPPED = std::is_same_v<FuncUnmapped, bool>;
  251. u64 used_page_size;
  252. u64 used_page_mask;
  253. u64 used_page_bits;
  254. if constexpr (is_big_pages) {
  255. used_page_size = big_page_size;
  256. used_page_mask = big_page_mask;
  257. used_page_bits = big_page_bits;
  258. } else {
  259. used_page_size = page_size;
  260. used_page_mask = page_mask;
  261. used_page_bits = page_bits;
  262. }
  263. std::size_t remaining_size{size};
  264. std::size_t page_index{gpu_src_addr >> used_page_bits};
  265. std::size_t page_offset{gpu_src_addr & used_page_mask};
  266. GPUVAddr current_address = gpu_src_addr;
  267. while (remaining_size > 0) {
  268. const std::size_t copy_amount{
  269. std::min(static_cast<std::size_t>(used_page_size) - page_offset, remaining_size)};
  270. auto entry = GetEntry<is_big_pages>(current_address);
  271. if (entry == EntryType::Mapped) [[likely]] {
  272. if constexpr (BOOL_BREAK_MAPPED) {
  273. if (func_mapped(page_index, page_offset, copy_amount)) {
  274. return;
  275. }
  276. } else {
  277. func_mapped(page_index, page_offset, copy_amount);
  278. }
  279. } else if (entry == EntryType::Reserved) {
  280. if constexpr (BOOL_BREAK_RESERVED) {
  281. if (func_reserved(page_index, page_offset, copy_amount)) {
  282. return;
  283. }
  284. } else {
  285. func_reserved(page_index, page_offset, copy_amount);
  286. }
  287. } else [[unlikely]] {
  288. if constexpr (BOOL_BREAK_UNMAPPED) {
  289. if (func_unmapped(page_index, page_offset, copy_amount)) {
  290. return;
  291. }
  292. } else {
  293. func_unmapped(page_index, page_offset, copy_amount);
  294. }
  295. }
  296. page_index++;
  297. page_offset = 0;
  298. remaining_size -= copy_amount;
  299. current_address += copy_amount;
  300. }
  301. }
  302. template <bool is_safe>
  303. void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer,
  304. std::size_t size) const {
  305. auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index,
  306. [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
  307. std::memset(dest_buffer, 0, copy_amount);
  308. dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
  309. };
  310. auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  311. const VAddr cpu_addr_base =
  312. (static_cast<VAddr>(page_table[page_index]) << cpu_page_bits) + offset;
  313. if constexpr (is_safe) {
  314. rasterizer->FlushRegion(cpu_addr_base, copy_amount);
  315. }
  316. u8* physical = memory.GetPointer(cpu_addr_base);
  317. std::memcpy(dest_buffer, physical, copy_amount);
  318. dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
  319. };
  320. auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  321. const VAddr cpu_addr_base =
  322. (static_cast<VAddr>(big_page_table_cpu[page_index]) << cpu_page_bits) + offset;
  323. if constexpr (is_safe) {
  324. rasterizer->FlushRegion(cpu_addr_base, copy_amount);
  325. }
  326. if (!IsBigPageContinous(page_index)) [[unlikely]] {
  327. memory.ReadBlockUnsafe(cpu_addr_base, dest_buffer, copy_amount);
  328. } else {
  329. u8* physical = memory.GetPointer(cpu_addr_base);
  330. std::memcpy(dest_buffer, physical, copy_amount);
  331. }
  332. dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
  333. };
  334. auto read_short_pages = [&](std::size_t page_index, std::size_t offset,
  335. std::size_t copy_amount) {
  336. GPUVAddr base = (page_index << big_page_bits) + offset;
  337. MemoryOperation<false>(base, copy_amount, mapped_normal, set_to_zero, set_to_zero);
  338. };
  339. MemoryOperation<true>(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages);
  340. }
  341. void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size) const {
  342. ReadBlockImpl<true>(gpu_src_addr, dest_buffer, size);
  343. }
  344. void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer,
  345. const std::size_t size) const {
  346. ReadBlockImpl<false>(gpu_src_addr, dest_buffer, size);
  347. }
  348. template <bool is_safe>
  349. void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer,
  350. std::size_t size) {
  351. auto just_advance = [&]([[maybe_unused]] std::size_t page_index,
  352. [[maybe_unused]] std::size_t offset, std::size_t copy_amount) {
  353. src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
  354. };
  355. auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  356. const VAddr cpu_addr_base =
  357. (static_cast<VAddr>(page_table[page_index]) << cpu_page_bits) + offset;
  358. if constexpr (is_safe) {
  359. rasterizer->InvalidateRegion(cpu_addr_base, copy_amount);
  360. }
  361. u8* physical = memory.GetPointer(cpu_addr_base);
  362. std::memcpy(physical, src_buffer, copy_amount);
  363. src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
  364. };
  365. auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  366. const VAddr cpu_addr_base =
  367. (static_cast<VAddr>(big_page_table_cpu[page_index]) << cpu_page_bits) + offset;
  368. if constexpr (is_safe) {
  369. rasterizer->InvalidateRegion(cpu_addr_base, copy_amount);
  370. }
  371. if (!IsBigPageContinous(page_index)) [[unlikely]] {
  372. memory.WriteBlockUnsafe(cpu_addr_base, src_buffer, copy_amount);
  373. } else {
  374. u8* physical = memory.GetPointer(cpu_addr_base);
  375. std::memcpy(physical, src_buffer, copy_amount);
  376. }
  377. src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
  378. };
  379. auto write_short_pages = [&](std::size_t page_index, std::size_t offset,
  380. std::size_t copy_amount) {
  381. GPUVAddr base = (page_index << big_page_bits) + offset;
  382. MemoryOperation<false>(base, copy_amount, mapped_normal, just_advance, just_advance);
  383. };
  384. MemoryOperation<true>(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages);
  385. }
  386. void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size) {
  387. WriteBlockImpl<true>(gpu_dest_addr, src_buffer, size);
  388. }
  389. void MemoryManager::WriteBlockUnsafe(GPUVAddr gpu_dest_addr, const void* src_buffer,
  390. std::size_t size) {
  391. WriteBlockImpl<false>(gpu_dest_addr, src_buffer, size);
  392. }
  393. void MemoryManager::FlushRegion(GPUVAddr gpu_addr, size_t size) const {
  394. auto do_nothing = [&]([[maybe_unused]] std::size_t page_index,
  395. [[maybe_unused]] std::size_t offset,
  396. [[maybe_unused]] std::size_t copy_amount) {};
  397. auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  398. const VAddr cpu_addr_base =
  399. (static_cast<VAddr>(page_table[page_index]) << cpu_page_bits) + offset;
  400. rasterizer->FlushRegion(cpu_addr_base, copy_amount);
  401. };
  402. auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  403. const VAddr cpu_addr_base =
  404. (static_cast<VAddr>(big_page_table_cpu[page_index]) << cpu_page_bits) + offset;
  405. rasterizer->FlushRegion(cpu_addr_base, copy_amount);
  406. };
  407. auto flush_short_pages = [&](std::size_t page_index, std::size_t offset,
  408. std::size_t copy_amount) {
  409. GPUVAddr base = (page_index << big_page_bits) + offset;
  410. MemoryOperation<false>(base, copy_amount, mapped_normal, do_nothing, do_nothing);
  411. };
  412. MemoryOperation<true>(gpu_addr, size, mapped_big, do_nothing, flush_short_pages);
  413. }
  414. void MemoryManager::CopyBlock(GPUVAddr gpu_dest_addr, GPUVAddr gpu_src_addr, std::size_t size) {
  415. std::vector<u8> tmp_buffer(size);
  416. ReadBlock(gpu_src_addr, tmp_buffer.data(), size);
  417. // The output block must be flushed in case it has data modified from the GPU.
  418. // Fixes NPC geometry in Zombie Panic in Wonderland DX
  419. FlushRegion(gpu_dest_addr, size);
  420. WriteBlock(gpu_dest_addr, tmp_buffer.data(), size);
  421. }
  422. bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const {
  423. if (GetEntry<true>(gpu_addr) == EntryType::Mapped) [[likely]] {
  424. size_t page_index = gpu_addr >> big_page_bits;
  425. if (IsBigPageContinous(page_index)) [[likely]] {
  426. const std::size_t page{(page_index & big_page_mask) + size};
  427. return page <= big_page_size;
  428. }
  429. const std::size_t page{(gpu_addr & Core::Memory::YUZU_PAGEMASK) + size};
  430. return page <= Core::Memory::YUZU_PAGESIZE;
  431. }
  432. if (GetEntry<false>(gpu_addr) != EntryType::Mapped) {
  433. return false;
  434. }
  435. const std::size_t page{(gpu_addr & Core::Memory::YUZU_PAGEMASK) + size};
  436. return page <= Core::Memory::YUZU_PAGESIZE;
  437. }
  438. bool MemoryManager::IsContinousRange(GPUVAddr gpu_addr, std::size_t size) const {
  439. std::optional<VAddr> old_page_addr{};
  440. bool result{true};
  441. auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
  442. std::size_t copy_amount) {
  443. result = false;
  444. return true;
  445. };
  446. auto short_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  447. const VAddr cpu_addr_base =
  448. (static_cast<VAddr>(page_table[page_index]) << cpu_page_bits) + offset;
  449. if (old_page_addr && *old_page_addr != cpu_addr_base) {
  450. result = false;
  451. return true;
  452. }
  453. old_page_addr = {cpu_addr_base + copy_amount};
  454. return false;
  455. };
  456. auto big_check = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  457. const VAddr cpu_addr_base =
  458. (static_cast<VAddr>(big_page_table_cpu[page_index]) << cpu_page_bits) + offset;
  459. if (old_page_addr && *old_page_addr != cpu_addr_base) {
  460. result = false;
  461. return true;
  462. }
  463. old_page_addr = {cpu_addr_base + copy_amount};
  464. return false;
  465. };
  466. auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
  467. std::size_t copy_amount) {
  468. GPUVAddr base = (page_index << big_page_bits) + offset;
  469. MemoryOperation<false>(base, copy_amount, short_check, fail, fail);
  470. return !result;
  471. };
  472. MemoryOperation<true>(gpu_addr, size, big_check, fail, check_short_pages);
  473. return result;
  474. }
  475. bool MemoryManager::IsFullyMappedRange(GPUVAddr gpu_addr, std::size_t size) const {
  476. std::optional<VAddr> old_page_addr{};
  477. bool result{true};
  478. auto fail = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
  479. [[maybe_unused]] std::size_t copy_amount) {
  480. result = false;
  481. return true;
  482. };
  483. auto pass = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset,
  484. [[maybe_unused]] std::size_t copy_amount) { return false; };
  485. auto check_short_pages = [&](std::size_t page_index, std::size_t offset,
  486. std::size_t copy_amount) {
  487. GPUVAddr base = (page_index << big_page_bits) + offset;
  488. MemoryOperation<false>(base, copy_amount, pass, pass, fail);
  489. return !result;
  490. };
  491. MemoryOperation<true>(gpu_addr, size, pass, fail, check_short_pages);
  492. return result;
  493. }
  494. std::vector<std::pair<GPUVAddr, std::size_t>> MemoryManager::GetSubmappedRange(
  495. GPUVAddr gpu_addr, std::size_t size) const {
  496. std::vector<std::pair<GPUVAddr, std::size_t>> result{};
  497. std::optional<std::pair<GPUVAddr, std::size_t>> last_segment{};
  498. std::optional<VAddr> old_page_addr{};
  499. const auto split = [&last_segment, &result]([[maybe_unused]] std::size_t page_index,
  500. [[maybe_unused]] std::size_t offset,
  501. [[maybe_unused]] std::size_t copy_amount) {
  502. if (last_segment) {
  503. result.push_back(*last_segment);
  504. last_segment = std::nullopt;
  505. }
  506. };
  507. const auto extend_size_big = [this, &split, &old_page_addr,
  508. &last_segment](std::size_t page_index, std::size_t offset,
  509. std::size_t copy_amount) {
  510. const VAddr cpu_addr_base =
  511. (static_cast<VAddr>(big_page_table_cpu[page_index]) << cpu_page_bits) + offset;
  512. if (old_page_addr) {
  513. if (*old_page_addr != cpu_addr_base) {
  514. split(0, 0, 0);
  515. }
  516. }
  517. old_page_addr = {cpu_addr_base + copy_amount};
  518. if (!last_segment) {
  519. const GPUVAddr new_base_addr = (page_index << big_page_bits) + offset;
  520. last_segment = {new_base_addr, copy_amount};
  521. } else {
  522. last_segment->second += copy_amount;
  523. }
  524. };
  525. const auto extend_size_short = [this, &split, &old_page_addr,
  526. &last_segment](std::size_t page_index, std::size_t offset,
  527. std::size_t copy_amount) {
  528. const VAddr cpu_addr_base =
  529. (static_cast<VAddr>(page_table[page_index]) << cpu_page_bits) + offset;
  530. if (old_page_addr) {
  531. if (*old_page_addr != cpu_addr_base) {
  532. split(0, 0, 0);
  533. }
  534. }
  535. old_page_addr = {cpu_addr_base + copy_amount};
  536. if (!last_segment) {
  537. const GPUVAddr new_base_addr = (page_index << page_bits) + offset;
  538. last_segment = {new_base_addr, copy_amount};
  539. } else {
  540. last_segment->second += copy_amount;
  541. }
  542. };
  543. auto do_short_pages = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) {
  544. GPUVAddr base = (page_index << big_page_bits) + offset;
  545. MemoryOperation<false>(base, copy_amount, extend_size_short, split, split);
  546. };
  547. MemoryOperation<true>(gpu_addr, size, extend_size_big, split, do_short_pages);
  548. split(0, 0, 0);
  549. return result;
  550. }
  551. } // namespace Tegra