k_page_table.cpp 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353
  1. // Copyright 2020 yuzu Emulator Project
  2. // Licensed under GPLv2 or any later version
  3. // Refer to the license.txt file included.
  4. #include "common/alignment.h"
  5. #include "common/assert.h"
  6. #include "common/literals.h"
  7. #include "common/scope_exit.h"
  8. #include "core/core.h"
  9. #include "core/hle/kernel/k_address_space_info.h"
  10. #include "core/hle/kernel/k_memory_block.h"
  11. #include "core/hle/kernel/k_memory_block_manager.h"
  12. #include "core/hle/kernel/k_page_linked_list.h"
  13. #include "core/hle/kernel/k_page_table.h"
  14. #include "core/hle/kernel/k_process.h"
  15. #include "core/hle/kernel/k_resource_limit.h"
  16. #include "core/hle/kernel/k_scoped_resource_reservation.h"
  17. #include "core/hle/kernel/k_system_control.h"
  18. #include "core/hle/kernel/kernel.h"
  19. #include "core/hle/kernel/svc_results.h"
  20. #include "core/memory.h"
  21. namespace Kernel {
  22. namespace {
  23. using namespace Common::Literals;
  24. constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) {
  25. switch (as_type) {
  26. case FileSys::ProgramAddressSpaceType::Is32Bit:
  27. case FileSys::ProgramAddressSpaceType::Is32BitNoMap:
  28. return 32;
  29. case FileSys::ProgramAddressSpaceType::Is36Bit:
  30. return 36;
  31. case FileSys::ProgramAddressSpaceType::Is39Bit:
  32. return 39;
  33. default:
  34. UNREACHABLE();
  35. return {};
  36. }
  37. }
  38. constexpr u64 GetAddressInRange(const KMemoryInfo& info, VAddr addr) {
  39. if (info.GetAddress() < addr) {
  40. return addr;
  41. }
  42. return info.GetAddress();
  43. }
  44. constexpr std::size_t GetSizeInRange(const KMemoryInfo& info, VAddr start, VAddr end) {
  45. std::size_t size{info.GetSize()};
  46. if (info.GetAddress() < start) {
  47. size -= start - info.GetAddress();
  48. }
  49. if (info.GetEndAddress() > end) {
  50. size -= info.GetEndAddress() - end;
  51. }
  52. return size;
  53. }
  54. } // namespace
  55. KPageTable::KPageTable(Core::System& system_) : system{system_} {}
  56. ResultCode KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type,
  57. bool enable_aslr, VAddr code_addr,
  58. std::size_t code_size, KMemoryManager::Pool pool) {
  59. const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) {
  60. return KAddressSpaceInfo::GetAddressSpaceStart(address_space_width, type);
  61. };
  62. const auto GetSpaceSize = [this](KAddressSpaceInfo::Type type) {
  63. return KAddressSpaceInfo::GetAddressSpaceSize(address_space_width, type);
  64. };
  65. // Set our width and heap/alias sizes
  66. address_space_width = GetAddressSpaceWidthFromType(as_type);
  67. const VAddr start = 0;
  68. const VAddr end{1ULL << address_space_width};
  69. std::size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)};
  70. std::size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)};
  71. ASSERT(start <= code_addr);
  72. ASSERT(code_addr < code_addr + code_size);
  73. ASSERT(code_addr + code_size - 1 <= end - 1);
  74. // Adjust heap/alias size if we don't have an alias region
  75. if (as_type == FileSys::ProgramAddressSpaceType::Is32BitNoMap) {
  76. heap_region_size += alias_region_size;
  77. alias_region_size = 0;
  78. }
  79. // Set code regions and determine remaining
  80. constexpr std::size_t RegionAlignment{2_MiB};
  81. VAddr process_code_start{};
  82. VAddr process_code_end{};
  83. std::size_t stack_region_size{};
  84. std::size_t kernel_map_region_size{};
  85. if (address_space_width == 39) {
  86. alias_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Alias);
  87. heap_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Heap);
  88. stack_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Stack);
  89. kernel_map_region_size = GetSpaceSize(KAddressSpaceInfo::Type::MapSmall);
  90. code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::Map39Bit);
  91. code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit);
  92. alias_code_region_start = code_region_start;
  93. alias_code_region_end = code_region_end;
  94. process_code_start = Common::AlignDown(code_addr, RegionAlignment);
  95. process_code_end = Common::AlignUp(code_addr + code_size, RegionAlignment);
  96. } else {
  97. stack_region_size = 0;
  98. kernel_map_region_size = 0;
  99. code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::MapSmall);
  100. code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::MapSmall);
  101. stack_region_start = code_region_start;
  102. alias_code_region_start = code_region_start;
  103. alias_code_region_end = GetSpaceStart(KAddressSpaceInfo::Type::MapLarge) +
  104. GetSpaceSize(KAddressSpaceInfo::Type::MapLarge);
  105. stack_region_end = code_region_end;
  106. kernel_map_region_start = code_region_start;
  107. kernel_map_region_end = code_region_end;
  108. process_code_start = code_region_start;
  109. process_code_end = code_region_end;
  110. }
  111. // Set other basic fields
  112. is_aslr_enabled = enable_aslr;
  113. address_space_start = start;
  114. address_space_end = end;
  115. is_kernel = false;
  116. // Determine the region we can place our undetermineds in
  117. VAddr alloc_start{};
  118. std::size_t alloc_size{};
  119. if ((process_code_start - code_region_start) >= (end - process_code_end)) {
  120. alloc_start = code_region_start;
  121. alloc_size = process_code_start - code_region_start;
  122. } else {
  123. alloc_start = process_code_end;
  124. alloc_size = end - process_code_end;
  125. }
  126. const std::size_t needed_size{
  127. (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)};
  128. if (alloc_size < needed_size) {
  129. UNREACHABLE();
  130. return ResultOutOfMemory;
  131. }
  132. const std::size_t remaining_size{alloc_size - needed_size};
  133. // Determine random placements for each region
  134. std::size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{};
  135. if (enable_aslr) {
  136. alias_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) *
  137. RegionAlignment;
  138. heap_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) *
  139. RegionAlignment;
  140. stack_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) *
  141. RegionAlignment;
  142. kmap_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) *
  143. RegionAlignment;
  144. }
  145. // Setup heap and alias regions
  146. alias_region_start = alloc_start + alias_rnd;
  147. alias_region_end = alias_region_start + alias_region_size;
  148. heap_region_start = alloc_start + heap_rnd;
  149. heap_region_end = heap_region_start + heap_region_size;
  150. if (alias_rnd <= heap_rnd) {
  151. heap_region_start += alias_region_size;
  152. heap_region_end += alias_region_size;
  153. } else {
  154. alias_region_start += heap_region_size;
  155. alias_region_end += heap_region_size;
  156. }
  157. // Setup stack region
  158. if (stack_region_size) {
  159. stack_region_start = alloc_start + stack_rnd;
  160. stack_region_end = stack_region_start + stack_region_size;
  161. if (alias_rnd < stack_rnd) {
  162. stack_region_start += alias_region_size;
  163. stack_region_end += alias_region_size;
  164. } else {
  165. alias_region_start += stack_region_size;
  166. alias_region_end += stack_region_size;
  167. }
  168. if (heap_rnd < stack_rnd) {
  169. stack_region_start += heap_region_size;
  170. stack_region_end += heap_region_size;
  171. } else {
  172. heap_region_start += stack_region_size;
  173. heap_region_end += stack_region_size;
  174. }
  175. }
  176. // Setup kernel map region
  177. if (kernel_map_region_size) {
  178. kernel_map_region_start = alloc_start + kmap_rnd;
  179. kernel_map_region_end = kernel_map_region_start + kernel_map_region_size;
  180. if (alias_rnd < kmap_rnd) {
  181. kernel_map_region_start += alias_region_size;
  182. kernel_map_region_end += alias_region_size;
  183. } else {
  184. alias_region_start += kernel_map_region_size;
  185. alias_region_end += kernel_map_region_size;
  186. }
  187. if (heap_rnd < kmap_rnd) {
  188. kernel_map_region_start += heap_region_size;
  189. kernel_map_region_end += heap_region_size;
  190. } else {
  191. heap_region_start += kernel_map_region_size;
  192. heap_region_end += kernel_map_region_size;
  193. }
  194. if (stack_region_size) {
  195. if (stack_rnd < kmap_rnd) {
  196. kernel_map_region_start += stack_region_size;
  197. kernel_map_region_end += stack_region_size;
  198. } else {
  199. stack_region_start += kernel_map_region_size;
  200. stack_region_end += kernel_map_region_size;
  201. }
  202. }
  203. }
  204. // Set heap members
  205. current_heap_end = heap_region_start;
  206. max_heap_size = 0;
  207. max_physical_memory_size = 0;
  208. // Ensure that we regions inside our address space
  209. auto IsInAddressSpace = [&](VAddr addr) {
  210. return address_space_start <= addr && addr <= address_space_end;
  211. };
  212. ASSERT(IsInAddressSpace(alias_region_start));
  213. ASSERT(IsInAddressSpace(alias_region_end));
  214. ASSERT(IsInAddressSpace(heap_region_start));
  215. ASSERT(IsInAddressSpace(heap_region_end));
  216. ASSERT(IsInAddressSpace(stack_region_start));
  217. ASSERT(IsInAddressSpace(stack_region_end));
  218. ASSERT(IsInAddressSpace(kernel_map_region_start));
  219. ASSERT(IsInAddressSpace(kernel_map_region_end));
  220. // Ensure that we selected regions that don't overlap
  221. const VAddr alias_start{alias_region_start};
  222. const VAddr alias_last{alias_region_end - 1};
  223. const VAddr heap_start{heap_region_start};
  224. const VAddr heap_last{heap_region_end - 1};
  225. const VAddr stack_start{stack_region_start};
  226. const VAddr stack_last{stack_region_end - 1};
  227. const VAddr kmap_start{kernel_map_region_start};
  228. const VAddr kmap_last{kernel_map_region_end - 1};
  229. ASSERT(alias_last < heap_start || heap_last < alias_start);
  230. ASSERT(alias_last < stack_start || stack_last < alias_start);
  231. ASSERT(alias_last < kmap_start || kmap_last < alias_start);
  232. ASSERT(heap_last < stack_start || stack_last < heap_start);
  233. ASSERT(heap_last < kmap_start || kmap_last < heap_start);
  234. current_heap_addr = heap_region_start;
  235. heap_capacity = 0;
  236. physical_memory_usage = 0;
  237. memory_pool = pool;
  238. page_table_impl.Resize(address_space_width, PageBits);
  239. return InitializeMemoryLayout(start, end);
  240. }
  241. ResultCode KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state,
  242. KMemoryPermission perm) {
  243. std::lock_guard lock{page_table_lock};
  244. const u64 size{num_pages * PageSize};
  245. if (!CanContain(addr, size, state)) {
  246. return ResultInvalidCurrentMemory;
  247. }
  248. if (IsRegionMapped(addr, size)) {
  249. return ResultInvalidCurrentMemory;
  250. }
  251. KPageLinkedList page_linked_list;
  252. CASCADE_CODE(
  253. system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool));
  254. CASCADE_CODE(Operate(addr, num_pages, page_linked_list, OperationType::MapGroup));
  255. block_manager->Update(addr, num_pages, state, perm);
  256. return ResultSuccess;
  257. }
  258. ResultCode KPageTable::MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) {
  259. std::lock_guard lock{page_table_lock};
  260. const std::size_t num_pages{size / PageSize};
  261. KMemoryState state{};
  262. KMemoryPermission perm{};
  263. CASCADE_CODE(CheckMemoryState(&state, &perm, nullptr, src_addr, size, KMemoryState::All,
  264. KMemoryState::Normal, KMemoryPermission::Mask,
  265. KMemoryPermission::ReadAndWrite, KMemoryAttribute::Mask,
  266. KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped));
  267. if (IsRegionMapped(dst_addr, size)) {
  268. return ResultInvalidCurrentMemory;
  269. }
  270. KPageLinkedList page_linked_list;
  271. AddRegionToPages(src_addr, num_pages, page_linked_list);
  272. {
  273. auto block_guard = detail::ScopeExit(
  274. [&] { Operate(src_addr, num_pages, perm, OperationType::ChangePermissions); });
  275. CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None,
  276. OperationType::ChangePermissions));
  277. CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::None));
  278. block_guard.Cancel();
  279. }
  280. block_manager->Update(src_addr, num_pages, state, KMemoryPermission::None,
  281. KMemoryAttribute::Locked);
  282. block_manager->Update(dst_addr, num_pages, KMemoryState::AliasCode);
  283. return ResultSuccess;
  284. }
  285. ResultCode KPageTable::UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) {
  286. std::lock_guard lock{page_table_lock};
  287. if (!size) {
  288. return ResultSuccess;
  289. }
  290. const std::size_t num_pages{size / PageSize};
  291. CASCADE_CODE(CheckMemoryState(nullptr, nullptr, nullptr, src_addr, size, KMemoryState::All,
  292. KMemoryState::Normal, KMemoryPermission::None,
  293. KMemoryPermission::None, KMemoryAttribute::Mask,
  294. KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped));
  295. KMemoryState state{};
  296. CASCADE_CODE(CheckMemoryState(
  297. &state, nullptr, nullptr, dst_addr, PageSize, KMemoryState::FlagCanCodeAlias,
  298. KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None,
  299. KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped));
  300. CASCADE_CODE(CheckMemoryState(dst_addr, size, KMemoryState::All, state, KMemoryPermission::None,
  301. KMemoryPermission::None, KMemoryAttribute::Mask,
  302. KMemoryAttribute::None));
  303. CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap));
  304. block_manager->Update(dst_addr, num_pages, KMemoryState::Free);
  305. block_manager->Update(src_addr, num_pages, KMemoryState::Normal,
  306. KMemoryPermission::ReadAndWrite);
  307. system.InvalidateCpuInstructionCacheRange(dst_addr, size);
  308. return ResultSuccess;
  309. }
  310. ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size,
  311. KPageTable& src_page_table, VAddr src_addr) {
  312. std::lock_guard lock{page_table_lock};
  313. const std::size_t num_pages{size / PageSize};
  314. // Check that the memory is mapped in the destination process.
  315. size_t num_allocator_blocks;
  316. R_TRY(CheckMemoryState(&num_allocator_blocks, dst_addr, size, KMemoryState::All,
  317. KMemoryState::SharedCode, KMemoryPermission::UserReadWrite,
  318. KMemoryPermission::UserReadWrite, KMemoryAttribute::All,
  319. KMemoryAttribute::None));
  320. // Check that the memory is mapped in the source process.
  321. R_TRY(src_page_table.CheckMemoryState(src_addr, size, KMemoryState::FlagCanMapProcess,
  322. KMemoryState::FlagCanMapProcess, KMemoryPermission::None,
  323. KMemoryPermission::None, KMemoryAttribute::All,
  324. KMemoryAttribute::None));
  325. CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap));
  326. // Apply the memory block update.
  327. block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None,
  328. KMemoryAttribute::None);
  329. return ResultSuccess;
  330. }
  331. void KPageTable::MapPhysicalMemory(KPageLinkedList& page_linked_list, VAddr start, VAddr end) {
  332. auto node{page_linked_list.Nodes().begin()};
  333. PAddr map_addr{node->GetAddress()};
  334. std::size_t src_num_pages{node->GetNumPages()};
  335. block_manager->IterateForRange(start, end, [&](const KMemoryInfo& info) {
  336. if (info.state != KMemoryState::Free) {
  337. return;
  338. }
  339. std::size_t dst_num_pages{GetSizeInRange(info, start, end) / PageSize};
  340. VAddr dst_addr{GetAddressInRange(info, start)};
  341. while (dst_num_pages) {
  342. if (!src_num_pages) {
  343. node = std::next(node);
  344. map_addr = node->GetAddress();
  345. src_num_pages = node->GetNumPages();
  346. }
  347. const std::size_t num_pages{std::min(src_num_pages, dst_num_pages)};
  348. Operate(dst_addr, num_pages, KMemoryPermission::ReadAndWrite, OperationType::Map,
  349. map_addr);
  350. dst_addr += num_pages * PageSize;
  351. map_addr += num_pages * PageSize;
  352. src_num_pages -= num_pages;
  353. dst_num_pages -= num_pages;
  354. }
  355. });
  356. }
  357. ResultCode KPageTable::MapPhysicalMemory(VAddr addr, std::size_t size) {
  358. std::lock_guard lock{page_table_lock};
  359. std::size_t mapped_size{};
  360. const VAddr end_addr{addr + size};
  361. block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) {
  362. if (info.state != KMemoryState::Free) {
  363. mapped_size += GetSizeInRange(info, addr, end_addr);
  364. }
  365. });
  366. if (mapped_size == size) {
  367. return ResultSuccess;
  368. }
  369. const std::size_t remaining_size{size - mapped_size};
  370. const std::size_t remaining_pages{remaining_size / PageSize};
  371. // Reserve the memory from the process resource limit.
  372. KScopedResourceReservation memory_reservation(
  373. system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory,
  374. remaining_size);
  375. if (!memory_reservation.Succeeded()) {
  376. LOG_ERROR(Kernel, "Could not reserve remaining {:X} bytes", remaining_size);
  377. return ResultLimitReached;
  378. }
  379. KPageLinkedList page_linked_list;
  380. CASCADE_CODE(
  381. system.Kernel().MemoryManager().Allocate(page_linked_list, remaining_pages, memory_pool));
  382. // We succeeded, so commit the memory reservation.
  383. memory_reservation.Commit();
  384. MapPhysicalMemory(page_linked_list, addr, end_addr);
  385. physical_memory_usage += remaining_size;
  386. const std::size_t num_pages{size / PageSize};
  387. block_manager->Update(addr, num_pages, KMemoryState::Free, KMemoryPermission::None,
  388. KMemoryAttribute::None, KMemoryState::Normal,
  389. KMemoryPermission::ReadAndWrite, KMemoryAttribute::None);
  390. return ResultSuccess;
  391. }
  392. ResultCode KPageTable::UnmapPhysicalMemory(VAddr addr, std::size_t size) {
  393. std::lock_guard lock{page_table_lock};
  394. const VAddr end_addr{addr + size};
  395. ResultCode result{ResultSuccess};
  396. std::size_t mapped_size{};
  397. // Verify that the region can be unmapped
  398. block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) {
  399. if (info.state == KMemoryState::Normal) {
  400. if (info.attribute != KMemoryAttribute::None) {
  401. result = ResultInvalidCurrentMemory;
  402. return;
  403. }
  404. mapped_size += GetSizeInRange(info, addr, end_addr);
  405. } else if (info.state != KMemoryState::Free) {
  406. result = ResultInvalidCurrentMemory;
  407. }
  408. });
  409. if (result.IsError()) {
  410. return result;
  411. }
  412. if (!mapped_size) {
  413. return ResultSuccess;
  414. }
  415. CASCADE_CODE(UnmapMemory(addr, size));
  416. auto process{system.Kernel().CurrentProcess()};
  417. process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size);
  418. physical_memory_usage -= mapped_size;
  419. return ResultSuccess;
  420. }
  421. ResultCode KPageTable::UnmapMemory(VAddr addr, std::size_t size) {
  422. std::lock_guard lock{page_table_lock};
  423. const VAddr end_addr{addr + size};
  424. ResultCode result{ResultSuccess};
  425. KPageLinkedList page_linked_list;
  426. // Unmap each region within the range
  427. block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) {
  428. if (info.state == KMemoryState::Normal) {
  429. const std::size_t block_size{GetSizeInRange(info, addr, end_addr)};
  430. const std::size_t block_num_pages{block_size / PageSize};
  431. const VAddr block_addr{GetAddressInRange(info, addr)};
  432. AddRegionToPages(block_addr, block_size / PageSize, page_linked_list);
  433. if (result = Operate(block_addr, block_num_pages, KMemoryPermission::None,
  434. OperationType::Unmap);
  435. result.IsError()) {
  436. return;
  437. }
  438. }
  439. });
  440. if (result.IsError()) {
  441. return result;
  442. }
  443. const std::size_t num_pages{size / PageSize};
  444. system.Kernel().MemoryManager().Free(page_linked_list, num_pages, memory_pool);
  445. block_manager->Update(addr, num_pages, KMemoryState::Free);
  446. return ResultSuccess;
  447. }
  448. ResultCode KPageTable::Map(VAddr dst_addr, VAddr src_addr, std::size_t size) {
  449. std::lock_guard lock{page_table_lock};
  450. KMemoryState src_state{};
  451. CASCADE_CODE(CheckMemoryState(
  452. &src_state, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias,
  453. KMemoryState::FlagCanAlias, KMemoryPermission::Mask, KMemoryPermission::ReadAndWrite,
  454. KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped));
  455. if (IsRegionMapped(dst_addr, size)) {
  456. return ResultInvalidCurrentMemory;
  457. }
  458. KPageLinkedList page_linked_list;
  459. const std::size_t num_pages{size / PageSize};
  460. AddRegionToPages(src_addr, num_pages, page_linked_list);
  461. {
  462. auto block_guard = detail::ScopeExit([&] {
  463. Operate(src_addr, num_pages, KMemoryPermission::ReadAndWrite,
  464. OperationType::ChangePermissions);
  465. });
  466. CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None,
  467. OperationType::ChangePermissions));
  468. CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::ReadAndWrite));
  469. block_guard.Cancel();
  470. }
  471. block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::None,
  472. KMemoryAttribute::Locked);
  473. block_manager->Update(dst_addr, num_pages, KMemoryState::Stack,
  474. KMemoryPermission::ReadAndWrite);
  475. return ResultSuccess;
  476. }
  477. ResultCode KPageTable::Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size) {
  478. std::lock_guard lock{page_table_lock};
  479. KMemoryState src_state{};
  480. CASCADE_CODE(CheckMemoryState(
  481. &src_state, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias,
  482. KMemoryState::FlagCanAlias, KMemoryPermission::Mask, KMemoryPermission::None,
  483. KMemoryAttribute::Mask, KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped));
  484. KMemoryPermission dst_perm{};
  485. CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, dst_addr, size, KMemoryState::All,
  486. KMemoryState::Stack, KMemoryPermission::None,
  487. KMemoryPermission::None, KMemoryAttribute::Mask,
  488. KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped));
  489. KPageLinkedList src_pages;
  490. KPageLinkedList dst_pages;
  491. const std::size_t num_pages{size / PageSize};
  492. AddRegionToPages(src_addr, num_pages, src_pages);
  493. AddRegionToPages(dst_addr, num_pages, dst_pages);
  494. if (!dst_pages.IsEqual(src_pages)) {
  495. return ResultInvalidMemoryRegion;
  496. }
  497. {
  498. auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); });
  499. CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap));
  500. CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::ReadAndWrite,
  501. OperationType::ChangePermissions));
  502. block_guard.Cancel();
  503. }
  504. block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::ReadAndWrite);
  505. block_manager->Update(dst_addr, num_pages, KMemoryState::Free);
  506. return ResultSuccess;
  507. }
  508. ResultCode KPageTable::MapPages(VAddr addr, const KPageLinkedList& page_linked_list,
  509. KMemoryPermission perm) {
  510. VAddr cur_addr{addr};
  511. for (const auto& node : page_linked_list.Nodes()) {
  512. if (const auto result{
  513. Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())};
  514. result.IsError()) {
  515. const std::size_t num_pages{(addr - cur_addr) / PageSize};
  516. ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap)
  517. .IsSuccess());
  518. return result;
  519. }
  520. cur_addr += node.GetNumPages() * PageSize;
  521. }
  522. return ResultSuccess;
  523. }
  524. ResultCode KPageTable::MapPages(VAddr addr, KPageLinkedList& page_linked_list, KMemoryState state,
  525. KMemoryPermission perm) {
  526. std::lock_guard lock{page_table_lock};
  527. const std::size_t num_pages{page_linked_list.GetNumPages()};
  528. const std::size_t size{num_pages * PageSize};
  529. if (!CanContain(addr, size, state)) {
  530. return ResultInvalidCurrentMemory;
  531. }
  532. if (IsRegionMapped(addr, num_pages * PageSize)) {
  533. return ResultInvalidCurrentMemory;
  534. }
  535. CASCADE_CODE(MapPages(addr, page_linked_list, perm));
  536. block_manager->Update(addr, num_pages, state, perm);
  537. return ResultSuccess;
  538. }
  539. ResultCode KPageTable::UnmapPages(VAddr addr, const KPageLinkedList& page_linked_list) {
  540. VAddr cur_addr{addr};
  541. for (const auto& node : page_linked_list.Nodes()) {
  542. const std::size_t num_pages{(addr - cur_addr) / PageSize};
  543. if (const auto result{
  544. Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap)};
  545. result.IsError()) {
  546. return result;
  547. }
  548. cur_addr += node.GetNumPages() * PageSize;
  549. }
  550. return ResultSuccess;
  551. }
  552. ResultCode KPageTable::UnmapPages(VAddr addr, KPageLinkedList& page_linked_list,
  553. KMemoryState state) {
  554. std::lock_guard lock{page_table_lock};
  555. const std::size_t num_pages{page_linked_list.GetNumPages()};
  556. const std::size_t size{num_pages * PageSize};
  557. if (!CanContain(addr, size, state)) {
  558. return ResultInvalidCurrentMemory;
  559. }
  560. if (IsRegionMapped(addr, num_pages * PageSize)) {
  561. return ResultInvalidCurrentMemory;
  562. }
  563. CASCADE_CODE(UnmapPages(addr, page_linked_list));
  564. block_manager->Update(addr, num_pages, state, KMemoryPermission::None);
  565. return ResultSuccess;
  566. }
  567. ResultCode KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size,
  568. KMemoryPermission perm) {
  569. std::lock_guard lock{page_table_lock};
  570. KMemoryState prev_state{};
  571. KMemoryPermission prev_perm{};
  572. CASCADE_CODE(CheckMemoryState(
  573. &prev_state, &prev_perm, nullptr, addr, size, KMemoryState::FlagCode,
  574. KMemoryState::FlagCode, KMemoryPermission::None, KMemoryPermission::None,
  575. KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped));
  576. KMemoryState state{prev_state};
  577. // Ensure state is mutable if permission allows write
  578. if ((perm & KMemoryPermission::Write) != KMemoryPermission::None) {
  579. if (prev_state == KMemoryState::Code) {
  580. state = KMemoryState::CodeData;
  581. } else if (prev_state == KMemoryState::AliasCode) {
  582. state = KMemoryState::AliasCodeData;
  583. } else {
  584. UNREACHABLE();
  585. }
  586. }
  587. // Return early if there is nothing to change
  588. if (state == prev_state && perm == prev_perm) {
  589. return ResultSuccess;
  590. }
  591. if ((prev_perm & KMemoryPermission::Execute) != (perm & KMemoryPermission::Execute)) {
  592. // Memory execution state is changing, invalidate CPU cache range
  593. system.InvalidateCpuInstructionCacheRange(addr, size);
  594. }
  595. const std::size_t num_pages{size / PageSize};
  596. const OperationType operation{(perm & KMemoryPermission::Execute) != KMemoryPermission::None
  597. ? OperationType::ChangePermissionsAndRefresh
  598. : OperationType::ChangePermissions};
  599. CASCADE_CODE(Operate(addr, num_pages, perm, operation));
  600. block_manager->Update(addr, num_pages, state, perm);
  601. return ResultSuccess;
  602. }
  603. KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) {
  604. std::lock_guard lock{page_table_lock};
  605. return block_manager->FindBlock(addr).GetMemoryInfo();
  606. }
  607. KMemoryInfo KPageTable::QueryInfo(VAddr addr) {
  608. if (!Contains(addr, 1)) {
  609. return {address_space_end, 0 - address_space_end, KMemoryState::Inaccessible,
  610. KMemoryPermission::None, KMemoryAttribute::None, KMemoryPermission::None};
  611. }
  612. return QueryInfoImpl(addr);
  613. }
  614. ResultCode KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) {
  615. std::lock_guard lock{page_table_lock};
  616. KMemoryState state{};
  617. KMemoryAttribute attribute{};
  618. CASCADE_CODE(CheckMemoryState(
  619. &state, nullptr, &attribute, addr, size,
  620. KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted,
  621. KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, KMemoryPermission::Mask,
  622. KMemoryPermission::ReadAndWrite, KMemoryAttribute::Mask, KMemoryAttribute::None,
  623. KMemoryAttribute::IpcAndDeviceMapped));
  624. block_manager->Update(addr, size / PageSize, state, perm, attribute | KMemoryAttribute::Locked);
  625. return ResultSuccess;
  626. }
  627. ResultCode KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) {
  628. std::lock_guard lock{page_table_lock};
  629. KMemoryState state{};
  630. CASCADE_CODE(
  631. CheckMemoryState(&state, nullptr, nullptr, addr, size,
  632. KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted,
  633. KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted,
  634. KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::Mask,
  635. KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped));
  636. block_manager->Update(addr, size / PageSize, state, KMemoryPermission::ReadAndWrite);
  637. return ResultSuccess;
  638. }
  639. ResultCode KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, KMemoryAttribute mask,
  640. KMemoryAttribute value) {
  641. std::lock_guard lock{page_table_lock};
  642. KMemoryState state{};
  643. KMemoryPermission perm{};
  644. KMemoryAttribute attribute{};
  645. CASCADE_CODE(CheckMemoryState(
  646. &state, &perm, &attribute, addr, size, KMemoryState::FlagCanChangeAttribute,
  647. KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None,
  648. KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None,
  649. KMemoryAttribute::DeviceSharedAndUncached));
  650. attribute = attribute & ~mask;
  651. attribute = attribute | (mask & value);
  652. block_manager->Update(addr, size / PageSize, state, perm, attribute);
  653. return ResultSuccess;
  654. }
  655. ResultCode KPageTable::SetHeapCapacity(std::size_t new_heap_capacity) {
  656. std::lock_guard lock{page_table_lock};
  657. heap_capacity = new_heap_capacity;
  658. return ResultSuccess;
  659. }
  660. ResultVal<VAddr> KPageTable::SetHeapSize(std::size_t size) {
  661. if (size > heap_region_end - heap_region_start) {
  662. return ResultOutOfMemory;
  663. }
  664. const u64 previous_heap_size{GetHeapSize()};
  665. UNIMPLEMENTED_IF_MSG(previous_heap_size > size, "Heap shrink is unimplemented");
  666. // Increase the heap size
  667. {
  668. std::lock_guard lock{page_table_lock};
  669. const u64 delta{size - previous_heap_size};
  670. // Reserve memory for the heap extension.
  671. KScopedResourceReservation memory_reservation(
  672. system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory,
  673. delta);
  674. if (!memory_reservation.Succeeded()) {
  675. LOG_ERROR(Kernel, "Could not reserve heap extension of size {:X} bytes", delta);
  676. return ResultLimitReached;
  677. }
  678. KPageLinkedList page_linked_list;
  679. const std::size_t num_pages{delta / PageSize};
  680. CASCADE_CODE(
  681. system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool));
  682. if (IsRegionMapped(current_heap_addr, delta)) {
  683. return ResultInvalidCurrentMemory;
  684. }
  685. CASCADE_CODE(
  686. Operate(current_heap_addr, num_pages, page_linked_list, OperationType::MapGroup));
  687. // Succeeded in allocation, commit the resource reservation
  688. memory_reservation.Commit();
  689. block_manager->Update(current_heap_addr, num_pages, KMemoryState::Normal,
  690. KMemoryPermission::ReadAndWrite);
  691. current_heap_addr = heap_region_start + size;
  692. }
  693. return heap_region_start;
  694. }
  695. ResultVal<VAddr> KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align,
  696. bool is_map_only, VAddr region_start,
  697. std::size_t region_num_pages, KMemoryState state,
  698. KMemoryPermission perm, PAddr map_addr) {
  699. std::lock_guard lock{page_table_lock};
  700. if (!CanContain(region_start, region_num_pages * PageSize, state)) {
  701. return ResultInvalidCurrentMemory;
  702. }
  703. if (region_num_pages <= needed_num_pages) {
  704. return ResultOutOfMemory;
  705. }
  706. const VAddr addr{
  707. AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)};
  708. if (!addr) {
  709. return ResultOutOfMemory;
  710. }
  711. if (is_map_only) {
  712. CASCADE_CODE(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr));
  713. } else {
  714. KPageLinkedList page_group;
  715. CASCADE_CODE(
  716. system.Kernel().MemoryManager().Allocate(page_group, needed_num_pages, memory_pool));
  717. CASCADE_CODE(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup));
  718. }
  719. block_manager->Update(addr, needed_num_pages, state, perm);
  720. return addr;
  721. }
  722. ResultCode KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) {
  723. std::lock_guard lock{page_table_lock};
  724. KMemoryPermission perm{};
  725. if (const ResultCode result{CheckMemoryState(
  726. nullptr, &perm, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute,
  727. KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None,
  728. KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None,
  729. KMemoryAttribute::DeviceSharedAndUncached)};
  730. result.IsError()) {
  731. return result;
  732. }
  733. block_manager->UpdateLock(
  734. addr, size / PageSize,
  735. [](KMemoryBlockManager::iterator block, KMemoryPermission permission) {
  736. block->ShareToDevice(permission);
  737. },
  738. perm);
  739. return ResultSuccess;
  740. }
  741. ResultCode KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) {
  742. std::lock_guard lock{page_table_lock};
  743. KMemoryPermission perm{};
  744. if (const ResultCode result{CheckMemoryState(
  745. nullptr, &perm, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute,
  746. KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None,
  747. KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None,
  748. KMemoryAttribute::DeviceSharedAndUncached)};
  749. result.IsError()) {
  750. return result;
  751. }
  752. block_manager->UpdateLock(
  753. addr, size / PageSize,
  754. [](KMemoryBlockManager::iterator block, KMemoryPermission permission) {
  755. block->UnshareToDevice(permission);
  756. },
  757. perm);
  758. return ResultSuccess;
  759. }
  760. ResultCode KPageTable::LockForCodeMemory(VAddr addr, std::size_t size) {
  761. std::lock_guard lock{page_table_lock};
  762. KMemoryPermission new_perm = KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite;
  763. KMemoryPermission old_perm{};
  764. if (const ResultCode result{CheckMemoryState(
  765. nullptr, &old_perm, nullptr, addr, size, KMemoryState::FlagCanCodeMemory,
  766. KMemoryState::FlagCanCodeMemory, KMemoryPermission::Mask,
  767. KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)};
  768. result.IsError()) {
  769. return result;
  770. }
  771. new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm;
  772. block_manager->UpdateLock(
  773. addr, size / PageSize,
  774. [](KMemoryBlockManager::iterator block, KMemoryPermission permission) {
  775. block->ShareToDevice(permission);
  776. },
  777. new_perm);
  778. return ResultSuccess;
  779. }
  780. ResultCode KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size) {
  781. std::lock_guard lock{page_table_lock};
  782. KMemoryPermission new_perm = KMemoryPermission::UserReadWrite;
  783. KMemoryPermission old_perm{};
  784. if (const ResultCode result{CheckMemoryState(
  785. nullptr, &old_perm, nullptr, addr, size, KMemoryState::FlagCanCodeMemory,
  786. KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None,
  787. KMemoryAttribute::All, KMemoryAttribute::Locked)};
  788. result.IsError()) {
  789. return result;
  790. }
  791. new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm;
  792. block_manager->UpdateLock(
  793. addr, size / PageSize,
  794. [](KMemoryBlockManager::iterator block, KMemoryPermission permission) {
  795. block->UnshareToDevice(permission);
  796. },
  797. new_perm);
  798. return ResultSuccess;
  799. }
  800. ResultCode KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) {
  801. block_manager = std::make_unique<KMemoryBlockManager>(start, end);
  802. return ResultSuccess;
  803. }
  804. bool KPageTable::IsRegionMapped(VAddr address, u64 size) {
  805. return CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free,
  806. KMemoryPermission::Mask, KMemoryPermission::None,
  807. KMemoryAttribute::Mask, KMemoryAttribute::None,
  808. KMemoryAttribute::IpcAndDeviceMapped)
  809. .IsError();
  810. }
  811. bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const {
  812. auto start_ptr = system.Memory().GetPointer(addr);
  813. for (u64 offset{}; offset < size; offset += PageSize) {
  814. if (start_ptr != system.Memory().GetPointer(addr + offset)) {
  815. return false;
  816. }
  817. start_ptr += PageSize;
  818. }
  819. return true;
  820. }
  821. void KPageTable::AddRegionToPages(VAddr start, std::size_t num_pages,
  822. KPageLinkedList& page_linked_list) {
  823. VAddr addr{start};
  824. while (addr < start + (num_pages * PageSize)) {
  825. const PAddr paddr{GetPhysicalAddr(addr)};
  826. if (!paddr) {
  827. UNREACHABLE();
  828. }
  829. page_linked_list.AddBlock(paddr, 1);
  830. addr += PageSize;
  831. }
  832. }
  833. VAddr KPageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_pages,
  834. u64 needed_num_pages, std::size_t align) {
  835. if (is_aslr_enabled) {
  836. UNIMPLEMENTED();
  837. }
  838. return block_manager->FindFreeArea(start, region_num_pages, needed_num_pages, align, 0,
  839. IsKernel() ? 1 : 4);
  840. }
  841. ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageLinkedList& page_group,
  842. OperationType operation) {
  843. std::lock_guard lock{page_table_lock};
  844. ASSERT(Common::IsAligned(addr, PageSize));
  845. ASSERT(num_pages > 0);
  846. ASSERT(num_pages == page_group.GetNumPages());
  847. for (const auto& node : page_group.Nodes()) {
  848. const std::size_t size{node.GetNumPages() * PageSize};
  849. switch (operation) {
  850. case OperationType::MapGroup:
  851. system.Memory().MapMemoryRegion(page_table_impl, addr, size, node.GetAddress());
  852. break;
  853. default:
  854. UNREACHABLE();
  855. }
  856. addr += size;
  857. }
  858. return ResultSuccess;
  859. }
  860. ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm,
  861. OperationType operation, PAddr map_addr) {
  862. std::lock_guard lock{page_table_lock};
  863. ASSERT(num_pages > 0);
  864. ASSERT(Common::IsAligned(addr, PageSize));
  865. ASSERT(ContainsPages(addr, num_pages));
  866. switch (operation) {
  867. case OperationType::Unmap:
  868. system.Memory().UnmapRegion(page_table_impl, addr, num_pages * PageSize);
  869. break;
  870. case OperationType::Map: {
  871. ASSERT(map_addr);
  872. ASSERT(Common::IsAligned(map_addr, PageSize));
  873. system.Memory().MapMemoryRegion(page_table_impl, addr, num_pages * PageSize, map_addr);
  874. break;
  875. }
  876. case OperationType::ChangePermissions:
  877. case OperationType::ChangePermissionsAndRefresh:
  878. break;
  879. default:
  880. UNREACHABLE();
  881. }
  882. return ResultSuccess;
  883. }
  884. constexpr VAddr KPageTable::GetRegionAddress(KMemoryState state) const {
  885. switch (state) {
  886. case KMemoryState::Free:
  887. case KMemoryState::Kernel:
  888. return address_space_start;
  889. case KMemoryState::Normal:
  890. return heap_region_start;
  891. case KMemoryState::Ipc:
  892. case KMemoryState::NonSecureIpc:
  893. case KMemoryState::NonDeviceIpc:
  894. return alias_region_start;
  895. case KMemoryState::Stack:
  896. return stack_region_start;
  897. case KMemoryState::Io:
  898. case KMemoryState::Static:
  899. case KMemoryState::ThreadLocal:
  900. return kernel_map_region_start;
  901. case KMemoryState::Shared:
  902. case KMemoryState::AliasCode:
  903. case KMemoryState::AliasCodeData:
  904. case KMemoryState::Transferred:
  905. case KMemoryState::SharedTransferred:
  906. case KMemoryState::SharedCode:
  907. case KMemoryState::GeneratedCode:
  908. case KMemoryState::CodeOut:
  909. return alias_code_region_start;
  910. case KMemoryState::Code:
  911. case KMemoryState::CodeData:
  912. return code_region_start;
  913. default:
  914. UNREACHABLE();
  915. return {};
  916. }
  917. }
  918. constexpr std::size_t KPageTable::GetRegionSize(KMemoryState state) const {
  919. switch (state) {
  920. case KMemoryState::Free:
  921. case KMemoryState::Kernel:
  922. return address_space_end - address_space_start;
  923. case KMemoryState::Normal:
  924. return heap_region_end - heap_region_start;
  925. case KMemoryState::Ipc:
  926. case KMemoryState::NonSecureIpc:
  927. case KMemoryState::NonDeviceIpc:
  928. return alias_region_end - alias_region_start;
  929. case KMemoryState::Stack:
  930. return stack_region_end - stack_region_start;
  931. case KMemoryState::Io:
  932. case KMemoryState::Static:
  933. case KMemoryState::ThreadLocal:
  934. return kernel_map_region_end - kernel_map_region_start;
  935. case KMemoryState::Shared:
  936. case KMemoryState::AliasCode:
  937. case KMemoryState::AliasCodeData:
  938. case KMemoryState::Transferred:
  939. case KMemoryState::SharedTransferred:
  940. case KMemoryState::SharedCode:
  941. case KMemoryState::GeneratedCode:
  942. case KMemoryState::CodeOut:
  943. return alias_code_region_end - alias_code_region_start;
  944. case KMemoryState::Code:
  945. case KMemoryState::CodeData:
  946. return code_region_end - code_region_start;
  947. default:
  948. UNREACHABLE();
  949. return {};
  950. }
  951. }
  952. bool KPageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) const {
  953. const VAddr end{addr + size};
  954. const VAddr last{end - 1};
  955. const VAddr region_start{GetRegionAddress(state)};
  956. const std::size_t region_size{GetRegionSize(state)};
  957. const bool is_in_region{region_start <= addr && addr < end &&
  958. last <= region_start + region_size - 1};
  959. const bool is_in_heap{!(end <= heap_region_start || heap_region_end <= addr)};
  960. const bool is_in_alias{!(end <= alias_region_start || alias_region_end <= addr)};
  961. switch (state) {
  962. case KMemoryState::Free:
  963. case KMemoryState::Kernel:
  964. return is_in_region;
  965. case KMemoryState::Io:
  966. case KMemoryState::Static:
  967. case KMemoryState::Code:
  968. case KMemoryState::CodeData:
  969. case KMemoryState::Shared:
  970. case KMemoryState::AliasCode:
  971. case KMemoryState::AliasCodeData:
  972. case KMemoryState::Stack:
  973. case KMemoryState::ThreadLocal:
  974. case KMemoryState::Transferred:
  975. case KMemoryState::SharedTransferred:
  976. case KMemoryState::SharedCode:
  977. case KMemoryState::GeneratedCode:
  978. case KMemoryState::CodeOut:
  979. return is_in_region && !is_in_heap && !is_in_alias;
  980. case KMemoryState::Normal:
  981. ASSERT(is_in_heap);
  982. return is_in_region && !is_in_alias;
  983. case KMemoryState::Ipc:
  984. case KMemoryState::NonSecureIpc:
  985. case KMemoryState::NonDeviceIpc:
  986. ASSERT(is_in_alias);
  987. return is_in_region && !is_in_heap;
  988. default:
  989. return false;
  990. }
  991. }
  992. constexpr ResultCode KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask,
  993. KMemoryState state, KMemoryPermission perm_mask,
  994. KMemoryPermission perm,
  995. KMemoryAttribute attr_mask,
  996. KMemoryAttribute attr) const {
  997. // Validate the states match expectation
  998. if ((info.state & state_mask) != state) {
  999. return ResultInvalidCurrentMemory;
  1000. }
  1001. if ((info.perm & perm_mask) != perm) {
  1002. return ResultInvalidCurrentMemory;
  1003. }
  1004. if ((info.attribute & attr_mask) != attr) {
  1005. return ResultInvalidCurrentMemory;
  1006. }
  1007. return ResultSuccess;
  1008. }
  1009. ResultCode KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm,
  1010. KMemoryAttribute* out_attr, VAddr addr, std::size_t size,
  1011. KMemoryState state_mask, KMemoryState state,
  1012. KMemoryPermission perm_mask, KMemoryPermission perm,
  1013. KMemoryAttribute attr_mask, KMemoryAttribute attr,
  1014. KMemoryAttribute ignore_attr) {
  1015. std::lock_guard lock{page_table_lock};
  1016. // Get information about the first block
  1017. const VAddr last_addr{addr + size - 1};
  1018. KMemoryBlockManager::const_iterator it{block_manager->FindIterator(addr)};
  1019. KMemoryInfo info{it->GetMemoryInfo()};
  1020. // Validate all blocks in the range have correct state
  1021. const KMemoryState first_state{info.state};
  1022. const KMemoryPermission first_perm{info.perm};
  1023. const KMemoryAttribute first_attr{info.attribute};
  1024. while (true) {
  1025. // Validate the current block
  1026. if (!(info.state == first_state)) {
  1027. return ResultInvalidCurrentMemory;
  1028. }
  1029. if (!(info.perm == first_perm)) {
  1030. return ResultInvalidCurrentMemory;
  1031. }
  1032. if (!((info.attribute | static_cast<KMemoryAttribute>(ignore_attr)) ==
  1033. (first_attr | static_cast<KMemoryAttribute>(ignore_attr)))) {
  1034. return ResultInvalidCurrentMemory;
  1035. }
  1036. // Validate against the provided masks
  1037. CASCADE_CODE(CheckMemoryState(info, state_mask, state, perm_mask, perm, attr_mask, attr));
  1038. // Break once we're done
  1039. if (last_addr <= info.GetLastAddress()) {
  1040. break;
  1041. }
  1042. // Advance our iterator
  1043. it++;
  1044. ASSERT(it != block_manager->cend());
  1045. info = it->GetMemoryInfo();
  1046. }
  1047. // Write output state
  1048. if (out_state) {
  1049. *out_state = first_state;
  1050. }
  1051. if (out_perm) {
  1052. *out_perm = first_perm;
  1053. }
  1054. if (out_attr) {
  1055. *out_attr = first_attr & static_cast<KMemoryAttribute>(~ignore_attr);
  1056. }
  1057. return ResultSuccess;
  1058. }
  1059. ResultCode KPageTable::CheckMemoryState(size_t* out_blocks_needed, VAddr addr, size_t size,
  1060. KMemoryState state_mask, KMemoryState state,
  1061. KMemoryPermission perm_mask, KMemoryPermission perm,
  1062. KMemoryAttribute attr_mask, KMemoryAttribute attr) const {
  1063. // Get information about the first block.
  1064. const VAddr last_addr = addr + size - 1;
  1065. KMemoryBlockManager::const_iterator it{block_manager->FindIterator(addr)};
  1066. KMemoryInfo info = it->GetMemoryInfo();
  1067. // If the start address isn't aligned, we need a block.
  1068. const size_t blocks_for_start_align =
  1069. (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0;
  1070. while (true) {
  1071. // Validate against the provided masks.
  1072. R_TRY(CheckMemoryState(info, state_mask, state, perm_mask, perm, attr_mask, attr));
  1073. // Break once we're done.
  1074. if (last_addr <= info.GetLastAddress()) {
  1075. break;
  1076. }
  1077. // Advance our iterator.
  1078. it++;
  1079. info = it->GetMemoryInfo();
  1080. }
  1081. // If the end address isn't aligned, we need a block.
  1082. const size_t blocks_for_end_align =
  1083. (Common::AlignUp(addr + size, PageSize) != info.GetEndAddress()) ? 1 : 0;
  1084. if (out_blocks_needed != nullptr) {
  1085. *out_blocks_needed = blocks_for_start_align + blocks_for_end_align;
  1086. }
  1087. return ResultSuccess;
  1088. }
  1089. } // namespace Kernel