scheduler.cpp 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783
  1. // Copyright 2018 yuzu emulator team
  2. // Licensed under GPLv2 or any later version
  3. // Refer to the license.txt file included.
  4. //
  5. // SelectThreads, Yield functions originally by TuxSH.
  6. // licensed under GPLv2 or later under exception provided by the author.
  7. #include <algorithm>
  8. #include <set>
  9. #include <unordered_set>
  10. #include <utility>
  11. #include "common/assert.h"
  12. #include "common/bit_util.h"
  13. #include "common/fiber.h"
  14. #include "common/logging/log.h"
  15. #include "core/arm/arm_interface.h"
  16. #include "core/core.h"
  17. #include "core/core_timing.h"
  18. #include "core/cpu_manager.h"
  19. #include "core/hle/kernel/kernel.h"
  20. #include "core/hle/kernel/physical_core.h"
  21. #include "core/hle/kernel/process.h"
  22. #include "core/hle/kernel/scheduler.h"
  23. #include "core/hle/kernel/time_manager.h"
  24. namespace Kernel {
  25. GlobalScheduler::GlobalScheduler(KernelCore& kernel) : kernel{kernel} {}
  26. GlobalScheduler::~GlobalScheduler() = default;
  27. void GlobalScheduler::AddThread(std::shared_ptr<Thread> thread) {
  28. global_list_guard.lock();
  29. thread_list.push_back(std::move(thread));
  30. global_list_guard.unlock();
  31. }
  32. void GlobalScheduler::RemoveThread(std::shared_ptr<Thread> thread) {
  33. global_list_guard.lock();
  34. thread_list.erase(std::remove(thread_list.begin(), thread_list.end(), thread),
  35. thread_list.end());
  36. global_list_guard.unlock();
  37. }
  38. u32 GlobalScheduler::SelectThreads() {
  39. ASSERT(is_locked);
  40. const auto update_thread = [](Thread* thread, Scheduler& sched) {
  41. sched.guard.lock();
  42. if (thread != sched.selected_thread_set.get()) {
  43. if (thread == nullptr) {
  44. ++sched.idle_selection_count;
  45. }
  46. sched.selected_thread_set = SharedFrom(thread);
  47. }
  48. const bool reschedule_pending =
  49. sched.is_context_switch_pending || (sched.selected_thread_set != sched.current_thread);
  50. sched.is_context_switch_pending = reschedule_pending;
  51. std::atomic_thread_fence(std::memory_order_seq_cst);
  52. sched.guard.unlock();
  53. return reschedule_pending;
  54. };
  55. if (!is_reselection_pending.load()) {
  56. return 0;
  57. }
  58. std::array<Thread*, Core::Hardware::NUM_CPU_CORES> top_threads{};
  59. u32 idle_cores{};
  60. // Step 1: Get top thread in schedule queue.
  61. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  62. Thread* top_thread =
  63. scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
  64. if (top_thread != nullptr) {
  65. // TODO(Blinkhawk): Implement Thread Pinning
  66. } else {
  67. idle_cores |= (1ul << core);
  68. }
  69. top_threads[core] = top_thread;
  70. }
  71. while (idle_cores != 0) {
  72. u32 core_id = Common::CountTrailingZeroes32(idle_cores);
  73. if (!suggested_queue[core_id].empty()) {
  74. std::array<s32, Core::Hardware::NUM_CPU_CORES> migration_candidates{};
  75. std::size_t num_candidates = 0;
  76. auto iter = suggested_queue[core_id].begin();
  77. Thread* suggested = nullptr;
  78. // Step 2: Try selecting a suggested thread.
  79. while (iter != suggested_queue[core_id].end()) {
  80. suggested = *iter;
  81. iter++;
  82. s32 suggested_core_id = suggested->GetProcessorID();
  83. Thread* top_thread =
  84. suggested_core_id > 0 ? top_threads[suggested_core_id] : nullptr;
  85. if (top_thread != suggested) {
  86. if (top_thread != nullptr &&
  87. top_thread->GetPriority() < THREADPRIO_MAX_CORE_MIGRATION) {
  88. suggested = nullptr;
  89. break;
  90. // There's a too high thread to do core migration, cancel
  91. }
  92. TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id), suggested);
  93. break;
  94. }
  95. suggested = nullptr;
  96. migration_candidates[num_candidates++] = suggested_core_id;
  97. }
  98. // Step 3: Select a suggested thread from another core
  99. if (suggested == nullptr) {
  100. for (std::size_t i = 0; i < num_candidates; i++) {
  101. s32 candidate_core = migration_candidates[i];
  102. suggested = top_threads[candidate_core];
  103. auto it = scheduled_queue[candidate_core].begin();
  104. it++;
  105. Thread* next = it != scheduled_queue[candidate_core].end() ? *it : nullptr;
  106. if (next != nullptr) {
  107. TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id),
  108. suggested);
  109. top_threads[candidate_core] = next;
  110. break;
  111. } else {
  112. suggested = nullptr;
  113. }
  114. }
  115. }
  116. top_threads[core_id] = suggested;
  117. }
  118. idle_cores &= ~(1ul << core_id);
  119. }
  120. u32 cores_needing_context_switch{};
  121. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  122. Scheduler& sched = kernel.Scheduler(core);
  123. if (update_thread(top_threads[core], sched)) {
  124. cores_needing_context_switch |= (1ul << core);
  125. }
  126. }
  127. return cores_needing_context_switch;
  128. }
  129. bool GlobalScheduler::YieldThread(Thread* yielding_thread) {
  130. ASSERT(is_locked);
  131. // Note: caller should use critical section, etc.
  132. const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID());
  133. const u32 priority = yielding_thread->GetPriority();
  134. // Yield the thread
  135. Reschedule(priority, core_id, yielding_thread);
  136. const Thread* const winner = scheduled_queue[core_id].front();
  137. if (kernel.GetCurrentHostThreadID() != core_id) {
  138. is_reselection_pending.store(true, std::memory_order_release);
  139. }
  140. return AskForReselectionOrMarkRedundant(yielding_thread, winner);
  141. }
  142. bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) {
  143. ASSERT(is_locked);
  144. // Note: caller should check if !thread.IsSchedulerOperationRedundant and use critical section,
  145. // etc.
  146. const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID());
  147. const u32 priority = yielding_thread->GetPriority();
  148. // Yield the thread
  149. Reschedule(priority, core_id, yielding_thread);
  150. std::array<Thread*, Core::Hardware::NUM_CPU_CORES> current_threads;
  151. for (std::size_t i = 0; i < current_threads.size(); i++) {
  152. current_threads[i] = scheduled_queue[i].empty() ? nullptr : scheduled_queue[i].front();
  153. }
  154. Thread* next_thread = scheduled_queue[core_id].front(priority);
  155. Thread* winner = nullptr;
  156. for (auto& thread : suggested_queue[core_id]) {
  157. const s32 source_core = thread->GetProcessorID();
  158. if (source_core >= 0) {
  159. if (current_threads[source_core] != nullptr) {
  160. if (thread == current_threads[source_core] ||
  161. current_threads[source_core]->GetPriority() < min_regular_priority) {
  162. continue;
  163. }
  164. }
  165. }
  166. if (next_thread->GetLastRunningTicks() >= thread->GetLastRunningTicks() ||
  167. next_thread->GetPriority() < thread->GetPriority()) {
  168. if (thread->GetPriority() <= priority) {
  169. winner = thread;
  170. break;
  171. }
  172. }
  173. }
  174. if (winner != nullptr) {
  175. if (winner != yielding_thread) {
  176. TransferToCore(winner->GetPriority(), s32(core_id), winner);
  177. }
  178. } else {
  179. winner = next_thread;
  180. }
  181. if (kernel.GetCurrentHostThreadID() != core_id) {
  182. is_reselection_pending.store(true, std::memory_order_release);
  183. }
  184. return AskForReselectionOrMarkRedundant(yielding_thread, winner);
  185. }
  186. bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread) {
  187. ASSERT(is_locked);
  188. // Note: caller should check if !thread.IsSchedulerOperationRedundant and use critical section,
  189. // etc.
  190. Thread* winner = nullptr;
  191. const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID());
  192. // Remove the thread from its scheduled mlq, put it on the corresponding "suggested" one instead
  193. TransferToCore(yielding_thread->GetPriority(), -1, yielding_thread);
  194. // If the core is idle, perform load balancing, excluding the threads that have just used this
  195. // function...
  196. if (scheduled_queue[core_id].empty()) {
  197. // Here, "current_threads" is calculated after the ""yield"", unlike yield -1
  198. std::array<Thread*, Core::Hardware::NUM_CPU_CORES> current_threads;
  199. for (std::size_t i = 0; i < current_threads.size(); i++) {
  200. current_threads[i] = scheduled_queue[i].empty() ? nullptr : scheduled_queue[i].front();
  201. }
  202. for (auto& thread : suggested_queue[core_id]) {
  203. const s32 source_core = thread->GetProcessorID();
  204. if (source_core < 0 || thread == current_threads[source_core]) {
  205. continue;
  206. }
  207. if (current_threads[source_core] == nullptr ||
  208. current_threads[source_core]->GetPriority() >= min_regular_priority) {
  209. winner = thread;
  210. }
  211. break;
  212. }
  213. if (winner != nullptr) {
  214. if (winner != yielding_thread) {
  215. TransferToCore(winner->GetPriority(), static_cast<s32>(core_id), winner);
  216. }
  217. } else {
  218. winner = yielding_thread;
  219. }
  220. } else {
  221. winner = scheduled_queue[i].front();
  222. }
  223. if (kernel.GetCurrentHostThreadID() != core_id) {
  224. is_reselection_pending.store(true, std::memory_order_release);
  225. }
  226. return AskForReselectionOrMarkRedundant(yielding_thread, winner);
  227. }
  228. void GlobalScheduler::PreemptThreads() {
  229. ASSERT(is_locked);
  230. for (std::size_t core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
  231. const u32 priority = preemption_priorities[core_id];
  232. if (scheduled_queue[core_id].size(priority) > 0) {
  233. if (scheduled_queue[core_id].size(priority) > 1) {
  234. scheduled_queue[core_id].front(priority)->IncrementYieldCount();
  235. }
  236. scheduled_queue[core_id].yield(priority);
  237. if (scheduled_queue[core_id].size(priority) > 1) {
  238. scheduled_queue[core_id].front(priority)->IncrementYieldCount();
  239. }
  240. }
  241. Thread* current_thread =
  242. scheduled_queue[core_id].empty() ? nullptr : scheduled_queue[core_id].front();
  243. Thread* winner = nullptr;
  244. for (auto& thread : suggested_queue[core_id]) {
  245. const s32 source_core = thread->GetProcessorID();
  246. if (thread->GetPriority() != priority) {
  247. continue;
  248. }
  249. if (source_core >= 0) {
  250. Thread* next_thread = scheduled_queue[source_core].empty()
  251. ? nullptr
  252. : scheduled_queue[source_core].front();
  253. if (next_thread != nullptr && next_thread->GetPriority() < 2) {
  254. break;
  255. }
  256. if (next_thread == thread) {
  257. continue;
  258. }
  259. }
  260. if (current_thread != nullptr &&
  261. current_thread->GetLastRunningTicks() >= thread->GetLastRunningTicks()) {
  262. winner = thread;
  263. break;
  264. }
  265. }
  266. if (winner != nullptr) {
  267. TransferToCore(winner->GetPriority(), s32(core_id), winner);
  268. current_thread =
  269. winner->GetPriority() <= current_thread->GetPriority() ? winner : current_thread;
  270. }
  271. if (current_thread != nullptr && current_thread->GetPriority() > priority) {
  272. for (auto& thread : suggested_queue[core_id]) {
  273. const s32 source_core = thread->GetProcessorID();
  274. if (thread->GetPriority() < priority) {
  275. continue;
  276. }
  277. if (source_core >= 0) {
  278. Thread* next_thread = scheduled_queue[source_core].empty()
  279. ? nullptr
  280. : scheduled_queue[source_core].front();
  281. if (next_thread != nullptr && next_thread->GetPriority() < 2) {
  282. break;
  283. }
  284. if (next_thread == thread) {
  285. continue;
  286. }
  287. }
  288. if (current_thread != nullptr &&
  289. current_thread->GetLastRunningTicks() >= thread->GetLastRunningTicks()) {
  290. winner = thread;
  291. break;
  292. }
  293. }
  294. if (winner != nullptr) {
  295. TransferToCore(winner->GetPriority(), s32(core_id), winner);
  296. current_thread = winner;
  297. }
  298. }
  299. is_reselection_pending.store(true, std::memory_order_release);
  300. }
  301. }
  302. void GlobalScheduler::EnableInterruptAndSchedule(u32 cores_pending_reschedule,
  303. Core::EmuThreadHandle global_thread) {
  304. u32 current_core = global_thread.host_handle;
  305. bool must_context_switch = global_thread.guest_handle != InvalidHandle &&
  306. (current_core < Core::Hardware::NUM_CPU_CORES);
  307. while (cores_pending_reschedule != 0) {
  308. u32 core = Common::CountTrailingZeroes32(cores_pending_reschedule);
  309. ASSERT(core < Core::Hardware::NUM_CPU_CORES);
  310. if (!must_context_switch || core != current_core) {
  311. auto& phys_core = kernel.PhysicalCore(core);
  312. phys_core.Interrupt();
  313. } else {
  314. must_context_switch = true;
  315. }
  316. cores_pending_reschedule &= ~(1ul << core);
  317. }
  318. if (must_context_switch) {
  319. auto& core_scheduler = kernel.CurrentScheduler();
  320. core_scheduler.TryDoContextSwitch();
  321. }
  322. }
  323. void GlobalScheduler::Suggest(u32 priority, std::size_t core, Thread* thread) {
  324. ASSERT(is_locked);
  325. suggested_queue[core].add(thread, priority);
  326. }
  327. void GlobalScheduler::Unsuggest(u32 priority, std::size_t core, Thread* thread) {
  328. ASSERT(is_locked);
  329. suggested_queue[core].remove(thread, priority);
  330. }
  331. void GlobalScheduler::Schedule(u32 priority, std::size_t core, Thread* thread) {
  332. ASSERT(is_locked);
  333. ASSERT_MSG(thread->GetProcessorID() == s32(core), "Thread must be assigned to this core.");
  334. scheduled_queue[core].add(thread, priority);
  335. }
  336. void GlobalScheduler::SchedulePrepend(u32 priority, std::size_t core, Thread* thread) {
  337. ASSERT(is_locked);
  338. ASSERT_MSG(thread->GetProcessorID() == s32(core), "Thread must be assigned to this core.");
  339. scheduled_queue[core].add(thread, priority, false);
  340. }
  341. void GlobalScheduler::Reschedule(u32 priority, std::size_t core, Thread* thread) {
  342. ASSERT(is_locked);
  343. scheduled_queue[core].remove(thread, priority);
  344. scheduled_queue[core].add(thread, priority);
  345. }
  346. void GlobalScheduler::Unschedule(u32 priority, std::size_t core, Thread* thread) {
  347. ASSERT(is_locked);
  348. scheduled_queue[core].remove(thread, priority);
  349. }
  350. void GlobalScheduler::TransferToCore(u32 priority, s32 destination_core, Thread* thread) {
  351. ASSERT(is_locked);
  352. const bool schedulable = thread->GetPriority() < THREADPRIO_COUNT;
  353. const s32 source_core = thread->GetProcessorID();
  354. if (source_core == destination_core || !schedulable) {
  355. return;
  356. }
  357. thread->SetProcessorID(destination_core);
  358. if (source_core >= 0) {
  359. Unschedule(priority, static_cast<u32>(source_core), thread);
  360. }
  361. if (destination_core >= 0) {
  362. Unsuggest(priority, static_cast<u32>(destination_core), thread);
  363. Schedule(priority, static_cast<u32>(destination_core), thread);
  364. }
  365. if (source_core >= 0) {
  366. Suggest(priority, static_cast<u32>(source_core), thread);
  367. }
  368. }
  369. bool GlobalScheduler::AskForReselectionOrMarkRedundant(Thread* current_thread,
  370. const Thread* winner) {
  371. if (current_thread == winner) {
  372. current_thread->IncrementYieldCount();
  373. return true;
  374. } else {
  375. is_reselection_pending.store(true, std::memory_order_release);
  376. return false;
  377. }
  378. }
  379. void GlobalScheduler::AdjustSchedulingOnStatus(Thread* thread, u32 old_flags) {
  380. if (old_flags == thread->scheduling_state) {
  381. return;
  382. }
  383. ASSERT(is_locked);
  384. if (old_flags == static_cast<u32>(ThreadSchedStatus::Runnable)) {
  385. // In this case the thread was running, now it's pausing/exitting
  386. if (thread->processor_id >= 0) {
  387. Unschedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
  388. }
  389. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  390. if (core != static_cast<u32>(thread->processor_id) &&
  391. ((thread->affinity_mask >> core) & 1) != 0) {
  392. Unsuggest(thread->current_priority, core, thread);
  393. }
  394. }
  395. } else if (thread->scheduling_state == static_cast<u32>(ThreadSchedStatus::Runnable)) {
  396. // The thread is now set to running from being stopped
  397. if (thread->processor_id >= 0) {
  398. Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
  399. }
  400. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  401. if (core != static_cast<u32>(thread->processor_id) &&
  402. ((thread->affinity_mask >> core) & 1) != 0) {
  403. Suggest(thread->current_priority, core, thread);
  404. }
  405. }
  406. }
  407. SetReselectionPending();
  408. }
  409. void GlobalScheduler::AdjustSchedulingOnPriority(Thread* thread, u32 old_priority) {
  410. if (thread->scheduling_state != static_cast<u32>(ThreadSchedStatus::Runnable)) {
  411. return;
  412. }
  413. ASSERT(is_locked);
  414. if (thread->processor_id >= 0) {
  415. Unschedule(old_priority, static_cast<u32>(thread->processor_id), thread);
  416. }
  417. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  418. if (core != static_cast<u32>(thread->processor_id) &&
  419. ((thread->affinity_mask >> core) & 1) != 0) {
  420. Unsuggest(old_priority, core, thread);
  421. }
  422. }
  423. if (thread->processor_id >= 0) {
  424. if (thread == kernel.CurrentScheduler().GetCurrentThread()) {
  425. SchedulePrepend(thread->current_priority, static_cast<u32>(thread->processor_id),
  426. thread);
  427. } else {
  428. Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
  429. }
  430. }
  431. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  432. if (core != static_cast<u32>(thread->processor_id) &&
  433. ((thread->affinity_mask >> core) & 1) != 0) {
  434. Suggest(thread->current_priority, core, thread);
  435. }
  436. }
  437. thread->IncrementYieldCount();
  438. SetReselectionPending();
  439. }
  440. void GlobalScheduler::AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask,
  441. s32 old_core) {
  442. if (thread->scheduling_state != static_cast<u32>(ThreadSchedStatus::Runnable) ||
  443. thread->current_priority >= THREADPRIO_COUNT) {
  444. return;
  445. }
  446. ASSERT(is_locked);
  447. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  448. if (((old_affinity_mask >> core) & 1) != 0) {
  449. if (core == static_cast<u32>(old_core)) {
  450. Unschedule(thread->current_priority, core, thread);
  451. } else {
  452. Unsuggest(thread->current_priority, core, thread);
  453. }
  454. }
  455. }
  456. for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  457. if (((thread->affinity_mask >> core) & 1) != 0) {
  458. if (core == static_cast<u32>(thread->processor_id)) {
  459. Schedule(thread->current_priority, core, thread);
  460. } else {
  461. Suggest(thread->current_priority, core, thread);
  462. }
  463. }
  464. }
  465. thread->IncrementYieldCount();
  466. SetReselectionPending();
  467. }
  468. void GlobalScheduler::Shutdown() {
  469. for (std::size_t core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
  470. scheduled_queue[core].clear();
  471. suggested_queue[core].clear();
  472. }
  473. thread_list.clear();
  474. }
  475. void GlobalScheduler::Lock() {
  476. Core::EmuThreadHandle current_thread = kernel.GetCurrentEmuThreadID();
  477. ASSERT(!current_thread.IsInvalid());
  478. if (current_thread == current_owner) {
  479. ++scope_lock;
  480. } else {
  481. inner_lock.lock();
  482. is_locked = true;
  483. current_owner = current_thread;
  484. ASSERT(current_owner != Core::EmuThreadHandle::InvalidHandle());
  485. scope_lock = 1;
  486. }
  487. }
  488. void GlobalScheduler::Unlock() {
  489. if (--scope_lock != 0) {
  490. ASSERT(scope_lock > 0);
  491. return;
  492. }
  493. u32 cores_pending_reschedule = SelectThreads();
  494. Core::EmuThreadHandle leaving_thread = current_owner;
  495. current_owner = Core::EmuThreadHandle::InvalidHandle();
  496. scope_lock = 1;
  497. is_locked = false;
  498. inner_lock.unlock();
  499. EnableInterruptAndSchedule(cores_pending_reschedule, leaving_thread);
  500. }
  501. Scheduler::Scheduler(Core::System& system, std::size_t core_id) : system(system), core_id(core_id) {
  502. switch_fiber = std::make_shared<Common::Fiber>(std::function<void(void*)>(OnSwitch), this);
  503. }
  504. Scheduler::~Scheduler() = default;
  505. bool Scheduler::HaveReadyThreads() const {
  506. return system.GlobalScheduler().HaveReadyThreads(core_id);
  507. }
  508. Thread* Scheduler::GetCurrentThread() const {
  509. if (current_thread) {
  510. return current_thread.get();
  511. }
  512. return idle_thread.get();
  513. }
  514. Thread* Scheduler::GetSelectedThread() const {
  515. return selected_thread.get();
  516. }
  517. u64 Scheduler::GetLastContextSwitchTicks() const {
  518. return last_context_switch_time;
  519. }
  520. void Scheduler::TryDoContextSwitch() {
  521. auto& phys_core = system.Kernel().CurrentPhysicalCore();
  522. if (phys_core.IsInterrupted()) {
  523. phys_core.ClearInterrupt();
  524. }
  525. guard.lock();
  526. if (is_context_switch_pending) {
  527. SwitchContext();
  528. } else {
  529. guard.unlock();
  530. }
  531. }
  532. void Scheduler::OnThreadStart() {
  533. SwitchContextStep2();
  534. }
  535. void Scheduler::SwitchContextStep2() {
  536. Thread* previous_thread = current_thread_prev.get();
  537. Thread* new_thread = selected_thread.get();
  538. // Load context of new thread
  539. Process* const previous_process =
  540. previous_thread != nullptr ? previous_thread->GetOwnerProcess() : nullptr;
  541. if (new_thread) {
  542. ASSERT_MSG(new_thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable,
  543. "Thread must be runnable.");
  544. // Cancel any outstanding wakeup events for this thread
  545. new_thread->SetIsRunning(true);
  546. auto* const thread_owner_process = current_thread->GetOwnerProcess();
  547. if (previous_process != thread_owner_process && thread_owner_process != nullptr) {
  548. system.Kernel().MakeCurrentProcess(thread_owner_process);
  549. }
  550. if (!new_thread->IsHLEThread()) {
  551. auto& cpu_core = system.ArmInterface(core_id);
  552. cpu_core.LoadContext(new_thread->GetContext32());
  553. cpu_core.LoadContext(new_thread->GetContext64());
  554. cpu_core.SetTlsAddress(new_thread->GetTLSAddress());
  555. cpu_core.SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
  556. cpu_core.ClearExclusiveState();
  557. }
  558. }
  559. TryDoContextSwitch();
  560. }
  561. void Scheduler::SwitchContext() {
  562. current_thread_prev = current_thread;
  563. selected_thread = selected_thread_set;
  564. Thread* previous_thread = current_thread_prev.get();
  565. Thread* new_thread = selected_thread.get();
  566. current_thread = selected_thread;
  567. is_context_switch_pending = false;
  568. if (new_thread == previous_thread) {
  569. guard.unlock();
  570. return;
  571. }
  572. Process* const previous_process = system.Kernel().CurrentProcess();
  573. UpdateLastContextSwitchTime(previous_thread, previous_process);
  574. // Save context for previous thread
  575. if (previous_thread) {
  576. if (!previous_thread->IsHLEThread()) {
  577. auto& cpu_core = system.ArmInterface(core_id);
  578. cpu_core.SaveContext(previous_thread->GetContext32());
  579. cpu_core.SaveContext(previous_thread->GetContext64());
  580. // Save the TPIDR_EL0 system register in case it was modified.
  581. previous_thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());
  582. cpu_core.ClearExclusiveState();
  583. }
  584. if (previous_thread->GetStatus() == ThreadStatus::Running) {
  585. previous_thread->SetStatus(ThreadStatus::Ready);
  586. }
  587. previous_thread->SetIsRunning(false);
  588. previous_thread->context_guard.unlock();
  589. }
  590. std::shared_ptr<Common::Fiber> old_context;
  591. if (previous_thread != nullptr) {
  592. old_context = previous_thread->GetHostContext();
  593. } else {
  594. old_context = idle_thread->GetHostContext();
  595. }
  596. guard.unlock();
  597. Common::Fiber::YieldTo(old_context, switch_fiber);
  598. /// When a thread wakes up, the scheduler may have changed to other in another core.
  599. auto& next_scheduler = system.Kernel().CurrentScheduler();
  600. next_scheduler.SwitchContextStep2();
  601. }
  602. void Scheduler::OnSwitch(void* this_scheduler) {
  603. Scheduler* sched = static_cast<Scheduler*>(this_scheduler);
  604. sched->SwitchToCurrent();
  605. }
  606. void Scheduler::SwitchToCurrent() {
  607. while (true) {
  608. guard.lock();
  609. selected_thread = selected_thread_set;
  610. current_thread = selected_thread;
  611. is_context_switch_pending = false;
  612. guard.unlock();
  613. while (!is_context_switch_pending) {
  614. if (current_thread != nullptr && !current_thread->IsHLEThread()) {
  615. current_thread->context_guard.lock();
  616. if (!current_thread->IsRunnable()) {
  617. current_thread->context_guard.unlock();
  618. break;
  619. }
  620. if (current_thread->GetProcessorID() != core_id) {
  621. current_thread->context_guard.unlock();
  622. break;
  623. }
  624. }
  625. std::shared_ptr<Common::Fiber> next_context;
  626. if (current_thread != nullptr) {
  627. next_context = current_thread->GetHostContext();
  628. } else {
  629. next_context = idle_thread->GetHostContext();
  630. }
  631. Common::Fiber::YieldTo(switch_fiber, next_context);
  632. }
  633. }
  634. }
  635. void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
  636. const u64 prev_switch_ticks = last_context_switch_time;
  637. const u64 most_recent_switch_ticks = system.CoreTiming().GetCPUTicks();
  638. const u64 update_ticks = most_recent_switch_ticks - prev_switch_ticks;
  639. if (thread != nullptr) {
  640. thread->UpdateCPUTimeTicks(update_ticks);
  641. }
  642. if (process != nullptr) {
  643. process->UpdateCPUTimeTicks(update_ticks);
  644. }
  645. last_context_switch_time = most_recent_switch_ticks;
  646. }
  647. void Scheduler::Initialize() {
  648. std::string name = "Idle Thread Id:" + std::to_string(core_id);
  649. std::function<void(void*)> init_func = system.GetCpuManager().GetIdleThreadStartFunc();
  650. void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
  651. ThreadType type = static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_IDLE);
  652. auto thread_res = Thread::Create(system, type, name, 0, 64, 0, static_cast<u32>(core_id), 0,
  653. nullptr, std::move(init_func), init_func_parameter);
  654. idle_thread = std::move(thread_res).Unwrap();
  655. }
  656. void Scheduler::Shutdown() {
  657. current_thread = nullptr;
  658. selected_thread = nullptr;
  659. }
  660. SchedulerLock::SchedulerLock(KernelCore& kernel) : kernel{kernel} {
  661. kernel.GlobalScheduler().Lock();
  662. }
  663. SchedulerLock::~SchedulerLock() {
  664. kernel.GlobalScheduler().Unlock();
  665. }
  666. SchedulerLockAndSleep::SchedulerLockAndSleep(KernelCore& kernel, Handle& event_handle,
  667. Thread* time_task, s64 nanoseconds)
  668. : SchedulerLock{kernel}, event_handle{event_handle}, time_task{time_task}, nanoseconds{
  669. nanoseconds} {
  670. event_handle = InvalidHandle;
  671. }
  672. SchedulerLockAndSleep::~SchedulerLockAndSleep() {
  673. if (sleep_cancelled) {
  674. return;
  675. }
  676. auto& time_manager = kernel.TimeManager();
  677. time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds);
  678. }
  679. void SchedulerLockAndSleep::Release() {
  680. if (sleep_cancelled) {
  681. return;
  682. }
  683. auto& time_manager = kernel.TimeManager();
  684. time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds);
  685. sleep_cancelled = true;
  686. }
  687. } // namespace Kernel