core: hle: kernel: Update init_slab_heap, use device memory, and add KThreadLocalPage and KPageBuffer.

- Refreshes our slab initialization code to latest known behavior.
- Moves all guest kernel slabs into emulated device memory.
- Adds KThreadLocalPage and KPageBuffer, which we will use for accurate TLS management.

src/core/hle/kernel/init/init_slab_setup.cpp

@@ -7,19 +7,23 @@
 #include "common/common_funcs.h"
 #include "common/common_types.h"
 #include "core/core.h"
+#include "core/device_memory.h"
 #include "core/hardware_properties.h"
 #include "core/hle/kernel/init/init_slab_setup.h"
 #include "core/hle/kernel/k_code_memory.h"
 #include "core/hle/kernel/k_event.h"
 #include "core/hle/kernel/k_memory_layout.h"
 #include "core/hle/kernel/k_memory_manager.h"
+#include "core/hle/kernel/k_page_buffer.h"
 #include "core/hle/kernel/k_port.h"
 #include "core/hle/kernel/k_process.h"
 #include "core/hle/kernel/k_resource_limit.h"
 #include "core/hle/kernel/k_session.h"
 #include "core/hle/kernel/k_shared_memory.h"
+#include "core/hle/kernel/k_shared_memory_info.h"
 #include "core/hle/kernel/k_system_control.h"
 #include "core/hle/kernel/k_thread.h"
+#include "core/hle/kernel/k_thread_local_page.h"
 #include "core/hle/kernel/k_transfer_memory.h"
 
 namespace Kernel::Init {
@@ -32,9 +36,13 @@ namespace Kernel::Init {
     HANDLER(KEvent, (SLAB_COUNT(KEvent)), ##__VA_ARGS__)                                           \
     HANDLER(KPort, (SLAB_COUNT(KPort)), ##__VA_ARGS__)                                             \
     HANDLER(KSharedMemory, (SLAB_COUNT(KSharedMemory)), ##__VA_ARGS__)                             \
+    HANDLER(KSharedMemoryInfo, (SLAB_COUNT(KSharedMemory) * 8), ##__VA_ARGS__)                     \
     HANDLER(KTransferMemory, (SLAB_COUNT(KTransferMemory)), ##__VA_ARGS__)                         \
     HANDLER(KCodeMemory, (SLAB_COUNT(KCodeMemory)), ##__VA_ARGS__)                                 \
     HANDLER(KSession, (SLAB_COUNT(KSession)), ##__VA_ARGS__)                                       \
+    HANDLER(KThreadLocalPage,                                                                      \
+            (SLAB_COUNT(KProcess) + (SLAB_COUNT(KProcess) + SLAB_COUNT(KThread)) / 8),             \
+            ##__VA_ARGS__)                                                                         \
     HANDLER(KResourceLimit, (SLAB_COUNT(KResourceLimit)), ##__VA_ARGS__)
 
 namespace {
@@ -50,38 +58,46 @@ enum KSlabType : u32 {
 // Constexpr counts.
 constexpr size_t SlabCountKProcess = 80;
 constexpr size_t SlabCountKThread = 800;
-constexpr size_t SlabCountKEvent = 700;
+constexpr size_t SlabCountKEvent = 900;
 constexpr size_t SlabCountKInterruptEvent = 100;
-constexpr size_t SlabCountKPort = 256 + 0x20; // Extra 0x20 ports over Nintendo for homebrew.
+constexpr size_t SlabCountKPort = 384;
 constexpr size_t SlabCountKSharedMemory = 80;
 constexpr size_t SlabCountKTransferMemory = 200;
 constexpr size_t SlabCountKCodeMemory = 10;
 constexpr size_t SlabCountKDeviceAddressSpace = 300;
-constexpr size_t SlabCountKSession = 933;
+constexpr size_t SlabCountKSession = 1133;
 constexpr size_t SlabCountKLightSession = 100;
 constexpr size_t SlabCountKObjectName = 7;
 constexpr size_t SlabCountKResourceLimit = 5;
 constexpr size_t SlabCountKDebug = Core::Hardware::NUM_CPU_CORES;
-constexpr size_t SlabCountKAlpha = 1;
-constexpr size_t SlabCountKBeta = 6;
+constexpr size_t SlabCountKIoPool = 1;
+constexpr size_t SlabCountKIoRegion = 6;
 
 constexpr size_t SlabCountExtraKThread = 160;
 
+/// Helper function to translate from the slab virtual address to the reserved location in physical
+/// memory.
+static PAddr TranslateSlabAddrToPhysical(KMemoryLayout& memory_layout, VAddr slab_addr) {
+    slab_addr -= memory_layout.GetSlabRegionAddress();
+    return slab_addr + Core::DramMemoryMap::SlabHeapBase;
+}
+
 template <typename T>
 VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAddr address,
                          size_t num_objects) {
-    // TODO(bunnei): This is just a place holder. We should initialize the appropriate KSlabHeap for
-    // kernel object type T with the backing kernel memory pointer once we emulate kernel memory.
 
     const size_t size = Common::AlignUp(sizeof(T) * num_objects, alignof(void*));
     VAddr start = Common::AlignUp(address, alignof(T));
 
-    // This is intentionally empty. Once KSlabHeap is fully implemented, we can replace this with
-    // the pointer to emulated memory to pass along. Until then, KSlabHeap will just allocate/free
-    // host memory.
-    void* backing_kernel_memory{};
+    // This should use the virtual memory address passed in, but currently, we do not setup the
+    // kernel virtual memory layout. Instead, we simply map these at a region of physical memory
+    // that we reserve for the slab heaps.
+    // TODO(bunnei): Fix this once we support the kernel virtual memory layout.
 
     if (size > 0) {
+        void* backing_kernel_memory{
+            system.DeviceMemory().GetPointer(TranslateSlabAddrToPhysical(memory_layout, start))};
+
         const KMemoryRegion* region = memory_layout.FindVirtual(start + size - 1);
         ASSERT(region != nullptr);
         ASSERT(region->IsDerivedFrom(KMemoryRegionType_KernelSlab));
@@ -109,8 +125,8 @@ KSlabResourceCounts KSlabResourceCounts::CreateDefault() {
         .num_KObjectName = SlabCountKObjectName,
         .num_KResourceLimit = SlabCountKResourceLimit,
         .num_KDebug = SlabCountKDebug,
-        .num_KAlpha = SlabCountKAlpha,
-        .num_KBeta = SlabCountKBeta,
+        .num_KIoPool = SlabCountKIoPool,
+        .num_KIoRegion = SlabCountKIoRegion,
     };
 }
 
@@ -121,6 +137,12 @@ void InitializeSlabResourceCounts(KernelCore& kernel) {
     }
 }
 
+size_t CalculateSlabHeapGapSize() {
+    constexpr size_t KernelSlabHeapGapSize = 2_MiB - 296_KiB;
+    static_assert(KernelSlabHeapGapSize <= KernelSlabHeapGapsSizeMax);
+    return KernelSlabHeapGapSize;
+}
+
 size_t CalculateTotalSlabHeapSize(const KernelCore& kernel) {
     size_t size = 0;
 
@@ -136,11 +158,34 @@ size_t CalculateTotalSlabHeapSize(const KernelCore& kernel) {
 #undef ADD_SLAB_SIZE
 
     // Add the reserved size.
-    size += KernelSlabHeapGapsSize;
+    size += CalculateSlabHeapGapSize();
 
     return size;
 }
 
+void InitializeKPageBufferSlabHeap(Core::System& system) {
+    auto& kernel = system.Kernel();
+
+    const auto& counts = kernel.SlabResourceCounts();
+    const size_t num_pages =
+        counts.num_KProcess + counts.num_KThread + (counts.num_KProcess + counts.num_KThread) / 8;
+    const size_t slab_size = num_pages * PageSize;
+
+    // Reserve memory from the system resource limit.
+    ASSERT(kernel.GetSystemResourceLimit()->Reserve(LimitableResource::PhysicalMemory, slab_size));
+
+    // Allocate memory for the slab.
+    constexpr auto AllocateOption = KMemoryManager::EncodeOption(
+        KMemoryManager::Pool::System, KMemoryManager::Direction::FromFront);
+    const PAddr slab_address =
+        kernel.MemoryManager().AllocateAndOpenContinuous(num_pages, 1, AllocateOption);
+    ASSERT(slab_address != 0);
+
+    // Initialize the slabheap.
+    KPageBuffer::InitializeSlabHeap(kernel, system.DeviceMemory().GetPointer(slab_address),
+                                    slab_size);
+}
+
 void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
     auto& kernel = system.Kernel();
 
@@ -160,13 +205,13 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
     }
 
     // Create an array to represent the gaps between the slabs.
-    const size_t total_gap_size = KernelSlabHeapGapsSize;
+    const size_t total_gap_size = CalculateSlabHeapGapSize();
     std::array<size_t, slab_types.size()> slab_gaps;
-    for (size_t i = 0; i < slab_gaps.size(); i++) {
+    for (auto& slab_gap : slab_gaps) {
         // Note: This is an off-by-one error from Nintendo's intention, because GenerateRandomRange
         // is inclusive. However, Nintendo also has the off-by-one error, and it's "harmless", so we
         // will include it ourselves.
-        slab_gaps[i] = KSystemControl::GenerateRandomRange(0, total_gap_size);
+        slab_gap = KSystemControl::GenerateRandomRange(0, total_gap_size);
     }
 
     // Sort the array, so that we can treat differences between values as offsets to the starts of
@@ -177,13 +222,21 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
         }
     }
 
-    for (size_t i = 0; i < slab_types.size(); i++) {
+    // Track the gaps, so that we can free them to the unused slab tree.
+    VAddr gap_start = address;
+    size_t gap_size = 0;
+
+    for (size_t i = 0; i < slab_gaps.size(); i++) {
         // Add the random gap to the address.
-        address += (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
+        const auto cur_gap = (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
+        address += cur_gap;
+        gap_size += cur_gap;
 
 #define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...)                                                     \
     case KSlabType_##NAME:                                                                         \
-        address = InitializeSlabHeap<NAME>(system, memory_layout, address, COUNT);                 \
+        if (COUNT > 0) {                                                                           \
+            address = InitializeSlabHeap<NAME>(system, memory_layout, address, COUNT);             \
+        }                                                                                          \
         break;
 
         // Initialize the slabheap.
@@ -192,7 +245,13 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
             FOREACH_SLAB_TYPE(INITIALIZE_SLAB_HEAP)
             // If we somehow get an invalid type, abort.
         default:
-            UNREACHABLE();
+            UNREACHABLE_MSG("Unknown slab type: {}", slab_types[i]);
+        }
+
+        // If we've hit the end of a gap, free it.
+        if (gap_start + gap_size != address) {
+            gap_start = address;
+            gap_size = 0;
         }
     }
 }

src/core/hle/kernel/init/init_slab_setup.h

@@ -32,12 +32,13 @@ struct KSlabResourceCounts {
     size_t num_KObjectName;
     size_t num_KResourceLimit;
     size_t num_KDebug;
-    size_t num_KAlpha;
-    size_t num_KBeta;
+    size_t num_KIoPool;
+    size_t num_KIoRegion;
 };
 
 void InitializeSlabResourceCounts(KernelCore& kernel);
 size_t CalculateTotalSlabHeapSize(const KernelCore& kernel);
+void InitializeKPageBufferSlabHeap(Core::System& system);
 void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout);
 
 } // namespace Kernel::Init

src/core/hle/kernel/kernel.cpp

@@ -76,7 +76,7 @@ struct KernelCore::Impl {
         // Initialize kernel memory and resources.
         InitializeSystemResourceLimit(kernel, system.CoreTiming());
         InitializeMemoryLayout();
-        InitializePageSlab();
+        Init::InitializeKPageBufferSlabHeap(system);
         InitializeSchedulers();
         InitializeSuspendThreads();
         InitializePreemption(kernel);
@@ -660,22 +660,6 @@ struct KernelCore::Impl {
                                     time_phys_addr, time_size, "Time:SharedMemory");
     }
 
-    void InitializePageSlab() {
-        // Allocate slab heaps
-        user_slab_heap_pages =
-            std::make_unique<KSlabHeap<Page>>(KSlabHeap<Page>::AllocationType::Guest);
-
-        // TODO(ameerj): This should be derived, not hardcoded within the kernel
-        constexpr u64 user_slab_heap_size{0x3de000};
-        // Reserve slab heaps
-        ASSERT(
-            system_resource_limit->Reserve(LimitableResource::PhysicalMemory, user_slab_heap_size));
-        // Initialize slab heap
-        user_slab_heap_pages->Initialize(
-            system.DeviceMemory().GetPointer(Core::DramMemoryMap::SlabHeapBase),
-            user_slab_heap_size);
-    }
-
     KClientPort* CreateNamedServicePort(std::string name) {
         auto search = service_interface_factory.find(name);
         if (search == service_interface_factory.end()) {
@@ -756,7 +740,6 @@ struct KernelCore::Impl {
 
     // Kernel memory management
     std::unique_ptr<KMemoryManager> memory_manager;
-    std::unique_ptr<KSlabHeap<Page>> user_slab_heap_pages;
 
     // Shared memory for services
     Kernel::KSharedMemory* hid_shared_mem{};
@@ -1031,14 +1014,6 @@ const KMemoryManager& KernelCore::MemoryManager() const {
     return *impl->memory_manager;
 }
 
-KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() {
-    return *impl->user_slab_heap_pages;
-}
-
-const KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() const {
-    return *impl->user_slab_heap_pages;
-}
-
 Kernel::KSharedMemory& KernelCore::GetHidSharedMem() {
     return *impl->hid_shared_mem;
 }

src/core/hle/kernel/kernel.h

@@ -43,6 +43,7 @@ class KHandleTable;
 class KLinkedListNode;
 class KMemoryLayout;
 class KMemoryManager;
+class KPageBuffer;
 class KPort;
 class KProcess;
 class KResourceLimit;
@@ -52,6 +53,7 @@ class KSession;
 class KSharedMemory;
 class KSharedMemoryInfo;
 class KThread;
+class KThreadLocalPage;
 class KTransferMemory;
 class KWorkerTaskManager;
 class KWritableEvent;
@@ -239,12 +241,6 @@ public:
     /// Gets the virtual memory manager for the kernel.
     const KMemoryManager& MemoryManager() const;
 
-    /// Gets the slab heap allocated for user space pages.
-    KSlabHeap<Page>& GetUserSlabHeapPages();
-
-    /// Gets the slab heap allocated for user space pages.
-    const KSlabHeap<Page>& GetUserSlabHeapPages() const;
-
     /// Gets the shared memory object for HID services.
     Kernel::KSharedMemory& GetHidSharedMem();
 
@@ -336,6 +332,10 @@ public:
             return slab_heap_container->writeable_event;
         } else if constexpr (std::is_same_v<T, KCodeMemory>) {
             return slab_heap_container->code_memory;
+        } else if constexpr (std::is_same_v<T, KPageBuffer>) {
+            return slab_heap_container->page_buffer;
+        } else if constexpr (std::is_same_v<T, KThreadLocalPage>) {
+            return slab_heap_container->thread_local_page;
         }
     }
 
@@ -397,6 +397,8 @@ private:
         KSlabHeap<KTransferMemory> transfer_memory;
         KSlabHeap<KWritableEvent> writeable_event;
         KSlabHeap<KCodeMemory> code_memory;
+        KSlabHeap<KPageBuffer> page_buffer;
+        KSlabHeap<KThreadLocalPage> thread_local_page;
     };
 
     std::unique_ptr<SlabHeapContainer> slab_heap_container;