From 7b32308388db815eb6caa0922c3b74a733ee1603 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 20 Dec 2022 17:19:22 -0700 Subject: [PATCH 01/43] Eliminate unnecessary setters for Thread --- System/Proc/kern_thread.cpp | 24 ------------------------ System/Proc/kern_thread.hpp | 8 +++----- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 89e3f35..8181c13 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -340,30 +340,6 @@ pantheon::Thread &pantheon::Thread::operator=(pantheon::Thread &&Other) noexcept return *this; } -void pantheon::Thread::SetKernelStackAddr(UINT64 Addr) -{ - OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("SetKernelStackAddr without lock"); - } - - this->Registers.SetSP(Addr); - this->KernelStackSpace = reinterpret_cast((CHAR*)Addr); -} - -void pantheon::Thread::SetUserStackAddr(UINT64 Addr) -{ - OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("SetUserStackAddr without lock"); - } - - this->Registers.SetSP(Addr); - this->UserStackSpace = reinterpret_cast((CHAR*)Addr); -} - void pantheon::Thread::SetProc(pantheon::Process *Proc) { OBJECT_SELF_ASSERT(); diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 238d6b1..1543984 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -104,11 +104,7 @@ class Thread : public pantheon::Object, public pantheon::Lockable VOID SetState(Thread::State State); VOID SetPriority(Thread::Priority Priority); - VOID SetEntryLocation(UINT64 IP, UINT64 SP, VOID* ThreadData); - CpuContext *GetRegisters(); - void SetUserStackAddr(UINT64 Addr); - void SetKernelStackAddr(UINT64 Addr); [[nodiscard]] pantheon::vmm::VirtualAddress GetThreadLocalAreaRegister() const { return this->LocalRegion; } Thread &operator=(const Thread &Other); @@ -125,7 +121,9 @@ class Thread : public pantheon::Object, public pantheon::Lockable void SetNext(pantheon::Thread *Item); ThreadLocalRegion *GetThreadLocalArea(); - + +private: + VOID SetEntryLocation(UINT64 IP, UINT64 SP, VOID* ThreadData); private: UINT64 TID; From ce6cd5a9496a1f556204fbc144751a9f1e9478ca Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 20 Dec 2022 22:26:09 -0700 Subject: [PATCH 02/43] Allow TLA to be read/write from userspace --- CMakeLists.txt | 2 +- System/Proc/kern_proc.hpp | 1 + System/Proc/kern_sched.cpp | 1 + System/Proc/kern_thread.cpp | 23 +++++++++++++++++++++++ System/Proc/kern_thread.hpp | 2 +- Userland/SDK/init.cpp | 7 +++++++ Userland/SDK/pantheon.h | 22 ++++++++++++++++++++++ Userland/SDK/sys.S | 2 +- Userland/sysm/sysm.cpp | 6 +++++- arch/aarch64/ints.cpp | 13 ++++++++----- 10 files changed, 70 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 074ab89..ff2fd79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ SET(PANTHEON_VERSION_MINOR 0) SET(PANTHEON_VERSION_PATCH 1) SET(POISON_MEMORY TRUE) -SET(FSANITIZE TRUE) +SET(FSANITIZE FALSE) CONFIGURE_FILE( "${CMAKE_SOURCE_DIR}/kern.h.in" diff --git a/System/Proc/kern_proc.hpp b/System/Proc/kern_proc.hpp index 2435990..ee6391e 100644 --- a/System/Proc/kern_proc.hpp +++ b/System/Proc/kern_proc.hpp @@ -95,6 +95,7 @@ class Process : public pantheon::Object, public pantheon::Lockable static const constexpr UINT64 StackPages = 16; static const constexpr pantheon::vmm::VirtualAddress StackAddr = 0xFFFFFFFFF000; + static const constexpr pantheon::vmm::VirtualAddress ThreadLocalBase = 0xFFFFFF000000; /** * @brief Obtains the process ID for this process diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index cb8993a..642c2bd 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -191,6 +191,7 @@ pantheon::Thread *pantheon::GlobalScheduler::CreateUserThreadCommon(pantheon::Pr { pantheon::Thread *T = pantheon::Thread::Create(); T->Initialize(Proc, StartAddr, ThreadData, Priority, TRUE); + T->SetupThreadLocalArea(); GlobalScheduler::ThreadList.PushFront(T); GlobalScheduler::AppendIntoReadyList(T); return GlobalScheduler::ThreadList.Front(); diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 8181c13..18ca4ed 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -427,4 +427,27 @@ pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() pantheon::vmm::PhysicalAddress PhyAddr = pantheon::vmm::VirtualToPhysicalAddress(Proc->GetPageTable(), this->LocalRegion); pantheon::vmm::VirtualAddress VirtAddr = pantheon::vmm::PhysicalToVirtualAddress(PhyAddr); return reinterpret_cast(VirtAddr); +} + +VOID pantheon::Thread::SetupThreadLocalArea() +{ + this->LocalRegion = pantheon::Process::ThreadLocalBase + (this->ThreadID() * 0x1000); + + /* Map in the TLR. */ + pantheon::vmm::PageTableEntry Entry; + Entry.SetBlock(TRUE); + Entry.SetMapped(TRUE); + Entry.SetUserNoExecute(TRUE); + Entry.SetKernelNoExecute(TRUE); + + /* Yuck. But we need to make page permissions less bad...*/ + UINT64 PagePermission = 0x1 << 6; + Entry.SetPagePermissions(PagePermission); + + Entry.SetSharable(pantheon::vmm::PAGE_SHARABLE_TYPE_INNER); + Entry.SetAccessor(pantheon::vmm::PAGE_MISC_ACCESSED); + Entry.SetMAIREntry(pantheon::vmm::MAIREntry_1); + + pantheon::ScopedLock _L(this->MyProc()); + this->MyProc()->MapAddress(this->LocalRegion, pantheon::PageAllocator::Alloc(), Entry); } \ No newline at end of file diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 1543984..4e445f3 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -89,7 +89,6 @@ class Thread : public pantheon::Object, public pantheon::Lockable void Initialize(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority, BOOL UserMode); [[nodiscard]] Process *MyProc() const; - [[nodiscard]] Thread::State MyState() const; [[nodiscard]] Thread::Priority MyPriority() const; @@ -121,6 +120,7 @@ class Thread : public pantheon::Object, public pantheon::Lockable void SetNext(pantheon::Thread *Item); ThreadLocalRegion *GetThreadLocalArea(); + VOID SetupThreadLocalArea(); private: VOID SetEntryLocation(UINT64 IP, UINT64 SP, VOID* ThreadData); diff --git a/Userland/SDK/init.cpp b/Userland/SDK/init.cpp index b474492..3ae7ded 100644 --- a/Userland/SDK/init.cpp +++ b/Userland/SDK/init.cpp @@ -7,4 +7,11 @@ extern "C" void _start() main(); svc_ExitProcess(); for (;;) {} +} + +ThreadLocalRegion *GetTLS(void) +{ + ThreadLocalRegion *RetVal = nullptr; + asm volatile ("mrs %0, tpidrro_el0\n" : "=r"(RetVal) ::); + return RetVal; } \ No newline at end of file diff --git a/Userland/SDK/pantheon.h b/Userland/SDK/pantheon.h index 0843163..2479241 100644 --- a/Userland/SDK/pantheon.h +++ b/Userland/SDK/pantheon.h @@ -24,4 +24,26 @@ extern "C" pantheon::Result svc_ReplyAndRecieve(UINT32 ContentSize, UINT32 *Repl extern "C" pantheon::Result svc_CloseHandle(INT32 Handle); extern "C" pantheon::Result svc_SendRequest(INT32 Handle); + + +typedef struct ThreadLocalHeader +{ + UINT16 Meta; + UINT16 Cmd; + UINT32 ClientPID; +}ThreadLocalHeader; + +typedef union ThreadLocalPayload +{ + UINT32 Data[1022]; +}ThreadLocalPayload; + +typedef struct ThreadLocalRegion +{ + ThreadLocalHeader Header; + ThreadLocalPayload Payload; +}ThreadLocalRegion; + +ThreadLocalRegion *GetTLS(void); + #endif \ No newline at end of file diff --git a/Userland/SDK/sys.S b/Userland/SDK/sys.S index 917af0e..cca386c 100644 --- a/Userland/SDK/sys.S +++ b/Userland/SDK/sys.S @@ -131,4 +131,4 @@ SVC_DEF svc_SendRequest mov w8, #19 svc #0 ret -SVC_END \ No newline at end of file +SVC_END diff --git a/Userland/sysm/sysm.cpp b/Userland/sysm/sysm.cpp index 98589e0..c2688bf 100644 --- a/Userland/sysm/sysm.cpp +++ b/Userland/sysm/sysm.cpp @@ -31,7 +31,11 @@ void sysm_Main() Status = svc_AcceptConnection(ServerPortRegistration, &ServerConnection); if (Status == pantheon::Result::SYS_OK) { - /* Reply with nothing, forever. */ + ThreadLocalRegion *Data = GetTLS(); + for (UINT16 Index = 0; Index < 1000; Index++) + { + Data->Payload.Data[Index] = 0xAA; + } svc_ReplyAndRecieve(0, nullptr, &ServerConnection); svc_LogText("[sysm] Replying to session"); } diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 7543d09..8b1ccfa 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -104,13 +104,16 @@ extern "C" void sync_handler_el0(pantheon::TrapFrame *Frame) pantheon::CallSyscall(SyscallNo, Frame); pantheon::CPU::CLI(); } - else if (ESR == 0x2000000) + else { SERIAL_LOG_UNSAFE("Bad sync handler el0: esr: 0x%lx far: 0x%lx elr: 0x%lx spsr: 0x%lx\n", ESR, FAR, ELR, SPSR); - SERIAL_LOG_UNSAFE("Process ID was %ld\n", pantheon::CPU::GetCurThread()->MyProc()->ProcessID()); - pantheon::vmm::PageTable *PT = pantheon::CPU::GetCurThread()->MyProc()->GetPageTable(); - pantheon::vmm::PrintPageTablesNoZeroes(PT); - } + SERIAL_LOG_UNSAFE("Process ID was %ld\n", pantheon::CPU::GetCurThread()->MyProc()->ProcessID()); + if (ESR == 0x2000000) + { + pantheon::vmm::PageTable *PT = pantheon::CPU::GetCurThread()->MyProc()->GetPageTable(); + pantheon::vmm::PrintPageTablesNoZeroes(PT); + } + } pantheon::CPU::GetCoreInfo()->CurFrame = nullptr; } From 09c47a4f9ccfbd4824ef19274a66b0da19f310d4 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 21 Dec 2022 19:33:13 -0700 Subject: [PATCH 03/43] Initial implementation of a skiplist --- Common/CMakeLists.txt | 3 + Common/Structures/kern_skiplist.hpp | 143 ++++++++++++++++++++++++++++ Common/kern_rand.cpp | 40 ++++++++ Common/kern_rand.hpp | 13 +++ System/Proc/kern_sched.cpp | 1 + 5 files changed, 200 insertions(+) create mode 100644 Common/Structures/kern_skiplist.hpp create mode 100644 Common/kern_rand.cpp create mode 100644 Common/kern_rand.hpp diff --git a/Common/CMakeLists.txt b/Common/CMakeLists.txt index 45a84b2..dfee9fb 100644 --- a/Common/CMakeLists.txt +++ b/Common/CMakeLists.txt @@ -6,6 +6,7 @@ LIST(APPEND COMMON_HEADERS kern_datatypes.hpp) LIST(APPEND COMMON_HEADERS kern_runtime.hpp) LIST(APPEND COMMON_HEADERS kern_string.hpp) LIST(APPEND COMMON_HEADERS kern_object.hpp) +LIST(APPEND COMMON_HEADERS kern_rand.hpp) LIST(APPEND COMMON_HEADERS Sync/kern_spinlock.hpp) LIST(APPEND COMMON_HEADERS Sync/kern_atomic.hpp) LIST(APPEND COMMON_HEADERS Structures/kern_optional.hpp) @@ -13,8 +14,10 @@ LIST(APPEND COMMON_HEADERS Structures/kern_bitmap.hpp) LIST(APPEND COMMON_HEADERS Structures/kern_rawbitmap.hpp) LIST(APPEND COMMON_HEADERS Structures/kern_slab.hpp) LIST(APPEND COMMON_HEADERS Structures/kern_linkedlist.hpp) +LIST(APPEND COMMON_HEADERS Structures/kern_skiplist.hpp) +LIST(APPEND COMMON_SOURCES kern_rand.cpp) LIST(APPEND COMMON_SOURCES kern_runtime.cpp) LIST(APPEND COMMON_SOURCES kern_basic_malloc.cpp) LIST(APPEND COMMON_SOURCES kern_status.cpp) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp new file mode 100644 index 0000000..b4275b8 --- /dev/null +++ b/Common/Structures/kern_skiplist.hpp @@ -0,0 +1,143 @@ +#include +#include + +#include +#include + +#ifndef _KERN_SKIPLIST_HPP_ +#define _KERN_SKIPLIST_HPP_ + +namespace pantheon +{ + +template +class SkipList +{ +public: + SkipList(UINT64 MaxLvl = 8) : Head(nullptr), MaxLevel(MaxLvl), Sz(0) {} + ~SkipList() = default; + + [[nodiscard]] UINT64 Size() const + { + return this->Sz; + } + + [[nodiscard]] BOOL Contains(K Key) const + { + return this->Get(Key).GetOkay(); + } + + [[nodiscard]] Optional Get(K Key) const + { + KVPair *Cur = this->Head; + while (Cur) + { + for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } + if (Cur->Next && Cur->Cur->Next->Key == Key) + { + return Optional(Cur->Value); + } + Cur = Cur->Down; + } + return Optional(); + } + + VOID Insert(K Key, V Value) + { + if (this->Head == nullptr) + { + this->Head = KVPair::Create(); + *this->Head = {Key, Value, nullptr, nullptr}; + return; + } + + KVPair *Cur = this->Head; + KVPair *Prev = nullptr; + + /* Iterate to the bottom */ + while (Cur) + { + for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } + Prev = Cur; + Cur = Cur->Down; + } + + /* Put a new node at the bottom */ + Cur = Prev; + UINT64 Level = this->RandLevel(); + + /* At every level, create a new KVPair */ + for (UINT64 Index = 0; Index < Level; Index++) + { + KVPair *Node = KVPair::Create(); + *Node = KVPair(Key, Value, Cur->Next, nullptr); + Cur->Next = Node; + + if (Index < Level - 1) + { + Head = Node; + Cur = Head; + } + } + this->Size++; + } + + BOOL Delete(K Key) + { + if (this->Head == nullptr) + { + return FALSE; + } + + KVPair *Cur = this->Head; + KVPair *Prev = nullptr; + + while (Cur) + { + for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } + Prev = Cur; + Cur = Cur->Down; + if (Prev->Next && Prev->Next->Key == Key) + { + Prev->Next = Cur->Next; + KVPair::Destroy(Cur); + this->Size--; + return TRUE; + } + } + return FALSE; + } + +private: + struct KVPair : public Allocatable + { + K Key; + V Value; + KVPair *Next; + KVPair *Down; + }; + + KVPair *Head; + UINT64 MaxLevel; + UINT64 Sz; + + UINT64 RandLevel() + { + int Level = 1; + + /* Rand() generates a 64-bit number. + * The probability of any one bit in particular being 0 or 1 + * is 50% for either case. We only care about one bit, + * so let's use the top bit for this. + */ + const UINT64 MAX_INT64 = 0x7FFFFFFFFFFFFFFF; + while (pantheon::Rand() < MAX_INT64 && Level < MaxLevel) + { + Level++; + } + return Level; + } +}; + +} +#endif \ No newline at end of file diff --git a/Common/kern_rand.cpp b/Common/kern_rand.cpp new file mode 100644 index 0000000..13a570a --- /dev/null +++ b/Common/kern_rand.cpp @@ -0,0 +1,40 @@ +#include +#include +#include + + +/** + * @brief An implementation of Wichmann-Hill random number generation + * @return A (cryptographically insecure) random number + */ +UINT64 pantheon::Rand() +{ + /* Arbitrary seeds. These were randomly generated. */ + static Atomic Seeds[3] = + { + 0x5648AD4DA64862EB, + 0xF7293DDAD5921465, + 0xC2743D545EDD6E10 + }; + + static const UINT64 Mults[3] = + { + 0x043186A3, + 0xA74CFF4D, + 0xDE834306, + }; + + static const UINT64 Additions[3] = + { + 0xC14A13C7025F69E3, + 0xD398EB8584DBF6d2, + 0x2C1AF76BB6FADB0E, + }; + + for (UINT8 Index = 0; Index < 3; Index++) + { + Seeds[0].Store(Mults[Index] * Seeds[Index].Load() + Additions[Index]); + } + + return Seeds[0].Load() ^ Seeds[1].Load() ^ Seeds[2].Load(); +} \ No newline at end of file diff --git a/Common/kern_rand.hpp b/Common/kern_rand.hpp new file mode 100644 index 0000000..d0dd139 --- /dev/null +++ b/Common/kern_rand.hpp @@ -0,0 +1,13 @@ +#include + +#ifndef _KERN_RAND_HPP_ +#define _KERN_RAND_HPP_ + +namespace pantheon +{ + +UINT64 Rand(); + +} + +#endif \ No newline at end of file diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 642c2bd..fa922e2 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #ifndef ONLY_TESTING From 4611de491f9ad9b02b6d771adeed5effa52d61e9 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 21 Dec 2022 20:05:56 -0700 Subject: [PATCH 04/43] Add MinKey and MaxKey --- Common/Structures/kern_skiplist.hpp | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index b4275b8..43b458c 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -42,6 +42,42 @@ class SkipList return Optional(); } + [[nodiscard]] Optional MinKey() const + { + if (this->Head == nullptr) + { + return Optional(); + } + + KVPair *Cur; + for (Cur = this->Head; Cur->Down != nullptr; Cur = Cur->Down) { } + return Optional(Cur->Key); + } + + [[nodiscard]] Optional MaxKey() const + { + if (this->Head == nullptr) + { + return Optional(); + } + + KVPair *Cur = this->Head; + while (Cur->Right || Cur->Down) + { + /* Go right as far as possible */ + if (Cur->Right) + { + Cur = Cur->Right; + } else { + /* Go down if we have to*/ + Cur = Cur->Down; + } + } + + /* If neither, we found the max value. */ + return Optional(Cur->Key); + } + VOID Insert(K Key, V Value) { if (this->Head == nullptr) From d115149c7bfc47977559448abe7fe66928b48512 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 21 Dec 2022 20:19:16 -0700 Subject: [PATCH 05/43] Convert GlobalScheduler to a namespace --- System/Proc/kern_sched.cpp | 23 ++++++++++++-------- System/Proc/kern_sched.hpp | 44 +++++++++++++------------------------- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index fa922e2..b006cc3 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -145,6 +145,20 @@ pantheon::Thread *pantheon::Scheduler::MyThread() extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); +namespace pantheon::GlobalScheduler +{ + pantheon::Atomic Okay; + pantheon::Spinlock AccessSpinlock; + + pantheon::LinkedList ProcessList; + pantheon::LinkedList ThreadList; + + pantheon::Thread *ReadyHead; + pantheon::Thread *ReadyTail; + + pantheon::Thread *CreateUserThreadCommon(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority); +} + /** * \~english @brief Creates a process, visible globally, from a name and address. * \~english @details A process is created, such that it can be run on any @@ -252,15 +266,6 @@ void pantheon::GlobalScheduler::Unlock() AccessSpinlock.Release(); } -pantheon::Atomic pantheon::GlobalScheduler::Okay; -pantheon::Spinlock pantheon::GlobalScheduler::AccessSpinlock; - -pantheon::LinkedList pantheon::GlobalScheduler::ProcessList; -pantheon::LinkedList pantheon::GlobalScheduler::ThreadList; - -pantheon::Thread *pantheon::GlobalScheduler::ReadyHead; -pantheon::Thread *pantheon::GlobalScheduler::ReadyTail; - static pantheon::Process IdleProc; VOID pantheon::GlobalScheduler::Init() { diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 300dc98..b99434a 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -33,43 +33,29 @@ class Scheduler Thread *IdleThread; }; -class GlobalScheduler +namespace GlobalScheduler { -public: - static void Init(); + void Init(); - static UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); - static pantheon::Thread *CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); + pantheon::Thread *CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - static pantheon::Thread *CreateUserThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - static pantheon::Thread *CreateUserThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + pantheon::Thread *CreateUserThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + pantheon::Thread *CreateUserThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - static UINT64 CountThreads(UINT64 PID); - static pantheon::Thread *CreateProcessorIdleThread(); + UINT64 CountThreads(UINT64 PID); + pantheon::Thread *CreateProcessorIdleThread(); - static BOOL RunProcess(UINT32 PID); - static BOOL SetState(UINT32 PID, pantheon::Process::State State); - static BOOL MapPages(UINT32 PID, pantheon::vmm::VirtualAddress *VAddresses, pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); + BOOL RunProcess(UINT32 PID); + BOOL SetState(UINT32 PID, pantheon::Process::State State); + BOOL MapPages(UINT32 PID, pantheon::vmm::VirtualAddress *VAddresses, pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); - static void AppendIntoReadyList(pantheon::Thread *Next); - static pantheon::Thread *PopFromReadyList(); + void AppendIntoReadyList(pantheon::Thread *Next); + pantheon::Thread *PopFromReadyList(); /* TODO: Inherit from Lockable... */ - static void Lock(); - static void Unlock(); - -private: - static Atomic Okay; - static Spinlock AccessSpinlock; - - static LinkedList ProcessList; - static LinkedList ThreadList; - - static Thread *ReadyHead; - static Thread *ReadyTail; - -private: - static Thread *CreateUserThreadCommon(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority); + void Lock(); + void Unlock(); }; class ScopedGlobalSchedulerLock From d3d37fb2e39974233c2304b10d51215cde308145 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 21 Dec 2022 20:42:40 -0700 Subject: [PATCH 06/43] Move unnecessary public functions into private scope --- System/Proc/kern_sched.cpp | 39 ++++++++++++++++++++++++------------- System/Proc/kern_sched.hpp | 9 +-------- System/Proc/kern_thread.hpp | 1 + 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index b006cc3..03a8113 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -34,6 +34,31 @@ static UINT64 USER_END = 0; * \~english @author Brian Schnepp */ + + +namespace pantheon::GlobalScheduler +{ + pantheon::Atomic Okay; + pantheon::Spinlock AccessSpinlock; + + pantheon::LinkedList ProcessList; + pantheon::LinkedList ThreadList; + + pantheon::Thread *ReadyHead; + pantheon::Thread *ReadyTail; + + pantheon::Thread *CreateProcessorIdleThread(); + + pantheon::Thread *CreateUserThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + pantheon::Thread *CreateUserThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + pantheon::Thread *CreateUserThreadCommon(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority); + + void AppendIntoReadyList(pantheon::Thread *Next); + pantheon::Thread *PopFromReadyList(); +} + + + /** * \~english @brief Initalizes an instance of a per-core scheduler. * \~english @author Brian Schnepp @@ -145,20 +170,6 @@ pantheon::Thread *pantheon::Scheduler::MyThread() extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); -namespace pantheon::GlobalScheduler -{ - pantheon::Atomic Okay; - pantheon::Spinlock AccessSpinlock; - - pantheon::LinkedList ProcessList; - pantheon::LinkedList ThreadList; - - pantheon::Thread *ReadyHead; - pantheon::Thread *ReadyTail; - - pantheon::Thread *CreateUserThreadCommon(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority); -} - /** * \~english @brief Creates a process, visible globally, from a name and address. * \~english @details A process is created, such that it can be run on any diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index b99434a..67a51ab 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -40,20 +41,12 @@ namespace GlobalScheduler UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); pantheon::Thread *CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - pantheon::Thread *CreateUserThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - pantheon::Thread *CreateUserThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - UINT64 CountThreads(UINT64 PID); - pantheon::Thread *CreateProcessorIdleThread(); BOOL RunProcess(UINT32 PID); BOOL SetState(UINT32 PID, pantheon::Process::State State); BOOL MapPages(UINT32 PID, pantheon::vmm::VirtualAddress *VAddresses, pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); - void AppendIntoReadyList(pantheon::Thread *Next); - pantheon::Thread *PopFromReadyList(); - - /* TODO: Inherit from Lockable... */ void Lock(); void Unlock(); }; diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 4e445f3..9e86f6a 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -64,6 +64,7 @@ class Thread : public pantheon::Object, public pantheon::Lockable STATE_INIT, STATE_RUNNING, STATE_WAITING, + STATE_BLOCKED, STATE_TERMINATED, STATE_MAX, }State; From f40c88acff0e41b4ed11b450f3d2e25b0d173d25 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 21 Dec 2022 23:05:42 -0700 Subject: [PATCH 07/43] Initial TryAcquire implementation --- Common/Sync/kern_spinlock.cpp | 24 ++++++++++++++++++++++++ Common/Sync/kern_spinlock.hpp | 1 + 2 files changed, 25 insertions(+) diff --git a/Common/Sync/kern_spinlock.cpp b/Common/Sync/kern_spinlock.cpp index 8def2f4..b58f24a 100644 --- a/Common/Sync/kern_spinlock.cpp +++ b/Common/Sync/kern_spinlock.cpp @@ -55,6 +55,30 @@ void pantheon::Spinlock::Acquire() __sync_synchronize(); } +BOOL pantheon::Spinlock::TryAcquire() +{ + if (this->DebugName == nullptr) + { + StopError("bad spinlock", this); + } + + pantheon::CPU::PUSHI(); + if (this->IsHolding()) + { + StopError(this->DebugName, this); + } + + if (__sync_lock_test_and_set(&this->Locked, TRUE) == FALSE) + { + this->CoreNo = pantheon::CPU::GetProcessorNumber(); + __sync_synchronize(); + return TRUE; + } + __sync_synchronize(); + pantheon::CPU::POPI(); + return FALSE; +} + void pantheon::Spinlock::Release() { if (!this->IsHolding()) diff --git a/Common/Sync/kern_spinlock.hpp b/Common/Sync/kern_spinlock.hpp index ba4cb34..3d6ba4f 100644 --- a/Common/Sync/kern_spinlock.hpp +++ b/Common/Sync/kern_spinlock.hpp @@ -15,6 +15,7 @@ class Spinlock void Acquire(); void Release(); + BOOL TryAcquire(); [[nodiscard]] UINT8 Holder() const; [[nodiscard]] BOOL IsLocked() const; From 25998be44e8613c589ad1a187427c55aa6a3dfa1 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 21 Dec 2022 23:22:16 -0700 Subject: [PATCH 08/43] Initial rewrite of scheduler code --- Common/Structures/kern_skiplist.hpp | 19 +- System/Exec/kern_initialprograms.cpp | 6 +- System/Proc/kern_cpu.cpp | 34 +-- System/Proc/kern_cpu.hpp | 10 +- System/Proc/kern_proc.cpp | 2 +- System/Proc/kern_sched.cpp | 440 ++++++--------------------- System/Proc/kern_sched.hpp | 46 +-- System/Proc/kern_thread.cpp | 2 +- System/Syscalls/Syscalls.cpp | 10 +- arch/aarch64/ints.cpp | 5 +- kern_main.cpp | 5 +- 11 files changed, 151 insertions(+), 428 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index 43b458c..6ae2732 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -33,7 +33,7 @@ class SkipList while (Cur) { for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } - if (Cur->Next && Cur->Cur->Next->Key == Key) + if (Cur->Next && Cur->Next->Key == Key) { return Optional(Cur->Value); } @@ -83,7 +83,10 @@ class SkipList if (this->Head == nullptr) { this->Head = KVPair::Create(); - *this->Head = {Key, Value, nullptr, nullptr}; + this->Head->Key = Key; + this->Head->Value = Value; + this->Head->Next = nullptr; + this->Head->Down = nullptr; return; } @@ -106,7 +109,11 @@ class SkipList for (UINT64 Index = 0; Index < Level; Index++) { KVPair *Node = KVPair::Create(); - *Node = KVPair(Key, Value, Cur->Next, nullptr); + Node->Key = Key; + Node->Value = Value; + Node->Next = Cur->Next; + Node->Down = nullptr; + Cur->Next = Node; if (Index < Level - 1) @@ -115,7 +122,7 @@ class SkipList Cur = Head; } } - this->Size++; + this->Sz++; } BOOL Delete(K Key) @@ -137,7 +144,7 @@ class SkipList { Prev->Next = Cur->Next; KVPair::Destroy(Cur); - this->Size--; + this->Sz--; return TRUE; } } @@ -159,7 +166,7 @@ class SkipList UINT64 RandLevel() { - int Level = 1; + UINT64 Level = 1; /* Rand() generates a 64-bit number. * The probability of any one bit in particular being 0 or 1 diff --git a/System/Exec/kern_initialprograms.cpp b/System/Exec/kern_initialprograms.cpp index 08beded..e441c22 100644 --- a/System/Exec/kern_initialprograms.cpp +++ b/System/Exec/kern_initialprograms.cpp @@ -81,7 +81,7 @@ static void RunElf(ELFFileHeader64 Header, const char *ElfLocation, UINT32 Proc, UEntry.SetAccessor(pantheon::vmm::PAGE_MISC_ACCESSED); UEntry.SetMAIREntry(pantheon::vmm::MAIREntry_1); - pantheon::GlobalScheduler::MapPages(Proc, &TargetVAddr, &NewPage, UEntry, 1); + pantheon::Scheduler::MapPages(Proc, &TargetVAddr, &NewPage, UEntry, 1); } } } @@ -100,9 +100,9 @@ static void RunExecutable(void *ElfLocation, const char *Name) } pantheon::vmm::VirtualAddress Base = pantheon::GenerateALSRBase(); - UINT32 PID = pantheon::GlobalScheduler::CreateProcess(Name, (void*)(Base + Header().e_entry)); + UINT32 PID = pantheon::Scheduler::CreateProcess(Name, (void*)(Base + Header().e_entry)); RunElf(Header(), (const char*)ElfLocation, PID, Base); - pantheon::GlobalScheduler::RunProcess(PID); + pantheon::Scheduler::RunProcess(PID); } void pantheon::UnpackInitPrograms() diff --git a/System/Proc/kern_cpu.cpp b/System/Proc/kern_cpu.cpp index 539089e..d48415e 100644 --- a/System/Proc/kern_cpu.cpp +++ b/System/Proc/kern_cpu.cpp @@ -21,40 +21,14 @@ pantheon::CPU::CoreInfo *pantheon::CPU::GetCoreInfo() return &(PerCoreInfo[pantheon::CPU::GetProcessorNumber()]); } -/** - * \~english @brief Initializes a CoreInfo structure. - * \~english @details Prepares a CoreInfo struct by initializing its - * basic variables to an idle state, signalling that it is ready to have - * a scheduler assigned to it to begin processing threads. - * - * \~english @author Brian Schnepp - */ -void pantheon::CPU::InitCoreInfo(UINT8 CoreNo) -{ - static pantheon::Scheduler Scheds[MAX_NUM_CPUS]; - ClearBuffer((CHAR*)&PerCoreInfo[CoreNo], sizeof(pantheon::CPU::CoreInfo)); - PerCoreInfo[CoreNo].CurSched = &Scheds[CoreNo]; - (*PerCoreInfo[CoreNo].CurSched) = pantheon::Scheduler(); -} - pantheon::Thread *pantheon::CPU::GetCurThread() { - pantheon::Scheduler *Sched = pantheon::CPU::GetCurSched(); - if (Sched == nullptr) - { - return nullptr; - } - return Sched->MyThread(); + return PerCoreInfo[pantheon::CPU::GetProcessorNumber()].CurThread; } pantheon::Process *pantheon::CPU::GetCurProcess() { - pantheon::Scheduler *Sched = pantheon::CPU::GetCurSched(); - if (Sched == nullptr) - { - return nullptr; - } - pantheon::Thread *CurThread = Sched->MyThread(); + pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); if (CurThread) { return CurThread->MyProc(); @@ -62,9 +36,9 @@ pantheon::Process *pantheon::CPU::GetCurProcess() return nullptr; } -pantheon::Scheduler *pantheon::CPU::GetCurSched() +UINT64 pantheon::CPU::GetJiffies() { - return pantheon::CPU::GetCoreInfo()->CurSched; + return pantheon::CPU::GetCoreInfo()->LocalJiffies; } pantheon::TrapFrame *pantheon::CPU::GetCurFrame() diff --git a/System/Proc/kern_cpu.hpp b/System/Proc/kern_cpu.hpp index 081c019..11f6593 100644 --- a/System/Proc/kern_cpu.hpp +++ b/System/Proc/kern_cpu.hpp @@ -12,7 +12,7 @@ namespace pantheon class Thread; class Process; -class Scheduler; +class LocalScheduler; namespace CPU { @@ -35,22 +35,24 @@ namespace CPU */ typedef struct CoreInfo { - pantheon::Scheduler *CurSched; + pantheon::Thread *CurThread; + pantheon::Process *CurProcess; pantheon::TrapFrame *CurFrame; + UINT64 LocalJiffies; UINT64 NOff; BOOL IntStatus; }CoreInfo; -void InitCoreInfo(UINT8 CoreNo); CoreInfo *GetCoreInfo(); UINT8 GetProcessorNumber(); pantheon::Thread *GetCurThread(); pantheon::Process *GetCurProcess(); -pantheon::Scheduler *GetCurSched(); pantheon::TrapFrame *GetCurFrame(); +UINT64 GetJiffies(); + VOID PUSHI(); VOID POPI(); UINT64 ICOUNT(); diff --git a/System/Proc/kern_proc.cpp b/System/Proc/kern_proc.cpp index 166352c..96d2763 100644 --- a/System/Proc/kern_proc.cpp +++ b/System/Proc/kern_proc.cpp @@ -63,7 +63,7 @@ void pantheon::Process::Initialize(const pantheon::ProcessCreateInfo &CreateInfo this->CurState = pantheon::Process::STATE_INIT; this->CurPriority = pantheon::Process::PRIORITY_NORMAL; this->ProcessString = CreateInfo.Name; - this->PID = pantheon::AcquireProcessID(); + this->PID = pantheon::Scheduler::AcquireProcessID(); this->TTBR0 = pantheon::PageAllocator::Alloc(); this->EntryPoint = CreateInfo.EntryPoint; this->HandTable.Clear(); diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 03a8113..14708a5 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -15,160 +15,93 @@ #include #include +#include #include #include -#ifndef ONLY_TESTING -extern "C" CHAR *USER_BEGIN; -extern "C" CHAR *USER_END; -#else -static UINT64 USER_BEGIN = 0; -static UINT64 USER_END = 0; -#endif - /** * @file Common/Proc/kern_sched.cpp - * \~english @brief Definitions for basic kernel scheduling data structures and - * algorithms. The pantheon kernel implements a basic round-robin style scheduling - * algorithm based on tick counts and a list of threads. + * \~english @brief Definitions for basic kernel scheduling data structures and algorithms. * \~english @author Brian Schnepp */ - -namespace pantheon::GlobalScheduler -{ - pantheon::Atomic Okay; - pantheon::Spinlock AccessSpinlock; - - pantheon::LinkedList ProcessList; - pantheon::LinkedList ThreadList; - - pantheon::Thread *ReadyHead; - pantheon::Thread *ReadyTail; - - pantheon::Thread *CreateProcessorIdleThread(); - - pantheon::Thread *CreateUserThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - pantheon::Thread *CreateUserThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); - pantheon::Thread *CreateUserThreadCommon(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority); - - void AppendIntoReadyList(pantheon::Thread *Next); - pantheon::Thread *PopFromReadyList(); -} - - - /** * \~english @brief Initalizes an instance of a per-core scheduler. * \~english @author Brian Schnepp */ -pantheon::Scheduler::Scheduler() -{ - this->IdleThread = pantheon::GlobalScheduler::CreateProcessorIdleThread(); - this->CurThread = this->IdleThread; -} - -pantheon::Scheduler::~Scheduler() +pantheon::LocalScheduler::LocalScheduler() : pantheon::Lockable("Scheduler") { - } +/* Provided by the arch/ subdir. These differ between hardware platforms. */ extern "C" void cpu_switch(pantheon::CpuContext *Old, pantheon::CpuContext *New, UINT64 RegOffset); +extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); -/** - * \~english @brief Changes the current thread of this core. - * \~english @details Forcefully changes the current thread by iterating to - * the next thread in the list of threads belonging to this core. The active - * count for the current thread is set to zero, and the new thread is run until - * either the core is forcefully rescheduled, or the timer indicates the current - * thread should quit. - * - * \~english @author Brian Schnepp - */ -void pantheon::Scheduler::Reschedule() -{ - OBJECT_SELF_ASSERT(); - /* Interrupts must be enabled before we can do anything. */ - if (pantheon::CPU::ICOUNT()) - { - return; - } - - GlobalScheduler::Lock(); - pantheon::Thread *Old = this->CurThread; - pantheon::Thread *New = GlobalScheduler::PopFromReadyList(); - /* If there is no next, just do the idle thread. */ - if (New == nullptr) +UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) +{ + UINT64 Res = 0; + pantheon::ScopedLock _L(this); + for (const auto &Item : this->Threads) { - New = this->IdleThread; + Res += 1ULL * (Item.MyProc()->ProcessID() == PID); } + return Res; +} - /* Don't bother trying to switching threads if we don't have to. */ - New->Lock(); - if (New == Old) +pantheon::Thread *pantheon::LocalScheduler::AcquireThread() +{ + pantheon::ScopedLock _L(this); + Optional LowestKey = this->LocalRunQueue.MinKey(); + if (!LowestKey.GetOkay()) { - New->Unlock(); - pantheon::GlobalScheduler::Unlock(); - return; + return nullptr; } - Old->Lock(); - /* If it's not currently waiting, definitely don't. */ - if (New->MyState() != pantheon::Thread::STATE_WAITING) + Optional Lowest = this->LocalRunQueue.Get(LowestKey.GetValue()); + if (!Lowest.GetOkay()) { - New->Unlock(); - Old->Unlock(); - pantheon::GlobalScheduler::Unlock(); - return; + return nullptr; } - pantheon::ScopedLocalSchedulerLock _L; - Old->SetState(pantheon::Thread::STATE_WAITING); - Old->RefreshTicks(); - - - pantheon::Process *NewProc = New->MyProc(); - pantheon::Process::Switch(NewProc); - this->CurThread = New; - this->CurThread->SetState(pantheon::Thread::STATE_RUNNING); - - pantheon::CpuContext *OldContext = Old->GetRegisters(); - pantheon::CpuContext *NewContext = New->GetRegisters(); - - /* Update the Thread Local Area register */ - pantheon::ipc::SetThreadLocalRegion(New->GetThreadLocalAreaRegister()); - - /* TODO: Make this better */ - pantheon::GlobalScheduler::AppendIntoReadyList(Old); - pantheon::GlobalScheduler::Unlock(); + this->LocalRunQueue.Delete(LowestKey.GetValue()); + return Lowest.GetValue(); - Old->Unlock(); - New->Unlock(); +} - pantheon::Sync::DSBISH(); - pantheon::Sync::ISB(); - cpu_switch(OldContext, NewContext, CpuIRegOffset); +static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInterval) +{ + return Jiffies + (PrioRatio * RRInterval); } -pantheon::Process *pantheon::Scheduler::MyProc() +/** + * \~english @brief Inserts a thread for this local scheduler + * \~english @param Thr The thread object to insert. Must be locked before calling. + */ +void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) { - OBJECT_SELF_ASSERT(); - if (this->CurThread) + pantheon::ScopedLock _L(this); + this->Threads.PushFront(Thr); + if (Thr->MyState() == pantheon::Thread::STATE_WAITING) { - return this->CurThread->MyProc(); + UINT64 Jiffies = pantheon::CPU::GetJiffies(); + UINT64 Prio = pantheon::Thread::PRIORITY_MAX - Thr->MyPriority(); + this->LocalRunQueue.Insert(CalculateDeadline(Jiffies, Prio, 6), Thr); } - return nullptr; } -pantheon::Thread *pantheon::Scheduler::MyThread() +pantheon::Spinlock SchedLock("Global Scheduler Lock"); + +void pantheon::Scheduler::Lock() { - OBJECT_SELF_ASSERT(); - return this->CurThread; + SchedLock.Acquire(); } -extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); +void pantheon::Scheduler::Unlock() +{ + SchedLock.Release(); +} /** * \~english @brief Creates a process, visible globally, from a name and address. @@ -182,151 +115,31 @@ extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); * \~english @return TRUE is the process was sucessfully created, false otherwise. * \~english @author Brian Schnepp */ -UINT32 pantheon::GlobalScheduler::CreateProcess(const pantheon::String &ProcStr, void *StartAddr) +UINT32 pantheon::Scheduler::CreateProcess(const pantheon::String &ProcStr, void *StartAddr) { - pantheon::ScopedGlobalSchedulerLock _L; - - pantheon::Process *NewProc = Process::Create(); - pantheon::ProcessCreateInfo Info = {}; - Info.Name = ProcStr; - Info.EntryPoint = (pantheon::vmm::VirtualAddress)StartAddr; - - if (NewProc == nullptr) - { - return 0; - } - - UINT32 Result = 0; - { - pantheon::ScopedLock _LL(NewProc); - NewProc->Initialize(Info); - Result = NewProc->ProcessID(); - } - - GlobalScheduler::ProcessList.PushFront(NewProc); - - if (NewProc != GlobalScheduler::ProcessList.Front()) - { - StopError("NewProc was not front."); - } - - return Result; + PANTHEON_UNUSED(ProcStr); + PANTHEON_UNUSED(StartAddr); + return 0; } -pantheon::Thread *pantheon::GlobalScheduler::CreateUserThreadCommon(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) +pantheon::Thread *pantheon::Scheduler::CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) { - pantheon::Thread *T = pantheon::Thread::Create(); - T->Initialize(Proc, StartAddr, ThreadData, Priority, TRUE); - T->SetupThreadLocalArea(); - GlobalScheduler::ThreadList.PushFront(T); - GlobalScheduler::AppendIntoReadyList(T); - return GlobalScheduler::ThreadList.Front(); -} - -pantheon::Thread *pantheon::GlobalScheduler::CreateUserThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) -{ - pantheon::ScopedGlobalSchedulerLock _L; - - pantheon::Process *SelProc = nullptr; - for (pantheon::Process &Proc : GlobalScheduler::ProcessList) - { - if (Proc.ProcessID() == PID) - { - SelProc = &Proc; - break; - } - } - - pantheon::Thread *Result = nullptr; - if (SelProc) - { - Result = GlobalScheduler::CreateUserThreadCommon(SelProc, StartAddr, ThreadData, Priority); - } - return Result; -} - -pantheon::Thread *pantheon::GlobalScheduler::CreateUserThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) -{ - pantheon::ScopedGlobalSchedulerLock _L; - pantheon::Thread *Result = pantheon::GlobalScheduler::CreateUserThreadCommon(Proc, StartAddr, ThreadData, Priority); - return Result; -} - -pantheon::Thread *pantheon::GlobalScheduler::CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) -{ - /* Assert that AccessSpinlock is locked. */ - if (GlobalScheduler::AccessSpinlock.IsLocked() == FALSE) - { - StopError("CreateThread without AccessSpinlock"); - } - - pantheon::Thread *T = pantheon::Thread::Create(); - T->Initialize(Proc, StartAddr, ThreadData, Priority, FALSE); - GlobalScheduler::ThreadList.PushFront(T); - GlobalScheduler::AppendIntoReadyList(T); - return GlobalScheduler::ThreadList.Front(); -} - -void pantheon::GlobalScheduler::Lock() -{ - AccessSpinlock.Acquire(); -} - -void pantheon::GlobalScheduler::Unlock() -{ - AccessSpinlock.Release(); -} - -static pantheon::Process IdleProc; -VOID pantheon::GlobalScheduler::Init() -{ - IdleProc = pantheon::Process(); - - GlobalScheduler::ReadyHead = nullptr; - GlobalScheduler::ReadyTail = nullptr; - - GlobalScheduler::ThreadList = LinkedList(); - GlobalScheduler::ProcessList = LinkedList(); - - GlobalScheduler::AccessSpinlock = Spinlock("access_spinlock"); - GlobalScheduler::ProcessList.PushFront(&IdleProc); - GlobalScheduler::Okay.Store(TRUE); -} - -pantheon::Thread *pantheon::GlobalScheduler::CreateProcessorIdleThread() -{ - while (!GlobalScheduler::Okay.Load()){} - - for (pantheon::Process &Proc : GlobalScheduler::ProcessList) - { - if (Proc.ProcessID() == 0) - { - /* Note the idle thread needs to have no meaningful data: it gets smashed on startup. */ - pantheon::Thread *CurThread = Thread::Create(); - CurThread->Initialize(&Proc, nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); - return CurThread; - } - } - GlobalScheduler::AccessSpinlock.Release(); + PANTHEON_UNUSED(Proc); + PANTHEON_UNUSED(StartAddr); + PANTHEON_UNUSED(ThreadData); + PANTHEON_UNUSED(Priority); return nullptr; } -UINT64 pantheon::GlobalScheduler::CountThreads(UINT64 PID) + +UINT64 pantheon::Scheduler::CountThreads(UINT64 PID) { UINT64 Count = 0; - pantheon::GlobalScheduler::Lock(); - for (const pantheon::Thread &T : GlobalScheduler::ThreadList) - { - if (T.MyProc()->ProcessID() == PID) - { - Count++; - } - } - pantheon::GlobalScheduler::Unlock(); + PANTHEON_UNUSED(PID); return Count; } -UINT32 pantheon::AcquireProcessID() +UINT32 pantheon::Scheduler::AcquireProcessID() { /* TODO: When we run out of IDs, go back and ensure we don't * reuse an ID already in use! @@ -339,7 +152,7 @@ UINT32 pantheon::AcquireProcessID() return RetVal; } -UINT64 pantheon::AcquireThreadID() +UINT64 pantheon::Scheduler::AcquireThreadID() { /* TODO: When we run out of IDs, go back and ensure we don't * reuse an ID already in use! @@ -351,9 +164,23 @@ UINT64 pantheon::AcquireThreadID() return RetVal; } -void pantheon::AttemptReschedule() +/** + * \~english @brief Changes the current thread of this core. + * \~english @details Forcefully changes the current thread by iterating to + * the next thread in the list of threads belonging to this core. The active + * count for the current thread is set to zero, and the new thread is run until + * either the core is forcefully rescheduled, or the timer indicates the current + * thread should quit. + * + * \~english @author Brian Schnepp + */ +void pantheon::Scheduler::Reschedule() +{ + +} + +void pantheon::Scheduler::AttemptReschedule() { - pantheon::Scheduler *CurSched = pantheon::CPU::GetCurSched(); pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); if (CurThread) @@ -377,7 +204,7 @@ void pantheon::AttemptReschedule() StopError("Interrupts not allowed for reschedule"); } - CurSched->Reschedule(); + pantheon::Scheduler::Reschedule(); } } @@ -386,117 +213,28 @@ extern "C" VOID FinishThread() pantheon::CPU::GetCurThread()->EnableScheduling(); } -BOOL pantheon::GlobalScheduler::MapPages(UINT32 PID, pantheon::vmm::VirtualAddress *VAddresses, pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages) +BOOL pantheon::Scheduler::MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages) { BOOL Success = FALSE; - pantheon::GlobalScheduler::Lock(); - for (pantheon::Process &Proc : GlobalScheduler::ProcessList) - { - if (Proc.ProcessID() == PID) - { - pantheon::ScopedLock L(&Proc); - for (UINT64 Index = 0; Index < NumPages; ++Index) - { - Proc.MapAddress(VAddresses[Index], PAddresses[Index], PageAttributes); - } - Success = TRUE; - break; - } - } - pantheon::GlobalScheduler::Unlock(); + PANTHEON_UNUSED(PID); + PANTHEON_UNUSED(VAddresses); + PANTHEON_UNUSED(PAddresses); + PANTHEON_UNUSED(PageAttributes); + PANTHEON_UNUSED(NumPages); return Success; } -BOOL pantheon::GlobalScheduler::RunProcess(UINT32 PID) +BOOL pantheon::Scheduler::RunProcess(UINT32 PID) { BOOL Success = FALSE; - pantheon::vmm::VirtualAddress Entry = 0x00; - pantheon::GlobalScheduler::Lock(); - for (pantheon::Process &Proc : GlobalScheduler::ProcessList) - { - if (Proc.ProcessID() == PID) - { - Proc.Lock(); - Proc.SetState(pantheon::Process::STATE_RUNNING); - Entry = Proc.GetEntryPoint(); - Success = TRUE; - Proc.Unlock(); - break; - } - } - pantheon::GlobalScheduler::Unlock(); - - if (Success) - { - pantheon::GlobalScheduler::CreateUserThread(PID, (void*)(Entry), nullptr); - } + PANTHEON_UNUSED(PID); return Success; } -BOOL pantheon::GlobalScheduler::SetState(UINT32 PID, pantheon::Process::State State) +BOOL pantheon::Scheduler::SetState(UINT32 PID, pantheon::Process::State State) { BOOL Success = FALSE; - pantheon::GlobalScheduler::Lock(); - for (pantheon::Process &Proc : GlobalScheduler::ProcessList) - { - if (Proc.ProcessID() == PID) - { - Proc.Lock(); - Proc.SetState(State); - Success = TRUE; - Proc.Unlock(); - break; - } - } - pantheon::GlobalScheduler::Unlock(); + PANTHEON_UNUSED(PID); + PANTHEON_UNUSED(State); return Success; } - -void pantheon::GlobalScheduler::AppendIntoReadyList(pantheon::Thread *Next) -{ - /* Make sure we're locked before doing this... */ - if (GlobalScheduler::AccessSpinlock.IsLocked() == FALSE) - { - StopError("Appending into readylist while not locked"); - } - - /* If this thread is null for whatever reason, don't bother. */ - if (Next == nullptr) - { - return; - } - - if (GlobalScheduler::ReadyTail) - { - GlobalScheduler::ReadyTail->SetNext(Next); - GlobalScheduler::ReadyTail = Next; - } - else - { - /* Only possible if the queue really is empty. */ - GlobalScheduler::ReadyTail = Next; - GlobalScheduler::ReadyHead = Next; - } - GlobalScheduler::ReadyTail->SetNext(nullptr); -} - -pantheon::Thread *pantheon::GlobalScheduler::PopFromReadyList() -{ - /* Make sure we're locked before doing this... */ - if (GlobalScheduler::AccessSpinlock.IsLocked() == FALSE) - { - StopError("Poping from readylist while not locked"); - } - - pantheon::Thread *Head = GlobalScheduler::ReadyHead; - if (Head) - { - GlobalScheduler::ReadyHead = GlobalScheduler::ReadyHead->Next(); - } - - if (GlobalScheduler::ReadyHead == nullptr) - { - GlobalScheduler::ReadyTail = nullptr; - } - return Head; -} \ No newline at end of file diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 67a51ab..3644292 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -5,10 +5,13 @@ #include #include +#include #include #include #include + +#include #include #ifndef _KERN_SCHED_HPP_ @@ -17,45 +20,51 @@ namespace pantheon { -class Scheduler +class LocalScheduler : public pantheon::Lockable { public: - Scheduler(); - ~Scheduler(); + LocalScheduler(); + ~LocalScheduler() override = default; + UINT64 CountThreads(UINT64 PID); - void Reschedule(); - Process *MyProc(); - Thread *MyThread(); + [[nodiscard]] UINT64 BusyRatio() const { return this->Threads.Size() / this->LocalRunQueue.Size(); } + + pantheon::Thread *AcquireThread(); + void InsertThread(pantheon::Thread *Thr); private: VOID PerformCpuSwitch(pantheon::CpuContext *Old, pantheon::CpuContext *New); - Thread *CurThread; - Thread *IdleThread; + pantheon::SkipList LocalRunQueue; + pantheon::LinkedList Threads; }; -namespace GlobalScheduler +namespace Scheduler { - void Init(); + BOOL RunProcess(UINT32 PID); + BOOL SetState(UINT32 PID, pantheon::Process::State State); + BOOL MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); pantheon::Thread *CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); UINT64 CountThreads(UINT64 PID); - BOOL RunProcess(UINT32 PID); - BOOL SetState(UINT32 PID, pantheon::Process::State State); - BOOL MapPages(UINT32 PID, pantheon::vmm::VirtualAddress *VAddresses, pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); - void Lock(); void Unlock(); + void Reschedule(); + + UINT32 AcquireProcessID(); + UINT64 AcquireThreadID(); + + void AttemptReschedule(); }; class ScopedGlobalSchedulerLock { public: - FORCE_INLINE ScopedGlobalSchedulerLock() { pantheon::GlobalScheduler::Lock(); } - FORCE_INLINE ~ScopedGlobalSchedulerLock() { pantheon::GlobalScheduler::Unlock(); } + FORCE_INLINE ScopedGlobalSchedulerLock() { pantheon::Scheduler::Lock(); } + FORCE_INLINE ~ScopedGlobalSchedulerLock() { pantheon::Scheduler::Unlock(); } }; class ScopedLocalSchedulerLock @@ -65,11 +74,6 @@ class ScopedLocalSchedulerLock FORCE_INLINE ~ScopedLocalSchedulerLock() { pantheon::CPU::GetCurThread()->EnableScheduling(); } }; -UINT32 AcquireProcessID(); -UINT64 AcquireThreadID(); - -void AttemptReschedule(); - } #endif \ No newline at end of file diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 18ca4ed..09d5669 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -64,7 +64,7 @@ pantheon::Thread::Thread(Process *OwningProcess, Priority Pri) : pantheon::Locka this->Registers.Wipe(); this->CurState = pantheon::Thread::STATE_INIT; - this->TID = AcquireThreadID(); + this->TID = pantheon::Scheduler::AcquireThreadID(); /* 45 for NORMAL, 30 for LOW, 15 for VERYLOW, etc. */ this->RefreshTicks(); diff --git a/System/Syscalls/Syscalls.cpp b/System/Syscalls/Syscalls.cpp index 56d5930..c4cd19d 100644 --- a/System/Syscalls/Syscalls.cpp +++ b/System/Syscalls/Syscalls.cpp @@ -103,7 +103,7 @@ VOID pantheon::SVCExitProcess(pantheon::TrapFrame *CurFrame) CurThread->SetState(pantheon::Thread::STATE_DEAD); CurThread->Unlock(); - pantheon::CPU::GetCoreInfo()->CurSched->Reschedule(); + pantheon::Scheduler::Reschedule(); } pantheon::Result pantheon::SVCForkProcess(pantheon::TrapFrame *CurFrame) @@ -354,13 +354,12 @@ pantheon::Result pantheon::SVCPollEvent(pantheon::TrapFrame *CurFrame) pantheon::Result pantheon::SVCYield(pantheon::TrapFrame *CurFrame) { PANTHEON_UNUSED(CurFrame); - pantheon::Scheduler *CurSched = pantheon::CPU::GetCurSched(); { pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::ScopedLock ScopeLockThread(CurThread); CurThread->SetTicks(0); } - CurSched->Reschedule(); + pantheon::Scheduler::Reschedule(); return pantheon::Result::SYS_OK; } @@ -369,18 +368,17 @@ pantheon::Result pantheon::SVCExitThread(pantheon::TrapFrame *CurFrame) PANTHEON_UNUSED(CurFrame); pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Process *CurProc = CurThread->MyProc(); - pantheon::Scheduler *CurSched = pantheon::CPU::GetCurSched(); CurProc->Lock(); CurThread->Lock(); CurThread->SetState(pantheon::Thread::STATE_TERMINATED); CurThread->Unlock(); - if (pantheon::GlobalScheduler::CountThreads(CurThread->MyProc()->ProcessID()) == 0) + if (pantheon::Scheduler::CountThreads(CurThread->MyProc()->ProcessID()) == 0) { CurProc->SetState(pantheon::Process::STATE_ZOMBIE); } CurProc->Unlock(); - CurSched->Reschedule(); + pantheon::Scheduler::Reschedule(); return pantheon::Result::SYS_OK; } diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 8b1ccfa..3934374 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -76,8 +76,11 @@ extern "C" void irq_handler_el1(pantheon::TrapFrame *Frame) pantheon::arm::GICAckInterrupt(IAR); if ((IAR & 0x3FF) == 30) { + pantheon::CPU::CoreInfo *CoreInfo = pantheon::CPU::GetCoreInfo(); + CoreInfo->LocalJiffies++; + pantheon::arm::RearmSystemTimer(); - pantheon::AttemptReschedule(); + pantheon::Scheduler::AttemptReschedule(); } pantheon::CPU::GetCoreInfo()->CurFrame = nullptr; } diff --git a/kern_main.cpp b/kern_main.cpp index ea6dfc3..42af8b2 100644 --- a/kern_main.cpp +++ b/kern_main.cpp @@ -35,7 +35,6 @@ static void kern_basic_init(InitialBootInfo *InitBootInfo) pantheon::ipc::ServerPort::Init(); pantheon::ipc::ClientPort::Init(); pantheon::ipc::Connection::Init(); - pantheon::GlobalScheduler::Init(); } static void kern_stage2_init() @@ -53,7 +52,6 @@ extern "C" void kern_init_core() /* Loop until core 0 finished essential kernel setup */ } - pantheon::CPU::InitCoreInfo(CpuNo); PerCoreInit(); while (pantheon::GetKernelStatus() < pantheon::KERNEL_STATUS_OK) @@ -69,8 +67,7 @@ extern "C" void kern_init_core() { if (CpuNo == 0) { - /* There's a few more races to deal with before enabling all-core scheduling */ - pantheon::CPU::GetCurSched()->Reschedule(); + pantheon::Scheduler::Reschedule(); } } } From 10c7a985ac0b0dd77a5f58b8d9f7934b534a325a Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 22 Dec 2022 18:57:07 -0700 Subject: [PATCH 09/43] Implement TryLock for Lockable --- Common/Sync/kern_lockable.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Common/Sync/kern_lockable.hpp b/Common/Sync/kern_lockable.hpp index e7d6dc7..db7744f 100644 --- a/Common/Sync/kern_lockable.hpp +++ b/Common/Sync/kern_lockable.hpp @@ -41,6 +41,12 @@ class Lockable this->ObjLock.Release(); } + BOOL TryLock() + { + OBJECT_SELF_ASSERT(); + return this->ObjLock.TryAcquire(); + } + [[nodiscard]] BOOL IsLocked() const { From 7f2e06e3fc5ee71872c5b45896cb51873bcb9869 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 22 Dec 2022 19:32:15 -0700 Subject: [PATCH 10/43] Associate LocalScheduler with CoreInfo --- System/Proc/kern_cpu.cpp | 22 ++++++++++++++++++++++ System/Proc/kern_cpu.hpp | 7 +++++++ System/Proc/kern_sched.cpp | 2 +- System/Proc/kern_sched.hpp | 2 +- kern_main.cpp | 1 + 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/System/Proc/kern_cpu.cpp b/System/Proc/kern_cpu.cpp index d48415e..1816200 100644 --- a/System/Proc/kern_cpu.cpp +++ b/System/Proc/kern_cpu.cpp @@ -14,8 +14,21 @@ #include /* Avoid having too high a number of cores to look through. */ +static UINT8 NoCPUs = 0; static pantheon::CPU::CoreInfo PerCoreInfo[MAX_NUM_CPUS]; +void pantheon::CPU::InitCore(UINT8 CoreNo) +{ + PerCoreInfo[CoreNo] = {}; + PerCoreInfo[CoreNo].LocalSched = pantheon::LocalScheduler::Create(); + NoCPUs++; +} + +UINT8 pantheon::CPU::GetNoCPUs() +{ + return NoCPUs; +} + pantheon::CPU::CoreInfo *pantheon::CPU::GetCoreInfo() { return &(PerCoreInfo[pantheon::CPU::GetProcessorNumber()]); @@ -46,6 +59,15 @@ pantheon::TrapFrame *pantheon::CPU::GetCurFrame() return pantheon::CPU::GetCoreInfo()->CurFrame; } +pantheon::LocalScheduler *GetLocalSched(UINT8 ProcNo) +{ + if (ProcNo >= pantheon::CPU::GetNoCPUs()) + { + return nullptr; + } + return PerCoreInfo[ProcNo].LocalSched; +} + void *pantheon::CPU::GetStackArea(UINT64 Core) { alignas(4096) static char StackArea[MAX_NUM_CPUS * DEFAULT_STACK_SIZE]; diff --git a/System/Proc/kern_cpu.hpp b/System/Proc/kern_cpu.hpp index 11f6593..762e8ba 100644 --- a/System/Proc/kern_cpu.hpp +++ b/System/Proc/kern_cpu.hpp @@ -38,11 +38,17 @@ typedef struct CoreInfo pantheon::Thread *CurThread; pantheon::Process *CurProcess; pantheon::TrapFrame *CurFrame; + pantheon::LocalScheduler *LocalSched; + + UINT64 LocalJiffies; UINT64 NOff; BOOL IntStatus; }CoreInfo; +UINT8 GetNoCPUs(); +void InitCore(UINT8 CoreNo); + CoreInfo *GetCoreInfo(); UINT8 GetProcessorNumber(); @@ -50,6 +56,7 @@ UINT8 GetProcessorNumber(); pantheon::Thread *GetCurThread(); pantheon::Process *GetCurProcess(); pantheon::TrapFrame *GetCurFrame(); +pantheon::LocalScheduler *GetLocalSched(UINT8 ProcNo); UINT64 GetJiffies(); diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 14708a5..725fe9e 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -30,7 +30,7 @@ * \~english @brief Initalizes an instance of a per-core scheduler. * \~english @author Brian Schnepp */ -pantheon::LocalScheduler::LocalScheduler() : pantheon::Lockable("Scheduler") +pantheon::LocalScheduler::LocalScheduler() : pantheon::Allocatable(), pantheon::Lockable("Scheduler") { } diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 3644292..0f16e7d 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -20,7 +20,7 @@ namespace pantheon { -class LocalScheduler : public pantheon::Lockable +class LocalScheduler : public pantheon::Allocatable, pantheon::Lockable { public: diff --git a/kern_main.cpp b/kern_main.cpp index 42af8b2..17c9be1 100644 --- a/kern_main.cpp +++ b/kern_main.cpp @@ -52,6 +52,7 @@ extern "C" void kern_init_core() /* Loop until core 0 finished essential kernel setup */ } + pantheon::CPU::InitCore(CpuNo); PerCoreInit(); while (pantheon::GetKernelStatus() < pantheon::KERNEL_STATUS_OK) From a3e73431b54cff69e308be06c12eab5e72bd9b25 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 22 Dec 2022 21:13:17 -0700 Subject: [PATCH 11/43] Initial implementation of CreateProcess under new scheduler --- Common/Structures/kern_skiplist.hpp | 18 ++++++ System/Exec/kern_initialprograms.cpp | 2 +- System/Proc/kern_cpu.cpp | 2 +- System/Proc/kern_proc.cpp | 2 +- System/Proc/kern_proc.hpp | 1 + System/Proc/kern_sched.cpp | 90 +++++++++++++++++++++------- System/Proc/kern_sched.hpp | 3 +- 7 files changed, 93 insertions(+), 25 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index 6ae2732..b98cc44 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -151,6 +151,24 @@ class SkipList return FALSE; } + + V& operator[](K Key) + { + KVPair *Cur = this->Head; + while (Cur) + { + for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } + if (Cur->Next && Cur->Next->Key == Key) + { + return Cur->Value; + } + Cur = Cur->Down; + + } + static V DefaultItem; + return DefaultItem; + } + private: struct KVPair : public Allocatable { diff --git a/System/Exec/kern_initialprograms.cpp b/System/Exec/kern_initialprograms.cpp index e441c22..fdcd17b 100644 --- a/System/Exec/kern_initialprograms.cpp +++ b/System/Exec/kern_initialprograms.cpp @@ -102,7 +102,7 @@ static void RunExecutable(void *ElfLocation, const char *Name) pantheon::vmm::VirtualAddress Base = pantheon::GenerateALSRBase(); UINT32 PID = pantheon::Scheduler::CreateProcess(Name, (void*)(Base + Header().e_entry)); RunElf(Header(), (const char*)ElfLocation, PID, Base); - pantheon::Scheduler::RunProcess(PID); + pantheon::Scheduler::SetState(PID, pantheon::Process::STATE_RUNNING); } void pantheon::UnpackInitPrograms() diff --git a/System/Proc/kern_cpu.cpp b/System/Proc/kern_cpu.cpp index 1816200..c77a596 100644 --- a/System/Proc/kern_cpu.cpp +++ b/System/Proc/kern_cpu.cpp @@ -59,7 +59,7 @@ pantheon::TrapFrame *pantheon::CPU::GetCurFrame() return pantheon::CPU::GetCoreInfo()->CurFrame; } -pantheon::LocalScheduler *GetLocalSched(UINT8 ProcNo) +pantheon::LocalScheduler *pantheon::CPU::GetLocalSched(UINT8 ProcNo) { if (ProcNo >= pantheon::CPU::GetNoCPUs()) { diff --git a/System/Proc/kern_proc.cpp b/System/Proc/kern_proc.cpp index 96d2763..dca41f3 100644 --- a/System/Proc/kern_proc.cpp +++ b/System/Proc/kern_proc.cpp @@ -63,7 +63,7 @@ void pantheon::Process::Initialize(const pantheon::ProcessCreateInfo &CreateInfo this->CurState = pantheon::Process::STATE_INIT; this->CurPriority = pantheon::Process::PRIORITY_NORMAL; this->ProcessString = CreateInfo.Name; - this->PID = pantheon::Scheduler::AcquireProcessID(); + this->PID = CreateInfo.ID; this->TTBR0 = pantheon::PageAllocator::Alloc(); this->EntryPoint = CreateInfo.EntryPoint; this->HandTable.Clear(); diff --git a/System/Proc/kern_proc.hpp b/System/Proc/kern_proc.hpp index ee6391e..eec3c4d 100644 --- a/System/Proc/kern_proc.hpp +++ b/System/Proc/kern_proc.hpp @@ -39,6 +39,7 @@ typedef enum ProcessCreateFlags typedef struct ProcessCreateInfo { + UINT32 ID; String Name; pantheon::vmm::VirtualAddress EntryPoint; diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 725fe9e..6a66d9b 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -22,6 +22,14 @@ /** * @file Common/Proc/kern_sched.cpp * \~english @brief Definitions for basic kernel scheduling data structures and algorithms. + * \~english @details Pantheon implements Con Kolivas' MuQSS scheduling algorithm, an efficient algorithm + * which takes priorities into account to provide a semi-"realtime" scheduling system to ensure + * fairness between processes. In contrast to BFS, lock contention is avoided by fairly distributing + * work between individual processors: in this case, threads are the fundamental schedulable unit, + * which is what is assigned virtual deadlines. Note that "realtime" is in quotes as to my knowledge, no definitive + * proof exists to guarantee that any given task will always meet it's virtual deadline, though algorithmic + * bounds assert that the worst case is not too bad anyway. + * @see http://ck.kolivas.org/patches/muqss/sched-MuQSS.txt * \~english @author Brian Schnepp */ @@ -39,6 +47,11 @@ extern "C" void cpu_switch(pantheon::CpuContext *Old, pantheon::CpuContext *New, extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); +/** + * \~english @brief Counts the number of threads under this scheduler which belong to a given PID. + * \~english @invariant This scheduler is not locked + * \~english @return The number of threads under this manager which belong to a given process + */ UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) { UINT64 Res = 0; @@ -50,9 +63,13 @@ UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) return Res; } +/** + * \~english @brief Obtains a thread from the local runqueue which can be run, and removes it from the local runqueue. + * \~english @invariant This scheduler is locked + * \~english @return A thread which can be run, if it exists. Nullptr otherwise. + */ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() { - pantheon::ScopedLock _L(this); Optional LowestKey = this->LocalRunQueue.MinKey(); if (!LowestKey.GetOkay()) { @@ -72,12 +89,15 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInterval) { + /* We don't necessarilly have a high resolution timer, so let's just use jiffies. */ return Jiffies + (PrioRatio * RRInterval); } /** * \~english @brief Inserts a thread for this local scheduler - * \~english @param Thr The thread object to insert. Must be locked before calling. + * \~english @invariant The thread to be inserted is locked before calling + * \~english @invariant This scheduler is not locked + * \~english @param Thr The thread object to insert. */ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) { @@ -103,6 +123,9 @@ void pantheon::Scheduler::Unlock() SchedLock.Release(); } + +static pantheon::LinkedList ProcessList; + /** * \~english @brief Creates a process, visible globally, from a name and address. * \~english @details A process is created, such that it can be run on any @@ -112,17 +135,35 @@ void pantheon::Scheduler::Unlock() * initial threaq should execute when scheduled. * \~english @param[in] ProcStr A human-readable name for the process * \~english @param[in] StartAddr The initial value of the program counter - * \~english @return TRUE is the process was sucessfully created, false otherwise. + * \~english @return New PID if the process was sucessfully created, 0 otherwise. * \~english @author Brian Schnepp */ UINT32 pantheon::Scheduler::CreateProcess(const pantheon::String &ProcStr, void *StartAddr) { - PANTHEON_UNUSED(ProcStr); - PANTHEON_UNUSED(StartAddr); - return 0; + pantheon::ScopedGlobalSchedulerLock _L; + + UINT32 NewID = pantheon::Scheduler::AcquireProcessID(); + + pantheon::Process *Proc = pantheon::Process::Create(); + if (Proc == nullptr) + { + return 0; + } + + pantheon::ScopedLock _SL(Proc); + + pantheon::ProcessCreateInfo CreateInfo{}; + CreateInfo.EntryPoint = reinterpret_cast(StartAddr); + CreateInfo.Name = ProcStr; + CreateInfo.ID = NewID; + Proc->Initialize(CreateInfo); + + ProcessList.PushFront(Proc); + + return NewID; } -pantheon::Thread *pantheon::Scheduler::CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) +pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) { PANTHEON_UNUSED(Proc); PANTHEON_UNUSED(StartAddr); @@ -134,8 +175,16 @@ pantheon::Thread *pantheon::Scheduler::CreateThread(pantheon::Process *Proc, voi UINT64 pantheon::Scheduler::CountThreads(UINT64 PID) { + pantheon::ScopedGlobalSchedulerLock _L; + UINT64 Count = 0; - PANTHEON_UNUSED(PID); + UINT8 MyCPU = pantheon::CPU::GetProcessorNumber(); + UINT8 NoCPUs = pantheon::CPU::GetNoCPUs(); + for (UINT8 Index = 0; Index < NoCPUs; Index++) + { + UINT8 LocalIndex = (MyCPU + Index) % NoCPUs; + Count += pantheon::CPU::GetLocalSched(LocalIndex)->CountThreads(PID); + } return Count; } @@ -176,7 +225,7 @@ UINT64 pantheon::Scheduler::AcquireThreadID() */ void pantheon::Scheduler::Reschedule() { - + /* NYI */ } void pantheon::Scheduler::AttemptReschedule() @@ -224,17 +273,18 @@ BOOL pantheon::Scheduler::MapPages(UINT32 PID, const pantheon::vmm::VirtualAddre return Success; } -BOOL pantheon::Scheduler::RunProcess(UINT32 PID) -{ - BOOL Success = FALSE; - PANTHEON_UNUSED(PID); - return Success; -} - BOOL pantheon::Scheduler::SetState(UINT32 PID, pantheon::Process::State State) { - BOOL Success = FALSE; - PANTHEON_UNUSED(PID); - PANTHEON_UNUSED(State); - return Success; + pantheon::ScopedGlobalSchedulerLock _L; + + for (pantheon::Process &Item : ProcessList) + { + if (Item.ProcessID() == PID) + { + pantheon::ScopedLock _L(&Item); + Item.SetState(State); + return TRUE; + } + } + return FALSE; } diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 0f16e7d..53e972e 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -41,12 +41,11 @@ class LocalScheduler : public pantheon::Allocatable, panthe namespace Scheduler { - BOOL RunProcess(UINT32 PID); BOOL SetState(UINT32 PID, pantheon::Process::State State); BOOL MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); - pantheon::Thread *CreateThread(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + pantheon::Thread *CreateThread(UINT32 Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); UINT64 CountThreads(UINT64 PID); From 8de0c8ec4b7271c2e3781b7d2774d274f8a15479 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 22 Dec 2022 21:33:01 -0700 Subject: [PATCH 12/43] Implement CreateThread for new scheduler --- System/Proc/kern_sched.cpp | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 6a66d9b..b99d910 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -125,6 +125,7 @@ void pantheon::Scheduler::Unlock() static pantheon::LinkedList ProcessList; +static pantheon::SkipList ThreadList; /** * \~english @brief Creates a process, visible globally, from a name and address. @@ -163,13 +164,30 @@ UINT32 pantheon::Scheduler::CreateProcess(const pantheon::String &ProcStr, void return NewID; } -pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) +pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) { - PANTHEON_UNUSED(Proc); - PANTHEON_UNUSED(StartAddr); - PANTHEON_UNUSED(ThreadData); - PANTHEON_UNUSED(Priority); - return nullptr; + pantheon::ScopedGlobalSchedulerLock _L; + + pantheon::Process *MyProc = nullptr; + for (pantheon::Process &Proc : ProcessList) + { + if (Proc.ProcessID() == PID) + { + MyProc = &Proc; + break; + } + } + + if(!MyProc) + { + /* Huh? You're asking for a PID that doesn't exist. */ + return nullptr; + } + + pantheon::Thread *Thr = pantheon::Thread::Create(); + Thr->Initialize(MyProc, StartAddr, ThreadData, Priority, TRUE); + ThreadList.Insert(Thr->ThreadID(), Thr); + return Thr; } @@ -264,6 +282,8 @@ extern "C" VOID FinishThread() BOOL pantheon::Scheduler::MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages) { + /* TODO: Move into a more sensible namespace like pantheon::mm or something */ + BOOL Success = FALSE; PANTHEON_UNUSED(PID); PANTHEON_UNUSED(VAddresses); From 7498af013cef5bb6ddf8fb93a295b90722f3edd2 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 22 Dec 2022 21:48:14 -0700 Subject: [PATCH 13/43] Add SetThreadState to Scheduler, re-add MapPages --- System/Proc/kern_cpu.cpp | 5 +++++ System/Proc/kern_cpu.hpp | 1 + System/Proc/kern_sched.cpp | 45 ++++++++++++++++++++++++++++++-------- System/Proc/kern_sched.hpp | 3 ++- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/System/Proc/kern_cpu.cpp b/System/Proc/kern_cpu.cpp index c77a596..d050b03 100644 --- a/System/Proc/kern_cpu.cpp +++ b/System/Proc/kern_cpu.cpp @@ -68,6 +68,11 @@ pantheon::LocalScheduler *pantheon::CPU::GetLocalSched(UINT8 ProcNo) return PerCoreInfo[ProcNo].LocalSched; } +pantheon::LocalScheduler *pantheon::CPU::GetMyLocalSched() +{ + return pantheon::CPU::GetLocalSched(pantheon::CPU::GetProcessorNumber()); +} + void *pantheon::CPU::GetStackArea(UINT64 Core) { alignas(4096) static char StackArea[MAX_NUM_CPUS * DEFAULT_STACK_SIZE]; diff --git a/System/Proc/kern_cpu.hpp b/System/Proc/kern_cpu.hpp index 762e8ba..41eaea0 100644 --- a/System/Proc/kern_cpu.hpp +++ b/System/Proc/kern_cpu.hpp @@ -57,6 +57,7 @@ pantheon::Thread *GetCurThread(); pantheon::Process *GetCurProcess(); pantheon::TrapFrame *GetCurFrame(); pantheon::LocalScheduler *GetLocalSched(UINT8 ProcNo); +pantheon::LocalScheduler *GetMyLocalSched(); UINT64 GetJiffies(); diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index b99d910..c081aeb 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -283,14 +283,20 @@ extern "C" VOID FinishThread() BOOL pantheon::Scheduler::MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages) { /* TODO: Move into a more sensible namespace like pantheon::mm or something */ - - BOOL Success = FALSE; - PANTHEON_UNUSED(PID); - PANTHEON_UNUSED(VAddresses); - PANTHEON_UNUSED(PAddresses); - PANTHEON_UNUSED(PageAttributes); - PANTHEON_UNUSED(NumPages); - return Success; + pantheon::ScopedGlobalSchedulerLock _L; + for (pantheon::Process &Item : ProcessList) + { + if (Item.ProcessID() == PID) + { + pantheon::ScopedLock _LL(&Item); + for (UINT64 Index = 0; Index < NumPages; ++Index) + { + Item.MapAddress(VAddresses[Index], PAddresses[Index], PageAttributes); + } + return TRUE; + } + } + return FALSE; } BOOL pantheon::Scheduler::SetState(UINT32 PID, pantheon::Process::State State) @@ -301,10 +307,31 @@ BOOL pantheon::Scheduler::SetState(UINT32 PID, pantheon::Process::State State) { if (Item.ProcessID() == PID) { - pantheon::ScopedLock _L(&Item); + pantheon::ScopedLock _LL(&Item); Item.SetState(State); return TRUE; } } return FALSE; } + +BOOL pantheon::Scheduler::SetThreadState(UINT64 TID, pantheon::Thread::State State) +{ + pantheon::ScopedGlobalSchedulerLock _L; + + if (!ThreadList.Contains(TID)) + { + return FALSE; + } + + pantheon::Thread *Thr = ThreadList[TID]; + pantheon::ScopedLock _LL(Thr); + + Thr->SetState(State); + + if (State == pantheon::Thread::STATE_WAITING) + { + pantheon::CPU::GetMyLocalSched()->InsertThread(Thr); + } + return TRUE; +} \ No newline at end of file diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 53e972e..238e65a 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -20,7 +20,7 @@ namespace pantheon { -class LocalScheduler : public pantheon::Allocatable, pantheon::Lockable +class LocalScheduler : public pantheon::Allocatable, public pantheon::Lockable { public: @@ -42,6 +42,7 @@ class LocalScheduler : public pantheon::Allocatable, panthe namespace Scheduler { BOOL SetState(UINT32 PID, pantheon::Process::State State); + BOOL SetThreadState(UINT64 TID, pantheon::Thread::State State); BOOL MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); From a23022d0f06ed2d2b9a241ee126bc5db9a81e81a Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 22 Dec 2022 23:14:36 -0700 Subject: [PATCH 14/43] Re-add CPU context switch --- Common/Structures/kern_skiplist.hpp | 23 +++-- System/Proc/kern_cpu.cpp | 5 + System/Proc/kern_sched.cpp | 137 ++++++++++++++++++++++++++-- System/Proc/kern_sched.hpp | 7 +- System/Proc/kern_thread.cpp | 2 +- kern_main.cpp | 2 + 6 files changed, 158 insertions(+), 18 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index b98cc44..86c2174 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -82,11 +82,12 @@ class SkipList { if (this->Head == nullptr) { - this->Head = KVPair::Create(); - this->Head->Key = Key; - this->Head->Value = Value; - this->Head->Next = nullptr; - this->Head->Down = nullptr; + KVPair *Head = new KVPair; + Head->Key = Key; + Head->Value = Value; + Head->Next = nullptr; + Head->Down = nullptr; + this->Head = Head; return; } @@ -108,7 +109,7 @@ class SkipList /* At every level, create a new KVPair */ for (UINT64 Index = 0; Index < Level; Index++) { - KVPair *Node = KVPair::Create(); + KVPair *Node = new KVPair; Node->Key = Key; Node->Value = Value; Node->Next = Cur->Next; @@ -143,7 +144,7 @@ class SkipList if (Prev->Next && Prev->Next->Key == Key) { Prev->Next = Cur->Next; - KVPair::Destroy(Cur); + delete Cur; this->Sz--; return TRUE; } @@ -170,14 +171,20 @@ class SkipList } private: - struct KVPair : public Allocatable + struct KVPair { K Key; V Value; KVPair *Next; KVPair *Down; + + /* To be implemented: this helps seeking!*/ + KVPair *Prev; + KVPair *Up; }; + static_assert(sizeof(KVPair) > 32); + KVPair *Head; UINT64 MaxLevel; UINT64 Sz; diff --git a/System/Proc/kern_cpu.cpp b/System/Proc/kern_cpu.cpp index d050b03..adb325b 100644 --- a/System/Proc/kern_cpu.cpp +++ b/System/Proc/kern_cpu.cpp @@ -22,6 +22,11 @@ void pantheon::CPU::InitCore(UINT8 CoreNo) PerCoreInfo[CoreNo] = {}; PerCoreInfo[CoreNo].LocalSched = pantheon::LocalScheduler::Create(); NoCPUs++; + + /* This is needed since kernel constructors happen to be ordered + * in a way that could cause a problem for our initialization order... + */ + PerCoreInfo[CoreNo].LocalSched->Setup(); } UINT8 pantheon::CPU::GetNoCPUs() diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index c081aeb..1c80847 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -27,19 +27,25 @@ * fairness between processes. In contrast to BFS, lock contention is avoided by fairly distributing * work between individual processors: in this case, threads are the fundamental schedulable unit, * which is what is assigned virtual deadlines. Note that "realtime" is in quotes as to my knowledge, no definitive - * proof exists to guarantee that any given task will always meet it's virtual deadline, though algorithmic - * bounds assert that the worst case is not too bad anyway. + * proof exists to guarantee that any given task will always meet it's virtual deadline, though all jobs should + * just by induction eventually all run anyway, even if they miss their deadlines. * @see http://ck.kolivas.org/patches/muqss/sched-MuQSS.txt * \~english @author Brian Schnepp */ +static pantheon::Spinlock SchedLock("Global Scheduler Lock"); +static pantheon::LinkedList ProcessList; +static pantheon::SkipList ThreadList; + + /** * \~english @brief Initalizes an instance of a per-core scheduler. * \~english @author Brian Schnepp */ pantheon::LocalScheduler::LocalScheduler() : pantheon::Allocatable(), pantheon::Lockable("Scheduler") { + this->IdleThread = nullptr; } /* Provided by the arch/ subdir. These differ between hardware platforms. */ @@ -87,6 +93,28 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() } +pantheon::Thread *pantheon::LocalScheduler::Idle() +{ + return this->IdleThread; +} + +void pantheon::LocalScheduler::Setup() +{ + pantheon::ScopedGlobalSchedulerLock _L; + + pantheon::Thread *Idle = pantheon::Thread::Create(); + for (pantheon::Process &Proc : ProcessList) + { + if (Proc.ProcessID() == 0) + { + Idle->Initialize(&Proc, nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); + } + } + + pantheon::CPU::GetCoreInfo()->CurThread = Idle; + this->IdleThread = Idle; +} + static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInterval) { /* We don't necessarilly have a high resolution timer, so let's just use jiffies. */ @@ -111,7 +139,11 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) } } -pantheon::Spinlock SchedLock("Global Scheduler Lock"); +void pantheon::Scheduler::Init() +{ + static pantheon::Process IdleProc; + ProcessList.PushFront(&IdleProc); +} void pantheon::Scheduler::Lock() { @@ -123,10 +155,6 @@ void pantheon::Scheduler::Unlock() SchedLock.Release(); } - -static pantheon::LinkedList ProcessList; -static pantheon::SkipList ThreadList; - /** * \~english @brief Creates a process, visible globally, from a name and address. * \~english @details A process is created, such that it can be run on any @@ -231,6 +259,77 @@ UINT64 pantheon::Scheduler::AcquireThreadID() return RetVal; } +static pantheon::Thread *GetNextThread() +{ + UINT8 MyCPU = pantheon::CPU::GetProcessorNumber(); + UINT8 NoCPUs = pantheon::CPU::GetNoCPUs(); + for (UINT8 Index = 0; Index < NoCPUs; Index++) + { + UINT8 LocalIndex = (MyCPU + Index) % NoCPUs; + pantheon::LocalScheduler *Sched = pantheon::CPU::GetLocalSched(LocalIndex); + if (Sched && Sched->TryLock()) + { + pantheon::Thread *Thr = Sched->AcquireThread(); + Sched->Unlock(); + if (Thr != nullptr) + { + return Thr; + } + } + } + return nullptr; +} + +struct SwapContext +{ + pantheon::CpuContext *Old; + pantheon::CpuContext *New; +}; + +static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) +{ + pantheon::Process *CurProcess = pantheon::CPU::GetCurProcess(); + + /* If this is the same thread, we've got a problem. Put it back and abort. */ + pantheon::ScopedLock _NL(NextThread); + if (NextThread == CurThread) + { + pantheon::CPU::GetMyLocalSched()->InsertThread(NextThread); + return {nullptr, nullptr}; + } + + /* If the new thread somehow isn't waiting, definitely don't run it. */ + if (NextThread->MyState() != pantheon::Thread::STATE_WAITING) + { + return {nullptr, nullptr}; + } + + pantheon::ScopedLock _OL(CurThread); + NextThread->RefreshTicks(); + + pantheon::ScopedLocalSchedulerLock _LSL; + pantheon::Scheduler::SetThreadState(CurThread->ThreadID(), pantheon::Thread::STATE_WAITING); + + /* Switch the current process */ + pantheon::Process *NextProc = NextThread->MyProc(); + if (NextProc != CurProcess) + { + pantheon::Process::Switch(NextProc); + } + + NextThread->SetState(pantheon::Thread::STATE_RUNNING); + pantheon::CPU::GetCoreInfo()->CurProcess = NextProc; + pantheon::CPU::GetCoreInfo()->CurThread = NextThread; + + pantheon::CpuContext *OldContext = CurThread->GetRegisters(); + pantheon::CpuContext *NewContext = NextThread->GetRegisters(); + + /* Switch the thread local region */ + pantheon::ipc::SetThreadLocalRegion(NextThread->GetThreadLocalAreaRegister()); + + return {OldContext, NewContext}; +} + /** * \~english @brief Changes the current thread of this core. * \~english @details Forcefully changes the current thread by iterating to @@ -243,7 +342,29 @@ UINT64 pantheon::Scheduler::AcquireThreadID() */ void pantheon::Scheduler::Reschedule() { - /* NYI */ + /* Don't reschedule if interrupts are off */ + if (pantheon::CPU::ICOUNT()) + { + return; + } + + pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); + pantheon::Thread *NextThread = GetNextThread(); + + /* If we have no next, give up and use the idle thread. */ + if (NextThread == nullptr) + { + NextThread = pantheon::CPU::GetMyLocalSched()->Idle(); + } + + SwapContext Con = SwapThreads(CurThread, NextThread); + + if (Con.New && Con.Old) + { + pantheon::Sync::DSBISH(); + pantheon::Sync::ISB(); + cpu_switch(Con.Old, Con.New, CpuIRegOffset); + } } void pantheon::Scheduler::AttemptReschedule() diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 238e65a..0559ffb 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -33,14 +33,19 @@ class LocalScheduler : public pantheon::Allocatable, public pantheon::Thread *AcquireThread(); void InsertThread(pantheon::Thread *Thr); + void Setup(); + pantheon::Thread *Idle(); + private: - VOID PerformCpuSwitch(pantheon::CpuContext *Old, pantheon::CpuContext *New); pantheon::SkipList LocalRunQueue; pantheon::LinkedList Threads; + + pantheon::Thread *IdleThread; }; namespace Scheduler { + void Init(); BOOL SetState(UINT32 PID, pantheon::Process::State State); BOOL SetThreadState(UINT64 TID, pantheon::Thread::State State); BOOL MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 09d5669..d202f60 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -233,7 +233,7 @@ VOID pantheon::Thread::RefreshTicks() { StopError("RefreshTicks without lock"); } - this->RemainingTicks = static_cast((this->CurPriority + 1)) * 3; + this->RemainingTicks = 6; } /** diff --git a/kern_main.cpp b/kern_main.cpp index 17c9be1..b26f011 100644 --- a/kern_main.cpp +++ b/kern_main.cpp @@ -35,6 +35,8 @@ static void kern_basic_init(InitialBootInfo *InitBootInfo) pantheon::ipc::ServerPort::Init(); pantheon::ipc::ClientPort::Init(); pantheon::ipc::Connection::Init(); + pantheon::Scheduler::Init(); + pantheon::LocalScheduler::Init(); } static void kern_stage2_init() From b218aa7a31be6e0ac5978bed4cc925e392248617 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Fri, 23 Dec 2022 15:53:28 -0700 Subject: [PATCH 15/43] Re-do SkipList to avoid weird linked lists for levels --- Common/Structures/kern_skiplist.hpp | 263 ++++++++++++++++++---------- 1 file changed, 172 insertions(+), 91 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index 86c2174..0ebf896 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -10,12 +10,21 @@ namespace pantheon { -template +template class SkipList { public: - SkipList(UINT64 MaxLvl = 8) : Head(nullptr), MaxLevel(MaxLvl), Sz(0) {} - ~SkipList() = default; + SkipList() + { + this->Head = nullptr; + this->Level = 0; + this->Sz = 0; + } + + ~SkipList() + { + /* NYI */ + } [[nodiscard]] UINT64 Size() const { @@ -29,17 +38,31 @@ class SkipList [[nodiscard]] Optional Get(K Key) const { - KVPair *Cur = this->Head; - while (Cur) + if (this->Head == nullptr) + { + return Optional(); + } + + INT16 Lvl = this->Level; + KVPair *Current = this->Head; + KVPair *Next = Current; + while (Lvl >= 0) { - for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } - if (Cur->Next && Cur->Next->Key == Key) + Next = Current->Next[Lvl]; + while (Next && (Next->Key <= Key)) { - return Optional(Cur->Value); + Current = Next; + Next = Current->Next[Lvl]; } - Cur = Cur->Down; + Lvl--; + } + + if (Current->Key == Key) + { + return Optional(Current->Value); } return Optional(); + } [[nodiscard]] Optional MinKey() const @@ -49,9 +72,12 @@ class SkipList return Optional(); } - KVPair *Cur; - for (Cur = this->Head; Cur->Down != nullptr; Cur = Cur->Down) { } - return Optional(Cur->Key); + KVPair *Current = this->Head->Next[0]; + if (Current && Current->Key != (K() - 1)) + { + return Optional(Current->Key); + } + return Optional(); } [[nodiscard]] Optional MaxKey() const @@ -61,93 +87,127 @@ class SkipList return Optional(); } - KVPair *Cur = this->Head; - while (Cur->Right || Cur->Down) + KVPair *Current = this->Head->Prev[0]; + if (Current && Current->Key != (K() - 1)) { - /* Go right as far as possible */ - if (Cur->Right) - { - Cur = Cur->Right; - } else { - /* Go down if we have to*/ - Cur = Cur->Down; - } + return Optional(Current->Key); } - - /* If neither, we found the max value. */ - return Optional(Cur->Key); + return Optional(); } VOID Insert(K Key, V Value) { - if (this->Head == nullptr) + this->Setup(); + + KVPair *Update[MaxLvl]; + INT16 Lvl = this->Level; + + KVPair *Current = this->Head; + KVPair *Next = Current; + while (Lvl >= 0) { - KVPair *Head = new KVPair; - Head->Key = Key; - Head->Value = Value; - Head->Next = nullptr; - Head->Down = nullptr; - this->Head = Head; - return; + Next = Current->Next[Lvl]; + while (Next && (Next->Key <= Key)) + { + Current = Next; + Next = Current->Next[Lvl]; + } + Update[Lvl] = Current; + Lvl--; } - KVPair *Cur = this->Head; - KVPair *Prev = nullptr; + /* This is where we'll need to be: this needs to match the current key. */ + if (Current->Key == Key) + { + Current->Value = Value; + return; + } - /* Iterate to the bottom */ - while (Cur) + /* Actually make the new entry */ + INT16 NewLvl = this->RandLevel(); + if (NewLvl > this->Level) { - for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } - Prev = Cur; - Cur = Cur->Down; + NewLvl = ++(this->Level); + Update[NewLvl] = this->Head; } - /* Put a new node at the bottom */ - Cur = Prev; - UINT64 Level = this->RandLevel(); + KVPair *NewNode = new KVPair(); + NewNode->Key = Key; + NewNode->Value = Value; + NewNode->Level = NewLvl; + for (UINT8 Index = 0; Index < MaxLvl; Index++) + { + NewNode->Next[Index] = nullptr; + NewNode->Prev[Index] = nullptr; + } - /* At every level, create a new KVPair */ - for (UINT64 Index = 0; Index < Level; Index++) + while (NewLvl > 0) { - KVPair *Node = new KVPair; - Node->Key = Key; - Node->Value = Value; - Node->Next = Cur->Next; - Node->Down = nullptr; + Current = Update[NewLvl]; - Cur->Next = Node; + NewNode->Next[NewLvl] = Current->Next[NewLvl]; + Current->Next[NewLvl] = NewNode; - if (Index < Level - 1) - { - Head = Node; - Cur = Head; - } + NewNode->Prev[NewLvl] = Current; + NewNode->Next[NewLvl]->Prev[NewLvl] = NewNode; + NewLvl--; } + this->Sz++; + } BOOL Delete(K Key) { - if (this->Head == nullptr) + this->Setup(); + + KVPair *Update[MaxLvl]; + INT16 Lvl = this->Level; + + KVPair *Current = this->Head; + KVPair *Next = Current; + while (Lvl >= 0) { - return FALSE; + Next = Current->Next[Lvl]; + while (Next && (Next->Key <= Key)) + { + Current = Next; + Next = Current->Next[Lvl]; + } + Update[Lvl] = Current; + Lvl--; } - KVPair *Cur = this->Head; - KVPair *Prev = nullptr; - - while (Cur) + /* This is where we'll need to be: this needs to match the current key. */ + if (Current->Key == Key) { - for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } - Prev = Cur; - Cur = Cur->Down; - if (Prev->Next && Prev->Next->Key == Key) + UINT8 CurLvl = Current->Level; + for (UINT8 Lvl = 0; Lvl < CurLvl; Lvl++) + { + Current->Prev[Lvl]->Next[Lvl] = Current->Next[Lvl]; + Current->Next[Lvl]->Prev[Lvl] = Current->Prev[Lvl]; + } + + /* If this was top, we might have to fix levels. */ + if (CurLvl == this->Level) { - Prev->Next = Cur->Next; - delete Cur; - this->Sz--; - return TRUE; + while (this->Head->Next[CurLvl] == this->Head && this->Head->Prev[CurLvl] == this->Head) + { + if (CurLvl > 0) + { + CurLvl--; + } else + { + break; + } + + } + this->Level = CurLvl; } + + this->Sz--; + delete Current; + return TRUE; } return FALSE; } @@ -155,16 +215,25 @@ class SkipList V& operator[](K Key) { - KVPair *Cur = this->Head; - while (Cur) + this->Setup(); + + INT16 Lvl = this->Level; + KVPair *Current = this->Head; + KVPair *Next = Current; + while (Lvl >= 0) { - for (; Cur->Next && Cur->Next->Key < Key; Cur = Cur->Next) { } - if (Cur->Next && Cur->Next->Key == Key) + Next = Current->Next[Lvl]; + while (Next && (Next->Key <= Key)) { - return Cur->Value; + Current = Next; + Next = Current->Next[Lvl]; } - Cur = Cur->Down; - + Lvl--; + } + + if (Current->Key == Key) + { + return Current->Value; } static V DefaultItem; return DefaultItem; @@ -175,35 +244,47 @@ class SkipList { K Key; V Value; - KVPair *Next; - KVPair *Down; - - /* To be implemented: this helps seeking!*/ - KVPair *Prev; - KVPair *Up; + UINT64 Level; + KVPair *Next[MaxLvl]; + KVPair *Prev[MaxLvl]; }; static_assert(sizeof(KVPair) > 32); KVPair *Head; - UINT64 MaxLevel; + INT16 Level; UINT64 Sz; - UINT64 RandLevel() + UINT8 RandLevel() { - UINT64 Level = 1; + UINT8 Level = 1; /* Rand() generates a 64-bit number. * The probability of any one bit in particular being 0 or 1 - * is 50% for either case. We only care about one bit, - * so let's use the top bit for this. + * is 25% for either case. We only care about two bits, + * so let's use the bottom 2 bits for this. */ - const UINT64 MAX_INT64 = 0x7FFFFFFFFFFFFFFF; - while (pantheon::Rand() < MAX_INT64 && Level < MaxLevel) + while ((pantheon::Rand() & 0x3) && Level < MaxLvl) { Level++; } - return Level; + return Level % MaxLvl; + } + + void Setup() + { + if (this->Head == nullptr) + { + this->Head = new KVPair(); + this->Level = 0; + this->Head->Key = K() - 1; /* HACK: Sets this at a huge value for integers only */ + this->Head->Value = V(); + for (UINT8 Lvl = 0; Lvl < MaxLvl; Lvl++) + { + this->Head->Next[Lvl] = this->Head; + this->Head->Prev[Lvl] = this->Head; + } + } } }; From f5b09929394376a1ac19037ede6ec862a1fe415a Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Fri, 23 Dec 2022 17:14:22 -0700 Subject: [PATCH 16/43] Use jiffies in Rand() to help avoid prediction --- Common/Sync/kern_atomic.hpp | 73 +++++++++++++++++++++++++++++++++++++ Common/kern_rand.cpp | 27 +++++++++++--- 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/Common/Sync/kern_atomic.hpp b/Common/Sync/kern_atomic.hpp index 7fe1b0f..f1ae904 100644 --- a/Common/Sync/kern_atomic.hpp +++ b/Common/Sync/kern_atomic.hpp @@ -69,7 +69,80 @@ class Atomic private: T Content; +}; + +template +class AtomicInteger +{ + +public: + AtomicInteger() + { + this->Content = T(); + } + AtomicInteger(T Item) + { + this->Store(Item); + } + + ~AtomicInteger() = default; + + [[nodiscard]] + T Load() const + { + T RetVal; + #ifndef ONLY_TESTS + __atomic_load(&(this->Content), &RetVal, __ATOMIC_SEQ_CST); + #else + RetVal = this->Content; + #endif + return RetVal; + } + + void Store(T Item) + { + #ifndef ONLY_TESTS + __atomic_store(&(this->Content), &Item, __ATOMIC_SEQ_CST); + #else + this->Content = Item; + #endif + } + + T Add(T Item) + { + #ifndef ONLY_TESTS + return __atomic_fetch_add(&(this->Content), Item, __ATOMIC_SEQ_CST); + #else + this->Content += Item; + return this->Content; + #endif + } + + [[nodiscard]] + bool operator==(const Atomic &Other) const + { + return this->Load() == Other.Load(); + } + + [[nodiscard]] + bool operator==(const T &Other) const + { + return this->Load() == Other; + } + + void operator=(const T &Thing) + { + this->Store(Thing); + } + + void operator=(T &&Thing) + { + this->Store(Thing); + } + +private: + T Content; }; } diff --git a/Common/kern_rand.cpp b/Common/kern_rand.cpp index 13a570a..ea51c40 100644 --- a/Common/kern_rand.cpp +++ b/Common/kern_rand.cpp @@ -1,7 +1,10 @@ #include + +#include + #include #include - +#include /** * @brief An implementation of Wichmann-Hill random number generation @@ -9,8 +12,13 @@ */ UINT64 pantheon::Rand() { + /* Do something about this at some point... */ + static pantheon::Spinlock RNGLock("Random Number Lock"); + + RNGLock.Acquire(); + /* Arbitrary seeds. These were randomly generated. */ - static Atomic Seeds[3] = + static UINT64 Seeds[3] = { 0x5648AD4DA64862EB, 0xF7293DDAD5921465, @@ -27,14 +35,21 @@ UINT64 pantheon::Rand() static const UINT64 Additions[3] = { 0xC14A13C7025F69E3, - 0xD398EB8584DBF6d2, + 0xD398EB8584DBF6D2, 0x2C1AF76BB6FADB0E, }; - for (UINT8 Index = 0; Index < 3; Index++) + /* Use jiffies to help make this more random */ + UINT64 Jiffies = pantheon::CPU::GetJiffies() % 16; + for (UINT64 Counter = 0; Counter < Jiffies; Counter++) { - Seeds[0].Store(Mults[Index] * Seeds[Index].Load() + Additions[Index]); + for (UINT8 Index = 0; Index < 3; Index++) + { + Seeds[Index] = Jiffies + (Mults[Index] * Seeds[Index] + Additions[Index]); + } } - return Seeds[0].Load() ^ Seeds[1].Load() ^ Seeds[2].Load(); + UINT64 Result = Seeds[0] ^ Seeds[1] ^ Seeds[2]; + RNGLock.Release(); + return Result; } \ No newline at end of file From 2c9edbd770e5f9aa4da0a563fadb4d1f4e3615cb Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sat, 24 Dec 2022 22:45:16 -0700 Subject: [PATCH 17/43] Cleanup unnecessary scheduling routines --- System/Proc/kern_sched.cpp | 81 +++++++++++++++++++++---------------- System/Proc/kern_sched.hpp | 8 ++-- System/Proc/kern_thread.cpp | 24 ++--------- System/Proc/kern_thread.hpp | 7 ++-- 4 files changed, 56 insertions(+), 64 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 1c80847..985d693 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -118,7 +118,15 @@ void pantheon::LocalScheduler::Setup() static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInterval) { /* We don't necessarilly have a high resolution timer, so let's just use jiffies. */ - return Jiffies + (PrioRatio * RRInterval); + UINT64 Deadline = Jiffies + (PrioRatio * RRInterval); + + /* However, we have to ban -1, since we're using a SkipList. */ + if ((Deadline & ~0ULL) == ~0ULL) + { + Deadline = 0; + } + + return Deadline; } /** @@ -129,6 +137,11 @@ static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInter */ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) { + if (Thr == nullptr) + { + return; + } + pantheon::ScopedLock _L(this); this->Threads.PushFront(Thr); if (Thr->MyState() == pantheon::Thread::STATE_WAITING) @@ -252,10 +265,16 @@ UINT64 pantheon::Scheduler::AcquireThreadID() /* TODO: When we run out of IDs, go back and ensure we don't * reuse an ID already in use! */ - UINT32 RetVal = 0; + UINT64 RetVal = 0; static UINT64 ThreadID = 0; /* A copy has to be made since we haven't unlocked the spinlock yet. */ RetVal = ThreadID++; + + /* Ban 0 and -1, since these are special values. */ + if (RetVal == 0 || RetVal == ~0ULL) + { + return pantheon::Scheduler::AcquireThreadID(); + } return RetVal; } @@ -267,8 +286,9 @@ static pantheon::Thread *GetNextThread() { UINT8 LocalIndex = (MyCPU + Index) % NoCPUs; pantheon::LocalScheduler *Sched = pantheon::CPU::GetLocalSched(LocalIndex); - if (Sched && Sched->TryLock()) + if (Sched) { + Sched->Lock(); /* This needs to be TryAcquire at some point. */ pantheon::Thread *Thr = Sched->AcquireThread(); Sched->Unlock(); if (Thr != nullptr) @@ -288,44 +308,45 @@ struct SwapContext static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { - pantheon::Process *CurProcess = pantheon::CPU::GetCurProcess(); + /* If we have no next, give up and use the idle thread. */ + if (NextThread == nullptr) + { + NextThread = pantheon::CPU::GetMyLocalSched()->Idle(); + } - /* If this is the same thread, we've got a problem. Put it back and abort. */ pantheon::ScopedLock _NL(NextThread); + + /* Don't try to swap to ourselves. */ if (NextThread == CurThread) { - pantheon::CPU::GetMyLocalSched()->InsertThread(NextThread); return {nullptr, nullptr}; } - /* If the new thread somehow isn't waiting, definitely don't run it. */ + pantheon::ScopedLock _OL(CurThread); + if (NextThread->MyState() != pantheon::Thread::STATE_WAITING) { - return {nullptr, nullptr}; + /* Huh? Why are we still here? */ + pantheon::StopErrorFmt("Thread %lx was on the runqueue, but wasn't waiting\n", NextThread->MyState()); } - pantheon::ScopedLock _OL(CurThread); - NextThread->RefreshTicks(); - - pantheon::ScopedLocalSchedulerLock _LSL; - pantheon::Scheduler::SetThreadState(CurThread->ThreadID(), pantheon::Thread::STATE_WAITING); + /* Okay, we have to do a weird little dance. We need this to prevent + * trying to reschedule and trip on ourselves. This gets restored + * when we context switch back here. + */ + pantheon::CPU::GetCurThread()->BlockScheduling(); - /* Switch the current process */ - pantheon::Process *NextProc = NextThread->MyProc(); - if (NextProc != CurProcess) - { - pantheon::Process::Switch(NextProc); - } + /* Change contexts */ + pantheon::Process::Switch(NextThread->MyProc()); + pantheon::ipc::SetThreadLocalRegion(NextThread->GetThreadLocalAreaRegister()); - NextThread->SetState(pantheon::Thread::STATE_RUNNING); - pantheon::CPU::GetCoreInfo()->CurProcess = NextProc; - pantheon::CPU::GetCoreInfo()->CurThread = NextThread; + pantheon::Scheduler::SetThreadState(CurThread->ThreadID(), pantheon::Thread::STATE_WAITING); + pantheon::Scheduler::SetThreadState(NextThread->ThreadID(), pantheon::Thread::STATE_RUNNING); pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - /* Switch the thread local region */ - pantheon::ipc::SetThreadLocalRegion(NextThread->GetThreadLocalAreaRegister()); + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); return {OldContext, NewContext}; } @@ -351,12 +372,6 @@ void pantheon::Scheduler::Reschedule() pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); - /* If we have no next, give up and use the idle thread. */ - if (NextThread == nullptr) - { - NextThread = pantheon::CPU::GetMyLocalSched()->Idle(); - } - SwapContext Con = SwapThreads(CurThread, NextThread); if (Con.New && Con.Old) @@ -364,6 +379,7 @@ void pantheon::Scheduler::Reschedule() pantheon::Sync::DSBISH(); pantheon::Sync::ISB(); cpu_switch(Con.Old, Con.New, CpuIRegOffset); + pantheon::CPU::GetCurThread()->EnableScheduling(); } } @@ -449,10 +465,5 @@ BOOL pantheon::Scheduler::SetThreadState(UINT64 TID, pantheon::Thread::State Sta pantheon::ScopedLock _LL(Thr); Thr->SetState(State); - - if (State == pantheon::Thread::STATE_WAITING) - { - pantheon::CPU::GetMyLocalSched()->InsertThread(Thr); - } return TRUE; } \ No newline at end of file diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 0559ffb..7f12708 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -28,8 +28,6 @@ class LocalScheduler : public pantheon::Allocatable, public ~LocalScheduler() override = default; UINT64 CountThreads(UINT64 PID); - [[nodiscard]] UINT64 BusyRatio() const { return this->Threads.Size() / this->LocalRunQueue.Size(); } - pantheon::Thread *AcquireThread(); void InsertThread(pantheon::Thread *Thr); @@ -72,11 +70,11 @@ class ScopedGlobalSchedulerLock FORCE_INLINE ~ScopedGlobalSchedulerLock() { pantheon::Scheduler::Unlock(); } }; -class ScopedLocalSchedulerLock +class ScopedRescheduleLock { public: - FORCE_INLINE ScopedLocalSchedulerLock() { pantheon::CPU::GetCurThread()->BlockScheduling(); } - FORCE_INLINE ~ScopedLocalSchedulerLock() { pantheon::CPU::GetCurThread()->EnableScheduling(); } + FORCE_INLINE ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->BlockScheduling(); } + FORCE_INLINE ~ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->EnableScheduling(); } }; } diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index d202f60..d92a19e 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -212,20 +212,6 @@ UINT64 pantheon::Thread::ThreadID() const return this->TID; } -/** - * \~english @brief Forcefully adds execution time to the current process. - * \~english @author Brian Schnepp - */ -VOID pantheon::Thread::AddTicks(UINT64 TickCount) -{ - OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("AddTicks without lock"); - } - this->RemainingTicks.Store(this->RemainingTicks.Load() + TickCount); -} - VOID pantheon::Thread::RefreshTicks() { OBJECT_SELF_ASSERT(); @@ -250,14 +236,14 @@ VOID pantheon::Thread::SetState(Thread::State St) this->CurState = St; } -VOID pantheon::Thread::SetTicks(UINT64 TickCount) +VOID pantheon::Thread::SetTicks(INT64 TickCount) { OBJECT_SELF_ASSERT(); if (this->IsLocked() == FALSE) { StopError("SetTicks without lock"); } - this->RemainingTicks = TickCount; + this->RemainingTicks = (TickCount < 0) ? 0 : TickCount; } /** @@ -354,18 +340,16 @@ void pantheon::Thread::BlockScheduling() { OBJECT_SELF_ASSERT(); pantheon::Sync::DSBISH(); - this->PreemptCount.Store(this->PreemptCount.Load() + 1); + this->PreemptCount.Add(1LL); pantheon::Sync::DSBISH(); } void pantheon::Thread::EnableScheduling() { OBJECT_SELF_ASSERT(); - this->Lock(); pantheon::Sync::DSBISH(); - this->PreemptCount.Store(this->PreemptCount.Load() - 1); + this->PreemptCount.Add(-1LL); pantheon::Sync::DSBISH(); - this->Unlock(); } extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 9e86f6a..deaf0c7 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -96,10 +96,9 @@ class Thread : public pantheon::Object, public pantheon::Lockable [[nodiscard]] UINT64 TicksLeft() const; [[nodiscard]] UINT64 ThreadID() const; - VOID AddTicks(UINT64 TickCount); VOID CountTick(); VOID RefreshTicks(); - VOID SetTicks(UINT64 TickCount); + VOID SetTicks(INT64 TickCount); VOID SetState(Thread::State State); VOID SetPriority(Thread::Priority Priority); @@ -135,8 +134,8 @@ class Thread : public pantheon::Object, public pantheon::Lockable Thread::State CurState; Thread::Priority CurPriority; - pantheon::Atomic PreemptCount; - pantheon::Atomic RemainingTicks; + pantheon::AtomicInteger PreemptCount; + pantheon::AtomicInteger RemainingTicks; void *KernelStackSpace; void *UserStackSpace; From 3fa52c9f19d5850908633b3c92ac9baf57f78139 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sat, 24 Dec 2022 23:58:28 -0700 Subject: [PATCH 18/43] Allow entry back into userland programs again --- Common/Structures/kern_skiplist.hpp | 2 +- System/Exec/kern_initialprograms.cpp | 3 +++ System/Proc/kern_sched.cpp | 9 ++++++--- System/Proc/kern_sched.hpp | 1 - kern_main.cpp | 29 ++++++++++++++++------------ 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index 0ebf896..f9f031b 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -141,7 +141,7 @@ class SkipList NewNode->Prev[Index] = nullptr; } - while (NewLvl > 0) + while (NewLvl >= 0) { Current = Update[NewLvl]; diff --git a/System/Exec/kern_initialprograms.cpp b/System/Exec/kern_initialprograms.cpp index fdcd17b..32c7c8d 100644 --- a/System/Exec/kern_initialprograms.cpp +++ b/System/Exec/kern_initialprograms.cpp @@ -103,6 +103,9 @@ static void RunExecutable(void *ElfLocation, const char *Name) UINT32 PID = pantheon::Scheduler::CreateProcess(Name, (void*)(Base + Header().e_entry)); RunElf(Header(), (const char*)ElfLocation, PID, Base); pantheon::Scheduler::SetState(PID, pantheon::Process::STATE_RUNNING); + + /* Create our initial thread */ + pantheon::Scheduler::CreateThread(PID, (void*)(Base + Header().e_entry), nullptr); } void pantheon::UnpackInitPrograms() diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 985d693..b32d04d 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -228,6 +228,9 @@ pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, pantheon::Thread *Thr = pantheon::Thread::Create(); Thr->Initialize(MyProc, StartAddr, ThreadData, Priority, TRUE); ThreadList.Insert(Thr->ThreadID(), Thr); + + pantheon::ScopedLock _STL(Thr); + pantheon::CPU::GetMyLocalSched()->InsertThread(Thr); return Thr; } @@ -327,7 +330,7 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne if (NextThread->MyState() != pantheon::Thread::STATE_WAITING) { /* Huh? Why are we still here? */ - pantheon::StopErrorFmt("Thread %lx was on the runqueue, but wasn't waiting\n", NextThread->MyState()); + return {nullptr, nullptr}; } /* Okay, we have to do a weird little dance. We need this to prevent @@ -340,8 +343,8 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne pantheon::Process::Switch(NextThread->MyProc()); pantheon::ipc::SetThreadLocalRegion(NextThread->GetThreadLocalAreaRegister()); - pantheon::Scheduler::SetThreadState(CurThread->ThreadID(), pantheon::Thread::STATE_WAITING); - pantheon::Scheduler::SetThreadState(NextThread->ThreadID(), pantheon::Thread::STATE_RUNNING); + CurThread->SetState(pantheon::Thread::STATE_WAITING); + NextThread->SetState(pantheon::Thread::STATE_RUNNING); pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 7f12708..f17835b 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -37,7 +37,6 @@ class LocalScheduler : public pantheon::Allocatable, public private: pantheon::SkipList LocalRunQueue; pantheon::LinkedList Threads; - pantheon::Thread *IdleThread; }; diff --git a/kern_main.cpp b/kern_main.cpp index b26f011..c14d825 100644 --- a/kern_main.cpp +++ b/kern_main.cpp @@ -45,7 +45,7 @@ static void kern_stage2_init() pantheon::UnpackInitPrograms(); } -extern "C" void kern_init_core() +static void kern_init_core() { UINT8 CpuNo = pantheon::CPU::GetProcessorNumber(); @@ -56,26 +56,20 @@ extern "C" void kern_init_core() pantheon::CPU::InitCore(CpuNo); PerCoreInit(); +} +static void kern_stage2_init_core() +{ + UINT8 CpuNo = pantheon::CPU::GetProcessorNumber(); while (pantheon::GetKernelStatus() < pantheon::KERNEL_STATUS_OK) { /* Loop until core 0 finished kernel setup */ } - SERIAL_LOG("Pantheon booted with core %hhu\n", CpuNo); pantheon::RearmSystemTimer(1000); pantheon::CPU::STI(); - - for (;;) - { - if (CpuNo == 0) - { - pantheon::Scheduler::Reschedule(); - } - } } - extern "C" void kern_init(InitialBootInfo *InitBootInfo) { if (pantheon::CPU::GetProcessorNumber() == 0) @@ -84,9 +78,20 @@ extern "C" void kern_init(InitialBootInfo *InitBootInfo) BoardRuntimeInit(); kern_basic_init(InitBootInfo); pantheon::SetKernelStatus(pantheon::KERNEL_STATUS_SECOND_STAGE); + } + + kern_init_core(); + + if (pantheon::CPU::GetProcessorNumber() == 0) + { kern_stage2_init(); pantheon::SetKernelStatus(pantheon::KERNEL_STATUS_OK); } - kern_init_core(); + + kern_stage2_init_core(); + for (;;) + { + pantheon::Scheduler::Reschedule(); + } pantheon::StopError("Broke out of reschedule loop\n"); } From f6ee85291999af06f535a96e9f2b847ee0e207ff Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Mon, 26 Dec 2022 03:17:53 -0700 Subject: [PATCH 19/43] Enforce consistency of state view --- System/Exec/kern_proc_alsr.cpp | 12 ++---------- System/Proc/kern_proc.hpp | 7 ++----- System/Proc/kern_sched.cpp | 1 + System/Syscalls/Syscalls.cpp | 36 ++++------------------------------ 4 files changed, 9 insertions(+), 47 deletions(-) diff --git a/System/Exec/kern_proc_alsr.cpp b/System/Exec/kern_proc_alsr.cpp index b33aec4..2bef97a 100644 --- a/System/Exec/kern_proc_alsr.cpp +++ b/System/Exec/kern_proc_alsr.cpp @@ -1,18 +1,10 @@ #include #include +#include #include pantheon::vmm::VirtualAddress pantheon::GenerateALSRBase() { - /* TODO: Grab some real randomness. This is still - * deterministic, but that's probably fine (for now). - */ - static pantheon::vmm::VirtualAddress Seed = pantheon::ALSRMaxAddress; - - /* Arbitrary constants */ - Seed *= 0xBF83525; - Seed += 0x8DD1FC7; - - return (Seed % pantheon::ALSRMaxAddress) & ~(pantheon::ALSRMask - 1); + return (pantheon::Rand() % pantheon::ALSRMaxAddress) & ~(pantheon::ALSRMask - 1); } \ No newline at end of file diff --git a/System/Proc/kern_proc.hpp b/System/Proc/kern_proc.hpp index eec3c4d..d567806 100644 --- a/System/Proc/kern_proc.hpp +++ b/System/Proc/kern_proc.hpp @@ -136,11 +136,8 @@ class Process : public pantheon::Object, public pantheon::Lockable return; } - pantheon::Process *CurProc = pantheon::CPU::GetCurProcess(); - if (CurProc && CurProc != Next) - { - pantheon::CPUReg::W_TTBR0_EL1(Next->GetTTBR0()); - } + pantheon::CPUReg::W_TTBR0_EL1(Next->GetTTBR0()); + pantheon::CPU::GetCoreInfo()->CurProcess = Next; } private: diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index b32d04d..364fb10 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -349,6 +349,7 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); + pantheon::CPU::GetCoreInfo()->CurThread = NextThread; pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); return {OldContext, NewContext}; diff --git a/System/Syscalls/Syscalls.cpp b/System/Syscalls/Syscalls.cpp index c4cd19d..7199dc7 100644 --- a/System/Syscalls/Syscalls.cpp +++ b/System/Syscalls/Syscalls.cpp @@ -24,34 +24,6 @@ static T ReadArgument(UINT64 Val) return reinterpret_cast(Val); } - -static const CHAR *ReadArgumentAsCharPointer(UINT64 Val) -{ - /* Don't allow reading of kernel memory */ - if (Val > pantheon::vmm::HigherHalfAddress) - { - return nullptr; - } - - pantheon::Process *CurProc = pantheon::CPU::GetCurProcess(); - if (CurProc->IsLocked() == FALSE) - { - pantheon::StopErrorFmt("Attempt to read userspace memory without CurProc lock (PID %hhu)\n", CurProc->ProcessID()); - } - - /* Make sure we translate this to something in kernel memory, - * just to be safe. */ - pantheon::vmm::VirtualAddress RawPtr = Val; - const CHAR *Result = nullptr; - - pantheon::vmm::PhysicalAddress PAddr = pantheon::vmm::VirtualToPhysicalAddress(CurProc->GetPageTable(), RawPtr); - pantheon::vmm::VirtualAddress VAddr = pantheon::vmm::PhysicalToVirtualAddress(PAddr); - - Result = reinterpret_cast(VAddr); - - return Result; -} - template static T *ReadArgumentAsPointer(UINT64 Val) { @@ -123,7 +95,7 @@ pantheon::Result pantheon::SVCLogText(pantheon::TrapFrame *CurFrame) const CHAR *Data = nullptr; - Data = ReadArgumentAsCharPointer(CurFrame->GetIntArgument(0)); + Data = ReadArgumentAsPointer(CurFrame->GetIntArgument(0)); if (Data == nullptr) { return pantheon::Result::SYS_FAIL; @@ -180,7 +152,7 @@ pantheon::Result pantheon::SVCCreateNamedEvent(pantheon::TrapFrame *CurFrame) pantheon::ScopedLock ScopeLockProc(Proc); const CHAR *Name = nullptr; - Name = ReadArgumentAsCharPointer(CurFrame->GetIntArgument(0)); + Name = ReadArgumentAsPointer(CurFrame->GetIntArgument(0)); INT32 *ReadHandle = nullptr; ReadHandle = ReadArgument(CurFrame->GetIntArgument(1)); @@ -434,7 +406,7 @@ pantheon::Result pantheon::SVCCreatePort(pantheon::TrapFrame *CurFrame) pantheon::ScopedLock _PL(CurProc); CHAR Buffer[pantheon::ipc::PortNameLength] = {0}; - const CHAR *Location = ReadArgumentAsCharPointer(CurFrame->GetIntArgument(0)); + const CHAR *Location = ReadArgumentAsPointer(CurFrame->GetIntArgument(0)); CopyString(Buffer, Location, pantheon::ipc::PortNameLength); pantheon::ipc::PortName Name; @@ -485,7 +457,7 @@ pantheon::Result pantheon::SVCConnectToNamedPort(pantheon::TrapFrame *CurFrame) pantheon::Process *CurProc = pantheon::CPU::GetCurProcess(); pantheon::ScopedLock _L(CurProc); - const char *Name = ReadArgumentAsCharPointer(CurFrame->GetIntArgument(0)); + const char *Name = ReadArgumentAsPointer(CurFrame->GetIntArgument(0)); if (Name == nullptr) { return pantheon::Result::SYS_FAIL; From 727d8dd13dc8ec9bb61443ff21f67fda6d1e9006 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 27 Dec 2022 16:59:13 -0700 Subject: [PATCH 20/43] Allow programs to switch again --- Common/Structures/kern_skiplist.hpp | 2 +- Common/Sync/kern_atomic.hpp | 10 ++++++ Common/Sync/kern_spinlock.cpp | 5 +++ System/Proc/kern_sched.cpp | 52 ++++++++++------------------- System/Proc/kern_sched.hpp | 2 +- System/Proc/kern_thread.cpp | 44 ++++++------------------ System/Proc/kern_thread.hpp | 5 +-- 7 files changed, 46 insertions(+), 74 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index f9f031b..61b0c66 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -182,7 +182,7 @@ class SkipList if (Current->Key == Key) { UINT8 CurLvl = Current->Level; - for (UINT8 Lvl = 0; Lvl < CurLvl; Lvl++) + for (UINT8 Lvl = 0; Lvl <= CurLvl; Lvl++) { Current->Prev[Lvl]->Next[Lvl] = Current->Next[Lvl]; Current->Next[Lvl]->Prev[Lvl] = Current->Prev[Lvl]; diff --git a/Common/Sync/kern_atomic.hpp b/Common/Sync/kern_atomic.hpp index f1ae904..4ae85bf 100644 --- a/Common/Sync/kern_atomic.hpp +++ b/Common/Sync/kern_atomic.hpp @@ -119,6 +119,16 @@ class AtomicInteger #endif } + T Sub(T Item) + { + #ifndef ONLY_TESTS + return __atomic_fetch_sub(&(this->Content), Item, __ATOMIC_SEQ_CST); + #else + this->Content -= Item; + return this->Content; + #endif + } + [[nodiscard]] bool operator==(const Atomic &Other) const { diff --git a/Common/Sync/kern_spinlock.cpp b/Common/Sync/kern_spinlock.cpp index b58f24a..b22c1c4 100644 --- a/Common/Sync/kern_spinlock.cpp +++ b/Common/Sync/kern_spinlock.cpp @@ -74,6 +74,11 @@ BOOL pantheon::Spinlock::TryAcquire() __sync_synchronize(); return TRUE; } + + while (__atomic_load_n(&this->Locked, __ATOMIC_RELAXED)) + { + pantheon::CPU::PAUSE(); + } __sync_synchronize(); pantheon::CPU::POPI(); return FALSE; diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 364fb10..bff1710 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -43,7 +43,7 @@ static pantheon::SkipList ThreadList; * \~english @brief Initalizes an instance of a per-core scheduler. * \~english @author Brian Schnepp */ -pantheon::LocalScheduler::LocalScheduler() : pantheon::Allocatable(), pantheon::Lockable("Scheduler") +pantheon::LocalScheduler::LocalScheduler() : pantheon::Lockable("Scheduler") { this->IdleThread = nullptr; } @@ -108,6 +108,9 @@ void pantheon::LocalScheduler::Setup() if (Proc.ProcessID() == 0) { Idle->Initialize(&Proc, nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); + Idle->Lock(); + Idle->SetState(pantheon::Thread::STATE_RUNNING); + Idle->Unlock(); } } @@ -148,7 +151,13 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) { UINT64 Jiffies = pantheon::CPU::GetJiffies(); UINT64 Prio = pantheon::Thread::PRIORITY_MAX - Thr->MyPriority(); - this->LocalRunQueue.Insert(CalculateDeadline(Jiffies, Prio, 6), Thr); + + UINT64 Deadline = CalculateDeadline(Jiffies, Prio, 6); + while (this->LocalRunQueue.Contains(Deadline)) + { + Deadline++; + } + this->LocalRunQueue.Insert(Deadline, Thr); } } @@ -258,8 +267,8 @@ UINT32 pantheon::Scheduler::AcquireProcessID() * 0 should be reserved for the generic idle process. */ UINT32 RetVal = 0; - static UINT32 ProcessID = 1; - RetVal = ProcessID++; + static pantheon::AtomicInteger ProcessID = 1; + RetVal = ProcessID.Add(1); return RetVal; } @@ -269,9 +278,8 @@ UINT64 pantheon::Scheduler::AcquireThreadID() * reuse an ID already in use! */ UINT64 RetVal = 0; - static UINT64 ThreadID = 0; - /* A copy has to be made since we haven't unlocked the spinlock yet. */ - RetVal = ThreadID++; + static pantheon::AtomicInteger ThreadID = 1; + RetVal = ThreadID.Add(1); /* Ban 0 and -1, since these are special values. */ if (RetVal == 0 || RetVal == ~0ULL) @@ -345,13 +353,13 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne CurThread->SetState(pantheon::Thread::STATE_WAITING); NextThread->SetState(pantheon::Thread::STATE_RUNNING); + NextThread->SetTicks(6); pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); pantheon::CPU::GetCoreInfo()->CurThread = NextThread; pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); - return {OldContext, NewContext}; } @@ -367,12 +375,6 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne */ void pantheon::Scheduler::Reschedule() { - /* Don't reschedule if interrupts are off */ - if (pantheon::CPU::ICOUNT()) - { - return; - } - pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); @@ -390,28 +392,10 @@ void pantheon::Scheduler::Reschedule() void pantheon::Scheduler::AttemptReschedule() { pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); + INT64 RemainingTicks = CurThread->CountTick(); - if (CurThread) + if (RemainingTicks <= 0 && !CurThread->Preempted()) { - CurThread->Lock(); - CurThread->CountTick(); - UINT64 RemainingTicks = CurThread->TicksLeft(); - - if (RemainingTicks > 0 || CurThread->Preempted()) - { - CurThread->Unlock(); - return; - } - - CurThread->SetTicks(0); - CurThread->Unlock(); - - - if (pantheon::CPU::ICOUNT() != 0) - { - StopError("Interrupts not allowed for reschedule"); - } - pantheon::Scheduler::Reschedule(); } } diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index f17835b..dd9f46b 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -20,7 +20,7 @@ namespace pantheon { -class LocalScheduler : public pantheon::Allocatable, public pantheon::Lockable +class LocalScheduler : public pantheon::Allocatable, public pantheon::Lockable { public: diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index d92a19e..525cd53 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -67,7 +67,6 @@ pantheon::Thread::Thread(Process *OwningProcess, Priority Pri) : pantheon::Locka this->TID = pantheon::Scheduler::AcquireThreadID(); /* 45 for NORMAL, 30 for LOW, 15 for VERYLOW, etc. */ - this->RefreshTicks(); this->Unlock(); } @@ -173,36 +172,20 @@ BOOL pantheon::Thread::Preempted() const return this->PreemptCount != 0; } -/** - * \~english @brief Gets the number of timer interrupts remaining for the thread - * \~english @author Brian Schnepp - * \~english @return The number of times the processor core will need to be - * interrupted before the thread is preempted and some other work resumes on - * the current processor core. - */ -UINT64 pantheon::Thread::TicksLeft() const -{ - OBJECT_SELF_ASSERT(); - return this->RemainingTicks.Load(); -} - /** * \~english @brief Counts down the number of timer interrupts for this thread * \~english @author Brian Schnepp */ -VOID pantheon::Thread::CountTick() +INT64 pantheon::Thread::CountTick() { OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("CountTicks without lock"); - } - - if (this->RemainingTicks == 0) + pantheon::ScopedLock _L(this); + if (this->RemainingTicks.Load() <= 0) { - return; + this->SetTicks(0); + return 0; } - this->RemainingTicks.Store(this->RemainingTicks.Load() - 1); + return this->RemainingTicks.Sub(1); } [[nodiscard]] @@ -212,16 +195,6 @@ UINT64 pantheon::Thread::ThreadID() const return this->TID; } -VOID pantheon::Thread::RefreshTicks() -{ - OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("RefreshTicks without lock"); - } - this->RemainingTicks = 6; -} - /** * \~english @brief Sets the state, such as running, to the current process. * \~english @author Brian Schnepp @@ -348,7 +321,7 @@ void pantheon::Thread::EnableScheduling() { OBJECT_SELF_ASSERT(); pantheon::Sync::DSBISH(); - this->PreemptCount.Add(-1LL); + this->PreemptCount.Sub(1LL); pantheon::Sync::DSBISH(); } @@ -363,6 +336,9 @@ static void true_drop_process(void *StartAddress, pantheon::vmm::VirtualAddress void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority, BOOL UserMode) { static constexpr UINT64 InitialThreadStackSize = pantheon::vmm::SmallestPageSize * pantheon::Thread::InitialNumStackPages; + this->TID = pantheon::Scheduler::AcquireThreadID(); + this->CurState = pantheon::Thread::STATE_DEAD; + Optional StackSpace = BasicMalloc(InitialThreadStackSize); if (StackSpace.GetOkay()) { diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index deaf0c7..a8aa79b 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -92,12 +92,9 @@ class Thread : public pantheon::Object, public pantheon::Lockable [[nodiscard]] Process *MyProc() const; [[nodiscard]] Thread::State MyState() const; [[nodiscard]] Thread::Priority MyPriority() const; - - [[nodiscard]] UINT64 TicksLeft() const; [[nodiscard]] UINT64 ThreadID() const; - VOID CountTick(); - VOID RefreshTicks(); + INT64 CountTick(); VOID SetTicks(INT64 TickCount); VOID SetState(Thread::State State); From f6a89d6904f902bc708cf5b35cf3e0849cb591ae Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 27 Dec 2022 19:18:02 -0700 Subject: [PATCH 21/43] Correct TLS setup --- Common/Sync/kern_atomic.hpp | 4 +- System/Proc/kern_sched.cpp | 88 ++++++++++++++++--------------------- System/Proc/kern_thread.cpp | 6 +++ 3 files changed, 45 insertions(+), 53 deletions(-) diff --git a/Common/Sync/kern_atomic.hpp b/Common/Sync/kern_atomic.hpp index 4ae85bf..50921d6 100644 --- a/Common/Sync/kern_atomic.hpp +++ b/Common/Sync/kern_atomic.hpp @@ -112,7 +112,7 @@ class AtomicInteger T Add(T Item) { #ifndef ONLY_TESTS - return __atomic_fetch_add(&(this->Content), Item, __ATOMIC_SEQ_CST); + return __atomic_fetch_add(&(this->Content), Item, __ATOMIC_SEQ_CST) + Item; #else this->Content += Item; return this->Content; @@ -122,7 +122,7 @@ class AtomicInteger T Sub(T Item) { #ifndef ONLY_TESTS - return __atomic_fetch_sub(&(this->Content), Item, __ATOMIC_SEQ_CST); + return __atomic_fetch_sub(&(this->Content), Item, __ATOMIC_SEQ_CST) - Item; #else this->Content -= Item; return this->Content; diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index bff1710..2a99b3a 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -35,7 +35,7 @@ static pantheon::Spinlock SchedLock("Global Scheduler Lock"); -static pantheon::LinkedList ProcessList; +static pantheon::SkipList ProcessList; static pantheon::SkipList ThreadList; @@ -103,17 +103,17 @@ void pantheon::LocalScheduler::Setup() pantheon::ScopedGlobalSchedulerLock _L; pantheon::Thread *Idle = pantheon::Thread::Create(); - for (pantheon::Process &Proc : ProcessList) + Optional MProc = ProcessList.Get(0); + if (!MProc.GetOkay()) { - if (Proc.ProcessID() == 0) - { - Idle->Initialize(&Proc, nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); - Idle->Lock(); - Idle->SetState(pantheon::Thread::STATE_RUNNING); - Idle->Unlock(); - } + pantheon::StopErrorFmt("No PID 0!\n"); } + Idle->Initialize(MProc.GetValue(), nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); + Idle->Lock(); + Idle->SetState(pantheon::Thread::STATE_RUNNING); + Idle->Unlock(); + pantheon::CPU::GetCoreInfo()->CurThread = Idle; this->IdleThread = Idle; } @@ -164,7 +164,7 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) void pantheon::Scheduler::Init() { static pantheon::Process IdleProc; - ProcessList.PushFront(&IdleProc); + ProcessList.Insert(0, &IdleProc); } void pantheon::Scheduler::Lock() @@ -209,7 +209,7 @@ UINT32 pantheon::Scheduler::CreateProcess(const pantheon::String &ProcStr, void CreateInfo.ID = NewID; Proc->Initialize(CreateInfo); - ProcessList.PushFront(Proc); + ProcessList.Insert(NewID, Proc); return NewID; } @@ -218,22 +218,13 @@ pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, { pantheon::ScopedGlobalSchedulerLock _L; - pantheon::Process *MyProc = nullptr; - for (pantheon::Process &Proc : ProcessList) - { - if (Proc.ProcessID() == PID) - { - MyProc = &Proc; - break; - } - } - - if(!MyProc) + if(!ProcessList.Contains(PID)) { /* Huh? You're asking for a PID that doesn't exist. */ return nullptr; } + pantheon::Process *MyProc = ProcessList.Get(PID).GetValue(); pantheon::Thread *Thr = pantheon::Thread::Create(); Thr->Initialize(MyProc, StartAddr, ThreadData, Priority, TRUE); ThreadList.Insert(Thr->ThreadID(), Thr); @@ -261,30 +252,26 @@ UINT64 pantheon::Scheduler::CountThreads(UINT64 PID) UINT32 pantheon::Scheduler::AcquireProcessID() { - /* TODO: When we run out of IDs, go back and ensure we don't - * reuse an ID already in use! - * - * 0 should be reserved for the generic idle process. - */ UINT32 RetVal = 0; static pantheon::AtomicInteger ProcessID = 1; RetVal = ProcessID.Add(1); + while (ProcessList.Contains(RetVal) || RetVal == 0) + { + RetVal = ProcessID.Add(1); + } return RetVal; } UINT64 pantheon::Scheduler::AcquireThreadID() { - /* TODO: When we run out of IDs, go back and ensure we don't - * reuse an ID already in use! - */ UINT64 RetVal = 0; static pantheon::AtomicInteger ThreadID = 1; RetVal = ThreadID.Add(1); /* Ban 0 and -1, since these are special values. */ - if (RetVal == 0 || RetVal == ~0ULL) + while (RetVal == 0 || RetVal == ~0ULL || ThreadList.Contains(RetVal)) { - return pantheon::Scheduler::AcquireThreadID(); + RetVal = ThreadID.Add(1); } return RetVal; } @@ -409,35 +396,34 @@ BOOL pantheon::Scheduler::MapPages(UINT32 PID, const pantheon::vmm::VirtualAddre { /* TODO: Move into a more sensible namespace like pantheon::mm or something */ pantheon::ScopedGlobalSchedulerLock _L; - for (pantheon::Process &Item : ProcessList) + + if (!ProcessList.Contains(PID)) { - if (Item.ProcessID() == PID) - { - pantheon::ScopedLock _LL(&Item); - for (UINT64 Index = 0; Index < NumPages; ++Index) - { - Item.MapAddress(VAddresses[Index], PAddresses[Index], PageAttributes); - } - return TRUE; - } + return FALSE; + } + + pantheon::Process *Proc = ProcessList.Get(PID).GetValue(); + pantheon::ScopedLock _LL(Proc); + for (UINT64 Index = 0; Index < NumPages; ++Index) + { + Proc->MapAddress(VAddresses[Index], PAddresses[Index], PageAttributes); } - return FALSE; + return TRUE; } BOOL pantheon::Scheduler::SetState(UINT32 PID, pantheon::Process::State State) { pantheon::ScopedGlobalSchedulerLock _L; - for (pantheon::Process &Item : ProcessList) + if (!ProcessList.Contains(PID)) { - if (Item.ProcessID() == PID) - { - pantheon::ScopedLock _LL(&Item); - Item.SetState(State); - return TRUE; - } + return FALSE; } - return FALSE; + + pantheon::Process *Proc = ProcessList.Get(PID).GetValue(); + pantheon::ScopedLock _LL(Proc); + Proc->SetState(State); + return TRUE; } BOOL pantheon::Scheduler::SetThreadState(UINT64 TID, pantheon::Thread::State State) diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 525cd53..5777562 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -361,6 +361,12 @@ void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void } this->SetState(pantheon::Thread::STATE_WAITING); this->SetPriority(Priority); + + /* Don't set up a TLS for idle proc. */ + if (Proc->ProcessID() != 0) + { + this->SetupThreadLocalArea(); + } this->Unlock(); } } From fbd22ad0c9eb0e6da27efaa6e1cd2b820caadf63 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 29 Dec 2022 15:56:36 -0700 Subject: [PATCH 22/43] Define constant for RR_INTERVAL --- System/Proc/kern_sched.cpp | 9 +++++---- System/Proc/kern_thread.cpp | 4 +--- System/Proc/kern_thread.hpp | 2 ++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 2a99b3a..de8a6c1 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -152,7 +152,7 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) UINT64 Jiffies = pantheon::CPU::GetJiffies(); UINT64 Prio = pantheon::Thread::PRIORITY_MAX - Thr->MyPriority(); - UINT64 Deadline = CalculateDeadline(Jiffies, Prio, 6); + UINT64 Deadline = CalculateDeadline(Jiffies, Prio, pantheon::Thread::RR_INTERVAL); while (this->LocalRunQueue.Contains(Deadline)) { Deadline++; @@ -340,7 +340,7 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne CurThread->SetState(pantheon::Thread::STATE_WAITING); NextThread->SetState(pantheon::Thread::STATE_RUNNING); - NextThread->SetTicks(6); + NextThread->SetTicks(pantheon::Thread::RR_INTERVAL); pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); @@ -365,12 +365,13 @@ void pantheon::Scheduler::Reschedule() pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); + pantheon::Sync::DSBISH(); + pantheon::Sync::ISB(); + SwapContext Con = SwapThreads(CurThread, NextThread); if (Con.New && Con.Old) { - pantheon::Sync::DSBISH(); - pantheon::Sync::ISB(); cpu_switch(Con.Old, Con.New, CpuIRegOffset); pantheon::CPU::GetCurThread()->EnableScheduling(); } diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 5777562..2598ce7 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -65,8 +65,6 @@ pantheon::Thread::Thread(Process *OwningProcess, Priority Pri) : pantheon::Locka this->CurState = pantheon::Thread::STATE_INIT; this->TID = pantheon::Scheduler::AcquireThreadID(); - - /* 45 for NORMAL, 30 for LOW, 15 for VERYLOW, etc. */ this->Unlock(); } @@ -136,7 +134,7 @@ pantheon::Process *pantheon::Thread::MyProc() const pantheon::Thread::State pantheon::Thread::MyState() const { OBJECT_SELF_ASSERT(); - UINT64 State = this->CurState; /* ubsan says this is a problem. */ + UINT64 State = this->CurState; if (State > pantheon::Thread::STATE_MAX) { StopErrorFmt("Invalid thread state: got 0x%lx\n", this->CurState); diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index a8aa79b..36cd6e0 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -79,6 +79,8 @@ class Thread : public pantheon::Object, public pantheon::Lockable PRIORITY_MAX, }Priority; + static constexpr UINT64 RR_INTERVAL = 6; + public: Thread(); Thread(Process *ParentProcess); From 2f1ec286efce541fba7cbea87619d9262b5bfa7c Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 29 Dec 2022 18:34:34 -0700 Subject: [PATCH 23/43] Move thread switch code to a static method --- System/Proc/kern_sched.cpp | 15 ++++----------- System/Proc/kern_thread.hpp | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index de8a6c1..3790d08 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -334,18 +334,12 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne */ pantheon::CPU::GetCurThread()->BlockScheduling(); - /* Change contexts */ + /* Change kernel view of contexts */ pantheon::Process::Switch(NextThread->MyProc()); - pantheon::ipc::SetThreadLocalRegion(NextThread->GetThreadLocalAreaRegister()); - - CurThread->SetState(pantheon::Thread::STATE_WAITING); - NextThread->SetState(pantheon::Thread::STATE_RUNNING); - NextThread->SetTicks(pantheon::Thread::RR_INTERVAL); + pantheon::Thread::Switch(NextThread); pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - - pantheon::CPU::GetCoreInfo()->CurThread = NextThread; pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); return {OldContext, NewContext}; } @@ -364,11 +358,10 @@ void pantheon::Scheduler::Reschedule() { pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); - - pantheon::Sync::DSBISH(); - pantheon::Sync::ISB(); SwapContext Con = SwapThreads(CurThread, NextThread); + pantheon::Sync::DSBISH(); + pantheon::Sync::ISB(); if (Con.New && Con.Old) { diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 36cd6e0..58342f4 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -121,6 +121,23 @@ class Thread : public pantheon::Object, public pantheon::Lockable ThreadLocalRegion *GetThreadLocalArea(); VOID SetupThreadLocalArea(); + /** + * @brief Switches thread context to another thread + */ + static void Switch(Thread *Next) + { + if (Next == nullptr) + { + return; + } + + pantheon::ipc::SetThreadLocalRegion(Next->LocalRegion); + pantheon::CPU::GetCurThread()->SetState(pantheon::Thread::STATE_WAITING); + Next->SetState(Thread::STATE_RUNNING); + Next->SetTicks(Thread::RR_INTERVAL); + pantheon::CPU::GetCoreInfo()->CurThread = Next; + } + private: VOID SetEntryLocation(UINT64 IP, UINT64 SP, VOID* ThreadData); From 8256435bdbfe46348d9505f51ba272ab40a58b70 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Thu, 29 Dec 2022 19:22:13 -0700 Subject: [PATCH 24/43] Make spinlock errors more descriptive --- Common/Sync/kern_spinlock.cpp | 14 +++++++------- Common/Sync/kern_spinlock.hpp | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Common/Sync/kern_spinlock.cpp b/Common/Sync/kern_spinlock.cpp index b22c1c4..f3dd6ef 100644 --- a/Common/Sync/kern_spinlock.cpp +++ b/Common/Sync/kern_spinlock.cpp @@ -31,13 +31,13 @@ void pantheon::Spinlock::Acquire() { if (this->DebugName == nullptr) { - StopError("bad spinlock", this); + StopErrorFmt("Bad Spinlock: was nullptr\n"); } pantheon::CPU::PUSHI(); if (this->IsHolding()) { - StopError(this->DebugName, this); + StopErrorFmt("Spinlock: %s (source %p), trying to acquire when already held", this->DebugName, this); } for (;;) @@ -59,7 +59,7 @@ BOOL pantheon::Spinlock::TryAcquire() { if (this->DebugName == nullptr) { - StopError("bad spinlock", this); + StopErrorFmt("Bad Spinlock: was nullptr\n"); } pantheon::CPU::PUSHI(); @@ -88,7 +88,7 @@ void pantheon::Spinlock::Release() { if (!this->IsHolding()) { - pantheon::StopError(this->DebugName, this); + StopErrorFmt("Spinlock: %s (source %p), trying to release when not held (holder is %hd)\n", this->DebugName, this, this->Holder()); } this->CoreNo = -1; __sync_synchronize(); @@ -111,12 +111,12 @@ BOOL pantheon::Spinlock::IsLocked() const /** * \~english @brief Gets the current holder of this lock, if it is held. - * \~english @details Returns 0 is this lock isn't held, otherwise gets the core number. + * \~english @details Returns -1 is this lock isn't held, otherwise gets the core number. * * \~english @author Brian Schnepp */ [[nodiscard]] -UINT8 pantheon::Spinlock::Holder() const +INT16 pantheon::Spinlock::Holder() const { - return (this->Locked ? this->CoreNo : 0); + return (this->Locked ? this->CoreNo : static_cast(-1)); } \ No newline at end of file diff --git a/Common/Sync/kern_spinlock.hpp b/Common/Sync/kern_spinlock.hpp index 3d6ba4f..f982af6 100644 --- a/Common/Sync/kern_spinlock.hpp +++ b/Common/Sync/kern_spinlock.hpp @@ -17,7 +17,7 @@ class Spinlock void Release(); BOOL TryAcquire(); - [[nodiscard]] UINT8 Holder() const; + [[nodiscard]] INT16 Holder() const; [[nodiscard]] BOOL IsLocked() const; void SetDebugName(const char *Name); @@ -26,7 +26,7 @@ class Spinlock private: [[nodiscard]] BOOL IsHolding() const; const char *DebugName; - UINT16 CoreNo; + INT16 CoreNo; BOOL Locked; }; From 049c3e181d9a90567ad93302c789db680b9cfc8c Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Fri, 30 Dec 2022 19:28:22 -0700 Subject: [PATCH 25/43] Enforce consistency of TIDs and PIDs --- Common/Sync/kern_spinlock.cpp | 4 +++- System/Proc/kern_sched.cpp | 31 ++++++++++++++++++++++--------- System/Proc/kern_thread.cpp | 14 ++++++++------ arch/aarch64/ints.cpp | 20 ++++++++++++++------ 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/Common/Sync/kern_spinlock.cpp b/Common/Sync/kern_spinlock.cpp index f3dd6ef..731ea2a 100644 --- a/Common/Sync/kern_spinlock.cpp +++ b/Common/Sync/kern_spinlock.cpp @@ -65,7 +65,7 @@ BOOL pantheon::Spinlock::TryAcquire() pantheon::CPU::PUSHI(); if (this->IsHolding()) { - StopError(this->DebugName, this); + StopErrorFmt("Spinlock: %s (source %p), trying to (try) acquire when already held", this->DebugName, this); } if (__sync_lock_test_and_set(&this->Locked, TRUE) == FALSE) @@ -86,6 +86,7 @@ BOOL pantheon::Spinlock::TryAcquire() void pantheon::Spinlock::Release() { + __sync_synchronize(); if (!this->IsHolding()) { StopErrorFmt("Spinlock: %s (source %p), trying to release when not held (holder is %hd)\n", this->DebugName, this, this->Holder()); @@ -94,6 +95,7 @@ void pantheon::Spinlock::Release() __sync_synchronize(); __sync_lock_release(&this->Locked); pantheon::CPU::POPI(); + __sync_synchronize(); } diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 3790d08..b82708d 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -253,26 +253,34 @@ UINT64 pantheon::Scheduler::CountThreads(UINT64 PID) UINT32 pantheon::Scheduler::AcquireProcessID() { UINT32 RetVal = 0; - static pantheon::AtomicInteger ProcessID = 1; - RetVal = ProcessID.Add(1); + static pantheon::Spinlock PIDLock("Process ID Lock"); + static UINT32 ProcessID = 1; + + PIDLock.Acquire(); + RetVal = ProcessID++; while (ProcessList.Contains(RetVal) || RetVal == 0) { - RetVal = ProcessID.Add(1); + RetVal = ProcessID++; } + PIDLock.Release(); + return RetVal; } UINT64 pantheon::Scheduler::AcquireThreadID() { UINT64 RetVal = 0; - static pantheon::AtomicInteger ThreadID = 1; - RetVal = ThreadID.Add(1); - + static pantheon::Spinlock TIDLock("Thread ID Lock"); + static UINT64 ThreadID = 1; + + TIDLock.Acquire(); + RetVal = ThreadID++; /* Ban 0 and -1, since these are special values. */ while (RetVal == 0 || RetVal == ~0ULL || ThreadList.Contains(RetVal)) { - RetVal = ThreadID.Add(1); + RetVal = ThreadID++; } + TIDLock.Release(); return RetVal; } @@ -340,7 +348,12 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); + + /* Don't put idle jobs on the runqueue */ + if (CurThread->MyProc()->ProcessID() != 0) + { + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); + } return {OldContext, NewContext}; } @@ -363,7 +376,7 @@ void pantheon::Scheduler::Reschedule() pantheon::Sync::DSBISH(); pantheon::Sync::ISB(); - if (Con.New && Con.Old) + if (Con.New && Con.Old && NextThread) { cpu_switch(Con.Old, Con.New, CpuIRegOffset); pantheon::CPU::GetCurThread()->EnableScheduling(); diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 2598ce7..571270e 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -64,7 +64,7 @@ pantheon::Thread::Thread(Process *OwningProcess, Priority Pri) : pantheon::Locka this->Registers.Wipe(); this->CurState = pantheon::Thread::STATE_INIT; - this->TID = pantheon::Scheduler::AcquireThreadID(); + this->TID = 0; this->Unlock(); } @@ -360,11 +360,7 @@ void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void this->SetState(pantheon::Thread::STATE_WAITING); this->SetPriority(Priority); - /* Don't set up a TLS for idle proc. */ - if (Proc->ProcessID() != 0) - { - this->SetupThreadLocalArea(); - } + this->SetupThreadLocalArea(); this->Unlock(); } } @@ -395,6 +391,12 @@ pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() VOID pantheon::Thread::SetupThreadLocalArea() { + /* Don't make a TLS for kernel idle threads */ + if (this->MyProc()->ProcessID() == 0) + { + return; + } + this->LocalRegion = pantheon::Process::ThreadLocalBase + (this->ThreadID() * 0x1000); /* Map in the TLR. */ diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 3934374..f238331 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -69,9 +69,18 @@ extern "C" void fiq_handler_el1(pantheon::TrapFrame *Frame) SERIAL_LOG_UNSAFE("%s\n", "ERR: FIQ HANDLER EL1"); } +class CurFrameMgr +{ +public: + CurFrameMgr(pantheon::TrapFrame *Frame) { pantheon::CPU::GetCoreInfo()->CurFrame = Frame; } + ~CurFrameMgr() { pantheon::CPU::GetCoreInfo()->CurFrame = nullptr; } +}; + + extern "C" void irq_handler_el1(pantheon::TrapFrame *Frame) { - pantheon::CPU::GetCoreInfo()->CurFrame = Frame; + CurFrameMgr _M(Frame); + UINT32 IAR = pantheon::arm::GICRecvInterrupt(); pantheon::arm::GICAckInterrupt(IAR); if ((IAR & 0x3FF) == 30) @@ -82,7 +91,6 @@ extern "C" void irq_handler_el1(pantheon::TrapFrame *Frame) pantheon::arm::RearmSystemTimer(); pantheon::Scheduler::AttemptReschedule(); } - pantheon::CPU::GetCoreInfo()->CurFrame = nullptr; } extern "C" void enable_interrupts(); @@ -90,7 +98,7 @@ extern "C" void disable_interrupts(); extern "C" void sync_handler_el0(pantheon::TrapFrame *Frame) { - pantheon::CPU::GetCoreInfo()->CurFrame = Frame; + CurFrameMgr _M(Frame); UINT64 ESR, FAR, ELR, SPSR; asm volatile( "mrs %0, esr_el1\n" @@ -109,15 +117,15 @@ extern "C" void sync_handler_el0(pantheon::TrapFrame *Frame) } else { + pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); SERIAL_LOG_UNSAFE("Bad sync handler el0: esr: 0x%lx far: 0x%lx elr: 0x%lx spsr: 0x%lx\n", ESR, FAR, ELR, SPSR); - SERIAL_LOG_UNSAFE("Process ID was %ld\n", pantheon::CPU::GetCurThread()->MyProc()->ProcessID()); + SERIAL_LOG_UNSAFE("Process ID was %ld\n", CurThread->MyProc()->ProcessID()); if (ESR == 0x2000000) { - pantheon::vmm::PageTable *PT = pantheon::CPU::GetCurThread()->MyProc()->GetPageTable(); + pantheon::vmm::PageTable *PT = CurThread->MyProc()->GetPageTable(); pantheon::vmm::PrintPageTablesNoZeroes(PT); } } - pantheon::CPU::GetCoreInfo()->CurFrame = nullptr; } extern "C" void err_handler_el0(pantheon::TrapFrame *Frame) From b4fabe9d2ff8bfb03cbfec6dbdf5329c46ddb477 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Fri, 30 Dec 2022 21:07:31 -0700 Subject: [PATCH 26/43] Cleanup unnecessary methods in Thread class --- System/Proc/kern_thread.cpp | 79 +++++++++++-------------------------- System/Proc/kern_thread.hpp | 9 +---- arch/aarch64/ints.cpp | 5 ++- 3 files changed, 28 insertions(+), 65 deletions(-) diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 571270e..067dc12 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -297,16 +297,6 @@ pantheon::Thread &pantheon::Thread::operator=(pantheon::Thread &&Other) noexcept return *this; } -void pantheon::Thread::SetProc(pantheon::Process *Proc) -{ - OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("SetProc without lock"); - } - this->ParentProcess = Proc; -} - void pantheon::Thread::BlockScheduling() { OBJECT_SELF_ASSERT(); @@ -337,6 +327,8 @@ void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void this->TID = pantheon::Scheduler::AcquireThreadID(); this->CurState = pantheon::Thread::STATE_DEAD; + this->Lock(); + Optional StackSpace = BasicMalloc(InitialThreadStackSize); if (StackSpace.GetOkay()) { @@ -346,7 +338,6 @@ void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void #endif IStackSpace += InitialThreadStackSize; this->ParentProcess = Proc; - this->Lock(); UINT64 IStartAddr = (UINT64)true_drop_process; UINT64 IThreadData = (UINT64)ThreadData; @@ -359,25 +350,32 @@ void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void } this->SetState(pantheon::Thread::STATE_WAITING); this->SetPriority(Priority); - - this->SetupThreadLocalArea(); - this->Unlock(); } -} -[[nodiscard]] BOOL pantheon::Thread::End() const -{ - return this->NextThread == nullptr; -} + /* Don't create a TLS for kernel threads */ + if (UserMode) + { + this->LocalRegion = pantheon::Process::ThreadLocalBase + (this->ThreadID() * 0x1000); -pantheon::Thread *pantheon::Thread::Next() -{ - return this->NextThread.Load(); -} + /* Map in the TLR. */ + pantheon::vmm::PageTableEntry Entry; + Entry.SetBlock(TRUE); + Entry.SetMapped(TRUE); + Entry.SetUserNoExecute(TRUE); + Entry.SetKernelNoExecute(TRUE); -void pantheon::Thread::SetNext(pantheon::Thread *Item) -{ - this->NextThread = Item; + /* Yuck. But we need to make page permissions less bad...*/ + UINT64 PagePermission = 0x1 << 6; + Entry.SetPagePermissions(PagePermission); + + Entry.SetSharable(pantheon::vmm::PAGE_SHARABLE_TYPE_INNER); + Entry.SetAccessor(pantheon::vmm::PAGE_MISC_ACCESSED); + Entry.SetMAIREntry(pantheon::vmm::MAIREntry_1); + + pantheon::ScopedLock _L(this->MyProc()); + this->MyProc()->MapAddress(this->LocalRegion, pantheon::PageAllocator::Alloc(), Entry); + } + this->Unlock(); } pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() @@ -387,33 +385,4 @@ pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() pantheon::vmm::PhysicalAddress PhyAddr = pantheon::vmm::VirtualToPhysicalAddress(Proc->GetPageTable(), this->LocalRegion); pantheon::vmm::VirtualAddress VirtAddr = pantheon::vmm::PhysicalToVirtualAddress(PhyAddr); return reinterpret_cast(VirtAddr); -} - -VOID pantheon::Thread::SetupThreadLocalArea() -{ - /* Don't make a TLS for kernel idle threads */ - if (this->MyProc()->ProcessID() == 0) - { - return; - } - - this->LocalRegion = pantheon::Process::ThreadLocalBase + (this->ThreadID() * 0x1000); - - /* Map in the TLR. */ - pantheon::vmm::PageTableEntry Entry; - Entry.SetBlock(TRUE); - Entry.SetMapped(TRUE); - Entry.SetUserNoExecute(TRUE); - Entry.SetKernelNoExecute(TRUE); - - /* Yuck. But we need to make page permissions less bad...*/ - UINT64 PagePermission = 0x1 << 6; - Entry.SetPagePermissions(PagePermission); - - Entry.SetSharable(pantheon::vmm::PAGE_SHARABLE_TYPE_INNER); - Entry.SetAccessor(pantheon::vmm::PAGE_MISC_ACCESSED); - Entry.SetMAIREntry(pantheon::vmm::MAIREntry_1); - - pantheon::ScopedLock _L(this->MyProc()); - this->MyProc()->MapAddress(this->LocalRegion, pantheon::PageAllocator::Alloc(), Entry); } \ No newline at end of file diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 58342f4..3847098 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -103,23 +103,16 @@ class Thread : public pantheon::Object, public pantheon::Lockable VOID SetPriority(Thread::Priority Priority); CpuContext *GetRegisters(); + ThreadLocalRegion *GetThreadLocalArea(); [[nodiscard]] pantheon::vmm::VirtualAddress GetThreadLocalAreaRegister() const { return this->LocalRegion; } Thread &operator=(const Thread &Other); Thread &operator=(Thread &&Other) noexcept; - void SetProc(pantheon::Process *Proc); - void BlockScheduling(); void EnableScheduling(); [[nodiscard]] BOOL Preempted() const; - [[nodiscard]] BOOL End() const; - Thread *Next(); - void SetNext(pantheon::Thread *Item); - - ThreadLocalRegion *GetThreadLocalArea(); - VOID SetupThreadLocalArea(); /** * @brief Switches thread context to another thread diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index f238331..14c2b9d 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -50,11 +50,12 @@ extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) "mov %4, sp\n" : "=r"(ESR), "=r"(FAR), "=r"(ELR), "=r"(SPSR), "=r"(SP)); + pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Process *CurProc = pantheon::CPU::GetCurProcess(); pantheon::StopErrorFmt( - "ERR: SYNC HANDLER EL1: esr: %lx far: %lx elr: %lx spsr: %lx, sp: %lx pid: %u\n", - ESR, FAR, ELR, SPSR, SP, CurProc->ProcessID()); + "ERR: SYNC HANDLER EL1: esr: %lx far: %lx elr: %lx spsr: %lx, sp: %lx pid: %u, tid: %lu\n", + ESR, FAR, ELR, SPSR, SP, CurProc->ProcessID(), CurThread->ThreadID()); } extern "C" void err_handler_el1(pantheon::TrapFrame *Frame) From 55fd3be7a0bd3372f0fc18a64df16a677334e8c5 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sat, 31 Dec 2022 15:13:50 -0700 Subject: [PATCH 27/43] Strengthen lock invariants --- Common/Sync/kern_spinlock.cpp | 6 ++++ Common/kern_rand.cpp | 6 ++-- Common/kern_runtime.cpp | 14 ++++---- System/Proc/kern_sched.cpp | 62 +++++++++++++++++------------------ System/Proc/kern_thread.cpp | 8 ++--- arch/aarch64/ints.cpp | 21 ++++++------ 6 files changed, 58 insertions(+), 59 deletions(-) diff --git a/Common/Sync/kern_spinlock.cpp b/Common/Sync/kern_spinlock.cpp index 731ea2a..126c783 100644 --- a/Common/Sync/kern_spinlock.cpp +++ b/Common/Sync/kern_spinlock.cpp @@ -29,6 +29,8 @@ pantheon::Spinlock::~Spinlock() void pantheon::Spinlock::Acquire() { + OBJECT_SELF_ASSERT(); + if (this->DebugName == nullptr) { StopErrorFmt("Bad Spinlock: was nullptr\n"); @@ -57,6 +59,8 @@ void pantheon::Spinlock::Acquire() BOOL pantheon::Spinlock::TryAcquire() { + OBJECT_SELF_ASSERT(); + if (this->DebugName == nullptr) { StopErrorFmt("Bad Spinlock: was nullptr\n"); @@ -86,6 +90,8 @@ BOOL pantheon::Spinlock::TryAcquire() void pantheon::Spinlock::Release() { + OBJECT_SELF_ASSERT(); + __sync_synchronize(); if (!this->IsHolding()) { diff --git a/Common/kern_rand.cpp b/Common/kern_rand.cpp index ea51c40..4bfe9c1 100644 --- a/Common/kern_rand.cpp +++ b/Common/kern_rand.cpp @@ -6,15 +6,15 @@ #include #include +/* Do something about this at some point... */ +static pantheon::Spinlock RNGLock("Random Number Lock"); + /** * @brief An implementation of Wichmann-Hill random number generation * @return A (cryptographically insecure) random number */ UINT64 pantheon::Rand() { - /* Do something about this at some point... */ - static pantheon::Spinlock RNGLock("Random Number Lock"); - RNGLock.Acquire(); /* Arbitrary seeds. These were randomly generated. */ diff --git a/Common/kern_runtime.cpp b/Common/kern_runtime.cpp index 74c3e1d..e647fa8 100644 --- a/Common/kern_runtime.cpp +++ b/Common/kern_runtime.cpp @@ -23,8 +23,8 @@ void SERIAL_LOG_UNSAFE(const char *Fmt, ...) va_end(Args); } -static pantheon::Spinlock PanicMutex; -static pantheon::Spinlock PrintMutex; +static pantheon::Spinlock PanicMutex("Panic Mutex"); +static pantheon::Spinlock PrintMutex("Print Mutex"); void SERIAL_LOG(const char *Fmt, ...) { @@ -50,16 +50,16 @@ void pantheon::StopError(const char *Reason, void *Source) { if (Source) { - SERIAL_LOG_UNSAFE("panic: %s [source: %lx, core %x]\n", Reason, Source, pantheon::CPU::GetProcessorNumber()); + SERIAL_LOG("panic: %s [source: %lx, core %x]\n", Reason, Source, pantheon::CPU::GetProcessorNumber()); } else { - SERIAL_LOG_UNSAFE("panic: %s [core %lx]\n", Reason, pantheon::CPU::GetProcessorNumber()); + SERIAL_LOG("panic: %s [core %lx]\n", Reason, pantheon::CPU::GetProcessorNumber()); } } else { - SERIAL_LOG_UNSAFE("panic: %s [core %lx]\n", "unknown reason", pantheon::CPU::GetProcessorNumber()); + SERIAL_LOG("panic: %s [core %lx]\n", "unknown reason", pantheon::CPU::GetProcessorNumber()); } /* TODO: stop other cores */ @@ -73,7 +73,7 @@ void pantheon::StopError(const char *Reason, void *Source) void pantheon::StopErrorFmt(const char *Fmt, ...) { PanicMutex.Acquire(); - SERIAL_LOG_UNSAFE("panic: "); + SERIAL_LOG("panic: "); va_list Args; va_start(Args, Fmt); vprintf(Fmt, Args); @@ -93,8 +93,6 @@ BOOL pantheon::Panicked() void pantheon::InitBasicRuntime() { - PrintMutex = pantheon::Spinlock("print_mutex"); - PanicMutex = pantheon::Spinlock("panic_mutex"); } void _putchar(char c) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index b82708d..702330a 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -163,6 +163,9 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) void pantheon::Scheduler::Init() { + /* This is probably unnecessary, but let's be safe... */ + ScopedGlobalSchedulerLock _L; + static pantheon::Process IdleProc; ProcessList.Insert(0, &IdleProc); } @@ -253,16 +256,18 @@ UINT64 pantheon::Scheduler::CountThreads(UINT64 PID) UINT32 pantheon::Scheduler::AcquireProcessID() { UINT32 RetVal = 0; - static pantheon::Spinlock PIDLock("Process ID Lock"); static UINT32 ProcessID = 1; - PIDLock.Acquire(); + if (!SchedLock.IsLocked()) + { + pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); + } + RetVal = ProcessID++; while (ProcessList.Contains(RetVal) || RetVal == 0) { RetVal = ProcessID++; } - PIDLock.Release(); return RetVal; } @@ -270,17 +275,19 @@ UINT32 pantheon::Scheduler::AcquireProcessID() UINT64 pantheon::Scheduler::AcquireThreadID() { UINT64 RetVal = 0; - static pantheon::Spinlock TIDLock("Thread ID Lock"); static UINT64 ThreadID = 1; - TIDLock.Acquire(); + if (!SchedLock.IsLocked()) + { + pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); + } + RetVal = ThreadID++; /* Ban 0 and -1, since these are special values. */ while (RetVal == 0 || RetVal == ~0ULL || ThreadList.Contains(RetVal)) { RetVal = ThreadID++; } - TIDLock.Release(); return RetVal; } @@ -292,9 +299,8 @@ static pantheon::Thread *GetNextThread() { UINT8 LocalIndex = (MyCPU + Index) % NoCPUs; pantheon::LocalScheduler *Sched = pantheon::CPU::GetLocalSched(LocalIndex); - if (Sched) + if (Sched && Sched->TryLock()) { - Sched->Lock(); /* This needs to be TryAcquire at some point. */ pantheon::Thread *Thr = Sched->AcquireThread(); Sched->Unlock(); if (Thr != nullptr) @@ -303,7 +309,9 @@ static pantheon::Thread *GetNextThread() } } } - return nullptr; + + /* If we have no next, give up and use the idle thread. */ + return pantheon::CPU::GetMyLocalSched()->Idle(); } struct SwapContext @@ -314,12 +322,6 @@ struct SwapContext static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { - /* If we have no next, give up and use the idle thread. */ - if (NextThread == nullptr) - { - NextThread = pantheon::CPU::GetMyLocalSched()->Idle(); - } - pantheon::ScopedLock _NL(NextThread); /* Don't try to swap to ourselves. */ @@ -336,11 +338,12 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne return {nullptr, nullptr}; } - /* Okay, we have to do a weird little dance. We need this to prevent - * trying to reschedule and trip on ourselves. This gets restored - * when we context switch back here. - */ - pantheon::CPU::GetCurThread()->BlockScheduling(); + /* Did we get swapped out from under ourselves? */ + if (CurThread != pantheon::CPU::GetCurThread()) + { + CurThread->EnableScheduling(); + return {nullptr, nullptr}; + } /* Change kernel view of contexts */ pantheon::Process::Switch(NextThread->MyProc()); @@ -349,11 +352,7 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - /* Don't put idle jobs on the runqueue */ - if (CurThread->MyProc()->ProcessID() != 0) - { - pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); - } + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); return {OldContext, NewContext}; } @@ -369,17 +368,18 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne */ void pantheon::Scheduler::Reschedule() { + pantheon::ScopedRescheduleLock _L; pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); + + /* Okay, we have to do a weird little dance. We need this to prevent + * trying to reschedule and trip on ourselves. This gets restored + * when we context switch back here. + */ pantheon::Thread *NextThread = GetNextThread(); - SwapContext Con = SwapThreads(CurThread, NextThread); - pantheon::Sync::DSBISH(); - pantheon::Sync::ISB(); - - if (Con.New && Con.Old && NextThread) + if (Con.New && Con.Old) { cpu_switch(Con.Old, Con.New, CpuIRegOffset); - pantheon::CPU::GetCurThread()->EnableScheduling(); } } diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 067dc12..d1f4914 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -202,7 +202,7 @@ VOID pantheon::Thread::SetState(Thread::State St) OBJECT_SELF_ASSERT(); if (this->IsLocked() == FALSE) { - StopError("SetState without lock"); + StopError("SetState without lock", this); } this->CurState = St; } @@ -212,7 +212,7 @@ VOID pantheon::Thread::SetTicks(INT64 TickCount) OBJECT_SELF_ASSERT(); if (this->IsLocked() == FALSE) { - StopError("SetTicks without lock"); + StopError("SetTicks without lock", this); } this->RemainingTicks = (TickCount < 0) ? 0 : TickCount; } @@ -300,17 +300,13 @@ pantheon::Thread &pantheon::Thread::operator=(pantheon::Thread &&Other) noexcept void pantheon::Thread::BlockScheduling() { OBJECT_SELF_ASSERT(); - pantheon::Sync::DSBISH(); this->PreemptCount.Add(1LL); - pantheon::Sync::DSBISH(); } void pantheon::Thread::EnableScheduling() { OBJECT_SELF_ASSERT(); - pantheon::Sync::DSBISH(); this->PreemptCount.Sub(1LL); - pantheon::Sync::DSBISH(); } extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 14c2b9d..65a783d 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -29,13 +29,13 @@ extern "C" void err_handler_el1_sp0(pantheon::TrapFrame *Frame) extern "C" void fiq_handler_el1_sp0(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: FIQ HANDLER EL1 SP0"); + SERIAL_LOG("%s\n", "ERR: FIQ HANDLER EL1 SP0"); } extern "C" void irq_handler_el1_sp0(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: IRQ HANDLER EL1 SP0"); + SERIAL_LOG("%s\n", "ERR: IRQ HANDLER EL1 SP0"); } extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) @@ -67,7 +67,7 @@ extern "C" void err_handler_el1(pantheon::TrapFrame *Frame) extern "C" void fiq_handler_el1(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: FIQ HANDLER EL1"); + SERIAL_LOG("%s\n", "ERR: FIQ HANDLER EL1"); } class CurFrameMgr @@ -119,8 +119,7 @@ extern "C" void sync_handler_el0(pantheon::TrapFrame *Frame) else { pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); - SERIAL_LOG_UNSAFE("Bad sync handler el0: esr: 0x%lx far: 0x%lx elr: 0x%lx spsr: 0x%lx\n", ESR, FAR, ELR, SPSR); - SERIAL_LOG_UNSAFE("Process ID was %ld\n", CurThread->MyProc()->ProcessID()); + SERIAL_LOG("PID %d: Bad sync handler el0: esr: 0x%lx far: 0x%lx elr: 0x%lx spsr: 0x%lx\n", CurThread->MyProc()->ProcessID(), ESR, FAR, ELR, SPSR); if (ESR == 0x2000000) { pantheon::vmm::PageTable *PT = CurThread->MyProc()->GetPageTable(); @@ -132,13 +131,13 @@ extern "C" void sync_handler_el0(pantheon::TrapFrame *Frame) extern "C" void err_handler_el0(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: ERR HANDLER EL0"); + SERIAL_LOG("%s\n", "ERR: ERR HANDLER EL0"); } extern "C" void fiq_handler_el0(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: FIQ HANDLER EL0"); + SERIAL_LOG("%s\n", "ERR: FIQ HANDLER EL0"); } extern "C" void irq_handler_el0(pantheon::TrapFrame *Frame) @@ -150,25 +149,25 @@ extern "C" void irq_handler_el0(pantheon::TrapFrame *Frame) extern "C" void sync_handler_el0_32(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: SYNC HANDLER EL0_32"); + SERIAL_LOG("%s\n", "ERR: SYNC HANDLER EL0_32"); } extern "C" void err_handler_el0_32(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: ERR HANDLER EL0_32"); + SERIAL_LOG("%s\n", "ERR: ERR HANDLER EL0_32"); } extern "C" void fiq_handler_el0_32(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: FIQ HANDLER EL0_32"); + SERIAL_LOG("%s\n", "ERR: FIQ HANDLER EL0_32"); } extern "C" void irq_handler_el0_32(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); - SERIAL_LOG_UNSAFE("%s\n", "ERR: IRQ HANDLER EL0_32"); + SERIAL_LOG("%s\n", "ERR: IRQ HANDLER EL0_32"); } VOID pantheon::arm::LoadInterruptTable(VOID *Table) From 0b6319cae212068d8c62e28ab85149e0b1636f48 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sat, 31 Dec 2022 18:51:07 -0700 Subject: [PATCH 28/43] Add 2SMP debug launch script --- buildscripts/test-gdb-2smp.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 buildscripts/test-gdb-2smp.sh diff --git a/buildscripts/test-gdb-2smp.sh b/buildscripts/test-gdb-2smp.sh new file mode 100755 index 0000000..683d9d4 --- /dev/null +++ b/buildscripts/test-gdb-2smp.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +rm -rf build +mkdir build +cd build +cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/qemu-aarch64.cmake -DCMAKE_C_FLAGS="-g3 -O0" -DCMAKE_CXX_FLAGS="-g3 -O0" +make -j`nproc` +qemu-img create -fqcow2 emmc.img 4G +qemu-system-aarch64 -machine virt,gic-version=2,secure=on -smp 2 -cpu cortex-a72 -m 8G -kernel ./kernel.img -device sdhci-pci -device sd-card,drive=flash -drive id=flash,if=none,format=qcow2,file=emmc.img -s -S -d int + From bbd0e40f7d2ab7bc3f2d17856ba675e6e89f1458 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sat, 31 Dec 2022 21:58:35 -0700 Subject: [PATCH 29/43] Cleanup Doxygen comments --- System/Proc/kern_sched.cpp | 67 +++++++++++++++++++++----------------- System/Proc/kern_sched.hpp | 26 +++++++++++++-- 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 702330a..bd0dd4a 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -20,7 +20,7 @@ #include /** - * @file Common/Proc/kern_sched.cpp + * @file System/Proc/kern_sched.cpp * \~english @brief Definitions for basic kernel scheduling data structures and algorithms. * \~english @details Pantheon implements Con Kolivas' MuQSS scheduling algorithm, an efficient algorithm * which takes priorities into account to provide a semi-"realtime" scheduling system to ensure @@ -38,26 +38,24 @@ static pantheon::Spinlock SchedLock("Global Scheduler Lock"); static pantheon::SkipList ProcessList; static pantheon::SkipList ThreadList; - -/** - * \~english @brief Initalizes an instance of a per-core scheduler. - * \~english @author Brian Schnepp - */ pantheon::LocalScheduler::LocalScheduler() : pantheon::Lockable("Scheduler") { this->IdleThread = nullptr; } -/* Provided by the arch/ subdir. These differ between hardware platforms. */ +/** + * @private + * @brief Provided by the arch/ subdir. Swaps the current context + */ extern "C" void cpu_switch(pantheon::CpuContext *Old, pantheon::CpuContext *New, UINT64 RegOffset); + +/** + * @private + * @brief Provided by the arch/ subdir. Drops a new thread to have a different PSTATE + */ extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); -/** - * \~english @brief Counts the number of threads under this scheduler which belong to a given PID. - * \~english @invariant This scheduler is not locked - * \~english @return The number of threads under this manager which belong to a given process - */ UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) { UINT64 Res = 0; @@ -69,11 +67,6 @@ UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) return Res; } -/** - * \~english @brief Obtains a thread from the local runqueue which can be run, and removes it from the local runqueue. - * \~english @invariant This scheduler is locked - * \~english @return A thread which can be run, if it exists. Nullptr otherwise. - */ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() { Optional LowestKey = this->LocalRunQueue.MinKey(); @@ -93,11 +86,19 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() } +/** + * @private + * @brief Obtains the idle thread + */ pantheon::Thread *pantheon::LocalScheduler::Idle() { return this->IdleThread; } +/** + * @private + * @brief Sets up a local scheduler + */ void pantheon::LocalScheduler::Setup() { pantheon::ScopedGlobalSchedulerLock _L; @@ -132,12 +133,6 @@ static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInter return Deadline; } -/** - * \~english @brief Inserts a thread for this local scheduler - * \~english @invariant The thread to be inserted is locked before calling - * \~english @invariant This scheduler is not locked - * \~english @param Thr The thread object to insert. - */ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) { if (Thr == nullptr) @@ -146,7 +141,6 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) } pantheon::ScopedLock _L(this); - this->Threads.PushFront(Thr); if (Thr->MyState() == pantheon::Thread::STATE_WAITING) { UINT64 Jiffies = pantheon::CPU::GetJiffies(); @@ -161,6 +155,11 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) } } + +/** + * @internal + * @brief Sets up a global scheduler + */ void pantheon::Scheduler::Init() { /* This is probably unnecessary, but let's be safe... */ @@ -170,6 +169,11 @@ void pantheon::Scheduler::Init() ProcessList.Insert(0, &IdleProc); } +/** + * \~english @brief Locks the global scheduler + * \~english @details Locks the global scheduler, so that it is safe to perform + * operations such as manipulating the global process table or global thread table. + */ void pantheon::Scheduler::Lock() { SchedLock.Acquire(); @@ -314,6 +318,10 @@ static pantheon::Thread *GetNextThread() return pantheon::CPU::GetMyLocalSched()->Idle(); } +/** + * @private + * @brief Struct to hold context registers + */ struct SwapContext { pantheon::CpuContext *Old; @@ -369,13 +377,10 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne void pantheon::Scheduler::Reschedule() { pantheon::ScopedRescheduleLock _L; - pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); - /* Okay, we have to do a weird little dance. We need this to prevent - * trying to reschedule and trip on ourselves. This gets restored - * when we context switch back here. - */ + pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); + SwapContext Con = SwapThreads(CurThread, NextThread); if (Con.New && Con.Old) { @@ -394,6 +399,10 @@ void pantheon::Scheduler::AttemptReschedule() } } +/** + * @private + * @brief Finish routine for jumping to userspace + */ extern "C" VOID FinishThread() { pantheon::CPU::GetCurThread()->EnableScheduling(); diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index dd9f46b..1f78e66 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -24,15 +24,37 @@ class LocalScheduler : public pantheon::Allocatable LocalRunQueue; From fcecb018f8f92120b3d096728e8d225000b454b8 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sat, 31 Dec 2022 22:13:14 -0700 Subject: [PATCH 30/43] Move ThreadID creation to kern_sched.cpp --- System/Proc/kern_sched.cpp | 118 ++++++++++++++++-------------------- System/Proc/kern_sched.hpp | 54 +++++++++++++---- System/Proc/kern_thread.cpp | 4 +- System/Proc/kern_thread.hpp | 2 +- 4 files changed, 98 insertions(+), 80 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index bd0dd4a..22d6630 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -55,6 +55,16 @@ extern "C" void cpu_switch(pantheon::CpuContext *Old, pantheon::CpuContext *New, */ extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); +/** + * @private + * @brief Prevent rescheduling under this current thread + */ +class ScopedRescheduleLock +{ +public: + FORCE_INLINE ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->BlockScheduling(); } + FORCE_INLINE ~ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->EnableScheduling(); } +}; UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) { @@ -95,6 +105,44 @@ pantheon::Thread *pantheon::LocalScheduler::Idle() return this->IdleThread; } +static UINT32 AcquireProcessID() +{ + UINT32 RetVal = 0; + static UINT32 ProcessID = 1; + + if (!SchedLock.IsLocked()) + { + pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); + } + + RetVal = ProcessID++; + while (ProcessList.Contains(RetVal) || RetVal == 0) + { + RetVal = ProcessID++; + } + + return RetVal; +} + +static UINT64 AcquireThreadID() +{ + UINT64 RetVal = 0; + static UINT64 ThreadID = 1; + + if (!SchedLock.IsLocked()) + { + pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); + } + + RetVal = ThreadID++; + /* Ban 0 and -1, since these are special values. */ + while (RetVal == 0 || RetVal == ~0ULL || ThreadList.Contains(RetVal)) + { + RetVal = ThreadID++; + } + return RetVal; +} + /** * @private * @brief Sets up a local scheduler @@ -110,7 +158,7 @@ void pantheon::LocalScheduler::Setup() pantheon::StopErrorFmt("No PID 0!\n"); } - Idle->Initialize(MProc.GetValue(), nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); + Idle->Initialize(AcquireThreadID(), MProc.GetValue(), nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); Idle->Lock(); Idle->SetState(pantheon::Thread::STATE_RUNNING); Idle->Unlock(); @@ -184,23 +232,11 @@ void pantheon::Scheduler::Unlock() SchedLock.Release(); } -/** - * \~english @brief Creates a process, visible globally, from a name and address. - * \~english @details A process is created, such that it can be run on any - * available processor on the system. The given process name identifies the - * process for debugging purposes and as a user-readable way to identify a - * process, and the start address is a pointer to the first instruction the - * initial threaq should execute when scheduled. - * \~english @param[in] ProcStr A human-readable name for the process - * \~english @param[in] StartAddr The initial value of the program counter - * \~english @return New PID if the process was sucessfully created, 0 otherwise. - * \~english @author Brian Schnepp - */ UINT32 pantheon::Scheduler::CreateProcess(const pantheon::String &ProcStr, void *StartAddr) { pantheon::ScopedGlobalSchedulerLock _L; - UINT32 NewID = pantheon::Scheduler::AcquireProcessID(); + UINT32 NewID = AcquireProcessID(); pantheon::Process *Proc = pantheon::Process::Create(); if (Proc == nullptr) @@ -233,7 +269,7 @@ pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, pantheon::Process *MyProc = ProcessList.Get(PID).GetValue(); pantheon::Thread *Thr = pantheon::Thread::Create(); - Thr->Initialize(MyProc, StartAddr, ThreadData, Priority, TRUE); + Thr->Initialize(AcquireThreadID(), MyProc, StartAddr, ThreadData, Priority, TRUE); ThreadList.Insert(Thr->ThreadID(), Thr); pantheon::ScopedLock _STL(Thr); @@ -257,44 +293,6 @@ UINT64 pantheon::Scheduler::CountThreads(UINT64 PID) return Count; } -UINT32 pantheon::Scheduler::AcquireProcessID() -{ - UINT32 RetVal = 0; - static UINT32 ProcessID = 1; - - if (!SchedLock.IsLocked()) - { - pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); - } - - RetVal = ProcessID++; - while (ProcessList.Contains(RetVal) || RetVal == 0) - { - RetVal = ProcessID++; - } - - return RetVal; -} - -UINT64 pantheon::Scheduler::AcquireThreadID() -{ - UINT64 RetVal = 0; - static UINT64 ThreadID = 1; - - if (!SchedLock.IsLocked()) - { - pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); - } - - RetVal = ThreadID++; - /* Ban 0 and -1, since these are special values. */ - while (RetVal == 0 || RetVal == ~0ULL || ThreadList.Contains(RetVal)) - { - RetVal = ThreadID++; - } - return RetVal; -} - static pantheon::Thread *GetNextThread() { UINT8 MyCPU = pantheon::CPU::GetProcessorNumber(); @@ -364,23 +362,13 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne return {OldContext, NewContext}; } -/** - * \~english @brief Changes the current thread of this core. - * \~english @details Forcefully changes the current thread by iterating to - * the next thread in the list of threads belonging to this core. The active - * count for the current thread is set to zero, and the new thread is run until - * either the core is forcefully rescheduled, or the timer indicates the current - * thread should quit. - * - * \~english @author Brian Schnepp - */ void pantheon::Scheduler::Reschedule() { - pantheon::ScopedRescheduleLock _L; + ScopedRescheduleLock _L; pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); - + SwapContext Con = SwapThreads(CurThread, NextThread); if (Con.New && Con.Old) { diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 1f78e66..7c1535f 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -64,11 +64,41 @@ class LocalScheduler : public pantheon::AllocatableBlockScheduling(); } - FORCE_INLINE ~ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->EnableScheduling(); } -}; - } #endif \ No newline at end of file diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index d1f4914..46cd05d 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -317,10 +317,10 @@ static void true_drop_process(void *StartAddress, pantheon::vmm::VirtualAddress drop_usermode((UINT64)StartAddress, 0x00, StackAddr); } -void pantheon::Thread::Initialize(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority, BOOL UserMode) +void pantheon::Thread::Initialize(UINT64 TID, pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority, BOOL UserMode) { static constexpr UINT64 InitialThreadStackSize = pantheon::vmm::SmallestPageSize * pantheon::Thread::InitialNumStackPages; - this->TID = pantheon::Scheduler::AcquireThreadID(); + this->TID = TID; this->CurState = pantheon::Thread::STATE_DEAD; this->Lock(); diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 3847098..e60a5d7 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -89,7 +89,7 @@ class Thread : public pantheon::Object, public pantheon::Lockable Thread(Thread &&Other) noexcept; ~Thread() override; - void Initialize(pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority, BOOL UserMode); + void Initialize(UINT64 TID, pantheon::Process *Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority, BOOL UserMode); [[nodiscard]] Process *MyProc() const; [[nodiscard]] Thread::State MyState() const; From 8bfddad4c36120ef9a653f80da527aca913b2ed9 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sun, 1 Jan 2023 00:52:44 -0700 Subject: [PATCH 31/43] Improve logging for errors under traps --- arch/aarch64/ints.cpp | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 65a783d..3cf58c8 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -14,6 +14,23 @@ static UINT64 TimerClock = 1000; +VOID TraceStack(UINT64 X29) +{ + struct Frame + { + Frame *Next; + UINT64 PC; + }; + + UINT64 Counter = 0; + Frame *Cur = reinterpret_cast(X29); + while (Cur && Cur->PC) + { + SERIAL_LOG("[Frame %lu]: %lx\n", Counter++, Cur->PC); + Cur = Cur->Next; + } +} + extern "C" void sync_handler_el1_sp0(pantheon::TrapFrame *Frame) { PANTHEON_UNUSED(Frame); @@ -40,7 +57,6 @@ extern "C" void irq_handler_el1_sp0(pantheon::TrapFrame *Frame) extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) { - PANTHEON_UNUSED(Frame); UINT64 ESR, FAR, ELR, SPSR, SP; asm volatile( "mrs %0, esr_el1\n" @@ -53,6 +69,13 @@ extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Process *CurProc = pantheon::CPU::GetCurProcess(); + TraceStack(Frame->Regs[29]); + for (UINT64 Index = 0; Index < 31; Index++) + { + SERIAL_LOG("x%lu: %lx ", Index, Frame->Regs[Index]); + } + SERIAL_LOG("\n"); + pantheon::StopErrorFmt( "ERR: SYNC HANDLER EL1: esr: %lx far: %lx elr: %lx spsr: %lx, sp: %lx pid: %u, tid: %lu\n", ESR, FAR, ELR, SPSR, SP, CurProc->ProcessID(), CurThread->ThreadID()); @@ -60,7 +83,7 @@ extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) extern "C" void err_handler_el1(pantheon::TrapFrame *Frame) { - PANTHEON_UNUSED(Frame); + TraceStack(Frame->Regs[29]); pantheon::StopError("ERR: ERR HANDLER EL1"); } From 4944886ad239410cd1d982a59cab4a864e320ce4 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sun, 1 Jan 2023 02:48:31 -0700 Subject: [PATCH 32/43] Cleanup thread-related initialization and tracing --- System/Proc/kern_sched.cpp | 37 +++++++++++++++++-------------------- System/Proc/kern_sched.hpp | 32 +++++++++++++++++++++++++++++--- System/Proc/kern_thread.cpp | 21 +++++++++++++++++++++ System/Proc/kern_thread.hpp | 14 +------------- arch/aarch64/ints.cpp | 7 ++++--- 5 files changed, 72 insertions(+), 39 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 22d6630..06abdfd 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -40,6 +40,9 @@ static pantheon::SkipList ThreadList; pantheon::LocalScheduler::LocalScheduler() : pantheon::Lockable("Scheduler") { + /* Because of the ordering required for init_array, the actual + * setup must be done with the Setup() function, not the constructor. + */ this->IdleThread = nullptr; } @@ -257,14 +260,14 @@ UINT32 pantheon::Scheduler::CreateProcess(const pantheon::String &ProcStr, void return NewID; } -pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) +UINT64 pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority) { pantheon::ScopedGlobalSchedulerLock _L; if(!ProcessList.Contains(PID)) { /* Huh? You're asking for a PID that doesn't exist. */ - return nullptr; + return 0; } pantheon::Process *MyProc = ProcessList.Get(PID).GetValue(); @@ -274,7 +277,7 @@ pantheon::Thread *pantheon::Scheduler::CreateThread(UINT32 PID, void *StartAddr, pantheon::ScopedLock _STL(Thr); pantheon::CPU::GetMyLocalSched()->InsertThread(Thr); - return Thr; + return Thr->ThreadID(); } @@ -328,6 +331,7 @@ struct SwapContext static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { + pantheon::ScopedGlobalSchedulerLock _L; pantheon::ScopedLock _NL(NextThread); /* Don't try to swap to ourselves. */ @@ -338,32 +342,23 @@ static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *Ne pantheon::ScopedLock _OL(CurThread); - if (NextThread->MyState() != pantheon::Thread::STATE_WAITING) - { - /* Huh? Why are we still here? */ - return {nullptr, nullptr}; - } - - /* Did we get swapped out from under ourselves? */ - if (CurThread != pantheon::CPU::GetCurThread()) - { - CurThread->EnableScheduling(); - return {nullptr, nullptr}; - } + pantheon::CpuContext *OldContext = CurThread->GetRegisters(); + pantheon::CpuContext *NewContext = NextThread->GetRegisters(); /* Change kernel view of contexts */ pantheon::Process::Switch(NextThread->MyProc()); pantheon::Thread::Switch(NextThread); - - pantheon::CpuContext *OldContext = CurThread->GetRegisters(); - pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - - pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); return {OldContext, NewContext}; } void pantheon::Scheduler::Reschedule() { + /* Interrupts must be enabled before we can do anything. */ + if (pantheon::CPU::ICOUNT()) + { + return; + } + ScopedRescheduleLock _L; pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); @@ -372,6 +367,8 @@ void pantheon::Scheduler::Reschedule() SwapContext Con = SwapThreads(CurThread, NextThread); if (Con.New && Con.Old) { + pantheon::Sync::DSBISH(); + pantheon::Sync::ISB(); cpu_switch(Con.Old, Con.New, CpuIRegOffset); } } diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 7c1535f..15e0f6b 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -80,11 +80,23 @@ namespace Scheduler * \~english @pre The global scheduler lock is not held by this processor * \~english @pre The thread lock is not held by this processor * \~english @param[in] TID The thread ID to modify - * \~english @param[in] State The state to set the process to + * \~english @param[in] State The state to set the thread to */ BOOL SetThreadState(UINT64 TID, pantheon::Thread::State State); - + /** + * \~english @brief Maps some pages into a given process' address space + * \~english @param[in] PID The process ID to map addresses into + * \~english @param[in] VAddresses The set of virtual addresses to map these pages as + * \~english @param[in] PAddresses The set of physical addresses to map these pages from + * \~english @param[in] PageAttributes The set of page attributes (permissions, cachability, etc.) to use for all of these pages + * \~english @param[in] NumPages The number of pages to map + * \~english @return TRUE on success, FALSE otherwise + * \~english @pre Length(VAddresses) == Length(PAddress) + * \~english @pre Length(VAddresses) >= NumPages + * \~english @pre Length(PAddresses) >= NumPages + * \~english @author Brian Schnepp + */ BOOL MapPages(UINT32 PID, const pantheon::vmm::VirtualAddress *VAddresses, const pantheon::vmm::PhysicalAddress *PAddresses, const pantheon::vmm::PageTableEntry &PageAttributes, UINT64 NumPages); /** @@ -100,11 +112,25 @@ namespace Scheduler * \~english @author Brian Schnepp */ UINT32 CreateProcess(const pantheon::String &ProcStr, void *StartAddr); - pantheon::Thread *CreateThread(UINT32 Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + UINT64 CreateThread(UINT32 Proc, void *StartAddr, void *ThreadData, pantheon::Thread::Priority Priority = pantheon::Thread::PRIORITY_NORMAL); + /** + * \~english @brief Counts the number of threads which belong to a given PID. + * \~english @pre Global scheduler is not locked + * \~english @return The number of threads which belong to a given process + */ UINT64 CountThreads(UINT64 PID); + /** + * \~english @brief Locks the global scheduler, preventing operations such as thread or process modification from proceeding on other threads + * @see ScopedGlobalSchedulerLock + */ void Lock(); + + /** + * \~english @brief Releases the global scheduler, allowing operations such as thread or process modification to proceed on other threads + * @see ScopedGlobalSchedulerLock + */ void Unlock(); /** diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 46cd05d..a44d203 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -381,4 +381,25 @@ pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() pantheon::vmm::PhysicalAddress PhyAddr = pantheon::vmm::VirtualToPhysicalAddress(Proc->GetPageTable(), this->LocalRegion); pantheon::vmm::VirtualAddress VirtAddr = pantheon::vmm::PhysicalToVirtualAddress(PhyAddr); return reinterpret_cast(VirtAddr); +} + +/** + * @brief Switches thread context to another thread + */ +void pantheon::Thread::Switch(pantheon::Thread *NextThread) +{ + if (NextThread == nullptr) + { + return; + } + + pantheon::Thread *CurThread = pantheon::CPU::GetCoreInfo()->CurThread; + pantheon::CPU::GetCurThread()->SetState(pantheon::Thread::STATE_WAITING); + NextThread->SetState(Thread::STATE_RUNNING); + NextThread->SetTicks(Thread::RR_INTERVAL); + + pantheon::ipc::SetThreadLocalRegion(NextThread->LocalRegion); + + pantheon::CPU::GetCoreInfo()->CurThread = NextThread; + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); } \ No newline at end of file diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index e60a5d7..3f0b1e1 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -117,19 +117,7 @@ class Thread : public pantheon::Object, public pantheon::Lockable /** * @brief Switches thread context to another thread */ - static void Switch(Thread *Next) - { - if (Next == nullptr) - { - return; - } - - pantheon::ipc::SetThreadLocalRegion(Next->LocalRegion); - pantheon::CPU::GetCurThread()->SetState(pantheon::Thread::STATE_WAITING); - Next->SetState(Thread::STATE_RUNNING); - Next->SetTicks(Thread::RR_INTERVAL); - pantheon::CPU::GetCoreInfo()->CurThread = Next; - } + static void Switch(Thread *Next); private: VOID SetEntryLocation(UINT64 IP, UINT64 SP, VOID* ThreadData); diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 3cf58c8..384c560 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -14,7 +14,7 @@ static UINT64 TimerClock = 1000; -VOID TraceStack(UINT64 X29) +VOID TraceStack(UINT64 PC, UINT64 X29) { struct Frame { @@ -24,6 +24,7 @@ VOID TraceStack(UINT64 X29) UINT64 Counter = 0; Frame *Cur = reinterpret_cast(X29); + SERIAL_LOG("[Frame %lu]: %lx\n", Counter++, PC); while (Cur && Cur->PC) { SERIAL_LOG("[Frame %lu]: %lx\n", Counter++, Cur->PC); @@ -69,7 +70,7 @@ extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Process *CurProc = pantheon::CPU::GetCurProcess(); - TraceStack(Frame->Regs[29]); + TraceStack(Frame->PC, Frame->Regs[29]); for (UINT64 Index = 0; Index < 31; Index++) { SERIAL_LOG("x%lu: %lx ", Index, Frame->Regs[Index]); @@ -83,7 +84,7 @@ extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) extern "C" void err_handler_el1(pantheon::TrapFrame *Frame) { - TraceStack(Frame->Regs[29]); + TraceStack(Frame->PC, Frame->Regs[29]); pantheon::StopError("ERR: ERR HANDLER EL1"); } From b40e66d2ff01e71cfec9c694c5d592fc2d63fcc8 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sun, 1 Jan 2023 03:14:24 -0700 Subject: [PATCH 33/43] Finish documentation for the Scheduler namespace --- System/Proc/kern_sched.cpp | 7 +------ System/Proc/kern_sched.hpp | 41 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 06abdfd..c59e182 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -208,7 +208,7 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) /** - * @internal + * @private * @brief Sets up a global scheduler */ void pantheon::Scheduler::Init() @@ -220,11 +220,6 @@ void pantheon::Scheduler::Init() ProcessList.Insert(0, &IdleProc); } -/** - * \~english @brief Locks the global scheduler - * \~english @details Locks the global scheduler, so that it is safe to perform - * operations such as manipulating the global process table or global thread table. - */ void pantheon::Scheduler::Lock() { SchedLock.Acquire(); diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 15e0f6b..5e4c368 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -20,6 +20,10 @@ namespace pantheon { +/** + * \~english @brief An instance of a per-core scheduler, including the local runqueue + * \~english @author Brian Schnepp + */ class LocalScheduler : public pantheon::Allocatable, public pantheon::Lockable { @@ -62,6 +66,10 @@ class LocalScheduler : public pantheon::Allocatable Date: Sun, 1 Jan 2023 19:20:33 -0700 Subject: [PATCH 34/43] Enforce additional consistency with the scheduler --- System/Proc/kern_cpu.cpp | 2 +- System/Proc/kern_sched.cpp | 57 ++++++++++++++++++------------------- System/Proc/kern_sched.hpp | 1 + System/Proc/kern_thread.cpp | 20 ++++--------- System/Proc/kern_thread.hpp | 4 +-- 5 files changed, 38 insertions(+), 46 deletions(-) diff --git a/System/Proc/kern_cpu.cpp b/System/Proc/kern_cpu.cpp index adb325b..4ee4a3a 100644 --- a/System/Proc/kern_cpu.cpp +++ b/System/Proc/kern_cpu.cpp @@ -14,7 +14,7 @@ #include /* Avoid having too high a number of cores to look through. */ -static UINT8 NoCPUs = 0; +static atomic_uint_fast8_t NoCPUs = 0; static pantheon::CPU::CoreInfo PerCoreInfo[MAX_NUM_CPUS]; void pantheon::CPU::InitCore(UINT8 CoreNo) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index c59e182..e8e73b0 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -44,6 +44,7 @@ pantheon::LocalScheduler::LocalScheduler() : pantheon::Lockable("Scheduler") * setup must be done with the Setup() function, not the constructor. */ this->IdleThread = nullptr; + this->Ready = FALSE; } /** @@ -71,6 +72,11 @@ class ScopedRescheduleLock UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) { + if (!this->Ready) + { + this->Setup(); + } + UINT64 Res = 0; pantheon::ScopedLock _L(this); for (const auto &Item : this->Threads) @@ -82,6 +88,11 @@ UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) pantheon::Thread *pantheon::LocalScheduler::AcquireThread() { + if (!this->Ready) + { + this->Setup(); + } + Optional LowestKey = this->LocalRunQueue.MinKey(); if (!LowestKey.GetOkay()) { @@ -105,6 +116,11 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() */ pantheon::Thread *pantheon::LocalScheduler::Idle() { + if (!this->Ready) + { + this->Setup(); + } + return this->IdleThread; } @@ -167,7 +183,10 @@ void pantheon::LocalScheduler::Setup() Idle->Unlock(); pantheon::CPU::GetCoreInfo()->CurThread = Idle; - this->IdleThread = Idle; + this->IdleThread = Idle; + this->Ready = TRUE; + + pantheon::CPU::GetMyLocalSched()->InsertThread(Idle); } static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInterval) @@ -314,58 +333,38 @@ static pantheon::Thread *GetNextThread() return pantheon::CPU::GetMyLocalSched()->Idle(); } -/** - * @private - * @brief Struct to hold context registers - */ -struct SwapContext +static void SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { - pantheon::CpuContext *Old; - pantheon::CpuContext *New; -}; - -static SwapContext SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) -{ - pantheon::ScopedGlobalSchedulerLock _L; - pantheon::ScopedLock _NL(NextThread); - /* Don't try to swap to ourselves. */ if (NextThread == CurThread) { - return {nullptr, nullptr}; + return; } - pantheon::ScopedLock _OL(CurThread); - pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); /* Change kernel view of contexts */ pantheon::Process::Switch(NextThread->MyProc()); - pantheon::Thread::Switch(NextThread); - return {OldContext, NewContext}; + pantheon::Thread::Switch(CurThread, NextThread); + + cpu_switch(OldContext, NewContext, CpuIRegOffset); } void pantheon::Scheduler::Reschedule() { + ScopedRescheduleLock _L; + /* Interrupts must be enabled before we can do anything. */ if (pantheon::CPU::ICOUNT()) { return; } - ScopedRescheduleLock _L; - pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::Thread *NextThread = GetNextThread(); - SwapContext Con = SwapThreads(CurThread, NextThread); - if (Con.New && Con.Old) - { - pantheon::Sync::DSBISH(); - pantheon::Sync::ISB(); - cpu_switch(Con.Old, Con.New, CpuIRegOffset); - } + SwapThreads(CurThread, NextThread); } void pantheon::Scheduler::AttemptReschedule() diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index 5e4c368..c1b016f 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -64,6 +64,7 @@ class LocalScheduler : public pantheon::Allocatable LocalRunQueue; pantheon::LinkedList Threads; pantheon::Thread *IdleThread; + BOOL Ready; }; /** diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index a44d203..0ab4d2e 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -161,10 +161,10 @@ pantheon::Thread::Priority pantheon::Thread::MyPriority() const /** * \~english @brief Checks if this thread is valid to interrupt at this moment * \~english @author Brian Schnepp - * \~english @return 0 if not doing system work currently, 1 otherwise. + * \~english @return 0 if not doing system work currently, positive integer otherwise. */ [[nodiscard]] -BOOL pantheon::Thread::Preempted() const +INT64 pantheon::Thread::Preempted() const { OBJECT_SELF_ASSERT(); return this->PreemptCount != 0; @@ -245,11 +245,6 @@ VOID pantheon::Thread::SetPriority(Thread::Priority Pri) pantheon::CpuContext *pantheon::Thread::GetRegisters() { OBJECT_SELF_ASSERT(); - if (this->IsLocked() == FALSE) - { - StopError("GetRegisters without lock"); - } - /* TODO: Copy the actual registers to the internal representation! */ return &this->Registers; } @@ -386,15 +381,12 @@ pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() /** * @brief Switches thread context to another thread */ -void pantheon::Thread::Switch(pantheon::Thread *NextThread) +void pantheon::Thread::Switch(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { - if (NextThread == nullptr) - { - return; - } + pantheon::ScopedLock _O(CurThread); + CurThread->SetState(pantheon::Thread::STATE_WAITING); - pantheon::Thread *CurThread = pantheon::CPU::GetCoreInfo()->CurThread; - pantheon::CPU::GetCurThread()->SetState(pantheon::Thread::STATE_WAITING); + pantheon::ScopedLock _N(NextThread); NextThread->SetState(Thread::STATE_RUNNING); NextThread->SetTicks(Thread::RR_INTERVAL); diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 3f0b1e1..7871514 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -111,13 +111,13 @@ class Thread : public pantheon::Object, public pantheon::Lockable void BlockScheduling(); void EnableScheduling(); - [[nodiscard]] BOOL Preempted() const; + [[nodiscard]] INT64 Preempted() const; /** * @brief Switches thread context to another thread */ - static void Switch(Thread *Next); + static void Switch(Thread *CurThread, Thread *NextThread); private: VOID SetEntryLocation(UINT64 IP, UINT64 SP, VOID* ThreadData); From 0210481615d584bd4f7c7a7d78f6beeda2eec280 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Sun, 1 Jan 2023 20:27:41 -0700 Subject: [PATCH 35/43] Enforce additiona locked requirement checks --- System/Proc/kern_sched.cpp | 12 +++++++++++- arch/aarch64/ints.cpp | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index e8e73b0..b886a5c 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -93,6 +93,11 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() this->Setup(); } + if (!this->IsLocked()) + { + pantheon::StopErrorFmt("Trying to AcquireThread without lock\n"); + } + Optional LowestKey = this->LocalRunQueue.MinKey(); if (!LowestKey.GetOkay()) { @@ -180,13 +185,13 @@ void pantheon::LocalScheduler::Setup() Idle->Initialize(AcquireThreadID(), MProc.GetValue(), nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); Idle->Lock(); Idle->SetState(pantheon::Thread::STATE_RUNNING); - Idle->Unlock(); pantheon::CPU::GetCoreInfo()->CurThread = Idle; this->IdleThread = Idle; this->Ready = TRUE; pantheon::CPU::GetMyLocalSched()->InsertThread(Idle); + Idle->Unlock(); } static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInterval) @@ -210,6 +215,11 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) return; } + if (!Thr->IsLocked()) + { + pantheon::StopErrorFmt("Attempting to insert thread which wasn't locked into runqueue\n"); + } + pantheon::ScopedLock _L(this); if (Thr->MyState() == pantheon::Thread::STATE_WAITING) { diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 384c560..06b720a 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -24,10 +24,10 @@ VOID TraceStack(UINT64 PC, UINT64 X29) UINT64 Counter = 0; Frame *Cur = reinterpret_cast(X29); - SERIAL_LOG("[Frame %lu]: %lx\n", Counter++, PC); + SERIAL_LOG("[Core No %hhu, Frame %lu]: %lx\n", pantheon::CPU::GetProcessorNumber(), Counter++, PC); while (Cur && Cur->PC) { - SERIAL_LOG("[Frame %lu]: %lx\n", Counter++, Cur->PC); + SERIAL_LOG("[Core No %hhu, Frame %lu]: %lx\n", pantheon::CPU::GetProcessorNumber(), Counter++, Cur->PC); Cur = Cur->Next; } } From ecc5b18a70bb6630b6b0da0d217a00f15a6a3a9a Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Mon, 2 Jan 2023 00:16:27 -0700 Subject: [PATCH 36/43] Stop crashing with SMP This doesn't allow for actual swapping of threads, for now though. --- System/Proc/kern_sched.cpp | 30 ++++++++++++++++++++++-------- System/Proc/kern_thread.cpp | 8 ++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index b886a5c..138520d 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -345,20 +345,17 @@ static pantheon::Thread *GetNextThread() static void SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { - /* Don't try to swap to ourselves. */ - if (NextThread == CurThread) - { - return; - } + pantheon::Thread::Switch(CurThread, NextThread); + pantheon::Process::Switch(NextThread->MyProc()); pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - /* Change kernel view of contexts */ - pantheon::Process::Switch(NextThread->MyProc()); - pantheon::Thread::Switch(CurThread, NextThread); + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); + NextThread->Unlock(); cpu_switch(OldContext, NewContext, CpuIRegOffset); + CurThread->Unlock(); } void pantheon::Scheduler::Reschedule() @@ -372,7 +369,24 @@ void pantheon::Scheduler::Reschedule() } pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); + CurThread->Lock(); + pantheon::Thread *NextThread = GetNextThread(); + if (NextThread == CurThread) + { + CurThread->Unlock(); + return; + } + + NextThread->Lock(); + + if (NextThread->MyState() == pantheon::Thread::STATE_RUNNING + || CurThread->MyState() != pantheon::Thread::STATE_RUNNING) + { + NextThread->Unlock(); + CurThread->Unlock(); + return; + } SwapThreads(CurThread, NextThread); } diff --git a/System/Proc/kern_thread.cpp b/System/Proc/kern_thread.cpp index 0ab4d2e..5dd86d6 100644 --- a/System/Proc/kern_thread.cpp +++ b/System/Proc/kern_thread.cpp @@ -383,15 +383,11 @@ pantheon::Thread::ThreadLocalRegion *pantheon::Thread::GetThreadLocalArea() */ void pantheon::Thread::Switch(pantheon::Thread *CurThread, pantheon::Thread *NextThread) { - pantheon::ScopedLock _O(CurThread); - CurThread->SetState(pantheon::Thread::STATE_WAITING); - - pantheon::ScopedLock _N(NextThread); NextThread->SetState(Thread::STATE_RUNNING); NextThread->SetTicks(Thread::RR_INTERVAL); pantheon::ipc::SetThreadLocalRegion(NextThread->LocalRegion); - pantheon::CPU::GetCoreInfo()->CurThread = NextThread; - pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); + + CurThread->SetState(pantheon::Thread::STATE_WAITING); } \ No newline at end of file From b7f9478a7929ca1c01fba27e176c61c69001d1df Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 3 Jan 2023 11:11:01 -0700 Subject: [PATCH 37/43] Implement O(k) task lookup algorithm --- System/Proc/kern_sched.cpp | 26 ++++++++++++++++++++++++++ System/Proc/kern_sched.hpp | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 138520d..c51053c 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -129,6 +129,13 @@ pantheon::Thread *pantheon::LocalScheduler::Idle() return this->IdleThread; } +[[nodiscard]] +UINT64 pantheon::LocalScheduler::GetLowestDeadline() const +{ + Optional Val = this->LocalRunQueue.MinKey(); + return Val.GetOkay() ? Val.GetValue() : -1; +} + static UINT32 AcquireProcessID() { UINT32 RetVal = 0; @@ -324,10 +331,29 @@ static pantheon::Thread *GetNextThread() { UINT8 MyCPU = pantheon::CPU::GetProcessorNumber(); UINT8 NoCPUs = pantheon::CPU::GetNoCPUs(); + + UINT64 LowestDeadline = -1; + UINT8 LowestScheduler = MyCPU; + for (UINT8 Index = 0; Index < NoCPUs; Index++) { UINT8 LocalIndex = (MyCPU + Index) % NoCPUs; pantheon::LocalScheduler *Sched = pantheon::CPU::GetLocalSched(LocalIndex); + if (Sched) + { + UINT64 Deadline = Sched->GetLowestDeadline(); + if (Deadline < LowestDeadline) + { + LowestScheduler = LocalIndex; + LowestDeadline = Deadline; + } + } + } + + for (UINT8 Index = 0; Index < NoCPUs; Index++) + { + UINT8 LocalIndex = (LowestScheduler + Index) % NoCPUs; + pantheon::LocalScheduler *Sched = pantheon::CPU::GetLocalSched(LocalIndex); if (Sched && Sched->TryLock()) { pantheon::Thread *Thr = Sched->AcquireThread(); diff --git a/System/Proc/kern_sched.hpp b/System/Proc/kern_sched.hpp index c1b016f..2f7ea17 100644 --- a/System/Proc/kern_sched.hpp +++ b/System/Proc/kern_sched.hpp @@ -57,6 +57,13 @@ class LocalScheduler : public pantheon::Allocatable Date: Tue, 3 Jan 2023 12:09:47 -0700 Subject: [PATCH 38/43] Don't drop ticks when issuing SVCYield() --- System/Proc/kern_sched.cpp | 8 ++++---- System/Syscalls/Syscalls.cpp | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index c51053c..9f292e4 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -23,12 +23,12 @@ * @file System/Proc/kern_sched.cpp * \~english @brief Definitions for basic kernel scheduling data structures and algorithms. * \~english @details Pantheon implements Con Kolivas' MuQSS scheduling algorithm, an efficient algorithm - * which takes priorities into account to provide a semi-"realtime" scheduling system to ensure + * which takes priorities into account to provide a fair scheduling system to ensure * fairness between processes. In contrast to BFS, lock contention is avoided by fairly distributing * work between individual processors: in this case, threads are the fundamental schedulable unit, - * which is what is assigned virtual deadlines. Note that "realtime" is in quotes as to my knowledge, no definitive - * proof exists to guarantee that any given task will always meet it's virtual deadline, though all jobs should - * just by induction eventually all run anyway, even if they miss their deadlines. + * which is what is assigned virtual deadlines. + * Some details are either simplified or ignored: this implementation ignores nice levels, and ignores cache locality, SMT thread groups, + * and does not use a high precision timer (relying on jiffies wherever the corresponding Linux patch uses niffies), and lacks any runtime tunable parameters. * @see http://ck.kolivas.org/patches/muqss/sched-MuQSS.txt * \~english @author Brian Schnepp */ diff --git a/System/Syscalls/Syscalls.cpp b/System/Syscalls/Syscalls.cpp index 7199dc7..c23ca83 100644 --- a/System/Syscalls/Syscalls.cpp +++ b/System/Syscalls/Syscalls.cpp @@ -329,7 +329,6 @@ pantheon::Result pantheon::SVCYield(pantheon::TrapFrame *CurFrame) { pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); pantheon::ScopedLock ScopeLockThread(CurThread); - CurThread->SetTicks(0); } pantheon::Scheduler::Reschedule(); return pantheon::Result::SYS_OK; From eafa44f97fedc3a0b6e7516e42be7c681e3bc9ac Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 3 Jan 2023 21:11:36 -0700 Subject: [PATCH 39/43] Increase scope of scheduler locks to cover more possible races --- Common/Sync/kern_atomic.hpp | 4 +-- System/Proc/kern_proc.hpp | 5 --- System/Proc/kern_sched.cpp | 71 ++++++++++++++++++------------------- 3 files changed, 37 insertions(+), 43 deletions(-) diff --git a/Common/Sync/kern_atomic.hpp b/Common/Sync/kern_atomic.hpp index 50921d6..4ae85bf 100644 --- a/Common/Sync/kern_atomic.hpp +++ b/Common/Sync/kern_atomic.hpp @@ -112,7 +112,7 @@ class AtomicInteger T Add(T Item) { #ifndef ONLY_TESTS - return __atomic_fetch_add(&(this->Content), Item, __ATOMIC_SEQ_CST) + Item; + return __atomic_fetch_add(&(this->Content), Item, __ATOMIC_SEQ_CST); #else this->Content += Item; return this->Content; @@ -122,7 +122,7 @@ class AtomicInteger T Sub(T Item) { #ifndef ONLY_TESTS - return __atomic_fetch_sub(&(this->Content), Item, __ATOMIC_SEQ_CST) - Item; + return __atomic_fetch_sub(&(this->Content), Item, __ATOMIC_SEQ_CST); #else this->Content -= Item; return this->Content; diff --git a/System/Proc/kern_proc.hpp b/System/Proc/kern_proc.hpp index d567806..90d5550 100644 --- a/System/Proc/kern_proc.hpp +++ b/System/Proc/kern_proc.hpp @@ -131,11 +131,6 @@ class Process : public pantheon::Object, public pantheon::Lockable */ static void Switch(Process *Next) { - if (Next == nullptr) - { - return; - } - pantheon::CPUReg::W_TTBR0_EL1(Next->GetTTBR0()); pantheon::CPU::GetCoreInfo()->CurProcess = Next; } diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 9f292e4..0b50bff 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -59,17 +59,6 @@ extern "C" void cpu_switch(pantheon::CpuContext *Old, pantheon::CpuContext *New, */ extern "C" VOID drop_usermode(UINT64 PC, UINT64 PSTATE, UINT64 SP); -/** - * @private - * @brief Prevent rescheduling under this current thread - */ -class ScopedRescheduleLock -{ -public: - FORCE_INLINE ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->BlockScheduling(); } - FORCE_INLINE ~ScopedRescheduleLock() { pantheon::CPU::GetCurThread()->EnableScheduling(); } -}; - UINT64 pantheon::LocalScheduler::CountThreads(UINT64 PID) { if (!this->Ready) @@ -110,8 +99,12 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() return nullptr; } - this->LocalRunQueue.Delete(LowestKey.GetValue()); - return Lowest.GetValue(); + UINT64 Key = LowestKey.GetValue(); + this->LocalRunQueue.Delete(Key); + + pantheon::Thread *Value = Lowest.GetValue(); + Value->Lock(); + return Value; } @@ -365,56 +358,62 @@ static pantheon::Thread *GetNextThread() } } - /* If we have no next, give up and use the idle thread. */ - return pantheon::CPU::GetMyLocalSched()->Idle(); -} - -static void SwapThreads(pantheon::Thread *CurThread, pantheon::Thread *NextThread) -{ - pantheon::Thread::Switch(CurThread, NextThread); - pantheon::Process::Switch(NextThread->MyProc()); - - pantheon::CpuContext *OldContext = CurThread->GetRegisters(); - pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - - pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); - NextThread->Unlock(); - - cpu_switch(OldContext, NewContext, CpuIRegOffset); - CurThread->Unlock(); + return nullptr; } void pantheon::Scheduler::Reschedule() { - ScopedRescheduleLock _L; - /* Interrupts must be enabled before we can do anything. */ if (pantheon::CPU::ICOUNT()) { return; } + pantheon::CPU::GetCurThread()->Lock(); + pantheon::CPU::GetCurThread()->BlockScheduling(); pantheon::Thread *CurThread = pantheon::CPU::GetCurThread(); - CurThread->Lock(); pantheon::Thread *NextThread = GetNextThread(); + if (NextThread == nullptr) + { + NextThread = pantheon::CPU::GetMyLocalSched()->Idle(); + if (NextThread->IsLocked()) + { + CurThread->Unlock(); + pantheon::CPU::GetCurThread()->EnableScheduling(); + return; + } + NextThread->Lock(); + } + if (NextThread == CurThread) { CurThread->Unlock(); + pantheon::CPU::GetCurThread()->EnableScheduling(); return; } - NextThread->Lock(); - if (NextThread->MyState() == pantheon::Thread::STATE_RUNNING || CurThread->MyState() != pantheon::Thread::STATE_RUNNING) { NextThread->Unlock(); CurThread->Unlock(); + pantheon::CPU::GetCurThread()->EnableScheduling(); return; } - SwapThreads(CurThread, NextThread); + pantheon::Thread::Switch(CurThread, NextThread); + pantheon::Process::Switch(NextThread->MyProc()); + + pantheon::CpuContext *OldContext = CurThread->GetRegisters(); + pantheon::CpuContext *NewContext = NextThread->GetRegisters(); + + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); + NextThread->Unlock(); + CurThread->Unlock(); + + cpu_switch(OldContext, NewContext, CpuIRegOffset); + pantheon::CPU::GetCurThread()->EnableScheduling(); } void pantheon::Scheduler::AttemptReschedule() From 1e8decebeb21fc040f7623ab7af5ab0902991b47 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Tue, 3 Jan 2023 23:48:21 -0700 Subject: [PATCH 40/43] Enforce memory poisoning for slab allocators --- Common/Structures/kern_allocatable.hpp | 1 - Common/Structures/kern_slab.hpp | 35 ++++++++++++++++++++------ Common/kern_runtime.hpp | 28 --------------------- 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/Common/Structures/kern_allocatable.hpp b/Common/Structures/kern_allocatable.hpp index 0a2e25e..945a662 100644 --- a/Common/Structures/kern_allocatable.hpp +++ b/Common/Structures/kern_allocatable.hpp @@ -18,7 +18,6 @@ class Allocatable static void Init() { AllocSpinlock = pantheon::Spinlock("Allocatable Lock"); - ClearBuffer((CHAR*)Items, sizeof(T) * Count); Allocator = pantheon::mm::SlabCache(Items, Count); } diff --git a/Common/Structures/kern_slab.hpp b/Common/Structures/kern_slab.hpp index 7a97285..7741d7f 100644 --- a/Common/Structures/kern_slab.hpp +++ b/Common/Structures/kern_slab.hpp @@ -21,9 +21,7 @@ template class SlabCache { public: - /* Implicitly assume that sizeof(T) >= sizeof(SlabCache) */ - - constexpr SlabCache() + SlabCache() { this->Size = 0; this->Used = 0; @@ -31,7 +29,7 @@ class SlabCache this->Area = nullptr; } - constexpr SlabCache(VOID *Area, UINT16 Count = 64) + SlabCache(VOID *Area, UINT16 Count = 64) { this->Area = reinterpret_cast(Area); #if POISON_MEMORY @@ -59,10 +57,7 @@ class SlabCache this->FreeList = reinterpret_cast*>(this->Area); } - constexpr ~SlabCache() - { - - } + constexpr ~SlabCache() = default; BOOL InRange(T *Ptr) { @@ -93,6 +88,16 @@ class SlabCache T* NewArea = reinterpret_cast(Current); this->FreeList = Next; this->Used++; +#ifdef POISON_MEMORY + for (UINT64 Address = (UINT64)Current + sizeof(SlabNext); Address < ((UINT64)Current + sizeof(T)); Address++) + { + UINT8 *AsBytes = (UINT8*)Address; + if (*AsBytes != 0xDF) + { + pantheon::StopErrorFmt("FOUND MEMORY CORRUPTION IN ALLOCATOR (%lx, address %lx)\n", this, Address); + } + } +#endif *NewArea = T(); return NewArea; } @@ -115,6 +120,17 @@ class SlabCache T* NewArea = reinterpret_cast(Current); this->FreeList = Next; this->Used++; + +#ifdef POISON_MEMORY + for (UINT64 Address = (UINT64)Current + sizeof(SlabNext); Address < ((UINT64)Current + sizeof(T)); Address++) + { + UINT8 *AsBytes = (UINT8*)Address; + if (*AsBytes != 0xDF) + { + pantheon::StopErrorFmt("FOUND MEMORY CORRUPTION IN ALLOCATOR (%lx, address %lx)\n", this, Address); + } + } +#endif return NewArea; } return nullptr; @@ -134,6 +150,9 @@ class SlabCache UINT64 BasePtr = Base + (Area * sizeof(T)); if (BasePtr == (UINT64)Ptr) { +#ifdef POISON_MEMORY + SetBufferBytes((UINT8*)Ptr, 0xDF, sizeof(T)); +#endif SlabNext *Next = reinterpret_cast*>(Ptr); Next->Next = FreeList; this->FreeList = Next; diff --git a/Common/kern_runtime.hpp b/Common/kern_runtime.hpp index 1e50ef4..c60275d 100644 --- a/Common/kern_runtime.hpp +++ b/Common/kern_runtime.hpp @@ -219,34 +219,6 @@ static inline void SetBufferBytes(UINT8 *Location, UINT8 Value, UINT32 Amount) } UINT64 Index = 0; - /* Enforce alignment */ - UINT64 ILocation = reinterpret_cast(Location); - if ((ILocation & 0x07) == 0) - { - UINT64 *AsUINT64 = (UINT64*)Location; - UINT64 NValue = ((UINT64)Value << 8*7) | ((UINT64)Value << 8*6) - | ((UINT64)Value << 8*5) | ((UINT64)Value << 8*4) - | ((UINT64)Value << 8*3) | ((UINT64)Value << 8*2) - | ((UINT64)Value << 8) | (UINT64)Value; - - for (Index = 0; Index < Amount; Index += 64) - { - AsUINT64[(Index + 0) / 8] = NValue; - AsUINT64[(Index + 1) / 8] = NValue; - AsUINT64[(Index + 2) / 8] = NValue; - AsUINT64[(Index + 3) / 8] = NValue; - AsUINT64[(Index + 4) / 8] = NValue; - AsUINT64[(Index + 5) / 8] = NValue; - AsUINT64[(Index + 6) / 8] = NValue; - AsUINT64[(Index + 7) / 8] = NValue; - } - - for (; Index < Amount; Index += 8) - { - AsUINT64[Index / 8] = NValue; - } - } - for (; Index < Amount; ++Index) { Location[Index] = Value; From 32463ed05b9e901932122d7d5e4efc58e9b55b19 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 4 Jan 2023 01:07:58 -0700 Subject: [PATCH 41/43] Implement and use SkipList Pop() function --- Common/Structures/kern_skiplist.hpp | 58 +++++++++++++++++++++++++++++ System/Proc/kern_sched.cpp | 11 ++---- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/Common/Structures/kern_skiplist.hpp b/Common/Structures/kern_skiplist.hpp index 61b0c66..8b54ff5 100644 --- a/Common/Structures/kern_skiplist.hpp +++ b/Common/Structures/kern_skiplist.hpp @@ -212,6 +212,64 @@ class SkipList return FALSE; } + Optional Pop(K Key) + { + this->Setup(); + + KVPair *Update[MaxLvl]; + INT16 Lvl = this->Level; + + KVPair *Current = this->Head; + KVPair *Next = Current; + while (Lvl >= 0) + { + Next = Current->Next[Lvl]; + while (Next && (Next->Key <= Key)) + { + Current = Next; + Next = Current->Next[Lvl]; + } + Update[Lvl] = Current; + Lvl--; + } + + /* This is where we'll need to be: this needs to match the current key. */ + if (Current->Key == Key) + { + UINT8 CurLvl = Current->Level; + for (UINT8 Lvl = 0; Lvl <= CurLvl; Lvl++) + { + Current->Prev[Lvl]->Next[Lvl] = Current->Next[Lvl]; + Current->Next[Lvl]->Prev[Lvl] = Current->Prev[Lvl]; + } + + /* If this was top, we might have to fix levels. */ + if (CurLvl == this->Level) + { + while (this->Head->Next[CurLvl] == this->Head && this->Head->Prev[CurLvl] == this->Head) + { + if (CurLvl > 0) + { + CurLvl--; + } else + { + break; + } + + } + this->Level = CurLvl; + } + + this->Sz--; + if (Current->Key == Key) + { + return Optional(Current->Value); + } + return Optional(); + } + return Optional(); + } + V& operator[](K Key) { diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 0b50bff..158a19c 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -93,15 +93,12 @@ pantheon::Thread *pantheon::LocalScheduler::AcquireThread() return nullptr; } - Optional Lowest = this->LocalRunQueue.Get(LowestKey.GetValue()); + Optional Lowest = this->LocalRunQueue.Pop(LowestKey.GetValue()); if (!Lowest.GetOkay()) { return nullptr; } - UINT64 Key = LowestKey.GetValue(); - this->LocalRunQueue.Delete(Key); - pantheon::Thread *Value = Lowest.GetValue(); Value->Lock(); return Value; @@ -183,14 +180,12 @@ void pantheon::LocalScheduler::Setup() } Idle->Initialize(AcquireThreadID(), MProc.GetValue(), nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); + Idle->Lock(); Idle->SetState(pantheon::Thread::STATE_RUNNING); - pantheon::CPU::GetCoreInfo()->CurThread = Idle; this->IdleThread = Idle; this->Ready = TRUE; - - pantheon::CPU::GetMyLocalSched()->InsertThread(Idle); Idle->Unlock(); } @@ -210,6 +205,7 @@ static UINT64 CalculateDeadline(UINT64 Jiffies, UINT64 PrioRatio, UINT64 RRInter void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) { + pantheon::ScopedLock _L(this); if (Thr == nullptr) { return; @@ -220,7 +216,6 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) pantheon::StopErrorFmt("Attempting to insert thread which wasn't locked into runqueue\n"); } - pantheon::ScopedLock _L(this); if (Thr->MyState() == pantheon::Thread::STATE_WAITING) { UINT64 Jiffies = pantheon::CPU::GetJiffies(); From 71167c084325eb398f778a8998bfd31e796a26ff Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 4 Jan 2023 16:05:15 -0700 Subject: [PATCH 42/43] Blend slab allocation in with BasicMalloc to avoid the explicit free list implementation where possible --- Common/kern_basic_malloc.cpp | 66 ++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/Common/kern_basic_malloc.cpp b/Common/kern_basic_malloc.cpp index 0aa2ebe..e9d5f22 100644 --- a/Common/kern_basic_malloc.cpp +++ b/Common/kern_basic_malloc.cpp @@ -4,6 +4,9 @@ #include #include + +#include + typedef struct FreeList { struct FreeList *Prev; @@ -12,7 +15,32 @@ typedef struct FreeList /* Try to align to 32-bytes, so that allocations can be bitpacked nicely. */ typedef UINT64 BlockHeader; -static constexpr UINT64 HeapSpace = 2ULL * 1024ULL * 1024ULL; + +typedef struct TinyContent +{ + UINT64 Data[4]; +}TinyContent; + +typedef struct SmallContent +{ + UINT64 Data[0x60]; +}SmallContent; + +typedef struct LargeContent +{ + UINT64 Data[0x1000]; +}LargeContent; + +static constexpr UINT64 HeapSpace = 512ULL * 1024ULL; + +alignas(0x1000) static TinyContent TinySlabCache[2 * 1024]; +alignas(0x1000) static SmallContent SmallSlabCache[256]; +alignas(0x1000) static LargeContent LargeSlabCache[8]; + +static pantheon::mm::SlabCache SlabTinyContent((VOID*)TinySlabCache, 2 * 1024); +static pantheon::mm::SlabCache SlabSmallContent((VOID*)SmallSlabCache, 256); +static pantheon::mm::SlabCache SlabLargeContent((VOID*)LargeSlabCache, 8); + static constexpr UINT64 MinBlockSize = sizeof(FreeList) + (2 * sizeof(BlockHeader)); COMPILER_ASSERT(MinBlockSize == 32); @@ -155,6 +183,23 @@ Optional BasicMalloc(UINT64 Amt) } AllocLock.Acquire(); + if (Amt <= sizeof(TinyContent) && !SlabTinyContent.Full()) + { + TinyContent *Content = SlabTinyContent.Allocate(); + AllocLock.Release(); + return Optional(Content); + } else if (Amt <= sizeof(SmallContent) && !SlabSmallContent.Full()) { + SmallContent *Content = SlabSmallContent.Allocate(); + AllocLock.Release(); + return Optional(Content); + } else if (Amt <= sizeof(LargeContent) && !SlabLargeContent.Full()) { + LargeContent *Content = SlabLargeContent.Allocate(); + AllocLock.Release(); + return Optional(Content); + } + + /* Give up and use original explicit free list implementation */ + UINT64 Amount = Max(Align(Amt, (sizeof(BlockHeader))), MinBlockSize); FreeList *Final = nullptr; @@ -200,7 +245,22 @@ void BasicFree(void *Addr) } AllocLock.Acquire(); - + + if (SlabTinyContent.InRange((TinyContent*)Addr)) + { + SlabTinyContent.Deallocate((TinyContent*)Addr); + AllocLock.Release(); + return; + } else if (SlabSmallContent.InRange((SmallContent*)Addr)) { + SlabSmallContent.Deallocate((SmallContent*)Addr); + AllocLock.Release(); + return; + } else if (SlabLargeContent.InRange((LargeContent*)Addr)) { + SlabLargeContent.Deallocate((LargeContent*)Addr); + AllocLock.Release(); + return; + } + /* Ensure the current block is consistent, ie, no double frees */ CHAR *ActualAddr = (CHAR*)Addr; if (GetAlloc(GetHeader((CHAR*)Addr)) == FALSE) @@ -251,5 +311,5 @@ void *operator new(UINT64 Sz) void operator delete(void *Ptr) { - PANTHEON_UNUSED(Ptr); + BasicFree(Ptr); } \ No newline at end of file From dba5eaa18ee3ee83a740dc2bd67d73c76769ba22 Mon Sep 17 00:00:00 2001 From: Brian Schnepp Date: Wed, 4 Jan 2023 22:55:36 -0700 Subject: [PATCH 43/43] Implement PANIC_ON macro --- Common/kern_runtime.hpp | 1 + System/Proc/kern_sched.cpp | 30 ++++++++++++------------------ System/Proc/kern_thread.hpp | 2 -- arch/aarch64/ints.cpp | 1 + 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/Common/kern_runtime.hpp b/Common/kern_runtime.hpp index c60275d..29d638e 100644 --- a/Common/kern_runtime.hpp +++ b/Common/kern_runtime.hpp @@ -297,5 +297,6 @@ BOOL Panicked(); } #define OBJECT_SELF_ASSERT() if ((this) == nullptr) { StopError("called method was nullptr"); } +#define PANIC_ON(cond, string) if (cond) { pantheon::StopError(string); } #endif \ No newline at end of file diff --git a/System/Proc/kern_sched.cpp b/System/Proc/kern_sched.cpp index 158a19c..8df469f 100644 --- a/System/Proc/kern_sched.cpp +++ b/System/Proc/kern_sched.cpp @@ -131,10 +131,7 @@ static UINT32 AcquireProcessID() UINT32 RetVal = 0; static UINT32 ProcessID = 1; - if (!SchedLock.IsLocked()) - { - pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); - } + PANIC_ON(!SchedLock.IsLocked(), "Acquire process ID without locking Global Scheduler\n"); RetVal = ProcessID++; while (ProcessList.Contains(RetVal) || RetVal == 0) @@ -150,10 +147,7 @@ static UINT64 AcquireThreadID() UINT64 RetVal = 0; static UINT64 ThreadID = 1; - if (!SchedLock.IsLocked()) - { - pantheon::StopErrorFmt("Acquire thread without locking Global Scheduler\n"); - } + PANIC_ON(!SchedLock.IsLocked(), "Acquire thread ID without locking Global Scheduler\n"); RetVal = ThreadID++; /* Ban 0 and -1, since these are special values. */ @@ -174,10 +168,7 @@ void pantheon::LocalScheduler::Setup() pantheon::Thread *Idle = pantheon::Thread::Create(); Optional MProc = ProcessList.Get(0); - if (!MProc.GetOkay()) - { - pantheon::StopErrorFmt("No PID 0!\n"); - } + PANIC_ON(!MProc.GetOkay(), "PID 0 does not exist!\n"); Idle->Initialize(AcquireThreadID(), MProc.GetValue(), nullptr, nullptr, pantheon::Thread::PRIORITY_VERYLOW, FALSE); @@ -211,10 +202,7 @@ void pantheon::LocalScheduler::InsertThread(pantheon::Thread *Thr) return; } - if (!Thr->IsLocked()) - { - pantheon::StopErrorFmt("Attempting to insert thread which wasn't locked into runqueue\n"); - } + PANIC_ON(!Thr->IsLocked(), "Attempting to insert thread which wasn't locked into runqueue\n"); if (Thr->MyState() == pantheon::Thread::STATE_WAITING) { @@ -345,6 +333,8 @@ static pantheon::Thread *GetNextThread() if (Sched && Sched->TryLock()) { pantheon::Thread *Thr = Sched->AcquireThread(); + PANIC_ON(Thr && !Thr->Preempted(), "Trying to acquire thread which was preempted\n"); + Sched->Unlock(); if (Thr != nullptr) { @@ -403,10 +393,14 @@ void pantheon::Scheduler::Reschedule() pantheon::CpuContext *OldContext = CurThread->GetRegisters(); pantheon::CpuContext *NewContext = NextThread->GetRegisters(); - pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); - NextThread->Unlock(); + if (CurThread->MyState() == pantheon::Thread::STATE_WAITING) + { + pantheon::CPU::GetMyLocalSched()->InsertThread(CurThread); + } CurThread->Unlock(); + NextThread->Unlock(); + cpu_switch(OldContext, NewContext, CpuIRegOffset); pantheon::CPU::GetCurThread()->EnableScheduling(); } diff --git a/System/Proc/kern_thread.hpp b/System/Proc/kern_thread.hpp index 7871514..d0fbdc7 100644 --- a/System/Proc/kern_thread.hpp +++ b/System/Proc/kern_thread.hpp @@ -140,8 +140,6 @@ class Thread : public pantheon::Object, public pantheon::Lockable pantheon::vmm::VirtualAddress LocalRegion; static constexpr UINT64 InitialNumStackPages = 4; - - pantheon::Atomic NextThread; }; } diff --git a/arch/aarch64/ints.cpp b/arch/aarch64/ints.cpp index 06b720a..21036ef 100644 --- a/arch/aarch64/ints.cpp +++ b/arch/aarch64/ints.cpp @@ -77,6 +77,7 @@ extern "C" void sync_handler_el1(pantheon::TrapFrame *Frame) } SERIAL_LOG("\n"); + SERIAL_LOG("CurThread Status: preempted: %ld MyState: %ld\n", CurThread->Preempted(), CurThread->MyState()); pantheon::StopErrorFmt( "ERR: SYNC HANDLER EL1: esr: %lx far: %lx elr: %lx spsr: %lx, sp: %lx pid: %u, tid: %lu\n", ESR, FAR, ELR, SPSR, SP, CurProc->ProcessID(), CurThread->ThreadID());