From 7079f11adda57fff5b7a319d7e4fa0da03899fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=97=B1=20PixelyIon?= Date: Wed, 16 Dec 2020 00:24:08 +0530 Subject: [PATCH] Implement PI-Mutexes + Optimize `InsertThread` --- app/src/main/cpp/skyline/kernel/scheduler.cpp | 76 +++++--- app/src/main/cpp/skyline/kernel/scheduler.h | 7 +- app/src/main/cpp/skyline/kernel/svc.cpp | 50 +++--- .../cpp/skyline/kernel/types/KProcess.cpp | 166 ++++++++++++------ .../main/cpp/skyline/kernel/types/KProcess.h | 29 +-- .../main/cpp/skyline/kernel/types/KThread.cpp | 2 +- .../main/cpp/skyline/kernel/types/KThread.h | 15 +- 7 files changed, 215 insertions(+), 130 deletions(-) diff --git a/app/src/main/cpp/skyline/kernel/scheduler.cpp b/app/src/main/cpp/skyline/kernel/scheduler.cpp index ad657fa8..22be52ca 100644 --- a/app/src/main/cpp/skyline/kernel/scheduler.cpp +++ b/app/src/main/cpp/skyline/kernel/scheduler.cpp @@ -30,7 +30,6 @@ namespace skyline::kernel { if (!currentCore->queue.empty() && thread->affinityMask.count() != 1) { // Select core where the current thread will be scheduled the earliest based off average timeslice durations for resident threads // There's a preference for the current core as migration isn't free - size_t minTimeslice{}; CoreContext *optimalCore{}; for (auto &candidateCore : cores) { @@ -63,7 +62,7 @@ namespace skyline::kernel { if (optimalCore != currentCore) { std::unique_lock coreLock(currentCore->mutex); currentCore->queue.erase(std::remove(currentCore->queue.begin(), currentCore->queue.end(), thread), currentCore->queue.end()); - currentCore->mutateCondition.notify_all(); + currentCore->frontCondition.notify_all(); thread->coreId = optimalCore->id; @@ -80,11 +79,14 @@ namespace skyline::kernel { return *currentCore; } - void Scheduler::InsertThread(bool loadBalance) { - auto &thread{state.thread}; + void Scheduler::InsertThread(const std::shared_ptr &thread, bool loadBalance) { auto &core{loadBalance ? LoadBalance() : cores.at(thread->coreId)}; - signal::SetSignalHandler({YieldSignal}, SignalHandler); + thread_local bool signalHandlerSetup{}; + if (!signalHandlerSetup) { + signal::SetSignalHandler({YieldSignal}, SignalHandler); + signalHandlerSetup = true; + } if (!thread->preemptionTimer) { struct sigevent event{ @@ -92,19 +94,34 @@ namespace skyline::kernel { .sigev_notify = SIGEV_THREAD_ID, .sigev_notify_thread_id = gettid(), }; - timer_create(CLOCK_THREAD_CPUTIME_ID, &event, &*thread->preemptionTimer); + timer_t timer; + if (timer_create(CLOCK_THREAD_CPUTIME_ID, &event, &timer)) + throw exception("timer_create has failed with '{}'", strerror(errno)); + thread->preemptionTimer.emplace(timer); } std::unique_lock lock(core.mutex); - auto nextThread{std::find_if(core.queue.begin(), core.queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; })}; - if (nextThread == core.queue.begin() && nextThread != core.queue.end()) { - // If the inserted thread has a higher priority than the currently running thread (and the queue isn't empty) - core.queue.front()->SendSignal(YieldSignal); - core.queue.insert(std::next(core.queue.begin()), thread); + auto nextThread{std::upper_bound(core.queue.begin(), core.queue.end(), thread->priority.load(), type::KThread::IsHigherPriority)}; + if (nextThread == core.queue.begin()) { + if (nextThread != core.queue.end()) { + // If the inserted thread has a higher priority than the currently running thread (and the queue isn't empty) + // We need to interrupt the currently scheduled thread and put this thread at the front instead + core.queue.insert(std::next(core.queue.begin()), thread); + if (state.thread != core.queue.front()) + core.queue.front()->SendSignal(YieldSignal); + else + YieldPending = true; + } else { + core.queue.push_front(thread); + } + if (thread != state.thread) { + lock.unlock(); // We should unlock this prior to waking all threads to not cause contention on the core lock + core.frontCondition.notify_all(); // We only want to trigger the conditional variable if the current thread isn't inserting itself + } } else { core.queue.insert(nextThread, thread); - core.mutateCondition.notify_all(); // We only want to trigger the conditional variable if the current thread isn't going to be scheduled next } + thread->needsReorder = true; // We need to reorder the thread from back to align it with other threads of it's priority and ensure strict ordering amongst priorities } @@ -115,13 +132,13 @@ namespace skyline::kernel { std::shared_lock lock(core->mutex); if (thread->affinityMask.count() > 1) { std::chrono::milliseconds loadBalanceThreshold{PreemptiveTimeslice * 2}; //!< The amount of time that needs to pass unscheduled for a thread to attempt load balancing - while (!core->mutateCondition.wait_for(lock, loadBalanceThreshold, [&]() { return core->queue.front() == thread; })) { + while (!core->frontCondition.wait_for(lock, loadBalanceThreshold, [&]() { return !core->queue.empty() && core->queue.front() == thread; })) { lock.unlock(); LoadBalance(); if (thread->coreId == core->id) { lock.lock(); } else { - InsertThread(false); + InsertThread(state.thread, false); core = &cores.at(thread->coreId); lock = std::shared_lock(core->mutex); } @@ -129,7 +146,7 @@ namespace skyline::kernel { loadBalanceThreshold *= 2; // We double the duration required for future load balancing for this invocation to minimize pointless load balancing } } else { - core->mutateCondition.wait(lock, [&]() { return core->queue.front() == thread; }); + core->frontCondition.wait(lock, [&]() { return !core->queue.empty() && core->queue.front() == thread; }); } if (thread->priority == core->preemptionPriority) { @@ -160,14 +177,15 @@ namespace skyline::kernel { auto foldingPoint{std::find_if(std::next(core.queue.begin()), core.queue.end(), [&](const std::shared_ptr &it) { return lastPriority > it->priority ? true : lastPriority = it->priority, false; })}; - core.queue.insert(std::find_if(foldingPoint, core.queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; }), thread); + core.queue.insert(std::upper_bound(foldingPoint, core.queue.end(), thread->priority.load(), type::KThread::IsHigherPriority), thread); thread->needsReorder = false; } else { core.queue.push_back(thread); thread->needsReorder = false; } - core.mutateCondition.notify_all(); + lock.unlock(); + core.frontCondition.notify_all(); if (cooperative && thread->isPreempted) { // If a preemptive thread did a cooperative yield then we need to disarm the preemptive timer @@ -178,7 +196,7 @@ namespace skyline::kernel { } } - void Scheduler::UpdatePriority(const std::shared_ptr& thread) { + void Scheduler::UpdatePriority(const std::shared_ptr &thread) { std::lock_guard migrationLock(thread->coreMigrationMutex); auto *core{&cores.at(thread->coreId)}; std::unique_lock coreLock(core->mutex); @@ -188,12 +206,18 @@ namespace skyline::kernel { // If the thread isn't in the queue then the new priority will be handled automatically on insertion return; if (currentIt == core->queue.begin()) { - // Alternatively, if it's currently running then we'd just want to update after it rotates + // Alternatively, if it's currently running then we'd just want to reorder after it rotates + if (!thread->isPreempted && thread->priority == core->preemptionPriority) { + // If the thread needs to be preempted due to the new priority then arm it's preemption timer + struct itimerspec spec{.it_value = {.tv_nsec = std::chrono::duration_cast(PreemptiveTimeslice).count()}}; + timer_settime(*thread->preemptionTimer, 0, &spec, nullptr); + thread->isPreempted = true; + } thread->needsReorder = true; return; } - auto targetIt{std::find_if(core->queue.begin(), core->queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; })}; + auto targetIt{std::upper_bound(core->queue.begin(), core->queue.end(), thread->priority.load(), type::KThread::IsHigherPriority)}; if (currentIt == targetIt) // If this thread's position isn't affected by the priority change then we have nothing to do return; @@ -206,13 +230,12 @@ namespace skyline::kernel { thread->isPreempted = false; } - targetIt = std::find_if(core->queue.begin(), core->queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; }); // Iterator invalidation + targetIt = std::upper_bound(core->queue.begin(), core->queue.end(), thread->priority.load(), type::KThread::IsHigherPriority); // Iterator invalidation if (targetIt == core->queue.begin() && targetIt != core->queue.end()) { - core->queue.front()->SendSignal(YieldSignal); core->queue.insert(std::next(core->queue.begin()), thread); + core->queue.front()->SendSignal(YieldSignal); } else { core->queue.insert(targetIt, thread); - core->mutateCondition.notify_all(); } thread->needsReorder = true; } @@ -221,8 +244,11 @@ namespace skyline::kernel { auto &thread{state.thread}; auto &core{cores.at(thread->coreId)}; - std::unique_lock lock(core.mutex); - core.queue.erase(std::remove(core.queue.begin(), core.queue.end(), thread), core.queue.end()); + { + std::unique_lock lock(core.mutex); + core.queue.erase(std::remove(core.queue.begin(), core.queue.end(), thread), core.queue.end()); + } + core.frontCondition.notify_all(); // We need to notify any threads in line to be scheduled if (thread->isPreempted) { struct itimerspec spec{}; diff --git a/app/src/main/cpp/skyline/kernel/scheduler.h b/app/src/main/cpp/skyline/kernel/scheduler.h index c9954e58..1123133c 100644 --- a/app/src/main/cpp/skyline/kernel/scheduler.h +++ b/app/src/main/cpp/skyline/kernel/scheduler.h @@ -46,7 +46,7 @@ namespace skyline { u8 id; u8 preemptionPriority; //!< The priority at which this core becomes preemptive as opposed to cooperative std::shared_mutex mutex; //!< Synchronizes all operations on the queue - std::condition_variable_any mutateCondition; //!< A conditional variable which is signalled on every mutation of the queue + std::condition_variable_any frontCondition; //!< A conditional variable which is signalled when the front of the queue has changed std::deque> queue; //!< A queue of threads which are running or to be run on this core CoreContext(u8 id, u8 preemptionPriority); @@ -74,10 +74,11 @@ namespace skyline { CoreContext& LoadBalance(); /** - * @brief Inserts the calling thread into the scheduler queue at the appropriate location based on it's priority + * @brief Inserts the specified thread into the scheduler queue at the appropriate location based on it's priority * @param loadBalance If to load balance or use the thread's current core (KThread::coreId) + * @note It isn't supported to load balance if the supplied thread isn't the calling thread, it'll lead to UB */ - void InsertThread(bool loadBalance = true); + void InsertThread(const std::shared_ptr& thread, bool loadBalance = true); /** * @brief Wait for the current thread to be scheduled on it's resident core diff --git a/app/src/main/cpp/skyline/kernel/svc.cpp b/app/src/main/cpp/skyline/kernel/svc.cpp index a8445737..faa178ec 100644 --- a/app/src/main/cpp/skyline/kernel/svc.cpp +++ b/app/src/main/cpp/skyline/kernel/svc.cpp @@ -276,7 +276,7 @@ namespace skyline::kernel::svc { } void ExitThread(const DeviceState &state) { - state.logger->Debug("svcExitThread: Exiting current thread: {}", state.thread->id); + state.logger->Debug("svcExitThread: Exiting current thread"); std::longjmp(state.thread->originalCtx, true); } @@ -330,9 +330,11 @@ namespace skyline::kernel::svc { } try { auto thread{state.process->GetHandle(handle)}; - state.logger->Debug("svcSetThreadPriority: Setting thread priority to {}", thread->id, priority); - thread->priority = priority; - state.scheduler->UpdatePriority(thread); + state.logger->Debug("svcSetThreadPriority: Setting thread priority to {}", priority); + if (thread->priority != priority) { + thread->priority = priority; + state.scheduler->UpdatePriority(thread); + } state.ctx->gpr.w0 = Result{}; } catch (const std::out_of_range &) { state.logger->Warn("svcSetThreadPriority: 'handle' invalid: 0x{:X}", handle); @@ -346,7 +348,7 @@ namespace skyline::kernel::svc { auto thread{state.process->GetHandle(handle)}; auto idealCore{thread->idealCore}; auto affinityMask{thread->affinityMask}; - state.logger->Debug("svcGetThreadCoreMask: Writing thread #{}'s Ideal Core ({}) + Affinity Mask ({})", thread->id, idealCore, affinityMask); + state.logger->Debug("svcGetThreadCoreMask: Setting Ideal Core ({}) + Affinity Mask ({})", idealCore, affinityMask); state.ctx->gpr.x2 = affinityMask.to_ullong(); state.ctx->gpr.w1 = idealCore; @@ -386,7 +388,7 @@ namespace skyline::kernel::svc { return; } - state.logger->Debug("svcSetThreadCoreMask: Setting thread #{}'s Ideal Core ({}) + Affinity Mask ({})", thread->id, idealCore, affinityMask); + state.logger->Debug("svcSetThreadCoreMask: Setting Ideal Core ({}) + Affinity Mask ({})", idealCore, affinityMask); thread->idealCore = idealCore; thread->affinityMask = affinityMask; @@ -396,7 +398,8 @@ namespace skyline::kernel::svc { state.scheduler->RemoveThread(); thread->coreId = idealCore; - state.scheduler->InsertThread(false); + state.scheduler->InsertThread(state.thread, false); + state.scheduler->WaitSchedule(); } state.ctx->gpr.w0 = Result{}; @@ -407,8 +410,9 @@ namespace skyline::kernel::svc { } void GetCurrentProcessorNumber(const DeviceState &state) { - state.logger->Debug("svcGetCurrentProcessorNumber: Writing current core for thread #{}: {}", state.thread->id, state.thread->coreId); - state.ctx->gpr.w0 = state.thread->coreId; + auto coreId{state.thread->coreId}; + state.logger->Debug("svcGetCurrentProcessorNumber: Writing current core: {}", coreId); + state.ctx->gpr.w0 = coreId; } void ClearEvent(const DeviceState &state) { @@ -611,12 +615,15 @@ namespace skyline::kernel::svc { state.logger->Debug("svcArbitrateLock: Locking mutex at 0x{:X}", pointer); - if (state.process->MutexLock(pointer, ownerHandle)) + auto result{state.process->MutexLock(pointer, ownerHandle)}; + if (result == Result{}) state.logger->Debug("svcArbitrateLock: Locked mutex at 0x{:X}", pointer); - else + else if (result == result::InvalidHandle) + state.logger->Warn("svcArbitrateLock: 'handle' invalid: 0x{:X}", ownerHandle); + else if (result == result::InvalidCurrentMemory) state.logger->Debug("svcArbitrateLock: Owner handle did not match current owner for mutex or didn't have waiter flag at 0x{:X}", pointer); - state.ctx->gpr.w0 = Result{}; + state.ctx->gpr.w0 = result; } void ArbitrateUnlock(const DeviceState &state) { @@ -629,13 +636,10 @@ namespace skyline::kernel::svc { state.logger->Debug("svcArbitrateUnlock: Unlocking mutex at 0x{:X}", mutex); - if (state.process->MutexUnlock(mutex)) { - state.logger->Debug("svcArbitrateUnlock: Unlocked mutex at 0x{:X}", mutex); - state.ctx->gpr.w0 = Result{}; - } else { - state.logger->Debug("svcArbitrateUnlock: A non-owner thread tried to release a mutex at 0x{:X}", mutex); - state.ctx->gpr.w0 = result::InvalidAddress; - } + state.process->MutexUnlock(mutex); + + state.logger->Debug("svcArbitrateUnlock: Unlocked mutex at 0x{:X}", mutex); + state.ctx->gpr.w0 = Result{}; } void WaitProcessWideKeyAtomic(const DeviceState &state) { @@ -651,11 +655,7 @@ namespace skyline::kernel::svc { if (handle != state.thread->handle) throw exception("svcWaitProcessWideKeyAtomic: Handle doesn't match current thread: 0x{:X} for thread 0x{:X}", handle, state.thread->handle); - if (!state.process->MutexUnlock(mutex)) { - state.logger->Debug("WaitProcessWideKeyAtomic: A non-owner thread tried to release a mutex at 0x{:X}", mutex); - state.ctx->gpr.w0 = result::InvalidAddress; - return; - } + state.process->MutexUnlock(mutex); u64 timeout{state.ctx->gpr.x3}; state.logger->Debug("svcWaitProcessWideKeyAtomic: Mutex: 0x{:X}, Conditional-Variable: 0x{:X}, Timeout: {} ns", mutex, conditional, timeout); @@ -731,7 +731,7 @@ namespace skyline::kernel::svc { if (debug.back() == '\n') debug.remove_suffix(1); - state.logger->Info("Debug Output: {}", debug); + state.logger->Info("svcOutputDebugString: {}", debug); state.ctx->gpr.w0 = Result{}; } diff --git a/app/src/main/cpp/skyline/kernel/types/KProcess.cpp b/app/src/main/cpp/skyline/kernel/types/KProcess.cpp index 87c1f530..07939789 100644 --- a/app/src/main/cpp/skyline/kernel/types/KProcess.cpp +++ b/app/src/main/cpp/skyline/kernel/types/KProcess.cpp @@ -3,13 +3,10 @@ #include #include +#include #include "KProcess.h" namespace skyline::kernel::type { - KProcess::WaitStatus::WaitStatus(u8 priority, KHandle handle) : priority(priority), handle(handle) {} - - KProcess::WaitStatus::WaitStatus(u8 priority, KHandle handle, u32 *mutex) : priority(priority), handle(handle), mutex(mutex) {} - KProcess::TlsPage::TlsPage(const std::shared_ptr &memory) : memory(memory) {} u8 *KProcess::TlsPage::ReserveSlot() { @@ -112,72 +109,135 @@ namespace skyline::kernel::type { return std::nullopt; } - bool KProcess::MutexLock(u32 *mutex, KHandle owner) { - std::unique_lock lock(mutexLock); + constexpr u32 HandleWaitMask{0x40000000}; //!< A mask of a bit which denotes if the handle has waiters or not - auto &mtxWaiters{mutexes[reinterpret_cast(mutex)]}; - if (mtxWaiters.empty()) { - u32 mtxExpected{}; - if (__atomic_compare_exchange_n(mutex, &mtxExpected, (constant::MtxOwnerMask & state.thread->handle), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) - return true; + + Result KProcess::MutexLock(u32 *mutex, KHandle ownerHandle) { + std::shared_ptr owner; + try { + owner = GetHandle(ownerHandle & ~HandleWaitMask); + } catch (const std::out_of_range &) { + return result::InvalidHandle; } - if (__atomic_load_n(mutex, __ATOMIC_SEQ_CST) != (owner | ~constant::MtxOwnerMask)) - return false; + bool isHighestPriority; + { + std::lock_guard lock(owner->waiterMutex); - std::shared_ptr status; - for (auto it{mtxWaiters.begin()};; it++) { - if (it != mtxWaiters.end() && (*it)->priority >= state.thread->priority) - continue; + u32 value{}; + if (__atomic_compare_exchange_n(mutex, &value, (HandleWaitMask & state.thread->handle), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) + // We try to do a CAS to get ownership of the mutex in the case that it's unoccupied + return {}; + if (value != (ownerHandle | HandleWaitMask)) + // We ensure that the mutex's value is the handle with the waiter bit set + return result::InvalidCurrentMemory; - status = std::make_shared(state.thread->priority, state.thread->handle); - mtxWaiters.insert(it, status); - break; + auto &waiters{owner->waiters}; + isHighestPriority = waiters.insert(std::upper_bound(waiters.begin(), waiters.end(), state.thread->priority.load(), KThread::IsHigherPriority), state.thread) == waiters.begin(); + state.scheduler->RemoveThread(); + + std::atomic_store(&state.thread->waitThread, owner); + state.thread->waitKey = mutex; } - lock.unlock(); - while (!status->flag); - lock.lock(); - status->flag = false; + if (isHighestPriority) { + // If we were the highest priority thread then we need to inherit priorities for all threads we're waiting on recursively + do { + u8 priority, ownerPriority; + do { + // Try to CAS the priority of the owner with the current thread + // If they're equivalent then we don't need to CAS as the priority won't be inherited + ownerPriority = owner->priority.load(); + priority = std::min(ownerPriority, state.thread->priority.load()); + } while (ownerPriority != priority && owner->priority.compare_exchange_strong(ownerPriority, priority)); - for (auto it{mtxWaiters.begin()}; it != mtxWaiters.end(); it++) { - if ((*it)->handle == state.thread->handle) { - mtxWaiters.erase(it); - break; - } + if (ownerPriority != priority) { + auto waitThread{std::atomic_load(&owner->waitThread)}; + while (waitThread) { + std::lock_guard lock(waitThread->waiterMutex); + + auto currentWaitThread{std::atomic_load(&owner->waitThread)}; + if (waitThread != currentWaitThread) { + waitThread = currentWaitThread; + continue; + } + + // We need to update the location of the owner thread in the waiter queue of the thread it's waiting on + auto &waiters{waitThread->waiters}; + waiters.erase(std::find(waiters.begin(), waiters.end(), waitThread)); + waiters.insert(std::upper_bound(waiters.begin(), waiters.end(), state.thread->priority.load(), KThread::IsHigherPriority), owner); + break; + } + + state.scheduler->UpdatePriority(owner); + + owner = waitThread; + } else { + break; + } + } while (owner); } - return true; + state.scheduler->WaitSchedule(); + + return {}; } - bool KProcess::MutexUnlock(u32 *mutex) { - std::unique_lock lock(mutexLock); + void KProcess::MutexUnlock(u32 *mutex) { + std::lock_guard lock(state.thread->waiterMutex); + auto &waiters{state.thread->waiters}; + auto nextOwnerIt{std::find_if(waiters.begin(), waiters.end(), [mutex](const std::shared_ptr &thread) { return thread->waitKey == mutex; })}; + if (nextOwnerIt != waiters.end()) { + auto nextOwner{*nextOwnerIt}; + std::lock_guard nextLock(nextOwner->waiterMutex); + std::atomic_store(&nextOwner->waitThread, std::shared_ptr{nullptr}); + nextOwner->waitKey = nullptr; - auto &mtxWaiters{mutexes[reinterpret_cast(mutex)]}; - u32 mtxDesired{}; - if (!mtxWaiters.empty()) - mtxDesired = (*mtxWaiters.begin())->handle | ((mtxWaiters.size() > 1) ? ~constant::MtxOwnerMask : 0); + // Move all threads waiting on this key to the next owner's waiter list + std::shared_ptr nextWaiter{}; + for (auto it{waiters.erase(nextOwnerIt)}; it != waiters.end(); it++) { + if ((*it)->waitKey == mutex) { + nextOwner->waiters.splice(std::upper_bound(nextOwner->waiters.begin(), nextOwner->waiters.end(), (*it)->priority.load(), KThread::IsHigherPriority), waiters, it); + std::atomic_store(&(*it)->waitThread, nextOwner); + if (!nextWaiter) + nextWaiter = *it; + } + } - u32 mtxExpected{(constant::MtxOwnerMask & state.thread->handle) | ~constant::MtxOwnerMask}; - if (!__atomic_compare_exchange_n(mutex, &mtxExpected, mtxDesired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) { - mtxExpected &= constant::MtxOwnerMask; + if (!waiters.empty()) { + // If there are threads still waiting on us then try to inherit their priority + auto highestPriority{waiters.front()}; + u8 priority, ownerPriority; + do { + ownerPriority = state.thread->priority.load(); + priority = std::min(ownerPriority, highestPriority->priority.load()); + } while (ownerPriority != priority && nextOwner->priority.compare_exchange_strong(ownerPriority, priority)); + state.scheduler->UpdatePriority(state.thread); + } - if (!__atomic_compare_exchange_n(mutex, &mtxExpected, mtxDesired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) - return false; + if (nextWaiter) { + // If there is a waiter on the new owner then try to inherit it's priority + u8 priority, ownerPriority; + do { + ownerPriority = nextOwner->priority.load(); + priority = std::min(ownerPriority, nextWaiter->priority.load()); + } while (ownerPriority != priority && nextOwner->priority.compare_exchange_strong(ownerPriority, priority)); + + __atomic_store_n(mutex, nextOwner->handle | HandleWaitMask, __ATOMIC_SEQ_CST); + } else { + __atomic_store_n(mutex, nextOwner->handle, __ATOMIC_SEQ_CST); + } + + // Finally, schedule the next owner accordingly + state.scheduler->InsertThread(nextOwner, false); + } else { + __atomic_store_n(mutex, 0, __ATOMIC_SEQ_CST); + return; } - - if (mtxDesired) { - auto status{(*mtxWaiters.begin())}; - status->flag = true; - lock.unlock(); - while (status->flag); - lock.lock(); - } - - return true; } bool KProcess::ConditionalVariableWait(void *conditional, u32 *mutex, u64 timeout) { + /* FIXME std::unique_lock lock(conditionalLock); auto &condWaiters{conditionals[reinterpret_cast(conditional)]}; @@ -216,9 +276,12 @@ namespace skyline::kernel::type { lock.unlock(); return !timedOut; + */ + return false; } void KProcess::ConditionalVariableSignal(void *conditional, u64 amount) { + /* FIXME std::unique_lock condLock(conditionalLock); auto &condWaiters{conditionals[reinterpret_cast(conditional)]}; @@ -282,5 +345,6 @@ namespace skyline::kernel::type { while (thread->flag); condLock.lock(); } + */ } } diff --git a/app/src/main/cpp/skyline/kernel/types/KProcess.h b/app/src/main/cpp/skyline/kernel/types/KProcess.h index 2f744111..1ab8816a 100644 --- a/app/src/main/cpp/skyline/kernel/types/KProcess.h +++ b/app/src/main/cpp/skyline/kernel/types/KProcess.h @@ -15,7 +15,6 @@ namespace skyline { constexpr u16 TlsSlotSize{0x200}; //!< The size of a single TLS slot constexpr u8 TlsSlots{PAGE_SIZE / TlsSlotSize}; //!< The amount of TLS slots in a single page constexpr KHandle BaseHandleIndex{0xD000}; //!< The index of the base handle - constexpr u32 MtxOwnerMask{0xBFFFFFFF}; //!< The mask of values which contain the owner of a mutex } namespace kernel::type { @@ -27,22 +26,6 @@ namespace skyline { MemoryManager memory; private: - struct WaitStatus { - std::atomic_bool flag{false}; - u8 priority; - KHandle handle; - u32 *mutex{}; - - WaitStatus(u8 priority, KHandle handle); - - WaitStatus(u8 priority, KHandle handle, u32 *mutex); - }; - - std::unordered_map>> mutexes; //!< A map from a mutex's address to a vector of Mutex objects for threads waiting on it - std::unordered_map>> conditionals; //!< A map from a conditional variable's address to a vector of threads waiting on it - std::mutex mutexLock; - std::mutex conditionalLock; - std::mutex threadMutex; //!< Synchronizes thread creation to prevent a race between thread creation and thread killing bool disableThreadCreation{}; //!< If to disable thread creation, we use this to prevent thread creation after all threads have been killed std::vector> threads; @@ -208,17 +191,15 @@ namespace skyline { } /** - * @brief Locks the Mutex at the specified address - * @param owner The handle of the current mutex owner - * @return If the mutex was successfully locked + * @brief Locks the mutex at the specified address + * @param ownerHandle The psuedo-handle of the current mutex owner */ - bool MutexLock(u32 *mutex, KHandle owner); + Result MutexLock(u32 *mutex, KHandle ownerHandle); /** - * @brief Unlocks the Mutex at the specified address - * @return If the mutex was successfully unlocked + * @brief Unlocks the mutex at the specified address */ - bool MutexUnlock(u32 *mutex); + void MutexUnlock(u32 *mutex); /** * @param timeout The amount of time to wait for the conditional variable diff --git a/app/src/main/cpp/skyline/kernel/types/KThread.cpp b/app/src/main/cpp/skyline/kernel/types/KThread.cpp index c5a3de8b..f36dbb06 100644 --- a/app/src/main/cpp/skyline/kernel/types/KThread.cpp +++ b/app/src/main/cpp/skyline/kernel/types/KThread.cpp @@ -58,7 +58,7 @@ namespace skyline::kernel::type { signal::SetSignalHandler({SIGINT, SIGILL, SIGTRAP, SIGBUS, SIGFPE, SIGSEGV}, nce::NCE::SignalHandler); try { - state.scheduler->InsertThread(); + state.scheduler->InsertThread(state.thread); state.scheduler->WaitSchedule(); asm volatile( diff --git a/app/src/main/cpp/skyline/kernel/types/KThread.h b/app/src/main/cpp/skyline/kernel/types/KThread.h index 33e50672..9fae4730 100644 --- a/app/src/main/cpp/skyline/kernel/types/KThread.h +++ b/app/src/main/cpp/skyline/kernel/types/KThread.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -38,16 +39,21 @@ namespace skyline { u64 entryArgument; void *stackTop; + std::atomic basePriority; //!< The priority of the thread for the scheduler without any priority-inheritance std::atomic priority; //!< The priority of the thread for the scheduler i8 idealCore; //!< The ideal CPU core for this thread to run on i8 coreId; //!< The CPU core on which this thread is running CoreMask affinityMask{}; //!< A mask of CPU cores this thread is allowed to run on + std::mutex coreMigrationMutex; //!< Synchronizes operations which depend on which core the thread is running on u64 timesliceStart{}; //!< Start of the scheduler timeslice u64 averageTimeslice{}; //!< A weighted average of the timeslice duration for this thread std::optional preemptionTimer{}; //!< A kernel timer used for preemption interrupts bool isPreempted{}; //!< If the preemption timer has been armed and will fire bool needsReorder{}; //!< If the thread needs to reorder itself during scheduler rotation - std::mutex coreMigrationMutex; //!< Synchronizes operations which depend on which core the thread is running on + std::mutex waiterMutex; //!< Synchronizes operations on mutation of the waiter members + u32* waitKey; //!< The key on which this thread is waiting on + std::shared_ptr waitThread; //!< The thread which this thread is waiting on + std::list> waiters; //!< A queue of threads waiting on this thread sorted by priority KThread(const DeviceState &state, KHandle handle, KProcess *parent, size_t id, void *entry, u64 argument, void *stackTop, u8 priority, i8 idealCore); @@ -69,6 +75,13 @@ namespace skyline { * @brief Sends a host OS signal to the thread which is running this KThread */ void SendSignal(int signal); + + /** + * @return If the supplied priority value is higher than the current thread + */ + static constexpr bool IsHigherPriority(const i8 priority, const std::shared_ptr &it) { + return priority < it->priority; + } }; } }