mirror of
https://github.com/skyline-emu/skyline.git
synced 2024-12-28 06:45:29 +03:00
Implement PI-Mutexes + Optimize InsertThread
This commit is contained in:
◱ PixelyIon
◱ Mark
parent
7ba7cd2394
commit
7079f11add
@ -30,7 +30,6 @@ namespace skyline::kernel {
|
||||
if (!currentCore->queue.empty() && thread->affinityMask.count() != 1) {
|
||||
// Select core where the current thread will be scheduled the earliest based off average timeslice durations for resident threads
|
||||
// There's a preference for the current core as migration isn't free
|
||||
|
||||
size_t minTimeslice{};
|
||||
CoreContext *optimalCore{};
|
||||
for (auto &candidateCore : cores) {
|
||||
@ -63,7 +62,7 @@ namespace skyline::kernel {
|
||||
if (optimalCore != currentCore) {
|
||||
std::unique_lock coreLock(currentCore->mutex);
|
||||
currentCore->queue.erase(std::remove(currentCore->queue.begin(), currentCore->queue.end(), thread), currentCore->queue.end());
|
||||
currentCore->mutateCondition.notify_all();
|
||||
currentCore->frontCondition.notify_all();
|
||||
|
||||
thread->coreId = optimalCore->id;
|
||||
|
||||
@ -80,11 +79,14 @@ namespace skyline::kernel {
|
||||
return *currentCore;
|
||||
}
|
||||
|
||||
void Scheduler::InsertThread(bool loadBalance) {
|
||||
auto &thread{state.thread};
|
||||
void Scheduler::InsertThread(const std::shared_ptr<type::KThread> &thread, bool loadBalance) {
|
||||
auto &core{loadBalance ? LoadBalance() : cores.at(thread->coreId)};
|
||||
|
||||
signal::SetSignalHandler({YieldSignal}, SignalHandler);
|
||||
thread_local bool signalHandlerSetup{};
|
||||
if (!signalHandlerSetup) {
|
||||
signal::SetSignalHandler({YieldSignal}, SignalHandler);
|
||||
signalHandlerSetup = true;
|
||||
}
|
||||
|
||||
if (!thread->preemptionTimer) {
|
||||
struct sigevent event{
|
||||
@ -92,19 +94,34 @@ namespace skyline::kernel {
|
||||
.sigev_notify = SIGEV_THREAD_ID,
|
||||
.sigev_notify_thread_id = gettid(),
|
||||
};
|
||||
timer_create(CLOCK_THREAD_CPUTIME_ID, &event, &*thread->preemptionTimer);
|
||||
timer_t timer;
|
||||
if (timer_create(CLOCK_THREAD_CPUTIME_ID, &event, &timer))
|
||||
throw exception("timer_create has failed with '{}'", strerror(errno));
|
||||
thread->preemptionTimer.emplace(timer);
|
||||
}
|
||||
|
||||
std::unique_lock lock(core.mutex);
|
||||
auto nextThread{std::find_if(core.queue.begin(), core.queue.end(), [&](const std::shared_ptr<type::KThread> &it) { return it->priority > thread->priority; })};
|
||||
if (nextThread == core.queue.begin() && nextThread != core.queue.end()) {
|
||||
// If the inserted thread has a higher priority than the currently running thread (and the queue isn't empty)
|
||||
core.queue.front()->SendSignal(YieldSignal);
|
||||
core.queue.insert(std::next(core.queue.begin()), thread);
|
||||
auto nextThread{std::upper_bound(core.queue.begin(), core.queue.end(), thread->priority.load(), type::KThread::IsHigherPriority)};
|
||||
if (nextThread == core.queue.begin()) {
|
||||
if (nextThread != core.queue.end()) {
|
||||
// If the inserted thread has a higher priority than the currently running thread (and the queue isn't empty)
|
||||
// We need to interrupt the currently scheduled thread and put this thread at the front instead
|
||||
core.queue.insert(std::next(core.queue.begin()), thread);
|
||||
if (state.thread != core.queue.front())
|
||||
core.queue.front()->SendSignal(YieldSignal);
|
||||
else
|
||||
YieldPending = true;
|
||||
} else {
|
||||
core.queue.push_front(thread);
|
||||
}
|
||||
if (thread != state.thread) {
|
||||
lock.unlock(); // We should unlock this prior to waking all threads to not cause contention on the core lock
|
||||
core.frontCondition.notify_all(); // We only want to trigger the conditional variable if the current thread isn't inserting itself
|
||||
}
|
||||
} else {
|
||||
core.queue.insert(nextThread, thread);
|
||||
core.mutateCondition.notify_all(); // We only want to trigger the conditional variable if the current thread isn't going to be scheduled next
|
||||
}
|
||||
|
||||
thread->needsReorder = true; // We need to reorder the thread from back to align it with other threads of it's priority and ensure strict ordering amongst priorities
|
||||
}
|
||||
|
||||
@ -115,13 +132,13 @@ namespace skyline::kernel {
|
||||
std::shared_lock lock(core->mutex);
|
||||
if (thread->affinityMask.count() > 1) {
|
||||
std::chrono::milliseconds loadBalanceThreshold{PreemptiveTimeslice * 2}; //!< The amount of time that needs to pass unscheduled for a thread to attempt load balancing
|
||||
while (!core->mutateCondition.wait_for(lock, loadBalanceThreshold, [&]() { return core->queue.front() == thread; })) {
|
||||
while (!core->frontCondition.wait_for(lock, loadBalanceThreshold, [&]() { return !core->queue.empty() && core->queue.front() == thread; })) {
|
||||
lock.unlock();
|
||||
LoadBalance();
|
||||
if (thread->coreId == core->id) {
|
||||
lock.lock();
|
||||
} else {
|
||||
InsertThread(false);
|
||||
InsertThread(state.thread, false);
|
||||
core = &cores.at(thread->coreId);
|
||||
lock = std::shared_lock(core->mutex);
|
||||
}
|
||||
@ -129,7 +146,7 @@ namespace skyline::kernel {
|
||||
loadBalanceThreshold *= 2; // We double the duration required for future load balancing for this invocation to minimize pointless load balancing
|
||||
}
|
||||
} else {
|
||||
core->mutateCondition.wait(lock, [&]() { return core->queue.front() == thread; });
|
||||
core->frontCondition.wait(lock, [&]() { return !core->queue.empty() && core->queue.front() == thread; });
|
||||
}
|
||||
|
||||
if (thread->priority == core->preemptionPriority) {
|
||||
@ -160,14 +177,15 @@ namespace skyline::kernel {
|
||||
auto foldingPoint{std::find_if(std::next(core.queue.begin()), core.queue.end(), [&](const std::shared_ptr<type::KThread> &it) {
|
||||
return lastPriority > it->priority ? true : lastPriority = it->priority, false;
|
||||
})};
|
||||
core.queue.insert(std::find_if(foldingPoint, core.queue.end(), [&](const std::shared_ptr<type::KThread> &it) { return it->priority > thread->priority; }), thread);
|
||||
core.queue.insert(std::upper_bound(foldingPoint, core.queue.end(), thread->priority.load(), type::KThread::IsHigherPriority), thread);
|
||||
thread->needsReorder = false;
|
||||
} else {
|
||||
core.queue.push_back(thread);
|
||||
thread->needsReorder = false;
|
||||
}
|
||||
|
||||
core.mutateCondition.notify_all();
|
||||
lock.unlock();
|
||||
core.frontCondition.notify_all();
|
||||
|
||||
if (cooperative && thread->isPreempted) {
|
||||
// If a preemptive thread did a cooperative yield then we need to disarm the preemptive timer
|
||||
@ -178,7 +196,7 @@ namespace skyline::kernel {
|
||||
}
|
||||
}
|
||||
|
||||
void Scheduler::UpdatePriority(const std::shared_ptr<type::KThread>& thread) {
|
||||
void Scheduler::UpdatePriority(const std::shared_ptr<type::KThread> &thread) {
|
||||
std::lock_guard migrationLock(thread->coreMigrationMutex);
|
||||
auto *core{&cores.at(thread->coreId)};
|
||||
std::unique_lock coreLock(core->mutex);
|
||||
@ -188,12 +206,18 @@ namespace skyline::kernel {
|
||||
// If the thread isn't in the queue then the new priority will be handled automatically on insertion
|
||||
return;
|
||||
if (currentIt == core->queue.begin()) {
|
||||
// Alternatively, if it's currently running then we'd just want to update after it rotates
|
||||
// Alternatively, if it's currently running then we'd just want to reorder after it rotates
|
||||
if (!thread->isPreempted && thread->priority == core->preemptionPriority) {
|
||||
// If the thread needs to be preempted due to the new priority then arm it's preemption timer
|
||||
struct itimerspec spec{.it_value = {.tv_nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(PreemptiveTimeslice).count()}};
|
||||
timer_settime(*thread->preemptionTimer, 0, &spec, nullptr);
|
||||
thread->isPreempted = true;
|
||||
}
|
||||
thread->needsReorder = true;
|
||||
return;
|
||||
}
|
||||
|
||||
auto targetIt{std::find_if(core->queue.begin(), core->queue.end(), [&](const std::shared_ptr<type::KThread> &it) { return it->priority > thread->priority; })};
|
||||
auto targetIt{std::upper_bound(core->queue.begin(), core->queue.end(), thread->priority.load(), type::KThread::IsHigherPriority)};
|
||||
if (currentIt == targetIt)
|
||||
// If this thread's position isn't affected by the priority change then we have nothing to do
|
||||
return;
|
||||
@ -206,13 +230,12 @@ namespace skyline::kernel {
|
||||
thread->isPreempted = false;
|
||||
}
|
||||
|
||||
targetIt = std::find_if(core->queue.begin(), core->queue.end(), [&](const std::shared_ptr<type::KThread> &it) { return it->priority > thread->priority; }); // Iterator invalidation
|
||||
targetIt = std::upper_bound(core->queue.begin(), core->queue.end(), thread->priority.load(), type::KThread::IsHigherPriority); // Iterator invalidation
|
||||
if (targetIt == core->queue.begin() && targetIt != core->queue.end()) {
|
||||
core->queue.front()->SendSignal(YieldSignal);
|
||||
core->queue.insert(std::next(core->queue.begin()), thread);
|
||||
core->queue.front()->SendSignal(YieldSignal);
|
||||
} else {
|
||||
core->queue.insert(targetIt, thread);
|
||||
core->mutateCondition.notify_all();
|
||||
}
|
||||
thread->needsReorder = true;
|
||||
}
|
||||
@ -221,8 +244,11 @@ namespace skyline::kernel {
|
||||
auto &thread{state.thread};
|
||||
auto &core{cores.at(thread->coreId)};
|
||||
|
||||
std::unique_lock lock(core.mutex);
|
||||
core.queue.erase(std::remove(core.queue.begin(), core.queue.end(), thread), core.queue.end());
|
||||
{
|
||||
std::unique_lock lock(core.mutex);
|
||||
core.queue.erase(std::remove(core.queue.begin(), core.queue.end(), thread), core.queue.end());
|
||||
}
|
||||
core.frontCondition.notify_all(); // We need to notify any threads in line to be scheduled
|
||||
|
||||
if (thread->isPreempted) {
|
||||
struct itimerspec spec{};
|
||||
|
||||
@ -46,7 +46,7 @@ namespace skyline {
|
||||
u8 id;
|
||||
u8 preemptionPriority; //!< The priority at which this core becomes preemptive as opposed to cooperative
|
||||
std::shared_mutex mutex; //!< Synchronizes all operations on the queue
|
||||
std::condition_variable_any mutateCondition; //!< A conditional variable which is signalled on every mutation of the queue
|
||||
std::condition_variable_any frontCondition; //!< A conditional variable which is signalled when the front of the queue has changed
|
||||
std::deque<std::shared_ptr<type::KThread>> queue; //!< A queue of threads which are running or to be run on this core
|
||||
|
||||
CoreContext(u8 id, u8 preemptionPriority);
|
||||
@ -74,10 +74,11 @@ namespace skyline {
|
||||
CoreContext& LoadBalance();
|
||||
|
||||
/**
|
||||
* @brief Inserts the calling thread into the scheduler queue at the appropriate location based on it's priority
|
||||
* @brief Inserts the specified thread into the scheduler queue at the appropriate location based on it's priority
|
||||
* @param loadBalance If to load balance or use the thread's current core (KThread::coreId)
|
||||
* @note It isn't supported to load balance if the supplied thread isn't the calling thread, it'll lead to UB
|
||||
*/
|
||||
void InsertThread(bool loadBalance = true);
|
||||
void InsertThread(const std::shared_ptr<type::KThread>& thread, bool loadBalance = true);
|
||||
|
||||
/**
|
||||
* @brief Wait for the current thread to be scheduled on it's resident core
|
||||
|
||||
@ -276,7 +276,7 @@ namespace skyline::kernel::svc {
|
||||
}
|
||||
|
||||
void ExitThread(const DeviceState &state) {
|
||||
state.logger->Debug("svcExitThread: Exiting current thread: {}", state.thread->id);
|
||||
state.logger->Debug("svcExitThread: Exiting current thread");
|
||||
std::longjmp(state.thread->originalCtx, true);
|
||||
}
|
||||
|
||||
@ -330,9 +330,11 @@ namespace skyline::kernel::svc {
|
||||
}
|
||||
try {
|
||||
auto thread{state.process->GetHandle<type::KThread>(handle)};
|
||||
state.logger->Debug("svcSetThreadPriority: Setting thread priority to {}", thread->id, priority);
|
||||
thread->priority = priority;
|
||||
state.scheduler->UpdatePriority(thread);
|
||||
state.logger->Debug("svcSetThreadPriority: Setting thread priority to {}", priority);
|
||||
if (thread->priority != priority) {
|
||||
thread->priority = priority;
|
||||
state.scheduler->UpdatePriority(thread);
|
||||
}
|
||||
state.ctx->gpr.w0 = Result{};
|
||||
} catch (const std::out_of_range &) {
|
||||
state.logger->Warn("svcSetThreadPriority: 'handle' invalid: 0x{:X}", handle);
|
||||
@ -346,7 +348,7 @@ namespace skyline::kernel::svc {
|
||||
auto thread{state.process->GetHandle<type::KThread>(handle)};
|
||||
auto idealCore{thread->idealCore};
|
||||
auto affinityMask{thread->affinityMask};
|
||||
state.logger->Debug("svcGetThreadCoreMask: Writing thread #{}'s Ideal Core ({}) + Affinity Mask ({})", thread->id, idealCore, affinityMask);
|
||||
state.logger->Debug("svcGetThreadCoreMask: Setting Ideal Core ({}) + Affinity Mask ({})", idealCore, affinityMask);
|
||||
|
||||
state.ctx->gpr.x2 = affinityMask.to_ullong();
|
||||
state.ctx->gpr.w1 = idealCore;
|
||||
@ -386,7 +388,7 @@ namespace skyline::kernel::svc {
|
||||
return;
|
||||
}
|
||||
|
||||
state.logger->Debug("svcSetThreadCoreMask: Setting thread #{}'s Ideal Core ({}) + Affinity Mask ({})", thread->id, idealCore, affinityMask);
|
||||
state.logger->Debug("svcSetThreadCoreMask: Setting Ideal Core ({}) + Affinity Mask ({})", idealCore, affinityMask);
|
||||
|
||||
thread->idealCore = idealCore;
|
||||
thread->affinityMask = affinityMask;
|
||||
@ -396,7 +398,8 @@ namespace skyline::kernel::svc {
|
||||
|
||||
state.scheduler->RemoveThread();
|
||||
thread->coreId = idealCore;
|
||||
state.scheduler->InsertThread(false);
|
||||
state.scheduler->InsertThread(state.thread, false);
|
||||
state.scheduler->WaitSchedule();
|
||||
}
|
||||
|
||||
state.ctx->gpr.w0 = Result{};
|
||||
@ -407,8 +410,9 @@ namespace skyline::kernel::svc {
|
||||
}
|
||||
|
||||
void GetCurrentProcessorNumber(const DeviceState &state) {
|
||||
state.logger->Debug("svcGetCurrentProcessorNumber: Writing current core for thread #{}: {}", state.thread->id, state.thread->coreId);
|
||||
state.ctx->gpr.w0 = state.thread->coreId;
|
||||
auto coreId{state.thread->coreId};
|
||||
state.logger->Debug("svcGetCurrentProcessorNumber: Writing current core: {}", coreId);
|
||||
state.ctx->gpr.w0 = coreId;
|
||||
}
|
||||
|
||||
void ClearEvent(const DeviceState &state) {
|
||||
@ -611,12 +615,15 @@ namespace skyline::kernel::svc {
|
||||
|
||||
state.logger->Debug("svcArbitrateLock: Locking mutex at 0x{:X}", pointer);
|
||||
|
||||
if (state.process->MutexLock(pointer, ownerHandle))
|
||||
auto result{state.process->MutexLock(pointer, ownerHandle)};
|
||||
if (result == Result{})
|
||||
state.logger->Debug("svcArbitrateLock: Locked mutex at 0x{:X}", pointer);
|
||||
else
|
||||
else if (result == result::InvalidHandle)
|
||||
state.logger->Warn("svcArbitrateLock: 'handle' invalid: 0x{:X}", ownerHandle);
|
||||
else if (result == result::InvalidCurrentMemory)
|
||||
state.logger->Debug("svcArbitrateLock: Owner handle did not match current owner for mutex or didn't have waiter flag at 0x{:X}", pointer);
|
||||
|
||||
state.ctx->gpr.w0 = Result{};
|
||||
state.ctx->gpr.w0 = result;
|
||||
}
|
||||
|
||||
void ArbitrateUnlock(const DeviceState &state) {
|
||||
@ -629,13 +636,10 @@ namespace skyline::kernel::svc {
|
||||
|
||||
state.logger->Debug("svcArbitrateUnlock: Unlocking mutex at 0x{:X}", mutex);
|
||||
|
||||
if (state.process->MutexUnlock(mutex)) {
|
||||
state.logger->Debug("svcArbitrateUnlock: Unlocked mutex at 0x{:X}", mutex);
|
||||
state.ctx->gpr.w0 = Result{};
|
||||
} else {
|
||||
state.logger->Debug("svcArbitrateUnlock: A non-owner thread tried to release a mutex at 0x{:X}", mutex);
|
||||
state.ctx->gpr.w0 = result::InvalidAddress;
|
||||
}
|
||||
state.process->MutexUnlock(mutex);
|
||||
|
||||
state.logger->Debug("svcArbitrateUnlock: Unlocked mutex at 0x{:X}", mutex);
|
||||
state.ctx->gpr.w0 = Result{};
|
||||
}
|
||||
|
||||
void WaitProcessWideKeyAtomic(const DeviceState &state) {
|
||||
@ -651,11 +655,7 @@ namespace skyline::kernel::svc {
|
||||
if (handle != state.thread->handle)
|
||||
throw exception("svcWaitProcessWideKeyAtomic: Handle doesn't match current thread: 0x{:X} for thread 0x{:X}", handle, state.thread->handle);
|
||||
|
||||
if (!state.process->MutexUnlock(mutex)) {
|
||||
state.logger->Debug("WaitProcessWideKeyAtomic: A non-owner thread tried to release a mutex at 0x{:X}", mutex);
|
||||
state.ctx->gpr.w0 = result::InvalidAddress;
|
||||
return;
|
||||
}
|
||||
state.process->MutexUnlock(mutex);
|
||||
|
||||
u64 timeout{state.ctx->gpr.x3};
|
||||
state.logger->Debug("svcWaitProcessWideKeyAtomic: Mutex: 0x{:X}, Conditional-Variable: 0x{:X}, Timeout: {} ns", mutex, conditional, timeout);
|
||||
@ -731,7 +731,7 @@ namespace skyline::kernel::svc {
|
||||
if (debug.back() == '\n')
|
||||
debug.remove_suffix(1);
|
||||
|
||||
state.logger->Info("Debug Output: {}", debug);
|
||||
state.logger->Info("svcOutputDebugString: {}", debug);
|
||||
state.ctx->gpr.w0 = Result{};
|
||||
}
|
||||
|
||||
|
||||
@ -3,13 +3,10 @@
|
||||
|
||||
#include <nce.h>
|
||||
#include <os.h>
|
||||
#include <kernel/results.h>
|
||||
#include "KProcess.h"
|
||||
|
||||
namespace skyline::kernel::type {
|
||||
KProcess::WaitStatus::WaitStatus(u8 priority, KHandle handle) : priority(priority), handle(handle) {}
|
||||
|
||||
KProcess::WaitStatus::WaitStatus(u8 priority, KHandle handle, u32 *mutex) : priority(priority), handle(handle), mutex(mutex) {}
|
||||
|
||||
KProcess::TlsPage::TlsPage(const std::shared_ptr<KPrivateMemory> &memory) : memory(memory) {}
|
||||
|
||||
u8 *KProcess::TlsPage::ReserveSlot() {
|
||||
@ -112,72 +109,135 @@ namespace skyline::kernel::type {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool KProcess::MutexLock(u32 *mutex, KHandle owner) {
|
||||
std::unique_lock lock(mutexLock);
|
||||
constexpr u32 HandleWaitMask{0x40000000}; //!< A mask of a bit which denotes if the handle has waiters or not
|
||||
|
||||
auto &mtxWaiters{mutexes[reinterpret_cast<u64>(mutex)]};
|
||||
if (mtxWaiters.empty()) {
|
||||
u32 mtxExpected{};
|
||||
if (__atomic_compare_exchange_n(mutex, &mtxExpected, (constant::MtxOwnerMask & state.thread->handle), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
|
||||
return true;
|
||||
|
||||
Result KProcess::MutexLock(u32 *mutex, KHandle ownerHandle) {
|
||||
std::shared_ptr<KThread> owner;
|
||||
try {
|
||||
owner = GetHandle<KThread>(ownerHandle & ~HandleWaitMask);
|
||||
} catch (const std::out_of_range &) {
|
||||
return result::InvalidHandle;
|
||||
}
|
||||
|
||||
if (__atomic_load_n(mutex, __ATOMIC_SEQ_CST) != (owner | ~constant::MtxOwnerMask))
|
||||
return false;
|
||||
bool isHighestPriority;
|
||||
{
|
||||
std::lock_guard lock(owner->waiterMutex);
|
||||
|
||||
std::shared_ptr<WaitStatus> status;
|
||||
for (auto it{mtxWaiters.begin()};; it++) {
|
||||
if (it != mtxWaiters.end() && (*it)->priority >= state.thread->priority)
|
||||
continue;
|
||||
u32 value{};
|
||||
if (__atomic_compare_exchange_n(mutex, &value, (HandleWaitMask & state.thread->handle), false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
|
||||
// We try to do a CAS to get ownership of the mutex in the case that it's unoccupied
|
||||
return {};
|
||||
if (value != (ownerHandle | HandleWaitMask))
|
||||
// We ensure that the mutex's value is the handle with the waiter bit set
|
||||
return result::InvalidCurrentMemory;
|
||||
|
||||
status = std::make_shared<WaitStatus>(state.thread->priority, state.thread->handle);
|
||||
mtxWaiters.insert(it, status);
|
||||
break;
|
||||
auto &waiters{owner->waiters};
|
||||
isHighestPriority = waiters.insert(std::upper_bound(waiters.begin(), waiters.end(), state.thread->priority.load(), KThread::IsHigherPriority), state.thread) == waiters.begin();
|
||||
state.scheduler->RemoveThread();
|
||||
|
||||
std::atomic_store(&state.thread->waitThread, owner);
|
||||
state.thread->waitKey = mutex;
|
||||
}
|
||||
|
||||
lock.unlock();
|
||||
while (!status->flag);
|
||||
lock.lock();
|
||||
status->flag = false;
|
||||
if (isHighestPriority) {
|
||||
// If we were the highest priority thread then we need to inherit priorities for all threads we're waiting on recursively
|
||||
do {
|
||||
u8 priority, ownerPriority;
|
||||
do {
|
||||
// Try to CAS the priority of the owner with the current thread
|
||||
// If they're equivalent then we don't need to CAS as the priority won't be inherited
|
||||
ownerPriority = owner->priority.load();
|
||||
priority = std::min(ownerPriority, state.thread->priority.load());
|
||||
} while (ownerPriority != priority && owner->priority.compare_exchange_strong(ownerPriority, priority));
|
||||
|
||||
for (auto it{mtxWaiters.begin()}; it != mtxWaiters.end(); it++) {
|
||||
if ((*it)->handle == state.thread->handle) {
|
||||
mtxWaiters.erase(it);
|
||||
break;
|
||||
}
|
||||
if (ownerPriority != priority) {
|
||||
auto waitThread{std::atomic_load(&owner->waitThread)};
|
||||
while (waitThread) {
|
||||
std::lock_guard lock(waitThread->waiterMutex);
|
||||
|
||||
auto currentWaitThread{std::atomic_load(&owner->waitThread)};
|
||||
if (waitThread != currentWaitThread) {
|
||||
waitThread = currentWaitThread;
|
||||
continue;
|
||||
}
|
||||
|
||||
// We need to update the location of the owner thread in the waiter queue of the thread it's waiting on
|
||||
auto &waiters{waitThread->waiters};
|
||||
waiters.erase(std::find(waiters.begin(), waiters.end(), waitThread));
|
||||
waiters.insert(std::upper_bound(waiters.begin(), waiters.end(), state.thread->priority.load(), KThread::IsHigherPriority), owner);
|
||||
break;
|
||||
}
|
||||
|
||||
state.scheduler->UpdatePriority(owner);
|
||||
|
||||
owner = waitThread;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} while (owner);
|
||||
}
|
||||
|
||||
return true;
|
||||
state.scheduler->WaitSchedule();
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
bool KProcess::MutexUnlock(u32 *mutex) {
|
||||
std::unique_lock lock(mutexLock);
|
||||
void KProcess::MutexUnlock(u32 *mutex) {
|
||||
std::lock_guard lock(state.thread->waiterMutex);
|
||||
auto &waiters{state.thread->waiters};
|
||||
auto nextOwnerIt{std::find_if(waiters.begin(), waiters.end(), [mutex](const std::shared_ptr<KThread> &thread) { return thread->waitKey == mutex; })};
|
||||
if (nextOwnerIt != waiters.end()) {
|
||||
auto nextOwner{*nextOwnerIt};
|
||||
std::lock_guard nextLock(nextOwner->waiterMutex);
|
||||
std::atomic_store(&nextOwner->waitThread, std::shared_ptr<KThread>{nullptr});
|
||||
nextOwner->waitKey = nullptr;
|
||||
|
||||
auto &mtxWaiters{mutexes[reinterpret_cast<u64>(mutex)]};
|
||||
u32 mtxDesired{};
|
||||
if (!mtxWaiters.empty())
|
||||
mtxDesired = (*mtxWaiters.begin())->handle | ((mtxWaiters.size() > 1) ? ~constant::MtxOwnerMask : 0);
|
||||
// Move all threads waiting on this key to the next owner's waiter list
|
||||
std::shared_ptr<KThread> nextWaiter{};
|
||||
for (auto it{waiters.erase(nextOwnerIt)}; it != waiters.end(); it++) {
|
||||
if ((*it)->waitKey == mutex) {
|
||||
nextOwner->waiters.splice(std::upper_bound(nextOwner->waiters.begin(), nextOwner->waiters.end(), (*it)->priority.load(), KThread::IsHigherPriority), waiters, it);
|
||||
std::atomic_store(&(*it)->waitThread, nextOwner);
|
||||
if (!nextWaiter)
|
||||
nextWaiter = *it;
|
||||
}
|
||||
}
|
||||
|
||||
u32 mtxExpected{(constant::MtxOwnerMask & state.thread->handle) | ~constant::MtxOwnerMask};
|
||||
if (!__atomic_compare_exchange_n(mutex, &mtxExpected, mtxDesired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
|
||||
mtxExpected &= constant::MtxOwnerMask;
|
||||
if (!waiters.empty()) {
|
||||
// If there are threads still waiting on us then try to inherit their priority
|
||||
auto highestPriority{waiters.front()};
|
||||
u8 priority, ownerPriority;
|
||||
do {
|
||||
ownerPriority = state.thread->priority.load();
|
||||
priority = std::min(ownerPriority, highestPriority->priority.load());
|
||||
} while (ownerPriority != priority && nextOwner->priority.compare_exchange_strong(ownerPriority, priority));
|
||||
state.scheduler->UpdatePriority(state.thread);
|
||||
}
|
||||
|
||||
if (!__atomic_compare_exchange_n(mutex, &mtxExpected, mtxDesired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))
|
||||
return false;
|
||||
if (nextWaiter) {
|
||||
// If there is a waiter on the new owner then try to inherit it's priority
|
||||
u8 priority, ownerPriority;
|
||||
do {
|
||||
ownerPriority = nextOwner->priority.load();
|
||||
priority = std::min(ownerPriority, nextWaiter->priority.load());
|
||||
} while (ownerPriority != priority && nextOwner->priority.compare_exchange_strong(ownerPriority, priority));
|
||||
|
||||
__atomic_store_n(mutex, nextOwner->handle | HandleWaitMask, __ATOMIC_SEQ_CST);
|
||||
} else {
|
||||
__atomic_store_n(mutex, nextOwner->handle, __ATOMIC_SEQ_CST);
|
||||
}
|
||||
|
||||
// Finally, schedule the next owner accordingly
|
||||
state.scheduler->InsertThread(nextOwner, false);
|
||||
} else {
|
||||
__atomic_store_n(mutex, 0, __ATOMIC_SEQ_CST);
|
||||
return;
|
||||
}
|
||||
|
||||
if (mtxDesired) {
|
||||
auto status{(*mtxWaiters.begin())};
|
||||
status->flag = true;
|
||||
lock.unlock();
|
||||
while (status->flag);
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool KProcess::ConditionalVariableWait(void *conditional, u32 *mutex, u64 timeout) {
|
||||
/* FIXME
|
||||
std::unique_lock lock(conditionalLock);
|
||||
|
||||
auto &condWaiters{conditionals[reinterpret_cast<u64>(conditional)]};
|
||||
@ -216,9 +276,12 @@ namespace skyline::kernel::type {
|
||||
lock.unlock();
|
||||
|
||||
return !timedOut;
|
||||
*/
|
||||
return false;
|
||||
}
|
||||
|
||||
void KProcess::ConditionalVariableSignal(void *conditional, u64 amount) {
|
||||
/* FIXME
|
||||
std::unique_lock condLock(conditionalLock);
|
||||
|
||||
auto &condWaiters{conditionals[reinterpret_cast<u64>(conditional)]};
|
||||
@ -282,5 +345,6 @@ namespace skyline::kernel::type {
|
||||
while (thread->flag);
|
||||
condLock.lock();
|
||||
}
|
||||
*/
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,7 +15,6 @@ namespace skyline {
|
||||
constexpr u16 TlsSlotSize{0x200}; //!< The size of a single TLS slot
|
||||
constexpr u8 TlsSlots{PAGE_SIZE / TlsSlotSize}; //!< The amount of TLS slots in a single page
|
||||
constexpr KHandle BaseHandleIndex{0xD000}; //!< The index of the base handle
|
||||
constexpr u32 MtxOwnerMask{0xBFFFFFFF}; //!< The mask of values which contain the owner of a mutex
|
||||
}
|
||||
|
||||
namespace kernel::type {
|
||||
@ -27,22 +26,6 @@ namespace skyline {
|
||||
MemoryManager memory;
|
||||
|
||||
private:
|
||||
struct WaitStatus {
|
||||
std::atomic_bool flag{false};
|
||||
u8 priority;
|
||||
KHandle handle;
|
||||
u32 *mutex{};
|
||||
|
||||
WaitStatus(u8 priority, KHandle handle);
|
||||
|
||||
WaitStatus(u8 priority, KHandle handle, u32 *mutex);
|
||||
};
|
||||
|
||||
std::unordered_map<u64, std::vector<std::shared_ptr<WaitStatus>>> mutexes; //!< A map from a mutex's address to a vector of Mutex objects for threads waiting on it
|
||||
std::unordered_map<u64, std::list<std::shared_ptr<WaitStatus>>> conditionals; //!< A map from a conditional variable's address to a vector of threads waiting on it
|
||||
std::mutex mutexLock;
|
||||
std::mutex conditionalLock;
|
||||
|
||||
std::mutex threadMutex; //!< Synchronizes thread creation to prevent a race between thread creation and thread killing
|
||||
bool disableThreadCreation{}; //!< If to disable thread creation, we use this to prevent thread creation after all threads have been killed
|
||||
std::vector<std::shared_ptr<KThread>> threads;
|
||||
@ -208,17 +191,15 @@ namespace skyline {
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Locks the Mutex at the specified address
|
||||
* @param owner The handle of the current mutex owner
|
||||
* @return If the mutex was successfully locked
|
||||
* @brief Locks the mutex at the specified address
|
||||
* @param ownerHandle The psuedo-handle of the current mutex owner
|
||||
*/
|
||||
bool MutexLock(u32 *mutex, KHandle owner);
|
||||
Result MutexLock(u32 *mutex, KHandle ownerHandle);
|
||||
|
||||
/**
|
||||
* @brief Unlocks the Mutex at the specified address
|
||||
* @return If the mutex was successfully unlocked
|
||||
* @brief Unlocks the mutex at the specified address
|
||||
*/
|
||||
bool MutexUnlock(u32 *mutex);
|
||||
void MutexUnlock(u32 *mutex);
|
||||
|
||||
/**
|
||||
* @param timeout The amount of time to wait for the conditional variable
|
||||
|
||||
@ -58,7 +58,7 @@ namespace skyline::kernel::type {
|
||||
signal::SetSignalHandler({SIGINT, SIGILL, SIGTRAP, SIGBUS, SIGFPE, SIGSEGV}, nce::NCE::SignalHandler);
|
||||
|
||||
try {
|
||||
state.scheduler->InsertThread();
|
||||
state.scheduler->InsertThread(state.thread);
|
||||
state.scheduler->WaitSchedule();
|
||||
|
||||
asm volatile(
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <list>
|
||||
#include <csetjmp>
|
||||
#include <nce/guest.h>
|
||||
#include <kernel/scheduler.h>
|
||||
@ -38,16 +39,21 @@ namespace skyline {
|
||||
u64 entryArgument;
|
||||
void *stackTop;
|
||||
|
||||
std::atomic<u8> basePriority; //!< The priority of the thread for the scheduler without any priority-inheritance
|
||||
std::atomic<u8> priority; //!< The priority of the thread for the scheduler
|
||||
i8 idealCore; //!< The ideal CPU core for this thread to run on
|
||||
i8 coreId; //!< The CPU core on which this thread is running
|
||||
CoreMask affinityMask{}; //!< A mask of CPU cores this thread is allowed to run on
|
||||
std::mutex coreMigrationMutex; //!< Synchronizes operations which depend on which core the thread is running on
|
||||
u64 timesliceStart{}; //!< Start of the scheduler timeslice
|
||||
u64 averageTimeslice{}; //!< A weighted average of the timeslice duration for this thread
|
||||
std::optional<timer_t> preemptionTimer{}; //!< A kernel timer used for preemption interrupts
|
||||
bool isPreempted{}; //!< If the preemption timer has been armed and will fire
|
||||
bool needsReorder{}; //!< If the thread needs to reorder itself during scheduler rotation
|
||||
std::mutex coreMigrationMutex; //!< Synchronizes operations which depend on which core the thread is running on
|
||||
std::mutex waiterMutex; //!< Synchronizes operations on mutation of the waiter members
|
||||
u32* waitKey; //!< The key on which this thread is waiting on
|
||||
std::shared_ptr<KThread> waitThread; //!< The thread which this thread is waiting on
|
||||
std::list<std::shared_ptr<type::KThread>> waiters; //!< A queue of threads waiting on this thread sorted by priority
|
||||
|
||||
KThread(const DeviceState &state, KHandle handle, KProcess *parent, size_t id, void *entry, u64 argument, void *stackTop, u8 priority, i8 idealCore);
|
||||
|
||||
@ -69,6 +75,13 @@ namespace skyline {
|
||||
* @brief Sends a host OS signal to the thread which is running this KThread
|
||||
*/
|
||||
void SendSignal(int signal);
|
||||
|
||||
/**
|
||||
* @return If the supplied priority value is higher than the current thread
|
||||
*/
|
||||
static constexpr bool IsHigherPriority(const i8 priority, const std::shared_ptr<type::KThread> &it) {
|
||||
return priority < it->priority;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user