diff --git a/app/src/main/cpp/skyline/kernel/scheduler.cpp b/app/src/main/cpp/skyline/kernel/scheduler.cpp index d12ab92e..ad657fa8 100644 --- a/app/src/main/cpp/skyline/kernel/scheduler.cpp +++ b/app/src/main/cpp/skyline/kernel/scheduler.cpp @@ -24,6 +24,7 @@ namespace skyline::kernel { Scheduler::CoreContext &Scheduler::LoadBalance() { auto &thread{state.thread}; + std::lock_guard migrationLock(thread->coreMigrationMutex); auto *currentCore{&cores.at(thread->coreId)}; if (!currentCore->queue.empty() && thread->affinityMask.count() != 1) { @@ -37,7 +38,7 @@ namespace skyline::kernel { u64 timeslice{}; if (!candidateCore.queue.empty()) { - std::shared_lock lock(candidateCore.mutex); + std::shared_lock coreLock(candidateCore.mutex); auto threadIterator{candidateCore.queue.cbegin()}; if (threadIterator != candidateCore.queue.cend()) { @@ -60,21 +61,21 @@ namespace skyline::kernel { } if (optimalCore != currentCore) { - std::unique_lock lock(currentCore->mutex); + std::unique_lock coreLock(currentCore->mutex); currentCore->queue.erase(std::remove(currentCore->queue.begin(), currentCore->queue.end(), thread), currentCore->queue.end()); currentCore->mutateCondition.notify_all(); thread->coreId = optimalCore->id; - state.logger->Debug("Load Balancing for #{}: C{} -> C{}", thread->id, currentCore->id, optimalCore->id); + state.logger->Debug("Load Balancing: C{} -> C{}", currentCore->id, optimalCore->id); } else { - state.logger->Debug("Load Balancing for #{}: C{} (Late)", thread->id, currentCore->id); + state.logger->Debug("Load Balancing: C{} (Late)", currentCore->id); } return *optimalCore; } - state.logger->Debug("Load Balancing for #{}: C{} (Early)", thread->id, currentCore->id); + state.logger->Debug("Load Balancing: C{} (Early)", currentCore->id); return *currentCore; } @@ -97,12 +98,14 @@ namespace skyline::kernel { std::unique_lock lock(core.mutex); auto nextThread{std::find_if(core.queue.begin(), core.queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; })}; if (nextThread == core.queue.begin() && nextThread != core.queue.end()) { + // If the inserted thread has a higher priority than the currently running thread (and the queue isn't empty) core.queue.front()->SendSignal(YieldSignal); core.queue.insert(std::next(core.queue.begin()), thread); } else { core.queue.insert(nextThread, thread); core.mutateCondition.notify_all(); // We only want to trigger the conditional variable if the current thread isn't going to be scheduled next } + thread->needsReorder = true; // We need to reorder the thread from back to align it with other threads of it's priority and ensure strict ordering amongst priorities } void Scheduler::WaitSchedule() { @@ -147,18 +150,71 @@ namespace skyline::kernel { thread->averageTimeslice = (thread->averageTimeslice / 4) + (3 * (util::GetTimeTicks() - thread->timesliceStart / 4)); // 0.25 * old timeslice duration + 0.75 * current timeslice duration core.queue.pop_front(); - core.queue.push_back(thread); + if (!thread->needsReorder) { + core.queue.push_back(thread); + } else if (core.queue.size() > 1 && core.queue.back()->priority > thread->priority) { + // If 'needsReorder' is set, the core queue isn't empty nor has only one member and the thread at the back of the queue has a lower priority than the current one + // We can attempt to reorder this thread, this is done by doing a priority-aware insert with the search starting at the "folding point" + // The folding point is where a thread has a lower priority than the one succeeding it in the queue, this is where a new "sequence" starts from highest to lowest priorities + u8 lastPriority{core.queue.front()->priority}; + auto foldingPoint{std::find_if(std::next(core.queue.begin()), core.queue.end(), [&](const std::shared_ptr &it) { + return lastPriority > it->priority ? true : lastPriority = it->priority, false; + })}; + core.queue.insert(std::find_if(foldingPoint, core.queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; }), thread); + thread->needsReorder = false; + } else { + core.queue.push_back(thread); + thread->needsReorder = false; + } core.mutateCondition.notify_all(); + + if (cooperative && thread->isPreempted) { + // If a preemptive thread did a cooperative yield then we need to disarm the preemptive timer + struct itimerspec spec{}; + timer_settime(*thread->preemptionTimer, 0, &spec, nullptr); + } + thread->isPreempted = false; + } + } + + void Scheduler::UpdatePriority(const std::shared_ptr& thread) { + std::lock_guard migrationLock(thread->coreMigrationMutex); + auto *core{&cores.at(thread->coreId)}; + std::unique_lock coreLock(core->mutex); + + auto currentIt{std::find(core->queue.begin(), core->queue.end(), thread)}; + if (currentIt == core->queue.end() || currentIt == core->queue.begin()) + // If the thread isn't in the queue then the new priority will be handled automatically on insertion + return; + if (currentIt == core->queue.begin()) { + // Alternatively, if it's currently running then we'd just want to update after it rotates + thread->needsReorder = true; + return; } - if (cooperative && thread->isPreempted) { - // If a preemptive thread did a cooperative yield then we need to disarm the preemptive timer + auto targetIt{std::find_if(core->queue.begin(), core->queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; })}; + if (currentIt == targetIt) + // If this thread's position isn't affected by the priority change then we have nothing to do + return; + + core->queue.erase(currentIt); + + if (thread->isPreempted && thread->priority != core->preemptionPriority) { struct itimerspec spec{}; timer_settime(*thread->preemptionTimer, 0, &spec, nullptr); + thread->isPreempted = false; } - thread->isPreempted = false; + targetIt = std::find_if(core->queue.begin(), core->queue.end(), [&](const std::shared_ptr &it) { return it->priority > thread->priority; }); // Iterator invalidation + if (targetIt == core->queue.begin() && targetIt != core->queue.end()) { + core->queue.front()->SendSignal(YieldSignal); + core->queue.insert(std::next(core->queue.begin()), thread); + } else { + core->queue.insert(targetIt, thread); + core->mutateCondition.notify_all(); + } + thread->needsReorder = true; } void Scheduler::RemoveThread() { diff --git a/app/src/main/cpp/skyline/kernel/scheduler.h b/app/src/main/cpp/skyline/kernel/scheduler.h index 094eb86c..c9954e58 100644 --- a/app/src/main/cpp/skyline/kernel/scheduler.h +++ b/app/src/main/cpp/skyline/kernel/scheduler.h @@ -91,6 +91,11 @@ namespace skyline { */ void Rotate(bool cooperative = true); + /** + * @brief Updates the placement of the supplied thread in it's resident core's queue according to it's new priority + */ + void UpdatePriority(const std::shared_ptr& thread); + /** * @brief Removes the calling thread from it's resident core queue */ diff --git a/app/src/main/cpp/skyline/kernel/svc.cpp b/app/src/main/cpp/skyline/kernel/svc.cpp index dd65f1b1..a8445737 100644 --- a/app/src/main/cpp/skyline/kernel/svc.cpp +++ b/app/src/main/cpp/skyline/kernel/svc.cpp @@ -309,7 +309,7 @@ namespace skyline::kernel::svc { KHandle handle{state.ctx->gpr.w1}; try { auto thread{state.process->GetHandle(handle)}; - auto priority{thread->priority}; + u8 priority{thread->priority}; state.logger->Debug("svcGetThreadPriority: Retrieving thread #{}'s priority: {}", thread->id, priority); state.ctx->gpr.w1 = priority; @@ -331,7 +331,8 @@ namespace skyline::kernel::svc { try { auto thread{state.process->GetHandle(handle)}; state.logger->Debug("svcSetThreadPriority: Setting thread priority to {}", thread->id, priority); - thread->UpdatePriority(static_cast(priority)); + thread->priority = priority; + state.scheduler->UpdatePriority(thread); state.ctx->gpr.w0 = Result{}; } catch (const std::out_of_range &) { state.logger->Warn("svcSetThreadPriority: 'handle' invalid: 0x{:X}", handle); @@ -358,18 +359,18 @@ namespace skyline::kernel::svc { void SetThreadCoreMask(const DeviceState &state) { KHandle handle{state.ctx->gpr.w0}; - i32 coreId{static_cast(state.ctx->gpr.w1)}; + i32 idealCore{static_cast(state.ctx->gpr.w1)}; CoreMask affinityMask{state.ctx->gpr.x2}; try { auto thread{state.process->GetHandle(handle)}; - if (coreId == IdealCoreUseProcessValue) { - coreId = state.process->npdm.meta.idealCore; - affinityMask.reset().set(coreId); - } else if (coreId == IdealCoreNoUpdate) { - coreId = thread->idealCore; - } else if (coreId == IdealCoreDontCare) { - coreId = __builtin_ctzll(affinityMask.to_ullong()); // The first enabled core in the affinity mask + if (idealCore == IdealCoreUseProcessValue) { + idealCore = state.process->npdm.meta.idealCore; + affinityMask.reset().set(idealCore); + } else if (idealCore == IdealCoreNoUpdate) { + idealCore = thread->idealCore; + } else if (idealCore == IdealCoreDontCare) { + idealCore = __builtin_ctzll(affinityMask.to_ullong()); // The first enabled core in the affinity mask } auto processMask{state.process->npdm.threadInfo.coreMask}; @@ -379,17 +380,25 @@ namespace skyline::kernel::svc { return; } - if (affinityMask.none() || !affinityMask.test(coreId)) { - state.logger->Warn("svcSetThreadCoreMask: 'affinityMask' invalid: {} (Ideal Core: {})", affinityMask, coreId); + if (affinityMask.none() || !affinityMask.test(idealCore)) { + state.logger->Warn("svcSetThreadCoreMask: 'affinityMask' invalid: {} (Ideal Core: {})", affinityMask, idealCore); state.ctx->gpr.w0 = result::InvalidCombination; return; } - state.logger->Debug("svcSetThreadCoreMask: Setting thread #{}'s Ideal Core ({}) + Affinity Mask ({})", thread->id, coreId, affinityMask); + state.logger->Debug("svcSetThreadCoreMask: Setting thread #{}'s Ideal Core ({}) + Affinity Mask ({})", thread->id, idealCore, affinityMask); - thread->idealCore = coreId; + thread->idealCore = idealCore; thread->affinityMask = affinityMask; + if (!affinityMask.test(thread->coreId)) { + state.logger->Debug("svcSetThreadCoreMask: Migrating to Ideal Core C{} -> C{}", thread->coreId, idealCore); + + state.scheduler->RemoveThread(); + thread->coreId = idealCore; + state.scheduler->InsertThread(false); + } + state.ctx->gpr.w0 = Result{}; } catch (const std::out_of_range &) { state.logger->Warn("svcSetThreadCoreMask: 'handle' invalid: 0x{:X}", handle); diff --git a/app/src/main/cpp/skyline/kernel/types/KProcess.cpp b/app/src/main/cpp/skyline/kernel/types/KProcess.cpp index 2e2b8ada..87c1f530 100644 --- a/app/src/main/cpp/skyline/kernel/types/KProcess.cpp +++ b/app/src/main/cpp/skyline/kernel/types/KProcess.cpp @@ -6,9 +6,9 @@ #include "KProcess.h" namespace skyline::kernel::type { - KProcess::WaitStatus::WaitStatus(i8 priority, KHandle handle) : priority(priority), handle(handle) {} + KProcess::WaitStatus::WaitStatus(u8 priority, KHandle handle) : priority(priority), handle(handle) {} - KProcess::WaitStatus::WaitStatus(i8 priority, KHandle handle, u32 *mutex) : priority(priority), handle(handle), mutex(mutex) {} + KProcess::WaitStatus::WaitStatus(u8 priority, KHandle handle, u32 *mutex) : priority(priority), handle(handle), mutex(mutex) {} KProcess::TlsPage::TlsPage(const std::shared_ptr &memory) : memory(memory) {} @@ -74,7 +74,7 @@ namespace skyline::kernel::type { return tlsPage->ReserveSlot(); } - std::shared_ptr KProcess::CreateThread(void *entry, u64 argument, void *stackTop, i8 priority, i8 idealCore) { + std::shared_ptr KProcess::CreateThread(void *entry, u64 argument, void *stackTop, std::optional priority, std::optional idealCore) { std::lock_guard guard(threadMutex); if (disableThreadCreation) return nullptr; @@ -84,7 +84,7 @@ namespace skyline::kernel::type { throw exception("Failed to create guard page for thread stack at 0x{:X}", mainThreadStack->ptr); stackTop = mainThreadStack->ptr + mainThreadStack->size; } - auto thread{NewHandle(this, threads.size(), entry, argument, stackTop, (priority == -1) ? state.process->npdm.meta.mainThreadPriority : priority, (idealCore == -1) ? state.process->npdm.meta.idealCore : idealCore).item}; + auto thread{NewHandle(this, threads.size(), entry, argument, stackTop, priority ? *priority : state.process->npdm.meta.mainThreadPriority, idealCore ? *idealCore : state.process->npdm.meta.idealCore).item}; threads.push_back(thread); return thread; } diff --git a/app/src/main/cpp/skyline/kernel/types/KProcess.h b/app/src/main/cpp/skyline/kernel/types/KProcess.h index 6eaca325..2f744111 100644 --- a/app/src/main/cpp/skyline/kernel/types/KProcess.h +++ b/app/src/main/cpp/skyline/kernel/types/KProcess.h @@ -29,13 +29,13 @@ namespace skyline { private: struct WaitStatus { std::atomic_bool flag{false}; - i8 priority; + u8 priority; KHandle handle; u32 *mutex{}; - WaitStatus(i8 priority, KHandle handle); + WaitStatus(u8 priority, KHandle handle); - WaitStatus(i8 priority, KHandle handle, u32 *mutex); + WaitStatus(u8 priority, KHandle handle, u32 *mutex); }; std::unordered_map>> mutexes; //!< A map from a mutex's address to a vector of Mutex objects for threads waiting on it @@ -105,7 +105,7 @@ namespace skyline { * @return A shared pointer to a KThread initialized with the specified values or nullptr, if thread creation has been disabled * @note The default values are for the main thread and will use values from the NPDM */ - std::shared_ptr CreateThread(void *entry, u64 argument = 0, void *stackTop = nullptr, i8 priority = -1, i8 idealCore = -1); + std::shared_ptr CreateThread(void *entry, u64 argument = 0, void *stackTop = nullptr, std::optional priority = std::nullopt, std::optional idealCore = std::nullopt); /** * @brief The output for functions that return created kernel objects diff --git a/app/src/main/cpp/skyline/kernel/types/KThread.cpp b/app/src/main/cpp/skyline/kernel/types/KThread.cpp index a8897644..c5a3de8b 100644 --- a/app/src/main/cpp/skyline/kernel/types/KThread.cpp +++ b/app/src/main/cpp/skyline/kernel/types/KThread.cpp @@ -10,9 +10,8 @@ #include "KThread.h" namespace skyline::kernel::type { - KThread::KThread(const DeviceState &state, KHandle handle, KProcess *parent, size_t id, void *entry, u64 argument, void *stackTop, i8 priority, i8 idealCore) : handle(handle), parent(parent), id(id), entry(entry), entryArgument(argument), stackTop(stackTop), idealCore(idealCore), coreId(idealCore), KSyncObject(state, KType::KThread) { + KThread::KThread(const DeviceState &state, KHandle handle, KProcess *parent, size_t id, void *entry, u64 argument, void *stackTop, u8 priority, i8 idealCore) : handle(handle), parent(parent), id(id), entry(entry), entryArgument(argument), stackTop(stackTop), priority(priority), idealCore(idealCore), coreId(idealCore), KSyncObject(state, KType::KThread) { affinityMask.set(coreId); - UpdatePriority(priority); } KThread::~KThread() { @@ -197,8 +196,4 @@ namespace skyline::kernel::type { if (running) pthread_kill(pthread, signal); } - - void KThread::UpdatePriority(i8 priority) { - this->priority = priority; - } } diff --git a/app/src/main/cpp/skyline/kernel/types/KThread.h b/app/src/main/cpp/skyline/kernel/types/KThread.h index 7e75627b..33e50672 100644 --- a/app/src/main/cpp/skyline/kernel/types/KThread.h +++ b/app/src/main/cpp/skyline/kernel/types/KThread.h @@ -38,7 +38,7 @@ namespace skyline { u64 entryArgument; void *stackTop; - i8 priority; + std::atomic priority; //!< The priority of the thread for the scheduler i8 idealCore; //!< The ideal CPU core for this thread to run on i8 coreId; //!< The CPU core on which this thread is running CoreMask affinityMask{}; //!< A mask of CPU cores this thread is allowed to run on @@ -46,8 +46,10 @@ namespace skyline { u64 averageTimeslice{}; //!< A weighted average of the timeslice duration for this thread std::optional preemptionTimer{}; //!< A kernel timer used for preemption interrupts bool isPreempted{}; //!< If the preemption timer has been armed and will fire + bool needsReorder{}; //!< If the thread needs to reorder itself during scheduler rotation + std::mutex coreMigrationMutex; //!< Synchronizes operations which depend on which core the thread is running on - KThread(const DeviceState &state, KHandle handle, KProcess *parent, size_t id, void *entry, u64 argument, void *stackTop, i8 priority, i8 idealCore); + KThread(const DeviceState &state, KHandle handle, KProcess *parent, size_t id, void *entry, u64 argument, void *stackTop, u8 priority, i8 idealCore); ~KThread(); @@ -67,8 +69,6 @@ namespace skyline { * @brief Sends a host OS signal to the thread which is running this KThread */ void SendSignal(int signal); - - void UpdatePriority(i8 priority); }; } }