summaryrefslogtreecommitdiffstats
path: root/src/core/hle/kernel
diff options
context:
space:
mode:
authorFernando Sahmkow <fsahmkow27@gmail.com>2020-02-25 03:04:12 +0100
committerFernando Sahmkow <fsahmkow27@gmail.com>2020-06-27 17:35:06 +0200
commite31425df3877636c098ec7426ebd2067920715cb (patch)
tree5c0fc518a4ebb8413c491b43a9fdd99450c7bd80 /src/core/hle/kernel
parentMerge pull request #3396 from FernandoS27/prometheus-1 (diff)
downloadyuzu-e31425df3877636c098ec7426ebd2067920715cb.tar
yuzu-e31425df3877636c098ec7426ebd2067920715cb.tar.gz
yuzu-e31425df3877636c098ec7426ebd2067920715cb.tar.bz2
yuzu-e31425df3877636c098ec7426ebd2067920715cb.tar.lz
yuzu-e31425df3877636c098ec7426ebd2067920715cb.tar.xz
yuzu-e31425df3877636c098ec7426ebd2067920715cb.tar.zst
yuzu-e31425df3877636c098ec7426ebd2067920715cb.zip
Diffstat (limited to 'src/core/hle/kernel')
-rw-r--r--src/core/hle/kernel/kernel.cpp84
-rw-r--r--src/core/hle/kernel/kernel.h19
-rw-r--r--src/core/hle/kernel/physical_core.cpp37
-rw-r--r--src/core/hle/kernel/physical_core.h21
-rw-r--r--src/core/hle/kernel/process.cpp17
-rw-r--r--src/core/hle/kernel/scheduler.cpp415
-rw-r--r--src/core/hle/kernel/scheduler.h94
-rw-r--r--src/core/hle/kernel/svc.cpp21
-rw-r--r--src/core/hle/kernel/thread.cpp232
-rw-r--r--src/core/hle/kernel/thread.h81
-rw-r--r--src/core/hle/kernel/time_manager.cpp2
11 files changed, 679 insertions, 344 deletions
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index 7655382fa..ba051a7d8 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -13,11 +13,13 @@
#include "common/assert.h"
#include "common/logging/log.h"
+#include "common/thread.h"
#include "core/arm/arm_interface.h"
#include "core/arm/exclusive_monitor.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/core_timing_util.h"
+#include "core/cpu_manager.h"
#include "core/device_memory.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/client_port.h"
@@ -117,7 +119,9 @@ struct KernelCore::Impl {
InitializeSystemResourceLimit(kernel);
InitializeMemoryLayout();
InitializeThreads();
- InitializePreemption();
+ InitializePreemption(kernel);
+ InitializeSchedulers();
+ InitializeSuspendThreads();
}
void Shutdown() {
@@ -155,6 +159,12 @@ struct KernelCore::Impl {
}
}
+ void InitializeSchedulers() {
+ for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+ cores[i].Scheduler().Initialize();
+ }
+ }
+
// Creates the default system resource limit
void InitializeSystemResourceLimit(KernelCore& kernel) {
system_resource_limit = ResourceLimit::Create(kernel);
@@ -178,10 +188,13 @@ struct KernelCore::Impl {
Core::Timing::CreateEvent("ThreadWakeupCallback", ThreadWakeupCallback);
}
- void InitializePreemption() {
- preemption_event =
- Core::Timing::CreateEvent("PreemptionCallback", [this](u64 userdata, s64 cycles_late) {
- global_scheduler.PreemptThreads();
+ void InitializePreemption(KernelCore& kernel) {
+ preemption_event = Core::Timing::CreateEvent(
+ "PreemptionCallback", [this, &kernel](u64 userdata, s64 cycles_late) {
+ {
+ SchedulerLock lock(kernel);
+ global_scheduler.PreemptThreads();
+ }
s64 time_interval = Core::Timing::msToCycles(std::chrono::milliseconds(10));
system.CoreTiming().ScheduleEvent(time_interval, preemption_event);
});
@@ -190,6 +203,20 @@ struct KernelCore::Impl {
system.CoreTiming().ScheduleEvent(time_interval, preemption_event);
}
+ void InitializeSuspendThreads() {
+ for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+ std::string name = "Suspend Thread Id:" + std::to_string(i);
+ std::function<void(void*)> init_func =
+ system.GetCpuManager().GetSuspendThreadStartFunc();
+ void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
+ ThreadType type =
+ static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_SUSPEND);
+ auto thread_res = Thread::Create(system, type, name, 0, 0, 0, static_cast<u32>(i), 0,
+ nullptr, std::move(init_func), init_func_parameter);
+ suspend_threads[i] = std::move(thread_res).Unwrap();
+ }
+ }
+
void MakeCurrentProcess(Process* process) {
current_process = process;
@@ -201,7 +228,10 @@ struct KernelCore::Impl {
core.SetIs64Bit(process->Is64BitProcess());
}
- system.Memory().SetCurrentPageTable(*process);
+ u32 core_id = GetCurrentHostThreadID();
+ if (core_id < Core::Hardware::NUM_CPU_CORES) {
+ system.Memory().SetCurrentPageTable(*process, core_id);
+ }
}
void RegisterCoreThread(std::size_t core_id) {
@@ -219,7 +249,9 @@ struct KernelCore::Impl {
std::unique_lock lock{register_thread_mutex};
const std::thread::id this_id = std::this_thread::get_id();
const auto it = host_thread_ids.find(this_id);
- ASSERT(it == host_thread_ids.end());
+ if (it != host_thread_ids.end()) {
+ return;
+ }
host_thread_ids[this_id] = registered_thread_ids++;
}
@@ -343,6 +375,8 @@ struct KernelCore::Impl {
std::shared_ptr<Kernel::SharedMemory> irs_shared_mem;
std::shared_ptr<Kernel::SharedMemory> time_shared_mem;
+ std::array<std::shared_ptr<Thread>, Core::Hardware::NUM_CPU_CORES> suspend_threads{};
+
// System context
Core::System& system;
};
@@ -412,6 +446,26 @@ const Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) const {
return impl->cores[id];
}
+Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() {
+ u32 core_id = impl->GetCurrentHostThreadID();
+ ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
+ return impl->cores[core_id];
+}
+
+const Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() const {
+ u32 core_id = impl->GetCurrentHostThreadID();
+ ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
+ return impl->cores[core_id];
+}
+
+Kernel::Scheduler& KernelCore::CurrentScheduler() {
+ return CurrentPhysicalCore().Scheduler();
+}
+
+const Kernel::Scheduler& KernelCore::CurrentScheduler() const {
+ return CurrentPhysicalCore().Scheduler();
+}
+
Kernel::Synchronization& KernelCore::Synchronization() {
return impl->synchronization;
}
@@ -557,4 +611,20 @@ const Kernel::SharedMemory& KernelCore::GetTimeSharedMem() const {
return *impl->time_shared_mem;
}
+void KernelCore::Suspend(bool in_suspention) {
+ const bool should_suspend = exception_exited || in_suspention;
+ {
+ SchedulerLock lock(*this);
+ ThreadStatus status = should_suspend ? ThreadStatus::Ready : ThreadStatus::WaitSleep;
+ for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+ impl->suspend_threads[i]->SetStatus(status);
+ }
+ }
+}
+
+void KernelCore::ExceptionalExit() {
+ exception_exited = true;
+ Suspend(true);
+}
+
} // namespace Kernel
diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h
index 83de1f542..5d32a8329 100644
--- a/src/core/hle/kernel/kernel.h
+++ b/src/core/hle/kernel/kernel.h
@@ -110,6 +110,18 @@ public:
/// Gets the an instance of the respective physical CPU core.
const Kernel::PhysicalCore& PhysicalCore(std::size_t id) const;
+ /// Gets the sole instance of the Scheduler at the current running core.
+ Kernel::Scheduler& CurrentScheduler();
+
+ /// Gets the sole instance of the Scheduler at the current running core.
+ const Kernel::Scheduler& CurrentScheduler() const;
+
+ /// Gets the an instance of the current physical CPU core.
+ Kernel::PhysicalCore& CurrentPhysicalCore();
+
+ /// Gets the an instance of the current physical CPU core.
+ const Kernel::PhysicalCore& CurrentPhysicalCore() const;
+
/// Gets the an instance of the Synchronization Interface.
Kernel::Synchronization& Synchronization();
@@ -191,6 +203,12 @@ public:
/// Gets the shared memory object for Time services.
const Kernel::SharedMemory& GetTimeSharedMem() const;
+ /// Suspend/unsuspend the OS.
+ void Suspend(bool in_suspention);
+
+ /// Exceptional exit the OS.
+ void ExceptionalExit();
+
private:
friend class Object;
friend class Process;
@@ -219,6 +237,7 @@ private:
struct Impl;
std::unique_ptr<Impl> impl;
+ bool exception_exited{};
};
} // namespace Kernel
diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp
index a15011076..69202540b 100644
--- a/src/core/hle/kernel/physical_core.cpp
+++ b/src/core/hle/kernel/physical_core.cpp
@@ -2,12 +2,15 @@
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
+#include "common/assert.h"
#include "common/logging/log.h"
+#include "common/spin_lock.h"
#include "core/arm/arm_interface.h"
#ifdef ARCHITECTURE_x86_64
#include "core/arm/dynarmic/arm_dynarmic_32.h"
#include "core/arm/dynarmic/arm_dynarmic_64.h"
#endif
+#include "core/arm/cpu_interrupt_handler.h"
#include "core/arm/exclusive_monitor.h"
#include "core/arm/unicorn/arm_unicorn.h"
#include "core/core.h"
@@ -19,21 +22,23 @@ namespace Kernel {
PhysicalCore::PhysicalCore(Core::System& system, std::size_t id,
Core::ExclusiveMonitor& exclusive_monitor)
- : core_index{id} {
+ : interrupt_handler{}, core_index{id} {
#ifdef ARCHITECTURE_x86_64
- arm_interface_32 =
- std::make_unique<Core::ARM_Dynarmic_32>(system, exclusive_monitor, core_index);
- arm_interface_64 =
- std::make_unique<Core::ARM_Dynarmic_64>(system, exclusive_monitor, core_index);
-
+ arm_interface_32 = std::make_unique<Core::ARM_Dynarmic_32>(system, interrupt_handler,
+ exclusive_monitor, core_index);
+ arm_interface_64 = std::make_unique<Core::ARM_Dynarmic_64>(system, interrupt_handler,
+ exclusive_monitor, core_index);
#else
using Core::ARM_Unicorn;
- arm_interface_32 = std::make_unique<ARM_Unicorn>(system, ARM_Unicorn::Arch::AArch32);
- arm_interface_64 = std::make_unique<ARM_Unicorn>(system, ARM_Unicorn::Arch::AArch64);
+ arm_interface_32 =
+ std::make_unique<ARM_Unicorn>(system, interrupt_handler, ARM_Unicorn::Arch::AArch32);
+ arm_interface_64 =
+ std::make_unique<ARM_Unicorn>(system, interrupt_handler, ARM_Unicorn::Arch::AArch64);
LOG_WARNING(Core, "CPU JIT requested, but Dynarmic not available");
#endif
scheduler = std::make_unique<Kernel::Scheduler>(system, core_index);
+ guard = std::make_unique<Common::SpinLock>();
}
PhysicalCore::~PhysicalCore() = default;
@@ -47,6 +52,10 @@ void PhysicalCore::Step() {
arm_interface->Step();
}
+void PhysicalCore::Idle() {
+ interrupt_handler.AwaitInterrupt();
+}
+
void PhysicalCore::Stop() {
arm_interface->PrepareReschedule();
}
@@ -63,4 +72,16 @@ void PhysicalCore::SetIs64Bit(bool is_64_bit) {
}
}
+void PhysicalCore::Interrupt() {
+ guard->lock();
+ interrupt_handler.SetInterrupt(true);
+ guard->unlock();
+}
+
+void PhysicalCore::ClearInterrupt() {
+ guard->lock();
+ interrupt_handler.SetInterrupt(false);
+ guard->unlock();
+}
+
} // namespace Kernel
diff --git a/src/core/hle/kernel/physical_core.h b/src/core/hle/kernel/physical_core.h
index 3269166be..c3da30b72 100644
--- a/src/core/hle/kernel/physical_core.h
+++ b/src/core/hle/kernel/physical_core.h
@@ -7,6 +7,12 @@
#include <cstddef>
#include <memory>
+#include "core/arm/cpu_interrupt_handler.h"
+
+namespace Common {
+ class SpinLock;
+}
+
namespace Kernel {
class Scheduler;
} // namespace Kernel
@@ -32,11 +38,24 @@ public:
/// Execute current jit state
void Run();
+ /// Set this core in IdleState.
+ void Idle();
/// Execute a single instruction in current jit.
void Step();
/// Stop JIT execution/exit
void Stop();
+ /// Interrupt this physical core.
+ void Interrupt();
+
+ /// Clear this core's interrupt
+ void ClearInterrupt();
+
+ /// Check if this core is interrupted
+ bool IsInterrupted() const {
+ return interrupt_handler.IsInterrupted();
+ }
+
// Shutdown this physical core.
void Shutdown();
@@ -71,11 +90,13 @@ public:
void SetIs64Bit(bool is_64_bit);
private:
+ Core::CPUInterruptHandler interrupt_handler;
std::size_t core_index;
std::unique_ptr<Core::ARM_Interface> arm_interface_32;
std::unique_ptr<Core::ARM_Interface> arm_interface_64;
std::unique_ptr<Kernel::Scheduler> scheduler;
Core::ARM_Interface* arm_interface{};
+ std::unique_ptr<Common::SpinLock> guard;
};
} // namespace Kernel
diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp
index c4c5199b1..7e26a54f4 100644
--- a/src/core/hle/kernel/process.cpp
+++ b/src/core/hle/kernel/process.cpp
@@ -30,14 +30,15 @@ namespace {
/**
* Sets up the primary application thread
*
+ * @param system The system instance to create the main thread under.
* @param owner_process The parent process for the main thread
- * @param kernel The kernel instance to create the main thread under.
* @param priority The priority to give the main thread
*/
-void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority, VAddr stack_top) {
+void SetupMainThread(Core::System& system, Process& owner_process, u32 priority, VAddr stack_top) {
const VAddr entry_point = owner_process.PageTable().GetCodeRegionStart();
- auto thread_res = Thread::Create(kernel, "main", entry_point, priority, 0,
- owner_process.GetIdealCore(), stack_top, owner_process);
+ ThreadType type = THREADTYPE_USER;
+ auto thread_res = Thread::Create(system, type, "main", entry_point, priority, 0,
+ owner_process.GetIdealCore(), stack_top, &owner_process);
std::shared_ptr<Thread> thread = std::move(thread_res).Unwrap();
@@ -48,8 +49,12 @@ void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority, V
thread->GetContext32().cpu_registers[1] = thread_handle;
thread->GetContext64().cpu_registers[1] = thread_handle;
+ auto& kernel = system.Kernel();
// Threads by default are dormant, wake up the main thread so it runs when the scheduler fires
- thread->ResumeFromWait();
+ {
+ SchedulerLock lock{kernel};
+ thread->SetStatus(ThreadStatus::Ready);
+ }
}
} // Anonymous namespace
@@ -294,7 +299,7 @@ void Process::Run(s32 main_thread_priority, u64 stack_size) {
ChangeStatus(ProcessStatus::Running);
- SetupMainThread(*this, kernel, main_thread_priority, main_thread_stack_top);
+ SetupMainThread(system, *this, main_thread_priority, main_thread_stack_top);
resource_limit->Reserve(ResourceType::Threads, 1);
resource_limit->Reserve(ResourceType::PhysicalMemory, main_thread_stack_size);
}
diff --git a/src/core/hle/kernel/scheduler.cpp b/src/core/hle/kernel/scheduler.cpp
index 1140c72a3..5166020a0 100644
--- a/src/core/hle/kernel/scheduler.cpp
+++ b/src/core/hle/kernel/scheduler.cpp
@@ -11,11 +11,15 @@
#include <utility>
#include "common/assert.h"
+#include "common/bit_util.h"
+#include "common/fiber.h"
#include "common/logging/log.h"
#include "core/arm/arm_interface.h"
#include "core/core.h"
#include "core/core_timing.h"
+#include "core/cpu_manager.h"
#include "core/hle/kernel/kernel.h"
+#include "core/hle/kernel/physical_core.h"
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/time_manager.h"
@@ -27,78 +31,108 @@ GlobalScheduler::GlobalScheduler(KernelCore& kernel) : kernel{kernel} {}
GlobalScheduler::~GlobalScheduler() = default;
void GlobalScheduler::AddThread(std::shared_ptr<Thread> thread) {
+ global_list_guard.lock();
thread_list.push_back(std::move(thread));
+ global_list_guard.unlock();
}
void GlobalScheduler::RemoveThread(std::shared_ptr<Thread> thread) {
+ global_list_guard.lock();
thread_list.erase(std::remove(thread_list.begin(), thread_list.end(), thread),
thread_list.end());
+ global_list_guard.unlock();
}
-void GlobalScheduler::UnloadThread(std::size_t core) {
- Scheduler& sched = kernel.Scheduler(core);
- sched.UnloadThread();
-}
-
-void GlobalScheduler::SelectThread(std::size_t core) {
+u32 GlobalScheduler::SelectThreads() {
const auto update_thread = [](Thread* thread, Scheduler& sched) {
+ sched.guard.lock();
if (thread != sched.selected_thread.get()) {
if (thread == nullptr) {
++sched.idle_selection_count;
}
sched.selected_thread = SharedFrom(thread);
}
- sched.is_context_switch_pending = sched.selected_thread != sched.current_thread;
+ const bool reschedule_pending = sched.selected_thread != sched.current_thread;
+ sched.is_context_switch_pending = reschedule_pending;
std::atomic_thread_fence(std::memory_order_seq_cst);
+ sched.guard.unlock();
+ return reschedule_pending;
};
- Scheduler& sched = kernel.Scheduler(core);
- Thread* current_thread = nullptr;
- // Step 1: Get top thread in schedule queue.
- current_thread = scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
- if (current_thread) {
- update_thread(current_thread, sched);
- return;
+ if (!is_reselection_pending.load()) {
+ return 0;
}
- // Step 2: Try selecting a suggested thread.
- Thread* winner = nullptr;
- std::set<s32> sug_cores;
- for (auto thread : suggested_queue[core]) {
- s32 this_core = thread->GetProcessorID();
- Thread* thread_on_core = nullptr;
- if (this_core >= 0) {
- thread_on_core = scheduled_queue[this_core].front();
- }
- if (this_core < 0 || thread != thread_on_core) {
- winner = thread;
- break;
+ std::array<Thread*, Core::Hardware::NUM_CPU_CORES> top_threads{};
+
+ u32 idle_cores{};
+
+ // Step 1: Get top thread in schedule queue.
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ Thread* top_thread =
+ scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
+ if (top_thread != nullptr) {
+ // TODO(Blinkhawk): Implement Thread Pinning
+ } else {
+ idle_cores |= (1ul << core);
}
- sug_cores.insert(this_core);
+ top_threads[core] = top_thread;
}
- // if we got a suggested thread, select it, else do a second pass.
- if (winner && winner->GetPriority() > 2) {
- if (winner->IsRunning()) {
- UnloadThread(static_cast<u32>(winner->GetProcessorID()));
+
+ while (idle_cores != 0) {
+ u32 core_id = Common::CountTrailingZeroes32(idle_cores);
+
+ if (!suggested_queue[core_id].empty()) {
+ std::array<s32, Core::Hardware::NUM_CPU_CORES> migration_candidates{};
+ std::size_t num_candidates = 0;
+ auto iter = suggested_queue[core_id].begin();
+ Thread* suggested = nullptr;
+ // Step 2: Try selecting a suggested thread.
+ while (iter != suggested_queue[core_id].end()) {
+ suggested = *iter;
+ iter++;
+ s32 suggested_core_id = suggested->GetProcessorID();
+ Thread* top_thread =
+ suggested_core_id > 0 ? top_threads[suggested_core_id] : nullptr;
+ if (top_thread != suggested) {
+ if (top_thread != nullptr &&
+ top_thread->GetPriority() < THREADPRIO_MAX_CORE_MIGRATION) {
+ suggested = nullptr;
+ break;
+ // There's a too high thread to do core migration, cancel
+ }
+ TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id), suggested);
+ break;
+ }
+ migration_candidates[num_candidates++] = suggested_core_id;
+ }
+ // Step 3: Select a suggested thread from another core
+ if (suggested == nullptr) {
+ for (std::size_t i = 0; i < num_candidates; i++) {
+ s32 candidate_core = migration_candidates[i];
+ suggested = top_threads[candidate_core];
+ auto it = scheduled_queue[candidate_core].begin();
+ it++;
+ Thread* next = it != scheduled_queue[candidate_core].end() ? *it : nullptr;
+ if (next != nullptr) {
+ TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id),
+ suggested);
+ top_threads[candidate_core] = next;
+ break;
+ }
+ }
+ }
+ top_threads[core_id] = suggested;
}
- TransferToCore(winner->GetPriority(), static_cast<s32>(core), winner);
- update_thread(winner, sched);
- return;
+
+ idle_cores &= ~(1ul << core_id);
}
- // Step 3: Select a suggested thread from another core
- for (auto& src_core : sug_cores) {
- auto it = scheduled_queue[src_core].begin();
- it++;
- if (it != scheduled_queue[src_core].end()) {
- Thread* thread_on_core = scheduled_queue[src_core].front();
- Thread* to_change = *it;
- if (thread_on_core->IsRunning() || to_change->IsRunning()) {
- UnloadThread(static_cast<u32>(src_core));
- }
- TransferToCore(thread_on_core->GetPriority(), static_cast<s32>(core), thread_on_core);
- current_thread = thread_on_core;
- break;
+ u32 cores_needing_context_switch{};
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ Scheduler& sched = kernel.Scheduler(core);
+ if (update_thread(top_threads[core], sched)) {
+ cores_needing_context_switch |= (1ul << core);
}
}
- update_thread(current_thread, sched);
+ return cores_needing_context_switch;
}
bool GlobalScheduler::YieldThread(Thread* yielding_thread) {
@@ -153,9 +187,6 @@ bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) {
if (winner != nullptr) {
if (winner != yielding_thread) {
- if (winner->IsRunning()) {
- UnloadThread(static_cast<u32>(winner->GetProcessorID()));
- }
TransferToCore(winner->GetPriority(), s32(core_id), winner);
}
} else {
@@ -195,9 +226,6 @@ bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread
}
if (winner != nullptr) {
if (winner != yielding_thread) {
- if (winner->IsRunning()) {
- UnloadThread(static_cast<u32>(winner->GetProcessorID()));
- }
TransferToCore(winner->GetPriority(), static_cast<s32>(core_id), winner);
}
} else {
@@ -213,7 +241,9 @@ void GlobalScheduler::PreemptThreads() {
const u32 priority = preemption_priorities[core_id];
if (scheduled_queue[core_id].size(priority) > 0) {
- scheduled_queue[core_id].front(priority)->IncrementYieldCount();
+ if (scheduled_queue[core_id].size(priority) > 1) {
+ scheduled_queue[core_id].front(priority)->IncrementYieldCount();
+ }
scheduled_queue[core_id].yield(priority);
if (scheduled_queue[core_id].size(priority) > 1) {
scheduled_queue[core_id].front(priority)->IncrementYieldCount();
@@ -247,9 +277,6 @@ void GlobalScheduler::PreemptThreads() {
}
if (winner != nullptr) {
- if (winner->IsRunning()) {
- UnloadThread(static_cast<u32>(winner->GetProcessorID()));
- }
TransferToCore(winner->GetPriority(), s32(core_id), winner);
current_thread =
winner->GetPriority() <= current_thread->GetPriority() ? winner : current_thread;
@@ -280,9 +307,6 @@ void GlobalScheduler::PreemptThreads() {
}
if (winner != nullptr) {
- if (winner->IsRunning()) {
- UnloadThread(static_cast<u32>(winner->GetProcessorID()));
- }
TransferToCore(winner->GetPriority(), s32(core_id), winner);
current_thread = winner;
}
@@ -292,6 +316,28 @@ void GlobalScheduler::PreemptThreads() {
}
}
+void GlobalScheduler::EnableInterruptAndSchedule(u32 cores_pending_reschedule,
+ Core::EmuThreadHandle global_thread) {
+ u32 current_core = global_thread.host_handle;
+ bool must_context_switch = global_thread.guest_handle != InvalidHandle &&
+ (current_core < Core::Hardware::NUM_CPU_CORES);
+ while (cores_pending_reschedule != 0) {
+ u32 core = Common::CountTrailingZeroes32(cores_pending_reschedule);
+ ASSERT(core < Core::Hardware::NUM_CPU_CORES);
+ if (!must_context_switch || core != current_core) {
+ auto& phys_core = kernel.PhysicalCore(core);
+ phys_core.Interrupt();
+ } else {
+ must_context_switch = true;
+ }
+ cores_pending_reschedule &= ~(1ul << core);
+ }
+ if (must_context_switch) {
+ auto& core_scheduler = kernel.CurrentScheduler();
+ core_scheduler.TryDoContextSwitch();
+ }
+}
+
void GlobalScheduler::Suggest(u32 priority, std::size_t core, Thread* thread) {
suggested_queue[core].add(thread, priority);
}
@@ -349,6 +395,108 @@ bool GlobalScheduler::AskForReselectionOrMarkRedundant(Thread* current_thread,
}
}
+void GlobalScheduler::AdjustSchedulingOnStatus(Thread* thread, u32 old_flags) {
+ if (old_flags == thread->scheduling_state) {
+ return;
+ }
+
+ if (static_cast<ThreadSchedStatus>(old_flags & static_cast<u32>(ThreadSchedMasks::LowMask)) ==
+ ThreadSchedStatus::Runnable) {
+ // In this case the thread was running, now it's pausing/exitting
+ if (thread->processor_id >= 0) {
+ Unschedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
+ }
+
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ if (core != static_cast<u32>(thread->processor_id) &&
+ ((thread->affinity_mask >> core) & 1) != 0) {
+ Unsuggest(thread->current_priority, core, thread);
+ }
+ }
+ } else if (thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable) {
+ // The thread is now set to running from being stopped
+ if (thread->processor_id >= 0) {
+ Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
+ }
+
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ if (core != static_cast<u32>(thread->processor_id) &&
+ ((thread->affinity_mask >> core) & 1) != 0) {
+ Suggest(thread->current_priority, core, thread);
+ }
+ }
+ }
+
+ SetReselectionPending();
+}
+
+void GlobalScheduler::AdjustSchedulingOnPriority(Thread* thread, u32 old_priority) {
+ if (thread->GetSchedulingStatus() != ThreadSchedStatus::Runnable) {
+ return;
+ }
+ if (thread->processor_id >= 0) {
+ Unschedule(old_priority, static_cast<u32>(thread->processor_id), thread);
+ }
+
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ if (core != static_cast<u32>(thread->processor_id) &&
+ ((thread->affinity_mask >> core) & 1) != 0) {
+ Unsuggest(old_priority, core, thread);
+ }
+ }
+
+ if (thread->processor_id >= 0) {
+ // TODO(Blinkhawk): compare it with current thread running on current core, instead of
+ // checking running
+ if (thread->IsRunning()) {
+ SchedulePrepend(thread->current_priority, static_cast<u32>(thread->processor_id),
+ thread);
+ } else {
+ Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
+ }
+ }
+
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ if (core != static_cast<u32>(thread->processor_id) &&
+ ((thread->affinity_mask >> core) & 1) != 0) {
+ Suggest(thread->current_priority, core, thread);
+ }
+ }
+ thread->IncrementYieldCount();
+ SetReselectionPending();
+}
+
+void GlobalScheduler::AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask,
+ s32 old_core) {
+ if (thread->GetSchedulingStatus() != ThreadSchedStatus::Runnable ||
+ thread->current_priority >= THREADPRIO_COUNT) {
+ return;
+ }
+
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ if (((old_affinity_mask >> core) & 1) != 0) {
+ if (core == static_cast<u32>(old_core)) {
+ Unschedule(thread->current_priority, core, thread);
+ } else {
+ Unsuggest(thread->current_priority, core, thread);
+ }
+ }
+ }
+
+ for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
+ if (((thread->affinity_mask >> core) & 1) != 0) {
+ if (core == static_cast<u32>(thread->processor_id)) {
+ Schedule(thread->current_priority, core, thread);
+ } else {
+ Suggest(thread->current_priority, core, thread);
+ }
+ }
+ }
+
+ thread->IncrementYieldCount();
+ SetReselectionPending();
+}
+
void GlobalScheduler::Shutdown() {
for (std::size_t core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
scheduled_queue[core].clear();
@@ -374,13 +522,12 @@ void GlobalScheduler::Unlock() {
ASSERT(scope_lock > 0);
return;
}
- for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
- SelectThread(i);
- }
+ u32 cores_pending_reschedule = SelectThreads();
+ Core::EmuThreadHandle leaving_thread = current_owner;
current_owner = Core::EmuThreadHandle::InvalidHandle();
scope_lock = 1;
inner_lock.unlock();
- // TODO(Blinkhawk): Setup the interrupts and change context on current core.
+ EnableInterruptAndSchedule(cores_pending_reschedule, leaving_thread);
}
Scheduler::Scheduler(Core::System& system, std::size_t core_id)
@@ -393,56 +540,83 @@ bool Scheduler::HaveReadyThreads() const {
}
Thread* Scheduler::GetCurrentThread() const {
- return current_thread.get();
+ if (current_thread) {
+ return current_thread.get();
+ }
+ return idle_thread.get();
}
Thread* Scheduler::GetSelectedThread() const {
return selected_thread.get();
}
-void Scheduler::SelectThreads() {
- system.GlobalScheduler().SelectThread(core_id);
-}
-
u64 Scheduler::GetLastContextSwitchTicks() const {
return last_context_switch_time;
}
void Scheduler::TryDoContextSwitch() {
+ auto& phys_core = system.Kernel().CurrentPhysicalCore();
+ if (phys_core.IsInterrupted()) {
+ phys_core.ClearInterrupt();
+ }
+ guard.lock();
if (is_context_switch_pending) {
SwitchContext();
+ } else {
+ guard.unlock();
}
}
-void Scheduler::UnloadThread() {
- Thread* const previous_thread = GetCurrentThread();
- Process* const previous_process = system.Kernel().CurrentProcess();
+void Scheduler::OnThreadStart() {
+ SwitchContextStep2();
+}
- UpdateLastContextSwitchTime(previous_thread, previous_process);
+void Scheduler::SwitchContextStep2() {
+ Thread* previous_thread = current_thread.get();
+ Thread* new_thread = selected_thread.get();
- // Save context for previous thread
- if (previous_thread) {
- system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32());
- system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64());
- // Save the TPIDR_EL0 system register in case it was modified.
- previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0());
+ // Load context of new thread
+ Process* const previous_process =
+ previous_thread != nullptr ? previous_thread->GetOwnerProcess() : nullptr;
- if (previous_thread->GetStatus() == ThreadStatus::Running) {
- // This is only the case when a reschedule is triggered without the current thread
- // yielding execution (i.e. an event triggered, system core time-sliced, etc)
- previous_thread->SetStatus(ThreadStatus::Ready);
+ if (new_thread) {
+ new_thread->context_guard.lock();
+ ASSERT_MSG(new_thread->GetProcessorID() == s32(this->core_id),
+ "Thread must be assigned to this core.");
+ ASSERT_MSG(new_thread->GetStatus() == ThreadStatus::Ready,
+ "Thread must be ready to become running.");
+
+ // Cancel any outstanding wakeup events for this thread
+ current_thread = SharedFrom(new_thread);
+ new_thread->SetStatus(ThreadStatus::Running);
+ new_thread->SetIsRunning(true);
+
+ auto* const thread_owner_process = current_thread->GetOwnerProcess();
+ if (previous_process != thread_owner_process && thread_owner_process != nullptr) {
+ system.Kernel().MakeCurrentProcess(thread_owner_process);
}
- previous_thread->SetIsRunning(false);
+ if (!new_thread->IsHLEThread()) {
+ auto& cpu_core = system.ArmInterface(core_id);
+ cpu_core.LoadContext(new_thread->GetContext32());
+ cpu_core.LoadContext(new_thread->GetContext64());
+ cpu_core.SetTlsAddress(new_thread->GetTLSAddress());
+ cpu_core.SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
+ }
+ } else {
+ current_thread = nullptr;
+ // Note: We do not reset the current process and current page table when idling because
+ // technically we haven't changed processes, our threads are just paused.
}
- current_thread = nullptr;
+ guard.unlock();
}
void Scheduler::SwitchContext() {
- Thread* const previous_thread = GetCurrentThread();
- Thread* const new_thread = GetSelectedThread();
+ Thread* previous_thread = current_thread.get();
+ Thread* new_thread = selected_thread.get();
is_context_switch_pending = false;
if (new_thread == previous_thread) {
+ guard.unlock();
return;
}
@@ -452,51 +626,44 @@ void Scheduler::SwitchContext() {
// Save context for previous thread
if (previous_thread) {
- system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32());
- system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64());
- // Save the TPIDR_EL0 system register in case it was modified.
- previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0());
+ if (!previous_thread->IsHLEThread()) {
+ auto& cpu_core = system.ArmInterface(core_id);
+ cpu_core.SaveContext(previous_thread->GetContext32());
+ cpu_core.SaveContext(previous_thread->GetContext64());
+ // Save the TPIDR_EL0 system register in case it was modified.
+ previous_thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());
+ }
if (previous_thread->GetStatus() == ThreadStatus::Running) {
- // This is only the case when a reschedule is triggered without the current thread
- // yielding execution (i.e. an event triggered, system core time-sliced, etc)
previous_thread->SetStatus(ThreadStatus::Ready);
}
previous_thread->SetIsRunning(false);
+ previous_thread->context_guard.unlock();
}
- // Load context of new thread
- if (new_thread) {
- ASSERT_MSG(new_thread->GetProcessorID() == s32(this->core_id),
- "Thread must be assigned to this core.");
- ASSERT_MSG(new_thread->GetStatus() == ThreadStatus::Ready,
- "Thread must be ready to become running.");
-
- // Cancel any outstanding wakeup events for this thread
- new_thread->CancelWakeupTimer();
- current_thread = SharedFrom(new_thread);
- new_thread->SetStatus(ThreadStatus::Running);
- new_thread->SetIsRunning(true);
-
- auto* const thread_owner_process = current_thread->GetOwnerProcess();
- if (previous_process != thread_owner_process) {
- system.Kernel().MakeCurrentProcess(thread_owner_process);
- }
+ std::shared_ptr<Common::Fiber> old_context;
+ if (previous_thread != nullptr) {
+ old_context = previous_thread->GetHostContext();
+ } else {
+ old_context = idle_thread->GetHostContext();
+ }
- system.ArmInterface(core_id).LoadContext(new_thread->GetContext32());
- system.ArmInterface(core_id).LoadContext(new_thread->GetContext64());
- system.ArmInterface(core_id).SetTlsAddress(new_thread->GetTLSAddress());
- system.ArmInterface(core_id).SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
+ std::shared_ptr<Common::Fiber> next_context;
+ if (new_thread != nullptr) {
+ next_context = new_thread->GetHostContext();
} else {
- current_thread = nullptr;
- // Note: We do not reset the current process and current page table when idling because
- // technically we haven't changed processes, our threads are just paused.
+ next_context = idle_thread->GetHostContext();
}
+
+ Common::Fiber::YieldTo(old_context, next_context);
+ /// When a thread wakes up, the scheduler may have changed to other in another core.
+ auto& next_scheduler = system.Kernel().CurrentScheduler();
+ next_scheduler.SwitchContextStep2();
}
void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
const u64 prev_switch_ticks = last_context_switch_time;
- const u64 most_recent_switch_ticks = system.CoreTiming().GetTicks();
+ const u64 most_recent_switch_ticks = system.CoreTiming().GetCPUTicks();
const u64 update_ticks = most_recent_switch_ticks - prev_switch_ticks;
if (thread != nullptr) {
@@ -510,6 +677,16 @@ void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
last_context_switch_time = most_recent_switch_ticks;
}
+void Scheduler::Initialize() {
+ std::string name = "Idle Thread Id:" + std::to_string(core_id);
+ std::function<void(void*)> init_func = system.GetCpuManager().GetIdleThreadStartFunc();
+ void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
+ ThreadType type = static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_IDLE);
+ auto thread_res = Thread::Create(system, type, name, 0, 64, 0, static_cast<u32>(core_id), 0,
+ nullptr, std::move(init_func), init_func_parameter);
+ idle_thread = std::move(thread_res).Unwrap();
+}
+
void Scheduler::Shutdown() {
current_thread = nullptr;
selected_thread = nullptr;
diff --git a/src/core/hle/kernel/scheduler.h b/src/core/hle/kernel/scheduler.h
index 07df33f9c..16655b03f 100644
--- a/src/core/hle/kernel/scheduler.h
+++ b/src/core/hle/kernel/scheduler.h
@@ -11,6 +11,7 @@
#include "common/common_types.h"
#include "common/multi_level_queue.h"
+#include "common/spin_lock.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/thread.h"
@@ -41,41 +42,17 @@ public:
return thread_list;
}
- /**
- * Add a thread to the suggested queue of a cpu core. Suggested threads may be
- * picked if no thread is scheduled to run on the core.
- */
- void Suggest(u32 priority, std::size_t core, Thread* thread);
-
- /**
- * Remove a thread to the suggested queue of a cpu core. Suggested threads may be
- * picked if no thread is scheduled to run on the core.
- */
- void Unsuggest(u32 priority, std::size_t core, Thread* thread);
-
- /**
- * Add a thread to the scheduling queue of a cpu core. The thread is added at the
- * back the queue in its priority level.
- */
- void Schedule(u32 priority, std::size_t core, Thread* thread);
+ /// Notify the scheduler a thread's status has changed.
+ void AdjustSchedulingOnStatus(Thread* thread, u32 old_flags);
- /**
- * Add a thread to the scheduling queue of a cpu core. The thread is added at the
- * front the queue in its priority level.
- */
- void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
+ /// Notify the scheduler a thread's priority has changed.
+ void AdjustSchedulingOnPriority(Thread* thread, u32 old_priority);
- /// Reschedule an already scheduled thread based on a new priority
- void Reschedule(u32 priority, std::size_t core, Thread* thread);
-
- /// Unschedules a thread.
- void Unschedule(u32 priority, std::size_t core, Thread* thread);
-
- /// Selects a core and forces it to unload its current thread's context
- void UnloadThread(std::size_t core);
+ /// Notify the scheduler a thread's core and/or affinity mask has changed.
+ void AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask, s32 old_core);
/**
- * Takes care of selecting the new scheduled thread in three steps:
+ * Takes care of selecting the new scheduled threads in three steps:
*
* 1. First a thread is selected from the top of the priority queue. If no thread
* is obtained then we move to step two, else we are done.
@@ -85,8 +62,10 @@ public:
*
* 3. Third is no suggested thread is found, we do a second pass and pick a running
* thread in another core and swap it with its current thread.
+ *
+ * returns the cores needing scheduling.
*/
- void SelectThread(std::size_t core);
+ u32 SelectThreads();
bool HaveReadyThreads(std::size_t core_id) const {
return !scheduled_queue[core_id].empty();
@@ -149,6 +128,39 @@ private:
/// Unlocks the scheduler, reselects threads, interrupts cores for rescheduling
/// and reschedules current core if needed.
void Unlock();
+
+ void EnableInterruptAndSchedule(u32 cores_pending_reschedule, Core::EmuThreadHandle global_thread);
+
+ /**
+ * Add a thread to the suggested queue of a cpu core. Suggested threads may be
+ * picked if no thread is scheduled to run on the core.
+ */
+ void Suggest(u32 priority, std::size_t core, Thread* thread);
+
+ /**
+ * Remove a thread to the suggested queue of a cpu core. Suggested threads may be
+ * picked if no thread is scheduled to run on the core.
+ */
+ void Unsuggest(u32 priority, std::size_t core, Thread* thread);
+
+ /**
+ * Add a thread to the scheduling queue of a cpu core. The thread is added at the
+ * back the queue in its priority level.
+ */
+ void Schedule(u32 priority, std::size_t core, Thread* thread);
+
+ /**
+ * Add a thread to the scheduling queue of a cpu core. The thread is added at the
+ * front the queue in its priority level.
+ */
+ void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
+
+ /// Reschedule an already scheduled thread based on a new priority
+ void Reschedule(u32 priority, std::size_t core, Thread* thread);
+
+ /// Unschedules a thread.
+ void Unschedule(u32 priority, std::size_t core, Thread* thread);
+
/**
* Transfers a thread into an specific core. If the destination_core is -1
* it will be unscheduled from its source code and added into its suggested
@@ -174,6 +186,8 @@ private:
std::atomic<s64> scope_lock{};
Core::EmuThreadHandle current_owner{Core::EmuThreadHandle::InvalidHandle()};
+ Common::SpinLock global_list_guard{};
+
/// Lists all thread ids that aren't deleted/etc.
std::vector<std::shared_ptr<Thread>> thread_list;
KernelCore& kernel;
@@ -190,12 +204,6 @@ public:
/// Reschedules to the next available thread (call after current thread is suspended)
void TryDoContextSwitch();
- /// Unloads currently running thread
- void UnloadThread();
-
- /// Select the threads in top of the scheduling multilist.
- void SelectThreads();
-
/// Gets the current running thread
Thread* GetCurrentThread() const;
@@ -209,15 +217,22 @@ public:
return is_context_switch_pending;
}
+ void Initialize();
+
/// Shutdowns the scheduler.
void Shutdown();
+ void OnThreadStart();
+
private:
friend class GlobalScheduler;
/// Switches the CPU's active thread context to that of the specified thread
void SwitchContext();
+ /// When a thread wakes up, it must run this through it's new scheduler
+ void SwitchContextStep2();
+
/**
* Called on every context switch to update the internal timestamp
* This also updates the running time ticks for the given thread and
@@ -233,12 +248,15 @@ private:
std::shared_ptr<Thread> current_thread = nullptr;
std::shared_ptr<Thread> selected_thread = nullptr;
+ std::shared_ptr<Thread> idle_thread = nullptr;
Core::System& system;
u64 last_context_switch_time = 0;
u64 idle_selection_count = 0;
const std::size_t core_id;
+ Common::SpinLock guard{};
+
bool is_context_switch_pending = false;
};
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 4ae4529f5..d7f0dcabd 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -863,9 +863,9 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha
if (same_thread && info_sub_id == 0xFFFFFFFFFFFFFFFF) {
const u64 thread_ticks = current_thread->GetTotalCPUTimeTicks();
- out_ticks = thread_ticks + (core_timing.GetTicks() - prev_ctx_ticks);
+ out_ticks = thread_ticks + (core_timing.GetCPUTicks() - prev_ctx_ticks);
} else if (same_thread && info_sub_id == system.CurrentCoreIndex()) {
- out_ticks = core_timing.GetTicks() - prev_ctx_ticks;
+ out_ticks = core_timing.GetCPUTicks() - prev_ctx_ticks;
}
*result = out_ticks;
@@ -1428,9 +1428,10 @@ static ResultCode CreateThread(Core::System& system, Handle* out_handle, VAddr e
ASSERT(kernel.CurrentProcess()->GetResourceLimit()->Reserve(ResourceType::Threads, 1));
+ ThreadType type = THREADTYPE_USER;
CASCADE_RESULT(std::shared_ptr<Thread> thread,
- Thread::Create(kernel, "", entry_point, priority, arg, processor_id, stack_top,
- *current_process));
+ Thread::Create(system, type, "", entry_point, priority, arg, processor_id, stack_top,
+ current_process));
const auto new_thread_handle = current_process->GetHandleTable().Create(thread);
if (new_thread_handle.Failed()) {
@@ -1513,13 +1514,6 @@ static void SleepThread(Core::System& system, s64 nanoseconds) {
} else {
current_thread->Sleep(nanoseconds);
}
-
- if (is_redundant) {
- // If it's redundant, the core is pretty much idle. Some games keep idling
- // a core while it's doing nothing, we advance timing to avoid costly continuous
- // calls.
- system.CoreTiming().AddTicks(2000);
- }
system.PrepareReschedule(current_thread->GetProcessorID());
}
@@ -1725,10 +1719,7 @@ static u64 GetSystemTick(Core::System& system) {
auto& core_timing = system.CoreTiming();
// Returns the value of cntpct_el0 (https://switchbrew.org/wiki/SVC#svcGetSystemTick)
- const u64 result{Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks())};
-
- // Advance time to defeat dumb games that busy-wait for the frame to end.
- core_timing.AddTicks(400);
+ const u64 result{system.CoreTiming().GetClockTicks()};
return result;
}
diff --git a/src/core/hle/kernel/thread.cpp b/src/core/hle/kernel/thread.cpp
index db7f379ac..8cb3593db 100644
--- a/src/core/hle/kernel/thread.cpp
+++ b/src/core/hle/kernel/thread.cpp
@@ -9,12 +9,14 @@
#include "common/assert.h"
#include "common/common_types.h"
+#include "common/fiber.h"
#include "common/logging/log.h"
#include "common/thread_queue_list.h"
#include "core/arm/arm_interface.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/core_timing_util.h"
+#include "core/cpu_manager.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/errors.h"
#include "core/hle/kernel/handle_table.h"
@@ -23,6 +25,7 @@
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/thread.h"
+#include "core/hle/kernel/time_manager.h"
#include "core/hle/result.h"
#include "core/memory.h"
@@ -44,6 +47,7 @@ Thread::Thread(KernelCore& kernel) : SynchronizationObject{kernel} {}
Thread::~Thread() = default;
void Thread::Stop() {
+ SchedulerLock lock(kernel);
// Cancel any outstanding wakeup events for this thread
Core::System::GetInstance().CoreTiming().UnscheduleEvent(kernel.ThreadWakeupCallbackEventType(),
global_handle);
@@ -71,9 +75,8 @@ void Thread::WakeAfterDelay(s64 nanoseconds) {
// This function might be called from any thread so we have to be cautious and use the
// thread-safe version of ScheduleEvent.
- const s64 cycles = Core::Timing::nsToCycles(std::chrono::nanoseconds{nanoseconds});
Core::System::GetInstance().CoreTiming().ScheduleEvent(
- cycles, kernel.ThreadWakeupCallbackEventType(), global_handle);
+ nanoseconds, kernel.ThreadWakeupCallbackEventType(), global_handle);
}
void Thread::CancelWakeupTimer() {
@@ -125,6 +128,16 @@ void Thread::ResumeFromWait() {
SetStatus(ThreadStatus::Ready);
}
+void Thread::OnWakeUp() {
+ SchedulerLock lock(kernel);
+ if (activity == ThreadActivity::Paused) {
+ SetStatus(ThreadStatus::Paused);
+ return;
+ }
+
+ SetStatus(ThreadStatus::Ready);
+}
+
void Thread::CancelWait() {
if (GetSchedulingStatus() != ThreadSchedStatus::Paused) {
is_sync_cancelled = true;
@@ -153,12 +166,29 @@ static void ResetThreadContext64(Core::ARM_Interface::ThreadContext64& context,
context.fpcr = 0;
}
-ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::string name,
- VAddr entry_point, u32 priority, u64 arg,
- s32 processor_id, VAddr stack_top,
- Process& owner_process) {
+std::shared_ptr<Common::Fiber> Thread::GetHostContext() const {
+ return host_context;
+}
+
+ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags,
+ std::string name, VAddr entry_point, u32 priority,
+ u64 arg, s32 processor_id, VAddr stack_top,
+ Process* owner_process) {
+ std::function<void(void*)> init_func = system.GetCpuManager().GetGuestThreadStartFunc();
+ void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
+ return Create(system, type_flags, name, entry_point, priority, arg, processor_id, stack_top,
+ owner_process, std::move(init_func), init_func_parameter);
+}
+
+ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags,
+ std::string name, VAddr entry_point, u32 priority,
+ u64 arg, s32 processor_id, VAddr stack_top,
+ Process* owner_process,
+ std::function<void(void*)>&& thread_start_func,
+ void* thread_start_parameter) {
+ auto& kernel = system.Kernel();
// Check if priority is in ranged. Lowest priority -> highest priority id.
- if (priority > THREADPRIO_LOWEST) {
+ if (priority > THREADPRIO_LOWEST && (type_flags & THREADTYPE_IDLE == 0)) {
LOG_ERROR(Kernel_SVC, "Invalid thread priority: {}", priority);
return ERR_INVALID_THREAD_PRIORITY;
}
@@ -168,11 +198,12 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
return ERR_INVALID_PROCESSOR_ID;
}
- auto& system = Core::System::GetInstance();
- if (!system.Memory().IsValidVirtualAddress(owner_process, entry_point)) {
- LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point);
- // TODO (bunnei): Find the correct error code to use here
- return RESULT_UNKNOWN;
+ if (owner_process) {
+ if (!system.Memory().IsValidVirtualAddress(*owner_process, entry_point)) {
+ LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point);
+ // TODO (bunnei): Find the correct error code to use here
+ return RESULT_UNKNOWN;
+ }
}
std::shared_ptr<Thread> thread = std::make_shared<Thread>(kernel);
@@ -183,7 +214,7 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
thread->stack_top = stack_top;
thread->tpidr_el0 = 0;
thread->nominal_priority = thread->current_priority = priority;
- thread->last_running_ticks = system.CoreTiming().GetTicks();
+ thread->last_running_ticks = 0;
thread->processor_id = processor_id;
thread->ideal_core = processor_id;
thread->affinity_mask = 1ULL << processor_id;
@@ -193,16 +224,27 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
thread->wait_handle = 0;
thread->name = std::move(name);
thread->global_handle = kernel.GlobalHandleTable().Create(thread).Unwrap();
- thread->owner_process = &owner_process;
- auto& scheduler = kernel.GlobalScheduler();
- scheduler.AddThread(thread);
- thread->tls_address = thread->owner_process->CreateTLSRegion();
-
- thread->owner_process->RegisterThread(thread.get());
-
- ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top),
- static_cast<u32>(entry_point), static_cast<u32>(arg));
- ResetThreadContext64(thread->context_64, stack_top, entry_point, arg);
+ thread->owner_process = owner_process;
+ thread->type = type_flags;
+ if ((type_flags & THREADTYPE_IDLE) == 0) {
+ auto& scheduler = kernel.GlobalScheduler();
+ scheduler.AddThread(thread);
+ }
+ if (owner_process) {
+ thread->tls_address = thread->owner_process->CreateTLSRegion();
+ thread->owner_process->RegisterThread(thread.get());
+ } else {
+ thread->tls_address = 0;
+ }
+ // TODO(peachum): move to ScheduleThread() when scheduler is added so selected core is used
+ // to initialize the context
+ if ((type_flags & THREADTYPE_HLE) == 0) {
+ ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top),
+ static_cast<u32>(entry_point), static_cast<u32>(arg));
+ ResetThreadContext64(thread->context_64, stack_top, entry_point, arg);
+ }
+ thread->host_context =
+ std::make_shared<Common::Fiber>(std::move(thread_start_func), thread_start_parameter);
return MakeResult<std::shared_ptr<Thread>>(std::move(thread));
}
@@ -258,7 +300,7 @@ void Thread::SetStatus(ThreadStatus new_status) {
}
if (status == ThreadStatus::Running) {
- last_running_ticks = Core::System::GetInstance().CoreTiming().GetTicks();
+ last_running_ticks = Core::System::GetInstance().CoreTiming().GetCPUTicks();
}
status = new_status;
@@ -375,38 +417,55 @@ void Thread::SetActivity(ThreadActivity value) {
}
void Thread::Sleep(s64 nanoseconds) {
- // Sleep current thread and check for next thread to schedule
- SetStatus(ThreadStatus::WaitSleep);
+ Handle event_handle{};
+ {
+ SchedulerLockAndSleep lock(kernel, event_handle, this, nanoseconds);
+ SetStatus(ThreadStatus::WaitSleep);
+ }
- // Create an event to wake the thread up after the specified nanosecond delay has passed
- WakeAfterDelay(nanoseconds);
+ if (event_handle != InvalidHandle) {
+ auto& time_manager = kernel.TimeManager();
+ time_manager.UnscheduleTimeEvent(event_handle);
+ }
}
bool Thread::YieldSimple() {
- auto& scheduler = kernel.GlobalScheduler();
- return scheduler.YieldThread(this);
+ bool result{};
+ {
+ SchedulerLock lock(kernel);
+ result = kernel.GlobalScheduler().YieldThread(this);
+ }
+ return result;
}
bool Thread::YieldAndBalanceLoad() {
- auto& scheduler = kernel.GlobalScheduler();
- return scheduler.YieldThreadAndBalanceLoad(this);
+ bool result{};
+ {
+ SchedulerLock lock(kernel);
+ result = kernel.GlobalScheduler().YieldThreadAndBalanceLoad(this);
+ }
+ return result;
}
bool Thread::YieldAndWaitForLoadBalancing() {
- auto& scheduler = kernel.GlobalScheduler();
- return scheduler.YieldThreadAndWaitForLoadBalancing(this);
+ bool result{};
+ {
+ SchedulerLock lock(kernel);
+ result = kernel.GlobalScheduler().YieldThreadAndWaitForLoadBalancing(this);
+ }
+ return result;
}
void Thread::SetSchedulingStatus(ThreadSchedStatus new_status) {
const u32 old_flags = scheduling_state;
scheduling_state = (scheduling_state & static_cast<u32>(ThreadSchedMasks::HighMask)) |
static_cast<u32>(new_status);
- AdjustSchedulingOnStatus(old_flags);
+ kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_flags);
}
void Thread::SetCurrentPriority(u32 new_priority) {
const u32 old_priority = std::exchange(current_priority, new_priority);
- AdjustSchedulingOnPriority(old_priority);
+ kernel.GlobalScheduler().AdjustSchedulingOnPriority(this, old_priority);
}
ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) {
@@ -443,111 +502,12 @@ ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) {
processor_id = ideal_core;
}
}
- AdjustSchedulingOnAffinity(old_affinity_mask, old_core);
+ kernel.GlobalScheduler().AdjustSchedulingOnAffinity(this, old_affinity_mask, old_core);
}
}
return RESULT_SUCCESS;
}
-void Thread::AdjustSchedulingOnStatus(u32 old_flags) {
- if (old_flags == scheduling_state) {
- return;
- }
-
- auto& scheduler = kernel.GlobalScheduler();
- if (static_cast<ThreadSchedStatus>(old_flags & static_cast<u32>(ThreadSchedMasks::LowMask)) ==
- ThreadSchedStatus::Runnable) {
- // In this case the thread was running, now it's pausing/exitting
- if (processor_id >= 0) {
- scheduler.Unschedule(current_priority, static_cast<u32>(processor_id), this);
- }
-
- for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
- if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
- scheduler.Unsuggest(current_priority, core, this);
- }
- }
- } else if (GetSchedulingStatus() == ThreadSchedStatus::Runnable) {
- // The thread is now set to running from being stopped
- if (processor_id >= 0) {
- scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this);
- }
-
- for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
- if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
- scheduler.Suggest(current_priority, core, this);
- }
- }
- }
-
- scheduler.SetReselectionPending();
-}
-
-void Thread::AdjustSchedulingOnPriority(u32 old_priority) {
- if (GetSchedulingStatus() != ThreadSchedStatus::Runnable) {
- return;
- }
- auto& scheduler = kernel.GlobalScheduler();
- if (processor_id >= 0) {
- scheduler.Unschedule(old_priority, static_cast<u32>(processor_id), this);
- }
-
- for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
- if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
- scheduler.Unsuggest(old_priority, core, this);
- }
- }
-
- // Add thread to the new priority queues.
- Thread* current_thread = GetCurrentThread();
-
- if (processor_id >= 0) {
- if (current_thread == this) {
- scheduler.SchedulePrepend(current_priority, static_cast<u32>(processor_id), this);
- } else {
- scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this);
- }
- }
-
- for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
- if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
- scheduler.Suggest(current_priority, core, this);
- }
- }
-
- scheduler.SetReselectionPending();
-}
-
-void Thread::AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core) {
- auto& scheduler = kernel.GlobalScheduler();
- if (GetSchedulingStatus() != ThreadSchedStatus::Runnable ||
- current_priority >= THREADPRIO_COUNT) {
- return;
- }
-
- for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
- if (((old_affinity_mask >> core) & 1) != 0) {
- if (core == static_cast<u32>(old_core)) {
- scheduler.Unschedule(current_priority, core, this);
- } else {
- scheduler.Unsuggest(current_priority, core, this);
- }
- }
- }
-
- for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
- if (((affinity_mask >> core) & 1) != 0) {
- if (core == static_cast<u32>(processor_id)) {
- scheduler.Schedule(current_priority, core, this);
- } else {
- scheduler.Suggest(current_priority, core, this);
- }
- }
- }
-
- scheduler.SetReselectionPending();
-}
-
////////////////////////////////////////////////////////////////////////////////////////////////////
/**
diff --git a/src/core/hle/kernel/thread.h b/src/core/hle/kernel/thread.h
index 23fdef8a4..33d340b47 100644
--- a/src/core/hle/kernel/thread.h
+++ b/src/core/hle/kernel/thread.h
@@ -9,23 +9,42 @@
#include <vector>
#include "common/common_types.h"
+#include "common/spin_lock.h"
#include "core/arm/arm_interface.h"
#include "core/hle/kernel/object.h"
#include "core/hle/kernel/synchronization_object.h"
#include "core/hle/result.h"
+namespace Common {
+class Fiber;
+}
+
+namespace Core {
+class System;
+}
+
namespace Kernel {
+class GlobalScheduler;
class KernelCore;
class Process;
class Scheduler;
enum ThreadPriority : u32 {
- THREADPRIO_HIGHEST = 0, ///< Highest thread priority
- THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps
- THREADPRIO_DEFAULT = 44, ///< Default thread priority for userland apps
- THREADPRIO_LOWEST = 63, ///< Lowest thread priority
- THREADPRIO_COUNT = 64, ///< Total number of possible thread priorities.
+ THREADPRIO_HIGHEST = 0, ///< Highest thread priority
+ THREADPRIO_MAX_CORE_MIGRATION = 2, ///< Highest priority for a core migration
+ THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps
+ THREADPRIO_DEFAULT = 44, ///< Default thread priority for userland apps
+ THREADPRIO_LOWEST = 63, ///< Lowest thread priority
+ THREADPRIO_COUNT = 64, ///< Total number of possible thread priorities.
+};
+
+enum ThreadType : u32 {
+ THREADTYPE_USER = 0x1,
+ THREADTYPE_KERNEL = 0x2,
+ THREADTYPE_HLE = 0x4,
+ THREADTYPE_IDLE = 0x8,
+ THREADTYPE_SUSPEND = 0x10,
};
enum ThreadProcessorId : s32 {
@@ -111,22 +130,43 @@ public:
std::function<bool(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
std::shared_ptr<SynchronizationObject> object, std::size_t index)>;
+ /**
+ * Creates and returns a new thread. The new thread is immediately scheduled
+ * @param system The instance of the whole system
+ * @param name The friendly name desired for the thread
+ * @param entry_point The address at which the thread should start execution
+ * @param priority The thread's priority
+ * @param arg User data to pass to the thread
+ * @param processor_id The ID(s) of the processors on which the thread is desired to be run
+ * @param stack_top The address of the thread's stack top
+ * @param owner_process The parent process for the thread, if null, it's a kernel thread
+ * @return A shared pointer to the newly created thread
+ */
+ static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags, std::string name,
+ VAddr entry_point, u32 priority, u64 arg,
+ s32 processor_id, VAddr stack_top,
+ Process* owner_process);
+
/**
* Creates and returns a new thread. The new thread is immediately scheduled
- * @param kernel The kernel instance this thread will be created under.
+ * @param system The instance of the whole system
* @param name The friendly name desired for the thread
* @param entry_point The address at which the thread should start execution
* @param priority The thread's priority
* @param arg User data to pass to the thread
* @param processor_id The ID(s) of the processors on which the thread is desired to be run
* @param stack_top The address of the thread's stack top
- * @param owner_process The parent process for the thread
+ * @param owner_process The parent process for the thread, if null, it's a kernel thread
+ * @param thread_start_func The function where the host context will start.
+ * @param thread_start_parameter The parameter which will passed to host context on init
* @return A shared pointer to the newly created thread
*/
- static ResultVal<std::shared_ptr<Thread>> Create(KernelCore& kernel, std::string name,
+ static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags, std::string name,
VAddr entry_point, u32 priority, u64 arg,
s32 processor_id, VAddr stack_top,
- Process& owner_process);
+ Process* owner_process,
+ std::function<void(void*)>&& thread_start_func,
+ void* thread_start_parameter);
std::string GetName() const override {
return name;
@@ -192,7 +232,9 @@ public:
}
/// Resumes a thread from waiting
- void ResumeFromWait();
+ void /* deprecated */ ResumeFromWait();
+
+ void OnWakeUp();
/// Cancels a waiting operation that this thread may or may not be within.
///
@@ -206,10 +248,10 @@ public:
* Schedules an event to wake up the specified thread after the specified delay
* @param nanoseconds The time this thread will be allowed to sleep for
*/
- void WakeAfterDelay(s64 nanoseconds);
+ void /* deprecated */ WakeAfterDelay(s64 nanoseconds);
/// Cancel any outstanding wakeup events for this thread
- void CancelWakeupTimer();
+ void /* deprecated */ CancelWakeupTimer();
/**
* Sets the result after the thread awakens (from svcWaitSynchronization)
@@ -290,6 +332,12 @@ public:
return context_64;
}
+ bool IsHLEThread() const {
+ return (type & THREADTYPE_HLE) != 0;
+ }
+
+ std::shared_ptr<Common::Fiber> GetHostContext() const;
+
ThreadStatus GetStatus() const {
return status;
}
@@ -467,16 +515,19 @@ public:
}
private:
+ friend class GlobalScheduler;
+ friend class Scheduler;
+
void SetSchedulingStatus(ThreadSchedStatus new_status);
void SetCurrentPriority(u32 new_priority);
ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask);
- void AdjustSchedulingOnStatus(u32 old_flags);
- void AdjustSchedulingOnPriority(u32 old_priority);
void AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core);
ThreadContext32 context_32{};
ThreadContext64 context_64{};
+ Common::SpinLock context_guard{};
+ std::shared_ptr<Common::Fiber> host_context{};
u64 thread_id = 0;
@@ -485,6 +536,8 @@ private:
VAddr entry_point = 0;
VAddr stack_top = 0;
+ ThreadType type;
+
/// Nominal thread priority, as set by the emulated application.
/// The nominal priority is the thread priority without priority
/// inheritance taken into account.
diff --git a/src/core/hle/kernel/time_manager.cpp b/src/core/hle/kernel/time_manager.cpp
index 21b290468..0b8f0d993 100644
--- a/src/core/hle/kernel/time_manager.cpp
+++ b/src/core/hle/kernel/time_manager.cpp
@@ -19,7 +19,7 @@ TimeManager::TimeManager(Core::System& system) : system{system} {
Handle proper_handle = static_cast<Handle>(thread_handle);
std::shared_ptr<Thread> thread =
this->system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle);
- thread->ResumeFromWait();
+ thread->OnWakeUp();
});
}