2018-03-07 20:57:14 +08:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2018 Intel Corporation. All rights reserved.
|
|
|
|
*
|
2018-05-26 01:49:13 +08:00
|
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
2018-03-07 20:57:14 +08:00
|
|
|
*/
|
|
|
|
|
2018-05-22 08:30:50 +08:00
|
|
|
#include <hypervisor.h>
|
2018-03-07 20:57:14 +08:00
|
|
|
#include <schedule.h>
|
|
|
|
|
|
|
|
static unsigned long pcpu_used_bitmap;
|
|
|
|
|
|
|
|
void init_scheduler(void)
|
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx;
|
2018-06-28 16:27:12 +08:00
|
|
|
uint32_t i;
|
2018-03-07 20:57:14 +08:00
|
|
|
|
2018-07-03 18:37:07 +08:00
|
|
|
for (i = 0U; i < phys_cpu_num; i++) {
|
2018-07-19 23:35:34 +08:00
|
|
|
ctx = &per_cpu(sched_ctx, i);
|
|
|
|
|
|
|
|
spinlock_init(&ctx->runqueue_lock);
|
|
|
|
spinlock_init(&ctx->scheduler_lock);
|
|
|
|
INIT_LIST_HEAD(&ctx->runqueue);
|
|
|
|
ctx->flags = 0UL;
|
|
|
|
ctx->curr_vcpu = NULL;
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
void get_schedule_lock(uint16_t pcpu_id)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
|
|
|
spinlock_obtain(&ctx->scheduler_lock);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
void release_schedule_lock(uint16_t pcpu_id)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
|
|
|
spinlock_release(&ctx->scheduler_lock);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-06-25 17:51:00 +08:00
|
|
|
uint16_t allocate_pcpu(void)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-06-25 17:51:00 +08:00
|
|
|
uint16_t i;
|
2018-03-07 20:57:14 +08:00
|
|
|
|
2018-07-03 18:37:07 +08:00
|
|
|
for (i = 0U; i < phys_cpu_num; i++) {
|
2018-07-25 09:11:26 +08:00
|
|
|
if (bitmap_test_and_set_lock(i, &pcpu_used_bitmap) == 0) {
|
2018-03-07 20:57:14 +08:00
|
|
|
return i;
|
2018-07-13 06:02:55 +08:00
|
|
|
}
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-07-02 14:05:58 +08:00
|
|
|
return INVALID_CPU_ID;
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
void set_pcpu_used(uint16_t pcpu_id)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-25 09:11:26 +08:00
|
|
|
bitmap_set_lock(pcpu_id, &pcpu_used_bitmap);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
void free_pcpu(uint16_t pcpu_id)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-25 09:11:26 +08:00
|
|
|
bitmap_clear_lock(pcpu_id, &pcpu_used_bitmap);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void add_vcpu_to_runqueue(struct vcpu *vcpu)
|
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
uint16_t pcpu_id = vcpu->pcpu_id;
|
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
2018-03-07 20:57:14 +08:00
|
|
|
|
2018-07-19 23:35:34 +08:00
|
|
|
spinlock_obtain(&ctx->runqueue_lock);
|
2018-07-13 06:02:55 +08:00
|
|
|
if (list_empty(&vcpu->run_list)) {
|
2018-07-19 23:35:34 +08:00
|
|
|
list_add_tail(&vcpu->run_list, &ctx->runqueue);
|
2018-07-13 06:02:55 +08:00
|
|
|
}
|
2018-07-19 23:35:34 +08:00
|
|
|
spinlock_release(&ctx->runqueue_lock);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
void remove_vcpu_from_runqueue(struct vcpu *vcpu)
|
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
uint16_t pcpu_id = vcpu->pcpu_id;
|
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
2018-03-07 20:57:14 +08:00
|
|
|
|
2018-07-19 23:35:34 +08:00
|
|
|
spinlock_obtain(&ctx->runqueue_lock);
|
2018-03-07 20:57:14 +08:00
|
|
|
list_del_init(&vcpu->run_list);
|
2018-07-19 23:35:34 +08:00
|
|
|
spinlock_release(&ctx->runqueue_lock);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
static struct vcpu *select_next_vcpu(uint16_t pcpu_id)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
2018-03-07 20:57:14 +08:00
|
|
|
struct vcpu *vcpu = NULL;
|
|
|
|
|
2018-07-19 23:35:34 +08:00
|
|
|
spinlock_obtain(&ctx->runqueue_lock);
|
|
|
|
if (!list_empty(&ctx->runqueue)) {
|
|
|
|
vcpu = get_first_item(&ctx->runqueue, struct vcpu, run_list);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
2018-07-19 23:35:34 +08:00
|
|
|
spinlock_release(&ctx->runqueue_lock);
|
2018-03-07 20:57:14 +08:00
|
|
|
|
|
|
|
return vcpu;
|
|
|
|
}
|
|
|
|
|
2018-10-23 13:36:25 +08:00
|
|
|
void make_reschedule_request(const struct vcpu *vcpu)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, vcpu->pcpu_id);
|
|
|
|
|
2018-07-25 09:11:26 +08:00
|
|
|
bitmap_set_lock(NEED_RESCHEDULE, &ctx->flags);
|
2018-08-10 14:12:16 +08:00
|
|
|
if (get_cpu_id() != vcpu->pcpu_id) {
|
|
|
|
send_single_ipi(vcpu->pcpu_id, VECTOR_NOTIFY_VCPU);
|
|
|
|
}
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
int need_reschedule(uint16_t pcpu_id)
|
2018-03-07 20:57:14 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
|
|
|
|
2018-07-25 09:11:26 +08:00
|
|
|
return bitmap_test_and_clear_lock(NEED_RESCHEDULE, &ctx->flags);
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static void context_switch_out(struct vcpu *vcpu)
|
|
|
|
{
|
|
|
|
/* if it's idle thread, no action for switch out */
|
2018-07-13 06:02:55 +08:00
|
|
|
if (vcpu == NULL) {
|
2018-03-07 20:57:14 +08:00
|
|
|
return;
|
2018-07-13 06:02:55 +08:00
|
|
|
}
|
2018-03-07 20:57:14 +08:00
|
|
|
|
2018-02-28 19:33:13 +08:00
|
|
|
/* cancel event(int, gp, nmi and exception) injection */
|
|
|
|
cancel_event_injection(vcpu);
|
|
|
|
|
2018-07-18 13:36:41 +08:00
|
|
|
atomic_store32(&vcpu->running, 0U);
|
2018-03-07 20:57:14 +08:00
|
|
|
/* do prev vcpu context switch out */
|
|
|
|
/* For now, we don't need to invalid ept.
|
|
|
|
* But if we have more than one vcpu on one pcpu,
|
|
|
|
* we need add ept invalid operation here.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
|
|
|
static void context_switch_in(struct vcpu *vcpu)
|
|
|
|
{
|
|
|
|
/* update current_vcpu */
|
|
|
|
get_cpu_var(sched_ctx).curr_vcpu = vcpu;
|
|
|
|
|
|
|
|
/* if it's idle thread, no action for switch out */
|
2018-07-13 06:02:55 +08:00
|
|
|
if (vcpu == NULL) {
|
2018-03-07 20:57:14 +08:00
|
|
|
return;
|
2018-07-13 06:02:55 +08:00
|
|
|
}
|
2018-03-07 20:57:14 +08:00
|
|
|
|
2018-07-18 13:36:41 +08:00
|
|
|
atomic_store32(&vcpu->running, 1U);
|
2018-03-07 20:57:14 +08:00
|
|
|
/* FIXME:
|
|
|
|
* Now, we don't need to load new vcpu VMCS because
|
|
|
|
* we only do switch between vcpu loop and idle loop.
|
|
|
|
* If we have more than one vcpu on on pcpu, need to
|
|
|
|
* add VMCS load operation here.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
void make_pcpu_offline(uint16_t pcpu_id)
|
2018-06-02 22:31:58 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
|
|
|
|
2018-07-25 09:11:26 +08:00
|
|
|
bitmap_set_lock(NEED_OFFLINE, &ctx->flags);
|
2018-08-10 14:12:16 +08:00
|
|
|
if (get_cpu_id() != pcpu_id) {
|
|
|
|
send_single_ipi(pcpu_id, VECTOR_NOTIFY_VCPU);
|
|
|
|
}
|
2018-06-02 22:31:58 +08:00
|
|
|
}
|
|
|
|
|
2018-06-20 15:42:52 +08:00
|
|
|
int need_offline(uint16_t pcpu_id)
|
2018-06-02 22:31:58 +08:00
|
|
|
{
|
2018-07-19 23:35:34 +08:00
|
|
|
struct sched_context *ctx = &per_cpu(sched_ctx, pcpu_id);
|
|
|
|
|
2018-07-25 09:11:26 +08:00
|
|
|
return bitmap_test_and_clear_lock(NEED_OFFLINE, &ctx->flags);
|
2018-06-02 22:31:58 +08:00
|
|
|
}
|
|
|
|
|
2018-03-07 20:57:14 +08:00
|
|
|
void default_idle(void)
|
|
|
|
{
|
2018-06-20 15:42:52 +08:00
|
|
|
uint16_t pcpu_id = get_cpu_id();
|
2018-03-07 20:57:14 +08:00
|
|
|
|
|
|
|
while (1) {
|
2018-07-13 06:02:55 +08:00
|
|
|
if (need_reschedule(pcpu_id) != 0) {
|
2018-03-07 20:57:14 +08:00
|
|
|
schedule();
|
2018-07-13 06:02:55 +08:00
|
|
|
} else if (need_offline(pcpu_id) != 0) {
|
2018-06-02 22:31:58 +08:00
|
|
|
cpu_dead(pcpu_id);
|
2018-08-12 20:56:30 +08:00
|
|
|
} else {
|
|
|
|
CPU_IRQ_ENABLE();
|
2018-09-25 15:05:15 +08:00
|
|
|
cpu_do_idle();
|
2018-08-12 20:56:30 +08:00
|
|
|
CPU_IRQ_DISABLE();
|
|
|
|
}
|
2018-03-07 20:57:14 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void switch_to(struct vcpu *curr)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* reset stack pointer here. Otherwise, schedule
|
|
|
|
* is recursive call and stack will overflow finally.
|
|
|
|
*/
|
2018-05-22 23:24:07 +08:00
|
|
|
uint64_t cur_sp = (uint64_t)&get_cpu_var(stack)[CONFIG_STACK_SIZE];
|
2018-03-07 20:57:14 +08:00
|
|
|
|
|
|
|
if (curr == NULL) {
|
|
|
|
asm volatile ("movq %1, %%rsp\n"
|
|
|
|
"movq $0, %%rdi\n"
|
2018-03-12 22:23:37 +08:00
|
|
|
"call 22f\n"
|
|
|
|
"11: \n"
|
|
|
|
"pause\n"
|
|
|
|
"jmp 11b\n"
|
|
|
|
"22:\n"
|
|
|
|
"mov %0, (%%rsp)\n"
|
|
|
|
"ret\n"
|
2018-03-07 20:57:14 +08:00
|
|
|
:
|
|
|
|
: "a"(default_idle), "r"(cur_sp)
|
|
|
|
: "memory");
|
|
|
|
} else {
|
|
|
|
asm volatile ("movq %2, %%rsp\n"
|
|
|
|
"movq %0, %%rdi\n"
|
2018-03-12 22:23:37 +08:00
|
|
|
"call 44f\n"
|
|
|
|
"33: \n"
|
|
|
|
"pause\n"
|
|
|
|
"jmp 33b\n"
|
|
|
|
"44:\n"
|
|
|
|
"mov %1, (%%rsp)\n"
|
|
|
|
"ret\n"
|
2018-03-07 20:57:14 +08:00
|
|
|
:
|
|
|
|
: "c"(curr), "a"(vcpu_thread), "r"(cur_sp)
|
|
|
|
: "memory");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void schedule(void)
|
|
|
|
{
|
2018-06-20 15:42:52 +08:00
|
|
|
uint16_t pcpu_id = get_cpu_id();
|
2018-03-07 20:57:14 +08:00
|
|
|
struct vcpu *next = NULL;
|
|
|
|
struct vcpu *prev = per_cpu(sched_ctx, pcpu_id).curr_vcpu;
|
|
|
|
|
|
|
|
get_schedule_lock(pcpu_id);
|
|
|
|
next = select_next_vcpu(pcpu_id);
|
|
|
|
|
|
|
|
if (prev == next) {
|
|
|
|
release_schedule_lock(pcpu_id);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
context_switch_out(prev);
|
|
|
|
context_switch_in(next);
|
|
|
|
release_schedule_lock(pcpu_id);
|
|
|
|
|
|
|
|
switch_to(next);
|
|
|
|
|
|
|
|
ASSERT(false, "Shouldn't go here");
|
|
|
|
}
|