OrgACRN
/
acrn-hypervisor
mirror of https://github.com/projectacrn/acrn-hypervisor.git
1
0
Fork 0
Code Issues Releases Wiki Activity
acrn-hypervisor/hypervisor/arch/x86/guest/vcpu.c

1059 lines
29 KiB
C
Executable File
Raw Permalink Blame History

/*
* Copyright (C) 2018-2022 Intel Corporation.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <types.h>
#include <errno.h>
#include <asm/guest/vcpu.h>
#include <asm/guest/virq.h>
#include <asm/lib/bits.h>
#include <asm/vmx.h>
#include <logmsg.h>
#include <asm/cpufeatures.h>
#include <asm/cpu_caps.h>
#include <asm/per_cpu.h>
#include <asm/init.h>
#include <asm/guest/vm.h>
#include <asm/guest/vmcs.h>
#include <asm/mmu.h>
#include <lib/sprintf.h>
#include <asm/lapic.h>
#include <asm/irq.h>
/* stack_frame is linked with the sequence of stack operation in arch_switch_to() */
struct stack_frame {
uint64_t rdi;
uint64_t r15;
uint64_t r14;
uint64_t r13;
uint64_t r12;
uint64_t rbp;
uint64_t rbx;
uint64_t rflag;
uint64_t rip;
uint64_t magic;
};
uint64_t vcpu_get_gpreg(const struct acrn_vcpu *vcpu, uint32_t reg)
{
const struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
return ctx->cpu_regs.longs[reg];
}
void vcpu_set_gpreg(struct acrn_vcpu *vcpu, uint32_t reg, uint64_t val)
{
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
ctx->cpu_regs.longs[reg] = val;
}
uint64_t vcpu_get_rip(struct acrn_vcpu *vcpu)
{
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
if (!bitmap_test(CPU_REG_RIP, &vcpu->reg_updated) &&
!bitmap_test_and_set_nolock(CPU_REG_RIP, &vcpu->reg_cached)) {
ctx->rip = exec_vmread(VMX_GUEST_RIP);
}
return ctx->rip;
}
void vcpu_set_rip(struct acrn_vcpu *vcpu, uint64_t val)
{
vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx.rip = val;
bitmap_set_nolock(CPU_REG_RIP, &vcpu->reg_updated);
}
uint64_t vcpu_get_rsp(const struct acrn_vcpu *vcpu)
{
const struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
return ctx->cpu_regs.regs.rsp;
}
void vcpu_set_rsp(struct acrn_vcpu *vcpu, uint64_t val)
{
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
ctx->cpu_regs.regs.rsp = val;
bitmap_set_nolock(CPU_REG_RSP, &vcpu->reg_updated);
}
uint64_t vcpu_get_efer(struct acrn_vcpu *vcpu)
{
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
return ctx->ia32_efer;
}
void vcpu_set_efer(struct acrn_vcpu *vcpu, uint64_t val)
{
vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx.ia32_efer
= val;
if (val == msr_read(MSR_IA32_EFER)) {
clear_vmcs_bit(VMX_ENTRY_CONTROLS, VMX_ENTRY_CTLS_LOAD_EFER);
clear_vmcs_bit(VMX_EXIT_CONTROLS, VMX_EXIT_CTLS_LOAD_EFER);
} else {
set_vmcs_bit(VMX_ENTRY_CONTROLS, VMX_ENTRY_CTLS_LOAD_EFER);
set_vmcs_bit(VMX_EXIT_CONTROLS, VMX_EXIT_CTLS_LOAD_EFER);
}
/* Write the new value to VMCS in either case */
bitmap_set_nolock(CPU_REG_EFER, &vcpu->reg_updated);
}
uint64_t vcpu_get_rflags(struct acrn_vcpu *vcpu)
{
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
if (!bitmap_test(CPU_REG_RFLAGS, &vcpu->reg_updated) &&
!bitmap_test_and_set_nolock(CPU_REG_RFLAGS, &vcpu->reg_cached) && vcpu->launched) {
ctx->rflags = exec_vmread(VMX_GUEST_RFLAGS);
}
return ctx->rflags;
}
void vcpu_set_rflags(struct acrn_vcpu *vcpu, uint64_t val)
{
vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx.rflags =
val;
bitmap_set_nolock(CPU_REG_RFLAGS, &vcpu->reg_updated);
}
uint64_t vcpu_get_guest_msr(const struct acrn_vcpu *vcpu, uint32_t msr)
{
uint32_t index = vmsr_get_guest_msr_index(msr);
uint64_t val = 0UL;
if (index < NUM_EMULATED_MSRS) {
val = vcpu->arch.guest_msrs[index];
}
return val;
}
void vcpu_set_guest_msr(struct acrn_vcpu *vcpu, uint32_t msr, uint64_t val)
{
uint32_t index = vmsr_get_guest_msr_index(msr);
if (index < NUM_EMULATED_MSRS) {
vcpu->arch.guest_msrs[index] = val;
}
}
/*
* Write the eoi_exit_bitmaps to VMCS fields
*/
void vcpu_set_vmcs_eoi_exit(const struct acrn_vcpu *vcpu)
{
pr_dbg("%s", __func__);
if (is_apicv_advanced_feature_supported()) {
exec_vmwrite64(VMX_EOI_EXIT0_FULL, vcpu->arch.eoi_exit_bitmap[0]);
exec_vmwrite64(VMX_EOI_EXIT1_FULL, vcpu->arch.eoi_exit_bitmap[1]);
exec_vmwrite64(VMX_EOI_EXIT2_FULL, vcpu->arch.eoi_exit_bitmap[2]);
exec_vmwrite64(VMX_EOI_EXIT3_FULL, vcpu->arch.eoi_exit_bitmap[3]);
}
}
/*
* Set the eoi_exit_bitmap bit for specific vector
* @pre vcpu != NULL && vector <= 255U
*/
void vcpu_set_eoi_exit_bitmap(struct acrn_vcpu *vcpu, uint32_t vector)
{
pr_dbg("%s", __func__);
if (!bitmap_test_and_set_lock((uint16_t)(vector & 0x3fU),
&(vcpu->arch.eoi_exit_bitmap[(vector & 0xffU) >> 6U]))) {
vcpu_make_request(vcpu, ACRN_REQUEST_EOI_EXIT_BITMAP_UPDATE);
}
}
void vcpu_clear_eoi_exit_bitmap(struct acrn_vcpu *vcpu, uint32_t vector)
{
pr_dbg("%s", __func__);
if (bitmap_test_and_clear_lock((uint16_t)(vector & 0x3fU),
&(vcpu->arch.eoi_exit_bitmap[(vector & 0xffU) >> 6U]))) {
vcpu_make_request(vcpu, ACRN_REQUEST_EOI_EXIT_BITMAP_UPDATE);
}
}
/*
* Reset all eoi_exit_bitmaps
*/
void vcpu_reset_eoi_exit_bitmaps(struct acrn_vcpu *vcpu)
{
pr_dbg("%s", __func__);
(void)memset((void *)(vcpu->arch.eoi_exit_bitmap), 0U, sizeof(vcpu->arch.eoi_exit_bitmap));
vcpu_make_request(vcpu, ACRN_REQUEST_EOI_EXIT_BITMAP_UPDATE);
}
static void init_iwkey(struct acrn_vcpu *vcpu)
{
/* Initial a random iwkey */
if (pcpu_has_cap(X86_FEATURE_KEYLOCKER)) {
vcpu->arch.IWKey.integrity_key[0] = get_random_value();
vcpu->arch.IWKey.integrity_key[1] = get_random_value();
vcpu->arch.IWKey.encryption_key[0] = get_random_value();
vcpu->arch.IWKey.encryption_key[1] = get_random_value();
vcpu->arch.IWKey.encryption_key[2] = get_random_value();
vcpu->arch.IWKey.encryption_key[3] = get_random_value();
/* It's always safe to clear whose_iwkey */
per_cpu(whose_iwkey, pcpuid_from_vcpu(vcpu)) = NULL;
}
}
void load_iwkey(struct acrn_vcpu *vcpu)
{
uint64_t xmm_save[6];
/* Only load IWKey with vCPU CR4 keylocker bit enabled */
if (pcpu_has_cap(X86_FEATURE_KEYLOCKER) && vcpu->arch.cr4_kl_enabled &&
(get_cpu_var(whose_iwkey) != vcpu)) {
/* Save/restore xmm0/xmm1/xmm2 during the process */
read_xmm_0_2(&xmm_save[0], &xmm_save[2], &xmm_save[4]);
write_xmm_0_2(&vcpu->arch.IWKey.integrity_key[0], &vcpu->arch.IWKey.encryption_key[0],
&vcpu->arch.IWKey.encryption_key[2]);
asm_loadiwkey(0);
write_xmm_0_2(&xmm_save[0], &xmm_save[2], &xmm_save[4]);
get_cpu_var(whose_iwkey) = vcpu;
}
}
/* As a vcpu reset internal API, DO NOT touch any vcpu state transition in this function. */
static void vcpu_reset_internal(struct acrn_vcpu *vcpu, enum reset_mode mode)
{
int32_t i;
struct acrn_vlapic *vlapic;
vcpu->launched = false;
vcpu->arch.nr_sipi = 0U;
vcpu->arch.exception_info.exception = VECTOR_INVALID;
vcpu->arch.cur_context = NORMAL_WORLD;
vcpu->arch.lapic_pt_enabled = false;
vcpu->arch.irq_window_enabled = false;
vcpu->arch.emulating_lock = false;
(void)memset((void *)vcpu->arch.vmcs, 0U, PAGE_SIZE);
for (i = 0; i < NR_WORLD; i++) {
(void)memset((void *)(&vcpu->arch.contexts[i]), 0U,
sizeof(struct run_context));
}
/* TODO: we may need to add one scheduler->reset_data to reset the thread_obj */
vcpu->thread_obj.notify_mode = SCHED_NOTIFY_IPI;
vlapic = vcpu_vlapic(vcpu);
vlapic_reset(vlapic, apicv_ops, mode);
reset_vcpu_regs(vcpu, mode);
for (i = 0; i < VCPU_EVENT_NUM; i++) {
reset_event(&vcpu->events[i]);
}
init_iwkey(vcpu);
vcpu->arch.iwkey_copy_status = 0UL;
}
struct acrn_vcpu *get_running_vcpu(uint16_t pcpu_id)
{
struct thread_object *curr = sched_get_current(pcpu_id);
struct acrn_vcpu *vcpu = NULL;
if ((curr != NULL) && (!is_idle_thread(curr))) {
vcpu = container_of(curr, struct acrn_vcpu, thread_obj);
}
return vcpu;
}
struct acrn_vcpu *get_ever_run_vcpu(uint16_t pcpu_id)
{
return per_cpu(ever_run_vcpu, pcpu_id);
}
static void set_vcpu_mode(struct acrn_vcpu *vcpu, uint32_t cs_attr, uint64_t ia32_efer,
uint64_t cr0)
{
if ((ia32_efer & MSR_IA32_EFER_LMA_BIT) != 0UL) {
if ((cs_attr & 0x2000U) != 0U) {
/* CS.L = 1 */
vcpu->arch.cpu_mode = CPU_MODE_64BIT;
} else {
vcpu->arch.cpu_mode = CPU_MODE_COMPATIBILITY;
}
} else if ((cr0 & CR0_PE) != 0UL) {
vcpu->arch.cpu_mode = CPU_MODE_PROTECTED;
} else {
vcpu->arch.cpu_mode = CPU_MODE_REAL;
}
}
static void init_xsave(struct acrn_vcpu *vcpu)
{
struct ext_context *ectx = &(vcpu->arch.contexts[vcpu->arch.cur_context].ext_ctx);
struct xsave_area *area = &ectx->xs_area;
/* if the HW has this cap, we need to prepare the buffer for potential save/restore.
* Guest may or may not enable XSAVE -- it doesn't matter.
*/
if (pcpu_has_cap(X86_FEATURE_XSAVE)) {
ectx->xcr0 = XSAVE_FPU;
(void)memset((void *)area, 0U, XSAVE_STATE_AREA_SIZE);
/* xsaves only support compacted format, so set it in xcomp_bv[63],
* keep the reset area in header area as zero.
*/
ectx->xs_area.xsave_hdr.hdr.xcomp_bv |= XSAVE_COMPACTED_FORMAT;
}
}
void set_vcpu_regs(struct acrn_vcpu *vcpu, struct acrn_regs *vcpu_regs)
{
struct ext_context *ectx;
struct run_context *ctx;
uint16_t *sel = &(vcpu_regs->cs_sel);
struct segment_sel *seg;
uint32_t limit, attr;
ectx = &(vcpu->arch.contexts[vcpu->arch.cur_context].ext_ctx);
ctx = &(vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx);
/* NOTE:
* This is to set the attr and limit to default value.
* If the set_vcpu_regs is used not only for vcpu state
* initialization, this part of code needs be revised.
*/
if ((vcpu_regs->cr0 & CR0_PE) != 0UL) {
attr = PROTECTED_MODE_DATA_SEG_AR;
limit = PROTECTED_MODE_SEG_LIMIT;
} else {
attr = REAL_MODE_DATA_SEG_AR;
limit = REAL_MODE_SEG_LIMIT;
}
for (seg = &(ectx->cs); seg <= &(ectx->gs); seg++) {
seg->base = 0UL;
seg->limit = limit;
seg->attr = attr;
seg->selector = *sel;
sel++;
}
/* override cs attr/base/limit */
ectx->cs.attr = vcpu_regs->cs_ar;
ectx->cs.base = vcpu_regs->cs_base;
ectx->cs.limit = vcpu_regs->cs_limit;
ectx->gdtr.base = vcpu_regs->gdt.base;
ectx->gdtr.limit = vcpu_regs->gdt.limit;
ectx->idtr.base = vcpu_regs->idt.base;
ectx->idtr.limit = vcpu_regs->idt.limit;
ectx->ldtr.selector = vcpu_regs->ldt_sel;
ectx->tr.selector = vcpu_regs->tr_sel;
/* NOTE:
* This is to set the ldtr and tr to default value.
* If the set_vcpu_regs is used not only for vcpu state
* initialization, this part of code needs be revised.
*/
ectx->ldtr.base = 0UL;
ectx->tr.base = 0UL;
ectx->ldtr.limit = 0xFFFFU;
ectx->tr.limit = 0xFFFFU;
ectx->ldtr.attr = LDTR_AR;
ectx->tr.attr = TR_AR;
(void)memcpy_s((void *)&(ctx->cpu_regs), sizeof(struct acrn_gp_regs),
(void *)&(vcpu_regs->gprs), sizeof(struct acrn_gp_regs));
vcpu_set_rip(vcpu, vcpu_regs->rip);
vcpu_set_efer(vcpu, vcpu_regs->ia32_efer);
vcpu_set_rsp(vcpu, vcpu_regs->gprs.rsp);
if (vcpu_regs->rflags == 0UL) {
vcpu_set_rflags(vcpu, 0x02UL);
} else {
vcpu_set_rflags(vcpu, vcpu_regs->rflags & ~(0x8d5UL));
}
/* cr0, cr3 and cr4 needs be set without using API.
* The real cr0/cr3/cr4 writing will be delayed to
* init_vmcs
*/
ctx->cr0 = vcpu_regs->cr0;
ectx->cr3 = vcpu_regs->cr3;
ctx->cr4 = vcpu_regs->cr4;
set_vcpu_mode(vcpu, vcpu_regs->cs_ar, vcpu_regs->ia32_efer,
vcpu_regs->cr0);
}
static struct acrn_regs realmode_init_vregs = {
.gdt = {
.limit = 0xFFFFU,
.base = 0UL,
},
.idt = {
.limit = 0xFFFFU,
.base = 0UL,
},
.cs_ar = REAL_MODE_CODE_SEG_AR,
.cs_sel = REAL_MODE_BSP_INIT_CODE_SEL,
.cs_base = 0xFFFF0000UL,
.cs_limit = 0xFFFFU,
.rip = 0xFFF0UL,
.cr0 = CR0_ET | CR0_NE,
.cr3 = 0UL,
.cr4 = 0UL,
};
static uint64_t init_vgdt[] = {
0x0UL,
0x0UL,
0x00CF9B000000FFFFUL, /* Linear Code */
0x00CF93000000FFFFUL, /* Linear Data */
};
static struct acrn_regs protect_mode_init_vregs = {
.cs_ar = PROTECTED_MODE_CODE_SEG_AR,
.cs_limit = PROTECTED_MODE_SEG_LIMIT,
.cs_sel = 0x10U,
.cr0 = CR0_ET | CR0_NE | CR0_PE,
.ds_sel = 0x18U,
.ss_sel = 0x18U,
.es_sel = 0x18U,
};
bool sanitize_cr0_cr4_pattern(void)
{
bool ret = false;
if (is_valid_cr0_cr4(realmode_init_vregs.cr0, realmode_init_vregs.cr4) &&
is_valid_cr0_cr4(protect_mode_init_vregs.cr0, protect_mode_init_vregs.cr4)) {
ret = true;
} else {
pr_err("Wrong CR0/CR4 pattern: real %lx %lx; protected %lx %lx\n", realmode_init_vregs.cr0,
realmode_init_vregs.cr4, protect_mode_init_vregs.cr0, protect_mode_init_vregs.cr4);
}
return ret;
}
void reset_vcpu_regs(struct acrn_vcpu *vcpu, enum reset_mode mode)
{
set_vcpu_regs(vcpu, &realmode_init_vregs);
/*
* According to SDM Vol3 "Table 9-1. IA-32 and Intel 64 Processor States Following Power-up, Reset, or INIT",
* for some registers, the state following INIT is different from the state following Power-up, Reset.
* (For all registers mentioned in Table 9-1, the state following Power-up and following Reset are same.)
*
* To distinguish this kind of case for vCPU:
* - If the state following INIT is same as the state following Power-up/Reset, handle it in
* set_vcpu_regs above.
* - Otherwise, handle it below.
*/
if (mode != INIT_RESET) {
struct ext_context *ectx = &(vcpu->arch.contexts[vcpu->arch.cur_context].ext_ctx);
/* IA32_TSC_AUX: 0 following Power-up/Reset, unchanged following INIT */
ectx->tsc_aux = 0UL;
}
}
void init_vcpu_protect_mode_regs(struct acrn_vcpu *vcpu, uint64_t vgdt_base_gpa)
{
struct acrn_regs vcpu_regs;
(void)memcpy_s((void *)&vcpu_regs, sizeof(struct acrn_regs),
(void *)&protect_mode_init_vregs, sizeof(struct acrn_regs));
vcpu_regs.gdt.base = vgdt_base_gpa;
vcpu_regs.gdt.limit = sizeof(init_vgdt) - 1U;
(void)copy_to_gpa(vcpu->vm, &init_vgdt, vgdt_base_gpa, sizeof(init_vgdt));
set_vcpu_regs(vcpu, &vcpu_regs);
}
void set_vcpu_startup_entry(struct acrn_vcpu *vcpu, uint64_t entry)
{
struct ext_context *ectx;
ectx = &(vcpu->arch.contexts[vcpu->arch.cur_context].ext_ctx);
ectx->cs.selector = (uint16_t)((entry >> 4U) & 0xFFFFU);
ectx->cs.base = ectx->cs.selector << 4U;
vcpu_set_rip(vcpu, 0UL);
}
/*
* @pre vm != NULL && rtn_vcpu_handle != NULL
*/
int32_t create_vcpu(uint16_t pcpu_id, struct acrn_vm *vm, struct acrn_vcpu **rtn_vcpu_handle)
{
struct acrn_vcpu *vcpu;
uint16_t vcpu_id;
int32_t ret;
pr_info("Creating VCPU working on PCPU%hu", pcpu_id);
/*
* vcpu->vcpu_id = vm->hw.created_vcpus;
* vm->hw.created_vcpus++;
*/
vcpu_id = vm->hw.created_vcpus;
if (vcpu_id < MAX_VCPUS_PER_VM) {
/* Allocate memory for VCPU */
vcpu = &(vm->hw.vcpu_array[vcpu_id]);
(void)memset((void *)vcpu, 0U, sizeof(struct acrn_vcpu));
/* Initialize CPU ID for this VCPU */
vcpu->vcpu_id = vcpu_id;
per_cpu(ever_run_vcpu, pcpu_id) = vcpu;
/* Initialize the parent VM reference */
vcpu->vm = vm;
/* Initialize the virtual ID for this VCPU */
/* FIXME:
* We have assumption that we always destroys vcpus in one
* shot (like when vm is destroyed). If we need to support
* specific vcpu destroy on fly, this vcpu_id assignment
* needs revise.
*/
pr_info("Create VM%d-VCPU%d, Role: %s",
vcpu->vm->vm_id, vcpu->vcpu_id,
is_vcpu_bsp(vcpu) ? "PRIMARY" : "SECONDARY");
/*
* If the logical processor is in VMX non-root operation and
* the "enable VPID" VM-execution control is 1, the current VPID
* is the value of the VPID VM-execution control field in the VMCS.
*
* This assignment guarantees a unique non-zero per vcpu vpid at runtime.
*/
vcpu->arch.vpid = ALLOCATED_MIN_L1_VPID + (vm->vm_id * MAX_VCPUS_PER_VM) + vcpu->vcpu_id;
/*
* ACRN uses the following approach to manage VT-d PI notification vectors:
* Allocate unique Activation Notification Vectors (ANV) for each vCPU that
* belongs to the same pCPU, the ANVs need only be unique within each pCPU,
* not across all vCPUs. The max numbers of vCPUs may be running on top of
* a pCPU is CONFIG_MAX_VM_NUM, since ACRN does not support 2 vCPUs of same
* VM running on top of same pCPU. This reduces # of pre-allocated ANVs for
* posted interrupts to CONFIG_MAX_VM_NUM, and enables ACRN to avoid switching
* between active and wake-up vector values in the posted interrupt descriptor
* on vCPU scheduling state changes.
*
* We maintain a per-pCPU array of vCPUs, and use vm_id as the index to the
* vCPU array
*/
per_cpu(vcpu_array, pcpu_id)[vm->vm_id] = vcpu;
/*
* Use vm_id as the index to indicate the posted interrupt IRQ/vector pair that are
* assigned to this vCPU:
* 0: first posted interrupt IRQs/vector pair (POSTED_INTR_IRQ/POSTED_INTR_VECTOR)
* ...
* CONFIG_MAX_VM_NUM-1: last posted interrupt IRQs/vector pair
* ((POSTED_INTR_IRQ + CONFIG_MAX_VM_NUM - 1U)/(POSTED_INTR_VECTOR + CONFIG_MAX_VM_NUM - 1U)
*/
vcpu->arch.pid.control.bits.nv = POSTED_INTR_VECTOR + vm->vm_id;
/* ACRN does not support vCPU migration, one vCPU always runs on
* the same pCPU, so PI's ndst is never changed after startup.
*/
vcpu->arch.pid.control.bits.ndst = per_cpu(lapic_id, pcpu_id);
/* Create per vcpu vlapic */
vlapic_create(vcpu, pcpu_id);
if (!vm_hide_mtrr(vm)) {
init_vmtrr(vcpu);
}
/* Populate the return handle */
*rtn_vcpu_handle = vcpu;
vcpu_set_state(vcpu, VCPU_INIT);
init_xsave(vcpu);
vcpu_reset_internal(vcpu, POWER_ON_RESET);
(void)memset((void *)&vcpu->req, 0U, sizeof(struct io_request));
vm->hw.created_vcpus++;
ret = 0;
} else {
pr_err("%s, vcpu id is invalid!\n", __func__);
ret = -EINVAL;
}
return ret;
}
/**
* @pre ctx != NULL
*/
static inline int32_t exec_vmentry(struct run_context *ctx, int32_t launch_type, int32_t ibrs_type)
{
#ifdef CONFIG_L1D_FLUSH_VMENTRY_ENABLED
cpu_l1d_flush();
#endif
/* Mitigation for MDS vulnerability, overwrite CPU internal buffers */
cpu_internal_buffers_clear();
return vmx_vmrun(ctx, launch_type, ibrs_type);
}
/*
* @pre vcpu != NULL
*/
static void write_cached_registers(struct acrn_vcpu *vcpu)
{
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
if (bitmap_test_and_clear_nolock(CPU_REG_RIP, &vcpu->reg_updated)) {
exec_vmwrite(VMX_GUEST_RIP, ctx->rip);
}
if (bitmap_test_and_clear_nolock(CPU_REG_RSP, &vcpu->reg_updated)) {
exec_vmwrite(VMX_GUEST_RSP, ctx->cpu_regs.regs.rsp);
}
if (bitmap_test_and_clear_nolock(CPU_REG_EFER, &vcpu->reg_updated)) {
exec_vmwrite64(VMX_GUEST_IA32_EFER_FULL, ctx->ia32_efer);
}
if (bitmap_test_and_clear_nolock(CPU_REG_RFLAGS, &vcpu->reg_updated)) {
exec_vmwrite(VMX_GUEST_RFLAGS, ctx->rflags);
}
/*
* Currently, updating CR0/CR4 here is only designed for world
* switching. There should no other module request updating
* CR0/CR4 here.
*/
if (bitmap_test_and_clear_nolock(CPU_REG_CR0, &vcpu->reg_updated)) {
vcpu_set_cr0(vcpu, ctx->cr0);
}
if (bitmap_test_and_clear_nolock(CPU_REG_CR4, &vcpu->reg_updated)) {
vcpu_set_cr4(vcpu, ctx->cr4);
}
}
/*
* @pre vcpu != NULL
*/
int32_t run_vcpu(struct acrn_vcpu *vcpu)
{
uint32_t cs_attr;
uint64_t ia32_efer, cr0;
struct run_context *ctx =
&vcpu->arch.contexts[vcpu->arch.cur_context].run_ctx;
int32_t status = 0;
int32_t ibrs_type = get_ibrs_type();
if (vcpu->reg_updated != 0UL) {
write_cached_registers(vcpu);
}
if (is_vcpu_in_l2_guest(vcpu)) {
int32_t launch_type;
if (vcpu->launched) {
/* for nested VM-exits that don't need to be reflected to L1 hypervisor */
launch_type = VM_RESUME;
} else {
/* for VMEntry case, VMCS02 was VMCLEARed by ACRN */
launch_type = VM_LAUNCH;
vcpu->launched = true;
}
status = exec_vmentry(ctx, launch_type, ibrs_type);
} else {
/* If this VCPU is not already launched, launch it */
if (!vcpu->launched) {
pr_info("VM %d Starting VCPU %hu",
vcpu->vm->vm_id, vcpu->vcpu_id);
if (vcpu->arch.vpid != 0U) {
exec_vmwrite16(VMX_VPID, vcpu->arch.vpid);
}
/*
* A power-up or a reset invalidates all linear mappings,
* guest-physical mappings, and combined mappings
*/
flush_vpid_global();
#ifdef CONFIG_HYPERV_ENABLED
if (is_vcpu_bsp(vcpu)) {
hyperv_init_time(vcpu->vm);
}
#endif
/* Set vcpu launched */
vcpu->launched = true;
/* avoid VMCS recycling RSB usage, set IBPB.
* NOTE: this should be done for any time vmcs got switch
* currently, there is no other place to do vmcs switch
* Please add IBPB set for future vmcs switch case(like trusty)
*/
if (ibrs_type == IBRS_RAW) {
msr_write(MSR_IA32_PRED_CMD, PRED_SET_IBPB);
}
/* Launch the VM */
status = exec_vmentry(ctx, VM_LAUNCH, ibrs_type);
/* See if VM launched successfully */
if (status == 0) {
if (is_vcpu_bsp(vcpu)) {
pr_info("VM %d VCPU %hu successfully launched",
vcpu->vm->vm_id, vcpu->vcpu_id);
}
}
} else {
/* This VCPU was already launched, check if the last guest
* instruction needs to be repeated and resume VCPU accordingly
*/
if (vcpu->arch.inst_len != 0U) {
exec_vmwrite(VMX_GUEST_RIP, vcpu_get_rip(vcpu) + vcpu->arch.inst_len);
}
/* Resume the VM */
status = exec_vmentry(ctx, VM_RESUME, ibrs_type);
}
cs_attr = exec_vmread32(VMX_GUEST_CS_ATTR);
ia32_efer = vcpu_get_efer(vcpu);
cr0 = vcpu_get_cr0(vcpu);
set_vcpu_mode(vcpu, cs_attr, ia32_efer, cr0);
}
vcpu->reg_cached = 0UL;
/* Obtain current VCPU instruction length */
vcpu->arch.inst_len = exec_vmread32(VMX_EXIT_INSTR_LEN);
/* Obtain VM exit reason */
vcpu->arch.exit_reason = exec_vmread32(VMX_EXIT_REASON);
if (status != 0) {
/* refer to 64-ia32 spec section 24.9.1 volume#3 */
if ((vcpu->arch.exit_reason & VMX_VMENTRY_FAIL) != 0U) {
pr_fatal("vmentry fail reason=%lx", vcpu->arch.exit_reason);
} else {
pr_fatal("vmexit fail err_inst=%x", exec_vmread32(VMX_INSTR_ERROR));
}
ASSERT(status == 0, "vm fail");
}
return status;
}
/*
* @pre vcpu != NULL
* @pre vcpu->state == VCPU_ZOMBIE
*/
void offline_vcpu(struct acrn_vcpu *vcpu)
{
vlapic_free(vcpu);
per_cpu(ever_run_vcpu, pcpuid_from_vcpu(vcpu)) = NULL;
/* This operation must be atomic to avoid contention with posted interrupt handler */
per_cpu(vcpu_array, pcpuid_from_vcpu(vcpu))[vcpu->vm->vm_id] = NULL;
vcpu_set_state(vcpu, VCPU_OFFLINE);
}
void kick_vcpu(struct acrn_vcpu *vcpu)
{
uint16_t pcpu_id = pcpuid_from_vcpu(vcpu);
if ((get_pcpu_id() != pcpu_id) &&
(per_cpu(vmcs_run, pcpu_id) == vcpu->arch.vmcs)) {
if (is_lapic_pt_enabled(vcpu)) {
/* For lapic-pt vCPUs */
send_single_init(pcpu_id);
} else {
send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
}
}
}
/*
* @pre (&vcpu->stack[CONFIG_STACK_SIZE] & (CPU_STACK_ALIGN - 1UL)) == 0
*/
static uint64_t build_stack_frame(struct acrn_vcpu *vcpu)
{
uint64_t stacktop = (uint64_t)&vcpu->stack[CONFIG_STACK_SIZE];
struct stack_frame *frame;
uint64_t *ret;
frame = (struct stack_frame *)stacktop;
frame -= 1;
frame->magic = SP_BOTTOM_MAGIC;
frame->rip = (uint64_t)vcpu->thread_obj.thread_entry; /*return address*/
frame->rflag = 0UL;
frame->rbx = 0UL;
frame->rbp = 0UL;
frame->r12 = 0UL;
frame->r13 = 0UL;
frame->r14 = 0UL;
frame->r15 = 0UL;
frame->rdi = (uint64_t)&vcpu->thread_obj;
ret = &frame->rdi;
return (uint64_t) ret;
}
/* NOTE:
* vcpu should be paused before call this function.
* @pre vcpu != NULL
* @pre vcpu->state == VCPU_ZOMBIE
*/
void reset_vcpu(struct acrn_vcpu *vcpu, enum reset_mode mode)
{
pr_dbg("vcpu%hu reset", vcpu->vcpu_id);
vcpu_reset_internal(vcpu, mode);
vcpu_set_state(vcpu, VCPU_INIT);
}
void zombie_vcpu(struct acrn_vcpu *vcpu, enum vcpu_state new_state)
{
enum vcpu_state prev_state;
uint16_t pcpu_id = pcpuid_from_vcpu(vcpu);
pr_dbg("vcpu%hu paused, new state: %d", vcpu->vcpu_id, new_state);
if (((vcpu->state == VCPU_RUNNING) || (vcpu->state == VCPU_INIT)) && (new_state == VCPU_ZOMBIE)) {
prev_state = vcpu->state;
vcpu_set_state(vcpu, new_state);
if (prev_state == VCPU_RUNNING) {
if (pcpu_id == get_pcpu_id()) {
sleep_thread(&vcpu->thread_obj);
} else {
sleep_thread_sync(&vcpu->thread_obj);
}
}
}
}
void save_xsave_area(__unused struct acrn_vcpu *vcpu, struct ext_context *ectx)
{
if (pcpu_has_cap(X86_FEATURE_XSAVES)) {
ectx->xcr0 = read_xcr(0);
write_xcr(0, ectx->xcr0 | XSAVE_SSE);
xsaves(&ectx->xs_area, UINT64_MAX);
}
}
void rstore_xsave_area(const struct acrn_vcpu *vcpu, const struct ext_context *ectx)
{
if (pcpu_has_cap(X86_FEATURE_XSAVES)) {
/*
* Restore XSAVE area if any of the following conditions is met:
* 1. "vcpu->launched" is false (state initialization for guest)
* 2. "vcpu->arch.xsave_enabled" is true (state restoring for guest)
*
* Before vCPU is launched, condition 1 is satisfied.
* After vCPU is launched, condition 2 is satisfied because
* that "vcpu->arch.xsave_enabled" is consistent with pcpu_has_cap(X86_FEATURE_XSAVES).
*
* Therefore, the check against "vcpu->launched" and "vcpu->arch.xsave_enabled" can be eliminated here.
*/
write_xcr(0, ectx->xcr0 | XSAVE_SSE);
msr_write(MSR_IA32_XSS, vcpu_get_guest_msr(vcpu, MSR_IA32_XSS));
xrstors(&ectx->xs_area, UINT64_MAX);
write_xcr(0, ectx->xcr0);
}
}
/* TODO:
* Now we have switch_out and switch_in callbacks for each thread_object, and schedule
* will call them every thread switch. We can implement lazy context swtich , which
* only do context swtich when really need.
*/
static void context_switch_out(struct thread_object *prev)
{
struct acrn_vcpu *vcpu = container_of(prev, struct acrn_vcpu, thread_obj);
struct ext_context *ectx = &(vcpu->arch.contexts[vcpu->arch.cur_context].ext_ctx);
/* We don't flush TLB as we assume each vcpu has different vpid */
ectx->ia32_star = msr_read(MSR_IA32_STAR);
ectx->ia32_cstar = msr_read(MSR_IA32_CSTAR);
ectx->ia32_lstar = msr_read(MSR_IA32_LSTAR);
ectx->ia32_fmask = msr_read(MSR_IA32_FMASK);
ectx->ia32_kernel_gs_base = msr_read(MSR_IA32_KERNEL_GS_BASE);
ectx->tsc_aux = msr_read(MSR_IA32_TSC_AUX);
save_xsave_area(vcpu, ectx);
}
static void context_switch_in(struct thread_object *next)
{
struct acrn_vcpu *vcpu = container_of(next, struct acrn_vcpu, thread_obj);
struct ext_context *ectx = &(vcpu->arch.contexts[vcpu->arch.cur_context].ext_ctx);
uint64_t vmsr_val;
load_vmcs(vcpu);
msr_write(MSR_IA32_STAR, ectx->ia32_star);
msr_write(MSR_IA32_CSTAR, ectx->ia32_cstar);
msr_write(MSR_IA32_LSTAR, ectx->ia32_lstar);
msr_write(MSR_IA32_FMASK, ectx->ia32_fmask);
msr_write(MSR_IA32_KERNEL_GS_BASE, ectx->ia32_kernel_gs_base);
msr_write(MSR_IA32_TSC_AUX, ectx->tsc_aux);
if (pcpu_has_cap(X86_FEATURE_WAITPKG)) {
vmsr_val = vcpu_get_guest_msr(vcpu, MSR_IA32_UMWAIT_CONTROL);
if (vmsr_val != msr_read(MSR_IA32_UMWAIT_CONTROL)) {
msr_write(MSR_IA32_UMWAIT_CONTROL, vmsr_val);
}
}
load_iwkey(vcpu);
rstore_xsave_area(vcpu, ectx);
}
/**
* @pre vcpu != NULL
* @pre vcpu->state == VCPU_INIT
*/
void launch_vcpu(struct acrn_vcpu *vcpu)
{
uint16_t pcpu_id = pcpuid_from_vcpu(vcpu);
pr_dbg("vcpu%hu scheduled on pcpu%hu", vcpu->vcpu_id, pcpu_id);
vcpu_set_state(vcpu, VCPU_RUNNING);
wake_thread(&vcpu->thread_obj);
}
/* help function for vcpu create */
int32_t prepare_vcpu(struct acrn_vm *vm, uint16_t pcpu_id)
{
int32_t ret, i;
struct acrn_vcpu *vcpu = NULL;
char thread_name[16];
ret = create_vcpu(pcpu_id, vm, &vcpu);
if (ret == 0) {
snprintf(thread_name, 16U, "vm%hu:vcpu%hu", vm->vm_id, vcpu->vcpu_id);
(void)strncpy_s(vcpu->thread_obj.name, 16U, thread_name, 16U);
vcpu->thread_obj.sched_ctl = &per_cpu(sched_ctl, pcpu_id);
vcpu->thread_obj.thread_entry = vcpu_thread;
vcpu->thread_obj.pcpu_id = pcpu_id;
/* vcpu->thread_obj.notify_mode is initialized in vcpu_reset_internal() when create vcpu */
vcpu->thread_obj.host_sp = build_stack_frame(vcpu);
vcpu->thread_obj.switch_out = context_switch_out;
vcpu->thread_obj.switch_in = context_switch_in;
vcpu->thread_obj.priority = get_vm_config(vm->vm_id)->vm_prio;
init_thread_data(&vcpu->thread_obj);
for (i = 0; i < VCPU_EVENT_NUM; i++) {
init_event(&vcpu->events[i]);
}
}
return ret;
}
/**
* @pre vcpu != NULL
*/
uint16_t pcpuid_from_vcpu(const struct acrn_vcpu *vcpu)
{
return sched_get_pcpuid(&vcpu->thread_obj);
}
uint64_t vcpumask2pcpumask(struct acrn_vm *vm, uint64_t vdmask)
{
uint16_t vcpu_id;
uint64_t dmask = 0UL;
struct acrn_vcpu *vcpu;
for (vcpu_id = 0U; vcpu_id < vm->hw.created_vcpus; vcpu_id++) {
if ((vdmask & (1UL << vcpu_id)) != 0UL) {
vcpu = vcpu_from_vid(vm, vcpu_id);
bitmap_set_nolock(pcpuid_from_vcpu(vcpu), &dmask);
}
}
return dmask;
}
/*
* @brief handle posted interrupts
*
* VT-d PI handler, find the corresponding vCPU for this IRQ,
* if the associated PID's bit ON is set, wake it up.
*
* shutdown_vm would unregister the devices before offline_vcpu is called,
* so spinlock is not needed to protect access to vcpu_array and vcpu.
*
* @pre (vcpu_index < CONFIG_MAX_VM_NUM) && (get_pi_desc(get_cpu_var(vcpu_array)[vcpu_index]) != NULL)
*/
void vcpu_handle_pi_notification(uint32_t vcpu_index)
{
struct acrn_vcpu *vcpu = get_cpu_var(vcpu_array)[vcpu_index];
ASSERT(vcpu_index < CONFIG_MAX_VM_NUM, "");
if (vcpu != NULL) {
struct pi_desc *pid = get_pi_desc(vcpu);
if (bitmap_test(POSTED_INTR_ON, &(pid->control.value))) {
/*
* Perform same as vlapic_accept_intr():
* Wake up the waiting thread, set the NEED_RESCHEDULE flag,
* at a point schedule() will be called to make scheduling decisions.
*
* Record this request as ACRN_REQUEST_EVENT,
* so that vlapic_inject_intr() will sync PIR to vIRR
*/
signal_event(&vcpu->events[VCPU_EVENT_VIRTUAL_INTERRUPT]);
vcpu_make_request(vcpu, ACRN_REQUEST_EVENT);
}
}
}
/*
* @brief Update the state of vCPU and state of vlapic
*
* The vlapic state of VM shall be updated for some vCPU
* state update cases, such as from VCPU_INIT to VCPU_RUNNING.
* @pre (vcpu != NULL)
*/
void vcpu_set_state(struct acrn_vcpu *vcpu, enum vcpu_state new_state)
{
vcpu->state = new_state;
update_vm_vlapic_state(vcpu->vm);
}
Reference in New Issue View Git Blame Copy Permalink