diff --git a/hypervisor/arch/x86/guest/vmcs.c b/hypervisor/arch/x86/guest/vmcs.c index 6e2d76692..1efa37db6 100644 --- a/hypervisor/arch/x86/guest/vmcs.c +++ b/hypervisor/arch/x86/guest/vmcs.c @@ -562,6 +562,7 @@ void switch_apicv_mode_x2apic(struct acrn_vcpu *vcpu) * Disable posted interrupt processing * update x2apic msr bitmap for pass-thru * enable inteception only for ICR + * enable NMI exit as we will use NMI to kick vCPU thread * disable pre-emption for TSC DEADLINE MSR * Disable Register Virtualization and virtual interrupt delivery * Disable "use TPR shadow" @@ -572,6 +573,16 @@ void switch_apicv_mode_x2apic(struct acrn_vcpu *vcpu) if (is_apicv_advanced_feature_supported()) { value32 &= ~VMX_PINBASED_CTLS_POST_IRQ; } + + /* + * ACRN hypervisor needs to kick vCPU off VMX non-root mode to do some + * operations in hypervisor, such as interrupt/exception injection, EPT + * flush etc. For non lapic-pt vCPUs, we can use IPI to do so. But, it + * doesn't work for lapic-pt vCPUs as the IPI will be injected to VMs + * directly without vmexit. So, here we enable NMI-exiting and use NMI + * as notification signal after passthroughing the lapic to vCPU. + */ + value32 |= VMX_PINBASED_CTLS_NMI_EXIT; exec_vmwrite32(VMX_PIN_VM_EXEC_CONTROLS, value32); value32 = exec_vmread32(VMX_EXIT_CONTROLS); diff --git a/hypervisor/common/schedule.c b/hypervisor/common/schedule.c index 2f0cd1540..da6c785be 100644 --- a/hypervisor/common/schedule.c +++ b/hypervisor/common/schedule.c @@ -123,7 +123,7 @@ struct thread_object *sched_get_current(uint16_t pcpu_id) } /** - * @pre delmode == DEL_MODE_IPI || delmode == DEL_MODE_INIT + * @pre delmode == DEL_MODE_IPI || delmode == DEL_MODE_INIT || delmode == DEL_MODE_NMI */ void make_reschedule_request(uint16_t pcpu_id, uint16_t delmode) { @@ -138,6 +138,9 @@ void make_reschedule_request(uint16_t pcpu_id, uint16_t delmode) case DEL_MODE_INIT: send_single_init(pcpu_id); break; + case DEL_MODE_NMI: + send_single_nmi(pcpu_id); + break; default: ASSERT(false, "Unknown delivery mode %u for pCPU%u", delmode, pcpu_id); break; @@ -235,10 +238,20 @@ void kick_thread(const struct thread_object *obj) obtain_schedule_lock(pcpu_id, &rflag); if (is_running(obj)) { if (get_pcpu_id() != pcpu_id) { - send_single_ipi(pcpu_id, VECTOR_NOTIFY_VCPU); + if (obj->notify_mode == SCHED_NOTIFY_IPI) { + send_single_ipi(pcpu_id, VECTOR_NOTIFY_VCPU); + } else { + /* For lapic-pt vCPUs */ + send_single_nmi(pcpu_id); + } } } else if (is_runnable(obj)) { - make_reschedule_request(pcpu_id, DEL_MODE_IPI); + if (obj->notify_mode == SCHED_NOTIFY_IPI) { + make_reschedule_request(pcpu_id, DEL_MODE_IPI); + } else { + /* For lapic-pt vCPUs */ + make_reschedule_request(pcpu_id, DEL_MODE_NMI); + } } else { /* do nothing */ } diff --git a/hypervisor/include/common/schedule.h b/hypervisor/include/common/schedule.h index 9d7c9d398..164e34410 100644 --- a/hypervisor/include/common/schedule.h +++ b/hypervisor/include/common/schedule.h @@ -14,6 +14,7 @@ #define DEL_MODE_INIT (1U) #define DEL_MODE_IPI (2U) +#define DEL_MODE_NMI (3U) #define THREAD_DATA_SIZE (256U)