HV: Reset physical core of lapic_pt vm when shutdown

The physical core of lapic_pt vm should be reset for security and
correctness when shutdown the vm.

Tracked-On: #2991
Signed-off-by: Kaige Fu <kaige.fu@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/arch/x86/cpu.c

@@ -332,6 +332,17 @@ bool start_cpus(uint64_t mask)
 	return ((pcpu_active_bitmap & mask) == mask);
 }
 
+void wait_pcpus_offline(uint64_t mask)
+{
+	uint32_t timeout;
+
+	timeout = CPU_DOWN_TIMEOUT * 1000U;
+	while (((pcpu_active_bitmap & mask) != 0UL) && (timeout != 0U)) {
+		udelay(10U);
+		timeout -= 10U;
+	}
+}
+
 void stop_cpus(void)
 {
 	uint16_t pcpu_id, expected_up;
@@ -390,13 +401,14 @@ void cpu_dead(void)
 	int32_t halt = 1;
 	uint16_t pcpu_id = get_cpu_id();
 
-	if (bitmap_test_and_clear_lock(pcpu_id, &pcpu_active_bitmap)) {
+	if (bitmap_test(pcpu_id, &pcpu_active_bitmap)) {
 		/* clean up native stuff */
 		vmx_off();
 		cache_flush_invalidate_all();
 
 		/* Set state to show CPU is dead */
 		cpu_set_current_state(pcpu_id, PCPU_STATE_DEAD);
+		bitmap_clear_nolock(pcpu_id, &pcpu_active_bitmap);
 
 		/* Halt the CPU */
 		do {

hypervisor/arch/x86/guest/vm.c

@@ -456,9 +456,10 @@ int32_t create_vm(uint16_t vm_id, struct acrn_vm_config *vm_config, struct acrn_
 int32_t shutdown_vm(struct acrn_vm *vm)
 {
 	uint16_t i;
+	uint64_t mask = 0UL;
 	struct acrn_vcpu *vcpu = NULL;
 	struct acrn_vm_config *vm_config = NULL;
-	int32_t ret;
+	int32_t ret = 0;
 
 	pause_vm(vm);
 
@@ -469,6 +470,18 @@ int32_t shutdown_vm(struct acrn_vm *vm)
 		foreach_vcpu(i, vm, vcpu) {
 			reset_vcpu(vcpu);
 			offline_vcpu(vcpu);
+
+			if (is_lapic_pt(vm)) {
+				bitmap_set_nolock(vcpu->pcpu_id, &mask);
+				make_pcpu_offline(vcpu->pcpu_id);
+			}
+		}
+
+		wait_pcpus_offline(mask);
+
+		if (is_lapic_pt(vm) && !start_cpus(mask)) {
+			pr_fatal("Failed to start all cpus in mask(0x%llx)", mask);
+			ret = -ETIMEDOUT;
 		}
 
 		vm_config = get_vm_config(vm->vm_id);

hypervisor/include/arch/x86/cpu.h

@@ -260,6 +260,7 @@ void load_cpu_state_data(void);
 void init_cpu_pre(uint16_t pcpu_id_args);
 void init_cpu_post(uint16_t pcpu_id);
 bool start_cpus(uint64_t mask);
+void wait_pcpus_offline(uint64_t mask);
 void stop_cpus(void);
 void wait_sync_change(uint64_t *sync, uint64_t wake_sync);
 

hypervisor/include/lib/errno.h

@@ -23,5 +23,7 @@
 #define ENODEV		19
 /** Indicates that argument is not valid. */
 #define EINVAL		22
+/** Indicates that timeout occurs. */
+#define ETIMEDOUT	110
 
 #endif /* ERRNO_H */