hv: vpci_cleanup for VMs of type POST_LAUNCHED_VM

ACRN cleans up the IOMMU domain and other data structures that
represents the state of device assigment to POST_LAUNCHED_VM. This is
with the help of hypercalls from SOS DM. Under scenarios where DM
execution can get terminated abruptly or due to bugs in DM, hypercalls
responsible for cleaning up ACRN cannot happen. This leaves ACRN
device representation/resource assignment in an incorrect state.
This patch cleans up the IOMMU resource and other data structures
upon shutdown of POST_LAUNCHED_VM.

Tracked-On: #2700
Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/common/hypercall.c

@@ -991,6 +991,7 @@ hcall_reset_ptdev_intr_info(struct acrn_vm *vm, uint16_t vmid, uint64_t param)
 		if (copy_from_gpa(vm, &irq, param, sizeof(irq)) != 0) {
 			pr_err("%s: Unable copy param to vm\n", __func__);
 		} else if (irq.type == IRQ_INTX) {
+			vpci_reset_ptdev_intr_info(target_vm, irq.virt_bdf, irq.phys_bdf);
 			ptirq_remove_intx_remapping(target_vm,
 					irq.is.intx.virt_pin,
 					irq.is.intx.pic_pin);

hypervisor/dm/vpci/vpci.c

@@ -250,8 +250,12 @@ void vpci_cleanup(const struct acrn_vm *vm)
 		sharing_mode_vpci_deinit(vm);
 		break;
 
+	case POST_LAUNCHED_VM:
+		post_launched_vm_vpci_deinit(vm);
+		break;
+
 	default:
-		/* Nothing to do for other vm types */
+		/* Unsupported VM type - Do nothing */
 		break;
 	}
 }
@@ -595,6 +599,52 @@ void sharing_mode_vpci_deinit(const struct acrn_vm *vm)
 	}
 }
 
+/**
+ * @pre vm != NULL
+ * @pre vm->vpci.pci_vdev_cnt <= CONFIG_MAX_PCI_DEV_NUM
+ * @pre is_postlaunched_vm(vm) == true
+ */
+void post_launched_vm_vpci_deinit(const struct acrn_vm *vm)
+{
+	struct acrn_vm *sos_vm;
+	uint32_t i;
+	struct pci_vdev *vdev;
+	int32_t ret;
+	/* PCI resources
+	 * 1) IOMMU domain switch
+	 * 2) Relese UOS MSI host IRQ/IRTE
+	 * 3) Update vdev info in SOS vdev
+	 * Cleanup mentioned above is  taken care when DM releases UOS resources
+	 * during a UOS reboot or shutdown
+	 * In the following cases, where DM does not get chance to cleanup
+	 * 1) DM crash/segfault
+	 * 2) SOS triple fault/hang
+	 * 3) SOS reboot before shutting down POST_LAUNCHED_VMs
+	 * ACRN must cleanup
+	 */
+	sos_vm = get_sos_vm();
+	for (i = 0U; i < sos_vm->vpci.pci_vdev_cnt; i++) {
+		vdev = (struct pci_vdev *)&(sos_vm->vpci.pci_vdevs[i]);
+
+		if (vdev->vpci->vm == vm) {
+			ret = move_pt_device(vm->iommu, sos_vm->iommu, (uint8_t)vdev->pdev->bdf.bits.b,
+					(uint8_t)(vdev->pdev->bdf.value & 0xFFU));
+			if (ret != 0) {
+				panic("failed to assign iommu device!");
+			}
+
+			vmsi_deinit(vdev);
+
+			vmsix_deinit(vdev);
+
+			/* Move vdev pointers back to SOS*/
+			vdev->vpci = (struct acrn_vpci *) &sos_vm->vpci;
+			/* vbdf equals to pbdf in sos */
+			vdev->vbdf.value = vdev->pdev->bdf.value;
+		}
+	}
+}
+
 /**
  * @pre target_vm != NULL
  */

hypervisor/dm/vpci/vpci_priv.h

@@ -114,5 +114,6 @@ void sharing_mode_cfgread(struct acrn_vpci *vpci, union pci_bdf bdf,
 void sharing_mode_cfgwrite(__unused struct acrn_vpci *vpci, union pci_bdf bdf,
 	uint32_t offset, uint32_t bytes, uint32_t val);
 void sharing_mode_vpci_deinit(const struct acrn_vm *vm);
+void post_launched_vm_vpci_deinit(const struct acrn_vm *vm);
 
 #endif /* VPCI_PRIV_H_ */