hv: release IOMMU irte when releasing ptirq remapping entries

IRTE is freed if ptirq entry is released from remove_msix_remapping() or
remove_intx_remapping(). But if it's called from ptdev_release_all_entries(),
e.g. SOS shutdown/reboot, IRTE is not freed.

This patch adds a release_cb() callback function to do any architectural
specific cleanup. In x86, it's used to release IRTE.

On VM shutdown, vpci_cleanup() needs to remove MSI/MSI-X remapping on
ptirq entries, thus it should be called before ptdev_release_all_entries().

Tracked-On: #2700
Signed-off-by: Sainath Grandhi <sainath.grandhi@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>

hypervisor/arch/x86/guest/assign.c

@@ -74,6 +74,24 @@ static uint32_t calculate_logical_dest_mask(uint64_t pdmask)
 	return dest_mask;
 }
 
+/**
+ * @pre entry != NULL
+ */
+static void ptirq_free_irte(const struct ptirq_remapping_info *entry)
+{
+	struct intr_source intr_src;
+
+	if (entry->intr_type == PTDEV_INTR_MSI) {
+		intr_src.is_msi = true;
+		intr_src.src.msi.value = entry->phys_sid.msi_id.bdf;
+	} else {
+		intr_src.is_msi = false;
+		intr_src.src.ioapic_id = ioapic_irq_to_ioapic_id(entry->allocated_pirq);
+	}
+
+	dmar_free_irte(intr_src, (uint16_t)entry->allocated_pirq);
+}
+
 static void ptirq_build_physical_msi(struct acrn_vm *vm, struct ptirq_msi_info *info,
 		const struct ptirq_remapping_info *entry, uint32_t vector)
 {
@@ -270,6 +288,8 @@ static struct ptirq_remapping_info *add_msix_remapping(struct acrn_vm *vm,
 			if (entry != NULL) {
 				entry->phys_sid.value = phys_sid.value;
 				entry->virt_sid.value = virt_sid.value;
+				entry->release_cb = ptirq_free_irte;
+
 				/* update msi source and active entry */
 				if (ptirq_activate_entry(entry, IRQ_INVALID) < 0) {
 					ptirq_release_entry(entry);
@@ -355,6 +375,7 @@ static struct ptirq_remapping_info *add_intx_remapping(struct acrn_vm *vm, uint3
 				if (entry != NULL) {
 					entry->phys_sid.value = phys_sid.value;
 					entry->virt_sid.value = virt_sid.value;
+					entry->release_cb = ptirq_free_irte;
 
 					/* activate entry */
 					if (ptirq_activate_entry(entry, phys_irq) < 0) {

hypervisor/arch/x86/guest/vm.c

@@ -486,10 +486,10 @@ int32_t shutdown_vm(struct acrn_vm *vm)
 		vm_config = get_vm_config(vm->vm_id);
 		vm_config->guest_flags &= ~DM_OWNED_GUEST_FLAG_MASK;
 
-		ptdev_release_all_entries(vm);
-
 		vpci_cleanup(vm);
 
+		ptdev_release_all_entries(vm);
+
 		/* Free iommu */
 		if (vm->iommu != NULL) {
 			destroy_iommu_domain(vm->iommu);

hypervisor/common/ptdev.c

@@ -198,6 +198,9 @@ void ptdev_release_all_entries(const struct acrn_vm *vm)
 		entry = &ptirq_entries[idx];
 		if ((entry->vm == vm) && is_entry_active(entry)) {
 			spinlock_obtain(&ptdev_lock);
+			if (entry->release_cb != NULL) {
+				entry->release_cb(entry);
+			}
 			ptirq_deactivate_entry(entry);
 			ptirq_release_entry(entry);
 			spinlock_release(&ptdev_lock);

hypervisor/include/common/ptdev.h

@@ -120,6 +120,9 @@ struct ptirq_msi_info {
 	int32_t is_msix;	/* 0-MSI, 1-MSIX */
 };
 
+struct ptirq_remapping_info;
+typedef void (*ptirq_arch_release_fn_t)(const struct ptirq_remapping_info *entry);
+
 /* entry per each allocated irq/vector
  * it represents a pass-thru device's remapping data entry which collecting
  * information related with its vm and msi/intx mapping & interaction nodes
@@ -139,6 +142,7 @@ struct ptirq_remapping_info {
 
 	uint64_t intr_count;
 	struct hv_timer intr_delay_timer; /* used for delay intr injection */
+	ptirq_arch_release_fn_t release_cb;
 };
 
 extern struct ptirq_remapping_info ptirq_entries[CONFIG_MAX_PT_IRQ_ENTRIES];