dm:gvt:update gvt bars before other pci devices write bar address

The current design has the following problem:
uos kernel may update gvt bars' regions,
but ACRN-DM doesn't know this update.
ACRN-DM only know out-of-date gvt bar regions,
For simplicity, mark these bar regions as OOD bar regions.
uos kernel may allocate OOD bar regions for
other pci devices, which will result in ACRN-DM
bar regions inconsistency with uos kernel.

The new design is the following:
When other pci device update bar regions
(1) ACRN-DM updates gvt bars' regions
provided by a system file.
(2) ACRN-DM updates this pci device bar regions

v5 -> v6:
	* add more comments

v4 -> v5:
	* remove & for callback func assignment

v3 -> v4:
	* compare gpu bar address to avoid unnecessary
	* unregistered/registered operation

v2 -> v3:
	* call unregister_bar and register_bar when update gvt bars
	* update gvt reserved regions when update gvt bars

Tracked-On: projectacrn#4005

Signed-off-by: Junming Liu <junming.liu@intel.com>
Reviewed-by: Zhao Yakui <yakui.zhao@intel.com>
Reviewed-by: Liu XinYun <xinyun.liu@intel.com>
Reviewed-by: Shuo A Liu <shuo.a.liu@intel.com>
Acked-by: Yu Wang <yu1.wang@intel.com>

devicemodel/hw/pci/core.c

@@ -710,6 +710,19 @@ update_bar_address(struct vmctx *ctx, struct pci_vdev *dev, uint64_t addr,
 	if (decode)
 		unregister_bar(dev, idx);
 
+	/* TODO:Currently, we only reserve gvt mmio regions,
+	 * so ignore PCIBAR_IO when adjust_bar_region_with_reserved_bars.
+	 * If other devices also use reserved bar regions later,
+	 * need remove pcibar_type != PCIBAR_IO condition
+	 */
+	if(type != PCIBAR_IO && ctx->gvt_enabled)
+		/* uos kernel may update gvt bars' value,
+		 * but ACRN-DM doesn't know this update.
+		 * When other pci devices write bar address,
+		 * ACRN-DM need update vgpu bars' info.
+		 */
+		ctx->update_gvt_bar(ctx);
+
 	switch (type) {
 	case PCIBAR_IO:
 	case PCIBAR_MEM32:
@@ -867,30 +880,36 @@ pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx, uint64_t hostbase,
 	return 0;
 }
 
+void
+pci_emul_free_bar(struct pci_vdev *pdi, int idx)
+{
+	bool enabled;
+
+	if ((pdi->bar[idx].type != PCIBAR_NONE) &&
+		(pdi->bar[idx].type != PCIBAR_MEMHI64)){
+		/*
+		 * Check whether the bar is enabled or not,
+		 * if it is disabled then it should have been
+		 * unregistered in pci_emul_cmdsts_write.
+		 */
+		if (pdi->bar[idx].type == PCIBAR_IO)
+			enabled = porten(pdi);
+		else
+			enabled = memen(pdi);
+
+		if (enabled)
+			unregister_bar(pdi, idx);
+		pdi->bar[idx].type = PCIBAR_NONE;
+	}
+}
+
 void
 pci_emul_free_bars(struct pci_vdev *pdi)
 {
 	int i;
-	bool enabled;
 
-	for (i = 0; i < PCI_BARMAX; i++) {
-		if ((pdi->bar[i].type != PCIBAR_NONE) &&
-			(pdi->bar[i].type != PCIBAR_MEMHI64)){
-			/*
-			 * Check whether the bar is enabled or not,
-			 * if it is disabled then it should have been
-			 * unregistered in pci_emul_cmdsts_write.
-			 */
-			if (pdi->bar[i].type == PCIBAR_IO)
-				enabled = porten(pdi);
-			else
-				enabled = memen(pdi);
-
-			if (enabled)
-				unregister_bar(pdi, i);
-			pdi->bar[i].type = PCIBAR_NONE;
-		}
-	}
+	for (i = 0; i < PCI_BARMAX; i++)
+		pci_emul_free_bar(pdi, i);
 }
 
 #define	CAP_START_OFFSET	0x40

devicemodel/hw/pci/gvt.c

@@ -85,6 +85,70 @@ pci_gvt_read(struct vmctx *ctx, int vcpu, struct pci_vdev *pi,
 	return 0;
 }
 
+void
+update_gvt_bar(struct vmctx *ctx)
+{
+	char bar_path[PATH_MAX];
+	int bar_fd;
+	int ret;
+	char resource[76];
+	char *next;
+	uint64_t bar0_start_addr, bar0_end_addr, bar2_start_addr, bar2_end_addr;
+	int i;
+
+	/* "/sys/kernel/gvt/vmx/vgpu_bar_info" exposes vgpu bar regions. */
+	snprintf(bar_path, sizeof(bar_path),
+		"/sys/kernel/gvt/vm%d/vgpu_bar_info",
+		ctx->vmid);
+
+	if(access(bar_path, F_OK) == -1)
+		return;
+
+	bar_fd = open(bar_path, O_RDONLY);
+	if(bar_fd == -1){
+		perror("failed to open sys bar info\n");
+		return;
+	}
+
+	ret = pread(bar_fd, resource, 76, 0);
+
+	close(bar_fd);
+
+	if (ret < 76) {
+		perror("failed to read sys bar info\n");
+		return;
+	}
+
+	next = resource;
+	bar0_start_addr = strtoull(next, &next, 16);
+	bar0_end_addr = strtoull(next, &next, 16) + bar0_start_addr -1;
+	bar2_start_addr = strtoull(next, &next, 16);
+	bar2_end_addr = strtoull(next, &next, 16) + bar2_start_addr -1;
+
+	for(i = 0; i < REGION_NUMS; i++){
+		if(reserved_bar_regions[i].vdev &&
+			reserved_bar_regions[i].vdev == gvt_dev){
+			pci_emul_free_bar(gvt_dev, reserved_bar_regions[i].idx);
+		}
+	}
+
+	destory_mmio_rsvd_rgns(gvt_dev);
+
+	ret = create_mmio_rsvd_rgn(bar0_start_addr,
+                        bar0_end_addr, 0, PCIBAR_MEM32, gvt_dev);
+	if(ret != 0)
+		return;
+	ret = create_mmio_rsvd_rgn(bar2_start_addr,
+                        bar2_end_addr, 2, PCIBAR_MEM32, gvt_dev);
+	if(ret != 0)
+		return;
+
+	pci_emul_alloc_bar(gvt_dev, 0, PCIBAR_MEM32,
+			bar0_end_addr - bar0_start_addr + 1);
+	pci_emul_alloc_bar(gvt_dev, 2, PCIBAR_MEM32,
+			bar2_end_addr - bar2_start_addr + 1);
+}
+
 static int
 gvt_init_config(struct pci_gvt *gvt)
 {
@@ -142,6 +206,7 @@ gvt_init_config(struct pci_gvt *gvt)
 	}
 
 	ctx->gvt_enabled = true;
+	ctx->update_gvt_bar = &update_gvt_bar;
 
 	/* In GVT-g design, it only use pci bar0 and bar2,
 	 * So we need reserve bar0 region and bar2 region only

devicemodel/include/pci_core.h

@@ -277,6 +277,7 @@ int	pci_emul_alloc_bar(struct pci_vdev *pdi, int idx,
 int	pci_emul_alloc_pbar(struct pci_vdev *pdi, int idx,
 			    uint64_t hostbase, enum pcibar_type type,
 			    uint64_t size);
+void	pci_emul_free_bar(struct pci_vdev *pdi, int idx);
 void	pci_emul_free_bars(struct pci_vdev *pdi);
 int	pci_emul_add_capability(struct pci_vdev *dev, u_char *capdata,
 				int caplen);

devicemodel/include/vmmapi.h

@@ -70,6 +70,8 @@ struct vmctx {
 
 	/* if gvt-g is enabled for current VM */
 	bool gvt_enabled;
+
+	void (*update_gvt_bar)(struct vmctx *ctx);
 };
 
 #define	PROT_RW		(PROT_READ | PROT_WRITE)