softirq: move softirq from hv_main to interrupt context

softirq shouldn't be bounded to vcpu thread. One issue for this
is shell (based on timer) can't work if we don't start any guest.

This change also is trying best to make softirq handler running
with irq enabled.

Also update the irq disable/enabel in vmexit handler to align
with the usage in vcpu_thread.

Tracked-On: #3387
Signed-off-by: Yin Fengwei <fengwei.yin@intel.com>
Acked-by: Anthony Xu <anthony.xu@intel.com>

hypervisor/arch/x86/guest/vmexit.c

@@ -224,9 +224,15 @@ int32_t vmexit_handler(struct acrn_vcpu *vcpu)
 			/* exit dispatch handling */
 			if (basic_exit_reason == VMX_EXIT_REASON_EXTERNAL_INTERRUPT) {
 				/* Handling external_interrupt should disable intr */
-				CPU_IRQ_DISABLE();
+				if (!is_lapic_pt_enabled(vcpu)) {
+					CPU_IRQ_DISABLE();
+				}
+
 				ret = dispatch->handler(vcpu);
-				CPU_IRQ_ENABLE();
+
+				if (!is_lapic_pt_enabled(vcpu)) {
+					CPU_IRQ_ENABLE();
+				}
 			} else {
 				ret = dispatch->handler(vcpu);
 			}

hypervisor/arch/x86/irq.c

@@ -361,6 +361,8 @@ void dispatch_interrupt(const struct intr_excp_ctx *ctx)
 	} else {
 		handle_spurious_interrupt(vr);
 	}
+
+	do_softirq();
 }
 
 void dispatch_exception(struct intr_excp_ctx *ctx)

hypervisor/common/hv_main.c

@@ -28,11 +28,7 @@ void vcpu_thread(struct sched_object *obj)
 		}
 
 		if (!is_lapic_pt_enabled(vcpu)) {
-			/* handle pending softirq when irq enable*/
-			do_softirq();
 			CPU_IRQ_DISABLE();
-			/* handle risk softirq when disabling irq*/
-			do_softirq();
 		}
 
 		/* Don't open interrupt window between here and vmentry */

hypervisor/common/softirq.c

@@ -32,9 +32,8 @@ void fire_softirq(uint16_t nr)
 	bitmap_set_lock(nr, &per_cpu(softirq_pending, get_pcpu_id()));
 }
 
-void do_softirq(void)
+static void do_softirq_internal(uint16_t cpu_id)
 {
-	uint16_t cpu_id = get_pcpu_id();
 	volatile uint64_t *softirq_pending_bitmap =
 			&per_cpu(softirq_pending, cpu_id);
 	uint16_t nr = ffs64(*softirq_pending_bitmap);
@@ -45,3 +44,22 @@ void do_softirq(void)
 		nr = ffs64(*softirq_pending_bitmap);
 	}
 }
+
+/*
+ * @pre: this function will only be called with irq disabled
+ */
+void do_softirq(void)
+{
+	uint16_t cpu_id = get_pcpu_id();
+
+	if (per_cpu(softirq_servicing, cpu_id) == 0U) {
+		per_cpu(softirq_servicing, cpu_id) = 1U;
+
+		CPU_IRQ_ENABLE();
+		do_softirq_internal(cpu_id);
+		CPU_IRQ_DISABLE();
+
+		do_softirq_internal(cpu_id);
+		per_cpu(softirq_servicing, cpu_id) = 0U;
+	}
+}

hypervisor/include/arch/x86/per_cpu.h

@@ -48,6 +48,7 @@ struct per_cpu_region {
 	uint8_t stack[CONFIG_STACK_SIZE] __aligned(16);
 	uint32_t lapic_id;
 	uint32_t lapic_ldr;
+	uint32_t softirq_servicing;
 	struct smp_call_info_data smp_call_info;
 #ifdef PROFILING_ON
 	struct profiling_info_wrapper profiling_info;