acrn-hypervisor/hypervisor/dm/io_req.c

727 lines
19 KiB
C

/*
* Copyright (C) 2019 Intel Corporation.
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <vm.h>
#include <irq.h>
#include <errno.h>
#include <logmsg.h>
#define DBG_LEVEL_IOREQ 6U
static uint32_t acrn_vhm_notification_vector = HYPERVISOR_CALLBACK_VHM_VECTOR;
#define MMIO_DEFAULT_VALUE_SIZE_1 (0xFFUL)
#define MMIO_DEFAULT_VALUE_SIZE_2 (0xFFFFUL)
#define MMIO_DEFAULT_VALUE_SIZE_4 (0xFFFFFFFFUL)
#define MMIO_DEFAULT_VALUE_SIZE_8 (0xFFFFFFFFFFFFFFFFUL)
#if defined(HV_DEBUG)
__unused static void acrn_print_request(uint16_t vcpu_id, const struct vhm_request *req)
{
switch (req->type) {
case REQ_MMIO:
dev_dbg(DBG_LEVEL_IOREQ, "[vcpu_id=%hu type=MMIO]", vcpu_id);
dev_dbg(DBG_LEVEL_IOREQ,
"gpa=0x%lx, R/W=%d, size=%ld value=0x%lx processed=%lx",
req->reqs.mmio.address,
req->reqs.mmio.direction,
req->reqs.mmio.size,
req->reqs.mmio.value,
req->processed);
break;
case REQ_PORTIO:
dev_dbg(DBG_LEVEL_IOREQ, "[vcpu_id=%hu type=PORTIO]", vcpu_id);
dev_dbg(DBG_LEVEL_IOREQ,
"IO=0x%lx, R/W=%d, size=%ld value=0x%lx processed=%lx",
req->reqs.pio.address,
req->reqs.pio.direction,
req->reqs.pio.size,
req->reqs.pio.value,
req->processed);
break;
default:
dev_dbg(DBG_LEVEL_IOREQ, "[vcpu_id=%hu type=%d] NOT support type",
vcpu_id, req->type);
break;
}
}
#endif
/**
* @brief Reset all IO requests status of the VM
*
* @param vm The VM whose IO requests to be reset
*
* @return None
*/
void reset_vm_ioreqs(struct acrn_vm *vm)
{
uint16_t i;
for (i = 0U; i < VHM_REQUEST_MAX; i++) {
set_vhm_req_state(vm, i, REQ_STATE_FREE);
}
}
static inline bool has_complete_ioreq(const struct acrn_vcpu *vcpu)
{
return (get_vhm_req_state(vcpu->vm, vcpu->vcpu_id) == REQ_STATE_COMPLETE);
}
/**
* @brief Deliver \p io_req to SOS and suspend \p vcpu till its completion
*
* @param vcpu The virtual CPU that triggers the MMIO access
* @param io_req The I/O request holding the details of the MMIO access
*
* @pre vcpu != NULL && io_req != NULL
*/
int32_t acrn_insert_request(struct acrn_vcpu *vcpu, const struct io_request *io_req)
{
union vhm_request_buffer *req_buf = NULL;
struct vhm_request *vhm_req;
bool is_polling = false;
int32_t ret = 0;
uint16_t cur;
if ((vcpu->vm->sw.io_shared_page != NULL)
&& (get_vhm_req_state(vcpu->vm, vcpu->vcpu_id) == REQ_STATE_FREE)) {
req_buf = (union vhm_request_buffer *)(vcpu->vm->sw.io_shared_page);
cur = vcpu->vcpu_id;
stac();
vhm_req = &req_buf->req_queue[cur];
/* ACRN insert request to VHM and inject upcall */
vhm_req->type = io_req->io_type;
(void)memcpy_s(&vhm_req->reqs, sizeof(union vhm_io_request),
&io_req->reqs, sizeof(union vhm_io_request));
if (vcpu->vm->sw.is_polling_ioreq) {
vhm_req->completion_polling = 1U;
is_polling = true;
}
clac();
/* Before updating the vhm_req state, enforce all fill vhm_req operations done */
cpu_write_memory_barrier();
/* Must clear the signal before we mark req as pending
* Once we mark it pending, VHM may process req and signal us
* before we perform upcall.
* because VHM can work in pulling mode without wait for upcall
*/
set_vhm_req_state(vcpu->vm, vcpu->vcpu_id, REQ_STATE_PENDING);
/* signal VHM */
arch_fire_vhm_interrupt();
/* Polling completion of the request in polling mode */
if (is_polling) {
while (true) {
if (has_complete_ioreq(vcpu)) {
/* we have completed ioreq pending */
break;
}
asm_pause();
if (need_reschedule(pcpuid_from_vcpu(vcpu))) {
schedule();
}
}
} else {
wait_event(&vcpu->events[VCPU_EVENT_IOREQ]);
}
} else {
ret = -EINVAL;
}
return ret;
}
uint32_t get_vhm_req_state(struct acrn_vm *vm, uint16_t vhm_req_id)
{
uint32_t state;
union vhm_request_buffer *req_buf = NULL;
struct vhm_request *vhm_req;
req_buf = (union vhm_request_buffer *)vm->sw.io_shared_page;
if (req_buf == NULL) {
state = 0xffffffffU;
} else {
stac();
vhm_req = &req_buf->req_queue[vhm_req_id];
state = vhm_req->processed;
clac();
}
return state;
}
void set_vhm_req_state(struct acrn_vm *vm, uint16_t vhm_req_id, uint32_t state)
{
union vhm_request_buffer *req_buf = NULL;
struct vhm_request *vhm_req;
req_buf = (union vhm_request_buffer *)vm->sw.io_shared_page;
if (req_buf != NULL) {
stac();
vhm_req = &req_buf->req_queue[vhm_req_id];
/*
* HV will only set processed to REQ_STATE_PENDING or REQ_STATE_FREE.
* we don't need to sfence here is that even if the SOS/DM sees the previous state,
* the only side effect is that it will defer the processing of the new IOReq.
* It won't lead wrong processing.
*/
vhm_req->processed = state;
clac();
}
}
void set_vhm_notification_vector(uint32_t vector)
{
acrn_vhm_notification_vector = vector;
}
uint32_t get_vhm_notification_vector(void)
{
return acrn_vhm_notification_vector;
}
/**
* @brief General complete-work for MMIO emulation
*
* @param vcpu The virtual CPU that triggers the MMIO access
* @param io_req The I/O request holding the details of the MMIO access
*
* @pre io_req->io_type == REQ_MMIO
*
* @remark This function must be called when \p io_req is completed, after
* either a previous call to emulate_io() returning 0 or the corresponding VHM
* request transferring to the COMPLETE state.
*/
static void emulate_mmio_complete(struct acrn_vcpu *vcpu, const struct io_request *io_req)
{
const struct mmio_request *mmio_req = &io_req->reqs.mmio;
if (mmio_req->direction == REQUEST_READ) {
/* Emulate instruction and update vcpu register set */
(void)emulate_instruction(vcpu);
}
}
static void complete_ioreq(struct acrn_vcpu *vcpu, struct io_request *io_req)
{
union vhm_request_buffer *req_buf = NULL;
struct vhm_request *vhm_req;
req_buf = (union vhm_request_buffer *)(vcpu->vm->sw.io_shared_page);
stac();
vhm_req = &req_buf->req_queue[vcpu->vcpu_id];
if (io_req != NULL) {
switch (vcpu->req.io_type) {
case REQ_PORTIO:
io_req->reqs.pio.value = vhm_req->reqs.pio.value;
break;
case REQ_MMIO:
io_req->reqs.mmio.value = vhm_req->reqs.mmio.value;
break;
default:
/*no actions are required for other cases.*/
break;
}
}
/*
* Only HV will check whether processed is REQ_STATE_FREE on per-vCPU before inject a ioreq.
* Only HV will set processed to REQ_STATE_FREE when ioreq is done.
*/
vhm_req->processed = REQ_STATE_FREE;
clac();
}
/**
* @brief Complete-work of VHM requests for port I/O emulation
*
* @pre vcpu->req.io_type == REQ_PORTIO
*
* @remark This function must be called after the VHM request corresponding to
* \p vcpu being transferred to the COMPLETE state.
*/
static void dm_emulate_pio_complete(struct acrn_vcpu *vcpu)
{
struct io_request *io_req = &vcpu->req;
complete_ioreq(vcpu, io_req);
emulate_pio_complete(vcpu, io_req);
}
/**
* @brief Complete-work of VHM requests for MMIO emulation
*
* @param vcpu The virtual CPU that triggers the MMIO access
*
* @pre vcpu->req.io_type == REQ_MMIO
*
* @remark This function must be called after the VHM request corresponding to
* \p vcpu being transferred to the COMPLETE state.
*/
static void dm_emulate_mmio_complete(struct acrn_vcpu *vcpu)
{
struct io_request *io_req = &vcpu->req;
complete_ioreq(vcpu, io_req);
emulate_mmio_complete(vcpu, io_req);
}
/**
* @brief General complete-work for all kinds of VHM requests for I/O emulation
*
* @param vcpu The virtual CPU that triggers the MMIO access
*/
static void dm_emulate_io_complete(struct acrn_vcpu *vcpu)
{
if (get_vhm_req_state(vcpu->vm, vcpu->vcpu_id) == REQ_STATE_COMPLETE) {
/*
* If vcpu is in Zombie state and will be destroyed soon. Just
* mark ioreq done and don't resume vcpu.
*/
if (vcpu->state == VCPU_ZOMBIE) {
complete_ioreq(vcpu, NULL);
} else {
switch (vcpu->req.io_type) {
case REQ_MMIO:
dm_emulate_mmio_complete(vcpu);
break;
case REQ_PORTIO:
case REQ_PCICFG:
/*
* REQ_PORTIO on 0xcf8 & 0xcfc may switch to REQ_PCICFG in some
* cases. It works to apply the post-work for REQ_PORTIO on
* REQ_PCICFG because the format of the first 28 bytes of
* REQ_PORTIO & REQ_PCICFG requests are exactly the same and
* post-work is mainly interested in the read value.
*/
dm_emulate_pio_complete(vcpu);
break;
default:
/*
* REQ_WP can only be triggered on writes which do not need
* post-work. Just mark the ioreq done.
*/
complete_ioreq(vcpu, NULL);
break;
}
}
}
}
/**
* @pre width < 8U
* @pre vcpu != NULL
* @pre vcpu->vm != NULL
*/
static bool pio_default_read(struct acrn_vcpu *vcpu,
__unused uint16_t addr, size_t width)
{
struct pio_request *pio_req = &vcpu->req.reqs.pio;
pio_req->value = (uint32_t)((1UL << (width * 8U)) - 1UL);
return true;
}
/**
* @pre width < 8U
* @pre vcpu != NULL
* @pre vcpu->vm != NULL
*/
static bool pio_default_write(__unused struct acrn_vcpu *vcpu, __unused uint16_t addr,
__unused size_t width, __unused uint32_t v)
{
return true; /* ignore write */
}
/**
* @pre (io_req->reqs.mmio.size == 1U) || (io_req->reqs.mmio.size == 2U) ||
* (io_req->reqs.mmio.size == 4U) || (io_req->reqs.mmio.size == 8U)
*/
static int32_t mmio_default_access_handler(struct io_request *io_req,
__unused void *handler_private_data)
{
struct mmio_request *mmio = &io_req->reqs.mmio;
if (mmio->direction == REQUEST_READ) {
switch (mmio->size) {
case 1U:
mmio->value = MMIO_DEFAULT_VALUE_SIZE_1;
break;
case 2U:
mmio->value = MMIO_DEFAULT_VALUE_SIZE_2;
break;
case 4U:
mmio->value = MMIO_DEFAULT_VALUE_SIZE_4;
break;
case 8U:
mmio->value = MMIO_DEFAULT_VALUE_SIZE_8;
break;
default:
/* This case is unreachable, this is guaranteed by the design. */
break;
}
}
return 0;
}
/**
* Try handling the given request by any port I/O handler registered in the
* hypervisor.
*
* @pre io_req->io_type == REQ_PORTIO
*
* @retval 0 Successfully emulated by registered handlers.
* @retval -ENODEV No proper handler found.
* @retval -EIO The request spans multiple devices and cannot be emulated.
*/
static int32_t
hv_emulate_pio(struct acrn_vcpu *vcpu, struct io_request *io_req)
{
int32_t status = -ENODEV;
uint16_t port, size;
uint32_t idx;
struct acrn_vm *vm = vcpu->vm;
struct pio_request *pio_req = &io_req->reqs.pio;
struct vm_io_handler_desc *handler;
io_read_fn_t io_read = NULL;
io_write_fn_t io_write = NULL;
if (is_sos_vm(vcpu->vm) || is_prelaunched_vm(vcpu->vm)) {
io_read = pio_default_read;
io_write = pio_default_write;
}
port = (uint16_t)pio_req->address;
size = (uint16_t)pio_req->size;
for (idx = 0U; idx < EMUL_PIO_IDX_MAX; idx++) {
handler = &(vm->emul_pio[idx]);
if ((port < handler->port_start) || (port >= handler->port_end)) {
continue;
}
if (handler->io_read != NULL) {
io_read = handler->io_read;
}
if (handler->io_write != NULL) {
io_write = handler->io_write;
}
break;
}
if ((pio_req->direction == REQUEST_WRITE) && (io_write != NULL)) {
if (io_write(vcpu, port, size, pio_req->value)) {
status = 0;
}
} else if ((pio_req->direction == REQUEST_READ) && (io_read != NULL)) {
if (io_read(vcpu, port, size)) {
status = 0;
}
} else {
/* do nothing */
}
pr_dbg("IO %s on port %04x, data %08x",
(pio_req->direction == REQUEST_READ) ? "read" : "write", port, pio_req->value);
return status;
}
/**
* Use registered MMIO handlers on the given request if it falls in the range of
* any of them.
*
* @pre io_req->io_type == REQ_MMIO
*
* @retval 0 Successfully emulated by registered handlers.
* @retval -ENODEV No proper handler found.
* @retval -EIO The request spans multiple devices and cannot be emulated.
*/
static int32_t
hv_emulate_mmio(struct acrn_vcpu *vcpu, struct io_request *io_req)
{
int32_t status = -ENODEV;
bool hold_lock = true;
uint16_t idx;
uint64_t address, size, base, end;
struct mmio_request *mmio_req = &io_req->reqs.mmio;
struct mem_io_node *mmio_handler = NULL;
hv_mem_io_handler_t read_write = NULL;
void *handler_private_data = NULL;
if (is_sos_vm(vcpu->vm) || is_prelaunched_vm(vcpu->vm)) {
read_write = mmio_default_access_handler;
}
address = mmio_req->address;
size = mmio_req->size;
spinlock_obtain(&vcpu->vm->emul_mmio_lock);
for (idx = 0U; idx <= vcpu->vm->max_emul_mmio_regions; idx++) {
mmio_handler = &(vcpu->vm->emul_mmio[idx]);
if (mmio_handler->read_write != NULL) {
base = mmio_handler->range_start;
end = mmio_handler->range_end;
if (((address + size) <= base) || (address >= end)) {
continue;
} else {
if ((address >= base) && ((address + size) <= end)) {
hold_lock = mmio_handler->hold_lock;
read_write = mmio_handler->read_write;
handler_private_data = mmio_handler->handler_private_data;
} else {
pr_fatal("Err MMIO, address:0x%lx, size:%x", address, size);
status = -EIO;
}
break;
}
}
}
if ((status == -ENODEV) && (read_write != NULL)) {
/* This mmio_handler will never modify once register, so we don't
* need to hold the lock when handling the MMIO access.
*/
if (!hold_lock) {
spinlock_release(&vcpu->vm->emul_mmio_lock);
}
status = read_write(io_req, handler_private_data);
if (!hold_lock) {
spinlock_obtain(&vcpu->vm->emul_mmio_lock);
}
}
spinlock_release(&vcpu->vm->emul_mmio_lock);
return status;
}
/**
* @brief Emulate \p io_req for \p vcpu
*
* Handle an I/O request by either invoking a hypervisor-internal handler or
* deliver to VHM.
*
* @pre vcpu != NULL
* @pre vcpu->vm != NULL
* @pre vcpu->vm->vm_id < CONFIG_MAX_VM_NUM
*
* @param vcpu The virtual CPU that triggers the MMIO access
* @param io_req The I/O request holding the details of the MMIO access
*
* @retval 0 Successfully emulated by registered handlers.
* @retval IOREQ_PENDING The I/O request is delivered to VHM.
* @retval -EIO The request spans multiple devices and cannot be emulated.
* @retval -EINVAL \p io_req has an invalid io_type.
* @retval <0 on other errors during emulation.
*/
int32_t
emulate_io(struct acrn_vcpu *vcpu, struct io_request *io_req)
{
int32_t status;
struct acrn_vm_config *vm_config;
vm_config = get_vm_config(vcpu->vm->vm_id);
switch (io_req->io_type) {
case REQ_PORTIO:
status = hv_emulate_pio(vcpu, io_req);
if (status == 0) {
emulate_pio_complete(vcpu, io_req);
}
break;
case REQ_MMIO:
case REQ_WP:
status = hv_emulate_mmio(vcpu, io_req);
if (status == 0) {
emulate_mmio_complete(vcpu, io_req);
}
break;
default:
/* Unknown I/O request io_type */
status = -EINVAL;
break;
}
if ((status == -ENODEV) && (vm_config->load_order == POST_LAUNCHED_VM)) {
/*
* No handler from HV side, search from VHM in Dom0
*
* ACRN insert request to VHM and inject upcall.
*/
status = acrn_insert_request(vcpu, io_req);
if (status == 0) {
dm_emulate_io_complete(vcpu);
} else {
/* here for both IO & MMIO, the direction, address,
* size definition is same
*/
struct pio_request *pio_req = &io_req->reqs.pio;
pr_fatal("%s Err: access dir %d, io_type %d, addr = 0x%lx, size=%lu", __func__,
pio_req->direction, io_req->io_type,
pio_req->address, pio_req->size);
}
}
return status;
}
/**
* @brief Register a port I/O handler
*
* @param vm The VM to which the port I/O handlers are registered
* @param pio_idx The emulated port io index
* @param range The emulated port io range
* @param io_read_fn_ptr The handler for emulating reads from the given range
* @param io_write_fn_ptr The handler for emulating writes to the given range
* @pre pio_idx < EMUL_PIO_IDX_MAX
*/
void register_pio_emulation_handler(struct acrn_vm *vm, uint32_t pio_idx,
const struct vm_io_range *range, io_read_fn_t io_read_fn_ptr, io_write_fn_t io_write_fn_ptr)
{
if (is_sos_vm(vm)) {
deny_guest_pio_access(vm, range->base, range->len);
}
vm->emul_pio[pio_idx].port_start = range->base;
vm->emul_pio[pio_idx].port_end = range->base + range->len;
vm->emul_pio[pio_idx].io_read = io_read_fn_ptr;
vm->emul_pio[pio_idx].io_write = io_write_fn_ptr;
}
/**
* @brief Find match MMIO node
*
* This API find match MMIO node from \p vm.
*
* @param vm The VM to which the MMIO node is belong to.
*
* @return If there's a match mmio_node return it, otherwise return NULL;
*/
static inline struct mem_io_node *find_match_mmio_node(struct acrn_vm *vm,
uint64_t start, uint64_t end)
{
bool found = false;
uint16_t idx;
struct mem_io_node *mmio_node;
for (idx = 0U; idx < CONFIG_MAX_EMULATED_MMIO_REGIONS; idx++) {
mmio_node = &(vm->emul_mmio[idx]);
if ((mmio_node->range_start == start) && (mmio_node->range_end == end)) {
found = true;
break;
}
}
if (!found) {
pr_fatal("%s, vm[%d] no match mmio region [0x%lx, 0x%lx] is found",
__func__, vm->vm_id, start, end);
mmio_node = NULL;
}
return mmio_node;
}
/**
* @brief Find a free MMIO node
*
* This API find a free MMIO node from \p vm.
*
* @param vm The VM to which the MMIO node is belong to.
*
* @return If there's a free mmio_node return it, otherwise return NULL;
*/
static inline struct mem_io_node *find_free_mmio_node(struct acrn_vm *vm)
{
uint16_t idx;
struct mem_io_node *mmio_node = find_match_mmio_node(vm, 0UL, 0UL);
if (mmio_node != NULL) {
idx = (uint16_t)(uint64_t)(mmio_node - &(vm->emul_mmio[0U]));
if (vm->max_emul_mmio_regions < idx) {
vm->max_emul_mmio_regions = idx;
}
}
return mmio_node;
}
/**
* @brief Register a MMIO handler
*
* This API registers a MMIO handler to \p vm
*
* @param vm The VM to which the MMIO handler is registered
* @param read_write The handler for emulating accesses to the given range
* @param start The base address of the range \p read_write can emulate
* @param end The end of the range (exclusive) \p read_write can emulate
* @param handler_private_data Handler-specific data which will be passed to \p read_write when called
*
* @return None
*/
void register_mmio_emulation_handler(struct acrn_vm *vm,
hv_mem_io_handler_t read_write, uint64_t start,
uint64_t end, void *handler_private_data, bool hold_lock)
{
struct mem_io_node *mmio_node;
/* Ensure both a read/write handler and range check function exist */
if ((read_write != NULL) && (end > start)) {
spinlock_obtain(&vm->emul_mmio_lock);
mmio_node = find_free_mmio_node(vm);
if (mmio_node != NULL) {
/* Fill in information for this node */
mmio_node->hold_lock = hold_lock;
mmio_node->read_write = read_write;
mmio_node->handler_private_data = handler_private_data;
mmio_node->range_start = start;
mmio_node->range_end = end;
}
spinlock_release(&vm->emul_mmio_lock);
}
}
/**
* @brief Unregister a MMIO handler
*
* This API unregisters a MMIO handler to \p vm
*
* @param vm The VM to which the MMIO handler is unregistered
* @param start The base address of the range which wants to unregister
* @param end The end of the range (exclusive) which wants to unregister
*
* @return None
*/
void unregister_mmio_emulation_handler(struct acrn_vm *vm,
uint64_t start, uint64_t end)
{
struct mem_io_node *mmio_node;
spinlock_obtain(&vm->emul_mmio_lock);
mmio_node = find_match_mmio_node(vm, start, end);
if (mmio_node != NULL) {
(void)memset(mmio_node, 0U, sizeof(struct mem_io_node));
}
spinlock_release(&vm->emul_mmio_lock);
}