acrn-hypervisor/hypervisor/lib/memory.c

455 lines
11 KiB
C

/*
* Copyright (C) 2018 Intel Corporation.
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <hypervisor.h>
/************************************************************************/
/* Memory pool declaration (block size = CONFIG_MALLOC_ALIGN) */
/************************************************************************/
#define __bss_noinit __attribute__((__section__(".bss_noinit")))
static uint8_t __bss_noinit
Malloc_Heap[CONFIG_HEAP_SIZE] __aligned(CONFIG_MALLOC_ALIGN);
#define MALLOC_HEAP_BUFF_SIZE CONFIG_MALLOC_ALIGN
#define MALLOC_HEAP_TOTAL_BUFF (CONFIG_HEAP_SIZE/MALLOC_HEAP_BUFF_SIZE)
#define MALLOC_HEAP_BITMAP_SIZE \
INT_DIV_ROUNDUP(MALLOC_HEAP_TOTAL_BUFF, BITMAP_WORD_SIZE)
static uint32_t Malloc_Heap_Bitmap[MALLOC_HEAP_BITMAP_SIZE];
static uint32_t Malloc_Heap_Contiguity_Bitmap[MALLOC_HEAP_BITMAP_SIZE];
static struct mem_pool Memory_Pool = {
.start_addr = Malloc_Heap,
.spinlock = {.head = 0U, .tail = 0U},
.size = CONFIG_HEAP_SIZE,
.buff_size = MALLOC_HEAP_BUFF_SIZE,
.total_buffs = MALLOC_HEAP_TOTAL_BUFF,
.bmp_size = MALLOC_HEAP_BITMAP_SIZE,
.bitmap = Malloc_Heap_Bitmap,
.contiguity_bitmap = Malloc_Heap_Contiguity_Bitmap
};
/************************************************************************/
/* Memory pool declaration (block size = CPU_PAGE_SIZE) */
/************************************************************************/
static uint8_t __bss_noinit
Paging_Heap[CONFIG_NUM_ALLOC_PAGES][CPU_PAGE_SIZE] __aligned(CPU_PAGE_SIZE);
#define PAGING_HEAP_BUFF_SIZE CPU_PAGE_SIZE
#define PAGING_HEAP_TOTAL_BUFF CONFIG_NUM_ALLOC_PAGES
#define PAGING_HEAP_BITMAP_SIZE \
INT_DIV_ROUNDUP(PAGING_HEAP_TOTAL_BUFF, BITMAP_WORD_SIZE)
static uint32_t Paging_Heap_Bitmap[PAGING_HEAP_BITMAP_SIZE];
static uint32_t Paging_Heap_Contiguity_Bitmap[MALLOC_HEAP_BITMAP_SIZE];
static struct mem_pool Paging_Memory_Pool = {
.start_addr = Paging_Heap,
.spinlock = {.head = 0U, .tail = 0U},
.size = CONFIG_NUM_ALLOC_PAGES * CPU_PAGE_SIZE,
.buff_size = PAGING_HEAP_BUFF_SIZE,
.total_buffs = PAGING_HEAP_TOTAL_BUFF,
.bmp_size = PAGING_HEAP_BITMAP_SIZE,
.bitmap = Paging_Heap_Bitmap,
.contiguity_bitmap = Paging_Heap_Contiguity_Bitmap
};
static void *allocate_mem(struct mem_pool *pool, unsigned int num_bytes)
{
void *memory = NULL;
uint32_t idx;
uint16_t bit_idx;
uint32_t requested_buffs;
/* Check if provided memory pool exists */
if (pool == NULL) {
return NULL;
}
/* Acquire the pool lock */
spinlock_obtain(&pool->spinlock);
/* Calculate number of buffers to be allocated from memory pool */
requested_buffs = INT_DIV_ROUNDUP(num_bytes, pool->buff_size);
for (idx = 0U; idx < pool->bmp_size; idx++) {
/* Find the first occurrence of requested_buffs number of free
* buffers. The 0th bit in bitmap represents a free buffer.
*/
for (bit_idx = ffz64(pool->bitmap[idx]);
bit_idx < BITMAP_WORD_SIZE; bit_idx++) {
/* Check if selected buffer is free */
if ((pool->bitmap[idx] & (1U << bit_idx)) != 0U) {
continue;
}
/* Declare temporary variables to be used locally in
* this block
*/
uint32_t i;
uint16_t tmp_bit_idx = bit_idx;
uint32_t tmp_idx = idx;
/* Check requested_buffs number of buffers availability
* in memory-pool right after selected buffer
*/
for (i = 1U; i < requested_buffs; i++) {
/* Check if tmp_bit_idx is out-of-range */
tmp_bit_idx++;
if (tmp_bit_idx == BITMAP_WORD_SIZE) {
/* Break the loop if tmp_idx is
* out-of-range
*/
tmp_idx++;
if (tmp_idx == pool->bmp_size) {
break;
}
/* Reset tmp_bit_idx */
tmp_bit_idx = 0U;
}
/* Break if selected buffer is not free */
if ((pool->bitmap[tmp_idx]
& (1U << tmp_bit_idx)) != 0U) {
break;
}
}
/* Check if requested_buffs number of free contiguous
* buffers are found in memory pool
*/
if (i == requested_buffs) {
/* Get start address of first buffer among
* selected free contiguous buffer in the
* memory pool
*/
memory = (char *)pool->start_addr +
(pool->buff_size *
((idx * BITMAP_WORD_SIZE) +
bit_idx));
/* Update allocation bitmaps information for
* selected buffers
*/
for (i = 0U; i < requested_buffs; i++) {
/* Set allocation bit in bitmap for
* this buffer
*/
pool->bitmap[idx] |= (1U << bit_idx);
/* Set contiguity information for this
* buffer in contiguity-bitmap
*/
if (i < (requested_buffs - 1)) {
/* Set contiguity bit to 1 if
* this buffer is not the last
* of selected contiguous
* buffers array
*/
pool->contiguity_bitmap[idx] |=
(1U << bit_idx);
} else {
/* Set contiguity bit to 0 if
* this buffer is not the last
* of selected contiguous
* buffers array
*/
pool->contiguity_bitmap[idx] &=
~(1U << bit_idx);
}
/* Check if bit_idx is out-of-range */
bit_idx++;
if (bit_idx == BITMAP_WORD_SIZE) {
/* Increment idx */
idx++;
/* Reset bit_idx */
bit_idx = 0U;
}
}
/* Release the pool lock. */
spinlock_release(&pool->spinlock);
return memory;
}
/* Update bit_idx and idx */
bit_idx = tmp_bit_idx;
idx = tmp_idx;
}
}
/* Release the pool lock. */
spinlock_release(&pool->spinlock);
return (void *)NULL;
}
static void deallocate_mem(struct mem_pool *pool, void *ptr)
{
uint32_t *bitmask, *contiguity_bitmask;
uint32_t bmp_idx, bit_idx, buff_idx;
if ((pool != NULL) && (ptr != NULL)) {
/* Acquire the pool lock */
spinlock_obtain(&pool->spinlock);
/* Map the buffer address to its index. */
buff_idx = ((char *)ptr - (char *)pool->start_addr) /
pool->buff_size;
/* De-allocate all allocated contiguous memory buffers */
while (buff_idx < pool->total_buffs) {
/* Translate the buffer index to bitmap index. */
bmp_idx = buff_idx / BITMAP_WORD_SIZE;
bit_idx = buff_idx % BITMAP_WORD_SIZE;
/* Get bitmap's reference for this buffer */
bitmask = &pool->bitmap[bmp_idx];
contiguity_bitmask = &pool->contiguity_bitmap[bmp_idx];
/* Mark the buffer as free */
if ((*bitmask & (1U << bit_idx)) != 0U) {
*bitmask ^= (1U << bit_idx);
} else {
break;
}
/* Reset the Contiguity bit of buffer */
if ((*contiguity_bitmask & (1U << bit_idx)) != 0U) {
*contiguity_bitmask ^= (1U << bit_idx);
} else {
break;
}
/* Increment buff_idx */
buff_idx++;
}
/* Release the pool lock. */
spinlock_release(&pool->spinlock);
}
}
/*
* The return address will be CPU_PAGE_SIZE aligned if 'num_bytes' is greater
* than CPU_PAGE_SIZE.
*/
void *malloc(unsigned int num_bytes)
{
void *memory = NULL;
/* Check if bytes requested extend page-size */
if (num_bytes < CPU_PAGE_SIZE) {
/*
* Request memory allocation from smaller segmented memory pool
*/
memory = allocate_mem(&Memory_Pool, num_bytes);
} else {
uint32_t page_num =
((num_bytes + CPU_PAGE_SIZE) - 1U) >> CPU_PAGE_SHIFT;
/* Request memory allocation through alloc_page */
memory = alloc_pages(page_num);
}
/* Check if memory allocation is successful */
if (memory == NULL) {
pr_err("%s: failed to alloc 0x%x Bytes", __func__, num_bytes);
}
/* Return memory pointer to caller */
return memory;
}
void *alloc_pages(unsigned int page_num)
{
void *memory = NULL;
/* Request memory allocation from Page-aligned memory pool */
memory = allocate_mem(&Paging_Memory_Pool, page_num * CPU_PAGE_SIZE);
/* Check if memory allocation is successful */
if (memory == NULL) {
pr_err("%s: failed to alloc %d pages", __func__, page_num);
}
return memory;
}
void *alloc_page(void)
{
return alloc_pages(1U);
}
void *calloc(unsigned int num_elements, unsigned int element_size)
{
void *memory = malloc(num_elements * element_size);
/* Determine if memory was allocated */
if (memory != NULL) {
/* Zero all the memory */
(void)memset(memory, 0U, num_elements * element_size);
}
/* Return pointer to memory */
return memory;
}
void free(void *ptr)
{
/* Check if ptr belongs to 16-Bytes aligned Memory Pool */
if ((Memory_Pool.start_addr < ptr) &&
(ptr < (Memory_Pool.start_addr +
(Memory_Pool.total_buffs * Memory_Pool.buff_size)))) {
/* Free buffer in 16-Bytes aligned Memory Pool */
deallocate_mem(&Memory_Pool, ptr);
}
/* Check if ptr belongs to page aligned Memory Pool */
else if ((Paging_Memory_Pool.start_addr < ptr) &&
(ptr < (Paging_Memory_Pool.start_addr +
(Paging_Memory_Pool.total_buffs *
Paging_Memory_Pool.buff_size)))) {
/* Free buffer in page aligned Memory Pool */
deallocate_mem(&Paging_Memory_Pool, ptr);
}
}
void *memchr(const void *void_s, int c, size_t n)
{
unsigned char val = (unsigned char)c;
unsigned char *ptr = (unsigned char *)void_s;
unsigned char *end = ptr + n;
while (ptr < end) {
if (*ptr == val) {
return ((void *)ptr);
}
ptr++;
}
return NULL;
}
/***********************************************************************
*
* FUNCTION
*
* memcpy_s
*
* DESCRIPTION
*
* Copies at most slen bytes from src address to dest address,
* up to dmax.
*
* INPUTS
*
* d pointer to Destination address
* dmax maximum length of dest
* s pointer to Source address
* slen maximum number of bytes of src to copy
*
* OUTPUTS
*
* void * pointer to destination address if successful,
* or else return null.
*
***********************************************************************/
void *memcpy_s(void *d, size_t dmax, const void *s, size_t slen_arg)
{
uint8_t *dest8;
uint8_t *src8;
size_t slen = slen_arg;
if ((slen == 0U) || (dmax == 0U) || (dmax < slen)) {
ASSERT(false);
}
if (((d > s) && (d <= ((s + slen) - 1U)))
|| ((d < s) && (s <= ((d + dmax) - 1U)))) {
ASSERT(false);
}
/* same memory block, no need to copy */
if (d == s) {
return d;
}
dest8 = (uint8_t *)d;
src8 = (uint8_t *)s;
/* small data block */
if (slen < 8U) {
while (slen != 0U) {
*dest8 = *src8;
dest8++;
src8++;
slen--;
}
return d;
}
/* make sure 8bytes-aligned for at least one addr. */
if ((!mem_aligned_check((uint64_t)src8, 8UL)) &&
(!mem_aligned_check((uint64_t)dest8, 8UL))) {
for (; (slen != 0U) && ((((uint64_t)src8) & 7UL) != 0UL);
slen--) {
*dest8 = *src8;
dest8++;
src8++;
}
}
/* copy main data blocks, with rep prefix */
if (slen > 8U) {
uint32_t ecx;
asm volatile ("cld; rep; movsq"
: "=&c"(ecx), "=&D"(dest8), "=&S"(src8)
: "0" (slen >> 3), "1" (dest8), "2" (src8)
: "memory");
slen = slen & 0x7U;
}
/* tail bytes */
while (slen != 0U) {
*dest8 = *src8;
dest8++;
src8++;
slen--;
}
return d;
}
void *memset(void *base, uint8_t v, size_t n)
{
uint8_t *dest_p;
size_t n_q;
size_t count;
dest_p = (uint8_t *)base;
if ((dest_p == NULL) || (n == 0U)) {
return NULL;
}
/* do the few bytes to get uint64_t alignment */
count = n;
for (; (count != 0U) && (((uint64_t)dest_p & 7UL) != 0UL); count--) {
*dest_p = v;
dest_p++;
}
/* 64-bit mode */
n_q = count >> 3U;
asm volatile("cld ; rep ; stosq ; movl %3,%%ecx ; rep ; stosb"
: "+c"(n_q), "+D"(dest_p)
: "a" (v * 0x0101010101010101U),
"r"((unsigned int)count & 7U));
return (void *)dest_p;
}