From 5118c6d330c6cdae02ff4b6232ce050d0d278d6b Mon Sep 17 00:00:00 2001
From: Marcin Maka <marcin.maka@linux.intel.com>
Date: Mon, 17 Sep 2018 14:54:00 +0200
Subject: [PATCH] memory: heap reorg for smp

All kernel level components/services allocate objects
from sys heap again. Separation from the runtime heap
provides more reliable, deterministic power flows.
It also presents an opportunity to implement kernel/app
memory protection on platforms which support it.

A better performance is expected due to much simpler
allocation approach.

In order to enable 'free()', slave cores allocate their
objects from their small dedicated sys heaps and when
the core goes down, the entire heap is reset at once.
Further memory power optimization is possible by moving
each sys heap to a separate memory bank that is powered
on/off along with the slave core.

Signed-off-by: Marcin Maka <marcin.maka@linux.intel.com>
---
 src/arch/xtensa/include/arch/task.h           | 23 +-----
 src/arch/xtensa/smp/cpu.c                     |  3 +
 src/arch/xtensa/smp/include/arch/alloc.h      |  3 +-
 src/arch/xtensa/smp/include/arch/idc.h        |  4 +-
 src/audio/eq_fir.c                            |  6 +-
 src/include/sof/alloc.h                       |  5 +-
 src/init/init.c                               |  2 +
 src/lib/alloc.c                               | 81 +++++++++++++------
 src/lib/interrupt.c                           |  3 +-
 src/lib/schedule.c                            |  4 +-
 src/lib/work.c                                |  4 +-
 src/library/include/platform/memory.h         |  2 -
 src/platform/apollolake/apollolake.x.in       |  8 +-
 .../apollolake/include/platform/memory.h      | 12 ++-
 .../baytrail/include/platform/memory.h        |  3 +
 src/platform/baytrail/memory.c                |  2 +-
 src/platform/cannonlake/cannonlake.x.in       |  8 +-
 .../cannonlake/include/platform/memory.h      | 20 ++++-
 .../haswell/include/platform/memory.h         |  3 +
 src/platform/haswell/memory.c                 |  2 +-
 src/platform/icelake/icelake.x.in             |  8 +-
 .../icelake/include/platform/memory.h         | 20 ++++-
 src/platform/intel/cavs/memory.c              | 33 ++++++--
 23 files changed, 165 insertions(+), 94 deletions(-)

diff --git a/src/arch/xtensa/include/arch/task.h b/src/arch/xtensa/include/arch/task.h
index fa5a372b3..180150f80 100644
--- a/src/arch/xtensa/include/arch/task.h
+++ b/src/arch/xtensa/include/arch/task.h
@@ -171,10 +171,7 @@ static inline int arch_allocate_tasks(void)
 {
 	/* irq low */
 	struct irq_task **low = task_irq_low_get();
-	*low = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(**low));
-
-	if (!*low)
-		return -ENOMEM;
+	*low = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(**low));
 
 	list_init(&((*low)->list));
 	spinlock_init(&((*low)->lock));
@@ -182,10 +179,7 @@ static inline int arch_allocate_tasks(void)
 
 	/* irq medium */
 	struct irq_task **med = task_irq_med_get();
-	*med = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(**med));
-
-	if (!*med)
-		return -ENOMEM;
+	*med = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(**med));
 
 	list_init(&((*med)->list));
 	spinlock_init(&((*med)->lock));
@@ -193,10 +187,7 @@ static inline int arch_allocate_tasks(void)
 
 	/* irq high */
 	struct irq_task **high = task_irq_high_get();
-	*high = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(**high));
-
-	if (!*high)
-		return -ENOMEM;
+	*high = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(**high));
 
 	list_init(&((*high)->list));
 	spinlock_init(&((*high)->lock));
@@ -211,7 +202,7 @@ static inline int arch_allocate_tasks(void)
 static inline void arch_free_tasks(void)
 {
 	uint32_t flags;
-
+/* TODO: do not want to free the tasks, just the entire heap */
 	/* free IRQ low task */
 	struct irq_task **low = task_irq_low_get();
 
@@ -221,8 +212,6 @@ static inline void arch_free_tasks(void)
 	list_item_del(&(*low)->list);
 	spin_unlock_irq(&(*low)->lock, flags);
 
-	rfree(*low);
-
 	/* free IRQ medium task */
 	struct irq_task **med = task_irq_med_get();
 
@@ -232,8 +221,6 @@ static inline void arch_free_tasks(void)
 	list_item_del(&(*med)->list);
 	spin_unlock_irq(&(*med)->lock, flags);
 
-	rfree(*med);
-
 	/* free IRQ high task */
 	struct irq_task **high = task_irq_high_get();
 
@@ -242,8 +229,6 @@ static inline void arch_free_tasks(void)
 	interrupt_unregister(PLATFORM_IRQ_TASK_HIGH);
 	list_item_del(&(*high)->list);
 	spin_unlock_irq(&(*high)->lock, flags);
-
-	rfree(*high);
 }
 
 /**
diff --git a/src/arch/xtensa/smp/cpu.c b/src/arch/xtensa/smp/cpu.c
index c7e549697..169db3618 100644
--- a/src/arch/xtensa/smp/cpu.c
+++ b/src/arch/xtensa/smp/cpu.c
@@ -111,6 +111,9 @@ void cpu_power_down_core(void)
 
 	dcache_writeback_invalidate_all();
 
+	/* free entire sys heap, an instance dedicated for this core */
+	free_heap(RZONE_SYS);
+
 	while (1)
 		arch_wait_for_interrupt(0);
 }
diff --git a/src/arch/xtensa/smp/include/arch/alloc.h b/src/arch/xtensa/smp/include/arch/alloc.h
index a36497d98..b17b3613a 100644
--- a/src/arch/xtensa/smp/include/arch/alloc.h
+++ b/src/arch/xtensa/smp/include/arch/alloc.h
@@ -51,7 +51,7 @@ static inline void alloc_core_context(int core)
 {
 	struct core_context *core_ctx;
 
-	core_ctx = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*core_ctx));
+	core_ctx = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(*core_ctx));
 	dcache_writeback_invalidate_region(core_ctx, sizeof(*core_ctx));
 
 	/* xtos_core_data is a big struct, so allocate it from system heap
@@ -83,7 +83,6 @@ static inline void alloc_core_context(int core)
  */
 static inline void free_core_context(int core)
 {
-	rfree(core_ctx_ptr[core]);
 	dcache_writeback_invalidate_region(core_ctx_ptr[core],
 					   sizeof(*core_ctx_ptr[core]));
 }
diff --git a/src/arch/xtensa/smp/include/arch/idc.h b/src/arch/xtensa/smp/include/arch/idc.h
index 6992c69c6..cd964db8f 100644
--- a/src/arch/xtensa/smp/include/arch/idc.h
+++ b/src/arch/xtensa/smp/include/arch/idc.h
@@ -301,7 +301,7 @@ static inline void arch_idc_init(void)
 
 	/* initialize idc data */
 	struct idc **idc = idc_get();
-	*idc = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(**idc));
+	*idc = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(**idc));
 	spinlock_init(&((*idc)->lock));
 	(*idc)->busy_bit_mask = idc_get_busy_bit_mask(core);
 	(*idc)->done_bit_mask = idc_get_done_bit_mask(core);
@@ -337,8 +337,6 @@ static inline void idc_free(void)
 		if (idctfc & IPC_IDCTFC_BUSY)
 			idc_write(IPC_IDCTFC(i), core, idctfc);
 	}
-
-	rfree(*idc_get());
 }
 
 #endif
diff --git a/src/audio/eq_fir.c b/src/audio/eq_fir.c
index b7ddadc91..da3eab8db 100644
--- a/src/audio/eq_fir.c
+++ b/src/audio/eq_fir.c
@@ -209,13 +209,11 @@ static int eq_fir_setup(struct fir_state_32x16 fir[],
 
 	trace_eq("all");
 	/* Allocate all FIR channels data in a big chunk and clear it */
-	fir_data = rballoc(RZONE_SYS, SOF_MEM_CAPS_RAM,
-		length_sum * sizeof(int32_t));
+	fir_data = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM,
+			   length_sum * sizeof(int32_t));
 	if (!fir_data)
 		return -ENOMEM;
 
-	memset(fir_data, 0, length_sum * sizeof(int32_t));
-
 	/* Initialize 2nd phase to set EQ delay lines pointers */
 	trace_eq("ini");
 	for (i = 0; i < nch; i++) {
diff --git a/src/include/sof/alloc.h b/src/include/sof/alloc.h
index 1df84d0a7..13e834bd6 100644
--- a/src/include/sof/alloc.h
+++ b/src/include/sof/alloc.h
@@ -100,7 +100,7 @@ struct mm_heap {
 /* heap block memory map */
 struct mm {
 	/* system heap - used during init cannot be freed */
-	struct mm_heap system;
+	struct mm_heap system[PLATFORM_HEAP_SYSTEM];
 	/* general heap for components */
 	struct mm_heap runtime[PLATFORM_HEAP_RUNTIME];
 	/* general component buffer heap */
@@ -132,6 +132,9 @@ int mm_pm_context_restore(struct dma_copy *dc, struct dma_sg_config *sg);
 /* heap initialisation */
 void init_heap(struct sof *sof);
 
+/* frees entire heap (supported for slave core system heap atm) */
+void free_heap(int zone);
+
 /* flush block map from cache to sram */
 static inline void flush_block_map(struct block_map *map)
 {
diff --git a/src/init/init.c b/src/init/init.c
index a1501c533..8d25982a4 100644
--- a/src/init/init.c
+++ b/src/init/init.c
@@ -88,6 +88,8 @@ int master_core_init(struct sof *sof)
 	return err;
 }
 
+/* TODO: should be compiled only on master/slave arch platforms */
+/* TODO: rename core -> cpu ? */
 int slave_core_init(struct sof *sof)
 {
 	int err;
diff --git a/src/lib/alloc.c b/src/lib/alloc.c
index 6d3ac3f6e..d25917941 100644
--- a/src/lib/alloc.c
+++ b/src/lib/alloc.c
@@ -106,26 +106,28 @@ static void alloc_memset_region(void *ptr, uint32_t bytes, uint32_t val)
 static void *rmalloc_sys(int zone, size_t bytes)
 {
 	void *ptr;
+	struct mm_heap *cpu_heap;
+	size_t alignment = 0;
 
-	/* system memory reserved only for master core */
-	if (cpu_get_id() != PLATFORM_MASTER_CORE_ID) {
-		trace_mem_error("eM0");
-		return NULL;
-	}
+	/* use the heap dedicated for the current core */
+	cpu_heap = memmap.system + cpu_get_id();
 
 	/* align address to dcache line size */
-	if (memmap.system.heap % PLATFORM_DCACHE_ALIGN)
-		memmap.system.heap += PLATFORM_DCACHE_ALIGN -
-			(memmap.system.heap % PLATFORM_DCACHE_ALIGN);
-
-	ptr = (void *)memmap.system.heap;
+	if (cpu_heap->info.used % PLATFORM_DCACHE_ALIGN)
+		alignment = PLATFORM_DCACHE_ALIGN -
+			(cpu_heap->info.used % PLATFORM_DCACHE_ALIGN);
 
 	/* always succeeds or panics */
-	memmap.system.heap += bytes;
-	if (memmap.system.heap >= HEAP_SYSTEM_BASE + HEAP_SYSTEM_SIZE) {
+	if (alignment + bytes > cpu_heap->info.free) {
 		trace_mem_error("eM1");
 		panic(SOF_IPC_PANIC_MEM);
 	}
+	cpu_heap->info.used += alignment;
+
+	ptr = (void *)(cpu_heap->heap + cpu_heap->info.used);
+
+	cpu_heap->info.used += bytes;
+	cpu_heap->info.free -= alignment + bytes;
 
 #if DEBUG_BLOCK_ALLOC
 	alloc_memset_region(ptr, bytes, DEBUG_BLOCK_ALLOC_VALUE);
@@ -528,18 +530,27 @@ uint32_t mm_pm_context_size(void)
 		heap = cache_to_uncache(&memmap.runtime[i]);
 		size += heap->info.used;
 	}
-	size += memmap.system.info.used;
+	for (i = 0; i < PLATFORM_HEAP_SYSTEM; i++) {
+		heap = cache_to_uncache(&memmap.system[i]);
+		size += heap->info.used;
+	}
 
 	/* add memory maps */
 	for (i = 0; i < PLATFORM_HEAP_BUFFER; i++)
 		size += heap_get_size(cache_to_uncache(&memmap.buffer[i]));
 	for (i = 0; i < PLATFORM_HEAP_RUNTIME; i++)
 		size += heap_get_size(cache_to_uncache(&memmap.runtime[i]));
-	size += heap_get_size(&memmap.system);
+	for (i = 0; i < PLATFORM_HEAP_SYSTEM; i++)
+		size += heap_get_size(cache_to_uncache(&memmap.system[i]));
 
+	memmap.total.free = 0;
+	memmap.total.used = 0;
 	/* recalc totals */
-	memmap.total.free = memmap.system.info.free;
-	memmap.total.used = memmap.system.info.used;
+	for (i = 0; i < PLATFORM_HEAP_SYSTEM; i++) {
+		heap = cache_to_uncache(&memmap.system[i]);
+		memmap.total.free += heap->info.free;
+		memmap.total.used += heap->info.used;
+	}
 
 	for (i = 0; i < PLATFORM_HEAP_BUFFER; i++) {
 		heap = cache_to_uncache(&memmap.buffer[i]);
@@ -579,10 +590,12 @@ int mm_pm_context_save(struct dma_copy *dc, struct dma_sg_config *sg)
 		return ret;
 
 	/* copy system memory contents to SG */
-	ret = dma_copy_to_host(dc, sg, offset + ret,
-		(void *)memmap.system.heap, (int32_t)(memmap.system.size));
-	if (ret < 0)
-		return ret;
+
+	/* TODO: does it work? heap ptr points to free location, not to base */
+//	ret = dma_copy_to_host(dc, sg, offset + ret,
+//		(void *)memmap.system.heap, (int32_t)(memmap.system.size));
+//	if (ret < 0)
+//		return ret;
 
 	/* copy module memory contents to SG */
 	// TODO: iterate over module block map and copy contents of each block
@@ -611,10 +624,10 @@ int mm_pm_context_restore(struct dma_copy *dc, struct dma_sg_config *sg)
 		return ret;
 
 	/* copy system memory contents from SG */
-	ret = dma_copy_to_host(dc, sg, offset + ret,
-		(void *)memmap.system.heap, (int32_t)(memmap.system.size));
-	if (ret < 0)
-		return ret;
+//	ret = dma_copy_to_host(dc, sg, offset + ret,
+//		(void *)memmap.system.heap, (int32_t)(memmap.system.size));
+//	if (ret < 0)
+//		return ret;
 
 	/* copy module memory contents from SG */
 	// TODO: iterate over module block map and copy contents of each block
@@ -627,6 +640,24 @@ int mm_pm_context_restore(struct dma_copy *dc, struct dma_sg_config *sg)
 	return 0;
 }
 
+void free_heap(int zone)
+{
+	struct mm_heap *cpu_heap;
+
+	/* to be called by slave cores only for sys heap,
+	 * otherwise this is critical flow issue.
+	 */
+	if (cpu_get_id() == PLATFORM_MASTER_CORE_ID ||
+	    zone != RZONE_SYS) {
+		trace_mem_error("eMf");
+		panic(SOF_IPC_PANIC_MEM);
+	}
+
+	cpu_heap = memmap.system + cpu_get_id();
+	cpu_heap->info.used = 0;
+	cpu_heap->info.free = cpu_heap->size;
+}
+
 /* initialise map */
 void init_heap(struct sof *sof)
 {
@@ -638,7 +669,7 @@ void init_heap(struct sof *sof)
 	int k;
 
 	/* sanity check for malformed images or loader issues */
-	if (memmap.system.heap != HEAP_SYSTEM_BASE)
+	if (memmap.system[0].heap != HEAP_SYSTEM_0_BASE)
 		panic(SOF_IPC_PANIC_MEM);
 
 	spinlock_init(&memmap.lock);
diff --git a/src/lib/interrupt.c b/src/lib/interrupt.c
index b40761bc6..49b00ef94 100644
--- a/src/lib/interrupt.c
+++ b/src/lib/interrupt.c
@@ -56,7 +56,7 @@ static int irq_register_child(struct irq_desc *parent, int irq,
 	spin_lock(&parent->lock);
 
 	/* init child */
-	child = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM,
+	child = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM,
 			sizeof(struct irq_desc));
 	if (!child) {
 		ret = -ENOMEM;
@@ -100,7 +100,6 @@ static void irq_unregister_child(struct irq_desc *parent, int irq)
 
 		if (SOF_IRQ_ID(irq) == child->id) {
 			list_item_del(&child->irq_list);
-			rfree(child);
 			parent->num_children--;
 		}
 	}
diff --git a/src/lib/schedule.c b/src/lib/schedule.c
index e5a220c0b..9e3448561 100644
--- a/src/lib/schedule.c
+++ b/src/lib/schedule.c
@@ -374,7 +374,7 @@ int scheduler_init(struct sof *sof)
 	trace_pipe("ScI");
 
 	struct schedule_data **sch = arch_schedule_get();
-	*sch = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(**sch));
+	*sch = rzalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(**sch));
 
 	if (!*sch)
 		return -ENOMEM;
@@ -413,6 +413,4 @@ void scheduler_free(void)
 	list_item_del(&(*sch)->list);
 
 	spin_unlock_irq(&(*sch)->lock, flags);
-
-	rfree(*sch);
 }
diff --git a/src/lib/work.c b/src/lib/work.c
index 898563502..5d8ac86b9 100644
--- a/src/lib/work.c
+++ b/src/lib/work.c
@@ -455,7 +455,7 @@ struct work_queue *work_new_queue(struct work_queue_timesource *ts)
 	struct work_queue *queue;
 
 	/* init work queue */
-	queue = rmalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*queue));
+	queue = rmalloc(RZONE_SYS, SOF_MEM_CAPS_RAM, sizeof(*queue));
 
 	list_init(&queue->work);
 	spinlock_init(&queue->lock);
@@ -499,6 +499,4 @@ void free_system_workq(void)
 	list_item_del(&(*queue)->work);
 
 	spin_unlock_irq(&(*queue)->lock, flags);
-
-	rfree(*queue);
 }
diff --git a/src/library/include/platform/memory.h b/src/library/include/platform/memory.h
index 4e5cc7009..bd4fdc6e8 100644
--- a/src/library/include/platform/memory.h
+++ b/src/library/include/platform/memory.h
@@ -38,8 +38,6 @@
 #endif
 
 #define HEAP_BUFFER_SIZE	(1024 * 128)
-#define PLATFORM_HEAP_RUNTIME           1
-#define PLATFORM_HEAP_BUFFER            3
 
 #if 0
 /* physical DSP addresses */
diff --git a/src/platform/apollolake/apollolake.x.in b/src/platform/apollolake/apollolake.x.in
index 503e2d1c3..39cb2948f 100644
--- a/src/platform/apollolake/apollolake.x.in
+++ b/src/platform/apollolake/apollolake.x.in
@@ -87,8 +87,8 @@ MEMORY
 	org = SOF_BSS_DATA_START,
         len = SOF_BSS_DATA_SIZE
   system_heap :
-        org = HEAP_SYSTEM_BASE,
-        len = HEAP_SYSTEM_SIZE
+        org = HEAP_SYSTEM_0_BASE,
+        len = HEAP_SYSTEM_T_SIZE
   runtime_heap :
         org = HEAP_RUNTIME_BASE,
         len = HEAP_RUNTIME_SIZE
@@ -477,7 +477,7 @@ SECTIONS
   __stack = SOF_STACK_BASE;
 
   /* System Heap */
-  _system_heap = HEAP_SYSTEM_BASE;
+  _system_heap = HEAP_SYSTEM_0_BASE;
 
 
   /* module heap */
@@ -542,7 +542,7 @@ SECTIONS
   {
     . = ALIGN (32);
     _system_heap_start = ABSOLUTE(.);
-    . = . + HEAP_SYSTEM_SIZE;
+    . = . + HEAP_SYSTEM_T_SIZE;
     _system_heap_end = ABSOLUTE(.);
   } >system_heap :system_heap_phdr
 
diff --git a/src/platform/apollolake/include/platform/memory.h b/src/platform/apollolake/include/platform/memory.h
index bba58c930..6909146f0 100644
--- a/src/platform/apollolake/include/platform/memory.h
+++ b/src/platform/apollolake/include/platform/memory.h
@@ -140,12 +140,17 @@
 #define L2_VECTOR_SIZE			0x1000
 
 /* Heap configuration */
-#define HEAP_SYSTEM_BASE \
+#define HEAP_SYSTEM_0_BASE \
 	(SOF_TEXT_BASE + SOF_TEXT_SIZE +\
 	SOF_DATA_SIZE + SOF_BSS_DATA_SIZE)
-#define HEAP_SYSTEM_SIZE		0x9000
+#define HEAP_SYSTEM_0_SIZE		0x8000
 
-#define HEAP_RUNTIME_BASE		(HEAP_SYSTEM_BASE + HEAP_SYSTEM_SIZE)
+#define HEAP_SYSTEM_1_BASE	(HEAP_SYSTEM_0_BASE + HEAP_SYSTEM_0_SIZE)
+#define HEAP_SYSTEM_1_SIZE	0x1000
+
+#define HEAP_SYSTEM_T_SIZE	(HEAP_SYSTEM_0_SIZE + HEAP_SYSTEM_1_SIZE)
+
+#define HEAP_RUNTIME_BASE	(HEAP_SYSTEM_1_BASE + HEAP_SYSTEM_1_SIZE)
 #define HEAP_RUNTIME_SIZE \
 	(HEAP_RT_COUNT64 * 64 + HEAP_RT_COUNT128 * 128 + \
 	HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512)
@@ -318,6 +323,7 @@
 #define HEAP_LP_BUFFER_COUNT \
 	(HEAP_LP_BUFFER_SIZE / HEAP_LP_BUFFER_BLOCK_SIZE)
 
+#define PLATFORM_HEAP_SYSTEM		2 /* one per core */
 #define PLATFORM_HEAP_RUNTIME		1
 #define PLATFORM_HEAP_BUFFER		3
 
diff --git a/src/platform/baytrail/include/platform/memory.h b/src/platform/baytrail/include/platform/memory.h
index a2fe3e108..05fa2cbce 100644
--- a/src/platform/baytrail/include/platform/memory.h
+++ b/src/platform/baytrail/include/platform/memory.h
@@ -114,6 +114,8 @@
 #define HEAP_SYSTEM_BASE		(DRAM0_BASE + SOF_DATA_SIZE)
 #define HEAP_SYSTEM_SIZE		0x2000
 
+#define HEAP_SYSTEM_0_BASE		HEAP_SYSTEM_BASE
+
 #define HEAP_RUNTIME_BASE		(HEAP_SYSTEM_BASE + HEAP_SYSTEM_SIZE)
 #define HEAP_RUNTIME_SIZE \
 	(HEAP_RT_COUNT8 * 8 + HEAP_RT_COUNT16 * 16 + \
@@ -129,6 +131,7 @@
 #define HEAP_BUFFER_BLOCK_SIZE		0x180
 #define HEAP_BUFFER_COUNT	(HEAP_BUFFER_SIZE / HEAP_BUFFER_BLOCK_SIZE)
 
+#define PLATFORM_HEAP_SYSTEM		1 /* one per core */
 #define PLATFORM_HEAP_RUNTIME		1
 #define PLATFORM_HEAP_BUFFER		1
 
diff --git a/src/platform/baytrail/memory.c b/src/platform/baytrail/memory.c
index e1eb7cffc..9a02abccf 100644
--- a/src/platform/baytrail/memory.c
+++ b/src/platform/baytrail/memory.c
@@ -59,7 +59,7 @@ static struct block_map buf_heap_map[] = {
 };
 
 struct mm memmap = {
-	.system = {
+	.system[0] = {
 		.heap = HEAP_SYSTEM_BASE,
 		.size = HEAP_SYSTEM_SIZE,
 		.info = {.free = HEAP_SYSTEM_SIZE,},
diff --git a/src/platform/cannonlake/cannonlake.x.in b/src/platform/cannonlake/cannonlake.x.in
index b73db3983..5f502f449 100644
--- a/src/platform/cannonlake/cannonlake.x.in
+++ b/src/platform/cannonlake/cannonlake.x.in
@@ -87,8 +87,8 @@ MEMORY
         org = SOF_BSS_DATA_START,
         len = SOF_BSS_DATA_SIZE
   system_heap :
-        org = HEAP_SYSTEM_BASE,
-        len = HEAP_SYSTEM_SIZE
+        org = HEAP_SYSTEM_0_BASE,
+        len = HEAP_SYSTEM_T_SIZE
   runtime_heap :
         org = HEAP_RUNTIME_BASE,
         len = HEAP_RUNTIME_SIZE
@@ -469,7 +469,7 @@ SECTIONS
   __stack = SOF_STACK_BASE;
 
   /* System Heap */
-  _system_heap = HEAP_SYSTEM_BASE;
+  _system_heap = HEAP_SYSTEM_0_BASE;
 
 
   /* module heap */
@@ -534,7 +534,7 @@ SECTIONS
   {
     . = ALIGN (32);
     _system_heap_start = ABSOLUTE(.);
-    . = . + HEAP_SYSTEM_SIZE;
+    . = . + HEAP_SYSTEM_T_SIZE;
     _system_heap_end = ABSOLUTE(.);
   } >system_heap :system_heap_phdr
 
diff --git a/src/platform/cannonlake/include/platform/memory.h b/src/platform/cannonlake/include/platform/memory.h
index 0cbfe996b..870d950e8 100644
--- a/src/platform/cannonlake/include/platform/memory.h
+++ b/src/platform/cannonlake/include/platform/memory.h
@@ -242,12 +242,24 @@
 #define SOF_BSS_DATA_SIZE	0x10900
 
 /* Heap configuration */
-#define HEAP_SYSTEM_BASE		(SOF_TEXT_BASE + SOF_TEXT_SIZE + \
+#define HEAP_SYSTEM_0_BASE		(SOF_TEXT_BASE + SOF_TEXT_SIZE + \
 					SOF_DATA_SIZE + SOF_BSS_DATA_SIZE)
 
-#define HEAP_SYSTEM_SIZE		0x11000
+#define HEAP_SYSTEM_0_SIZE		0xe000
 
-#define HEAP_RUNTIME_BASE		(HEAP_SYSTEM_BASE + HEAP_SYSTEM_SIZE)
+#define HEAP_SYSTEM_1_BASE	(HEAP_SYSTEM_0_BASE + HEAP_SYSTEM_0_SIZE)
+#define HEAP_SYSTEM_1_SIZE	0x1000
+
+#define HEAP_SYSTEM_2_BASE	(HEAP_SYSTEM_1_BASE + HEAP_SYSTEM_1_SIZE)
+#define HEAP_SYSTEM_2_SIZE	0x1000
+
+#define HEAP_SYSTEM_3_BASE	(HEAP_SYSTEM_2_BASE + HEAP_SYSTEM_2_SIZE)
+#define HEAP_SYSTEM_3_SIZE	0x1000
+
+#define HEAP_SYSTEM_T_SIZE	(HEAP_SYSTEM_0_SIZE + HEAP_SYSTEM_1_SIZE + \
+				 HEAP_SYSTEM_2_SIZE + HEAP_SYSTEM_3_SIZE)
+
+#define HEAP_RUNTIME_BASE	(HEAP_SYSTEM_3_BASE + HEAP_SYSTEM_3_SIZE)
 #define HEAP_RUNTIME_SIZE \
 	(HEAP_RT_COUNT64 * 64 + HEAP_RT_COUNT128 * 128 + \
 	HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512)
@@ -319,7 +331,7 @@
 #define HEAP_LP_BUFFER_BLOCK_SIZE		0x180
 #define HEAP_LP_BUFFER_COUNT	(HEAP_LP_BUFFER_SIZE / HEAP_LP_BUFFER_BLOCK_SIZE)
 
-
+#define PLATFORM_HEAP_SYSTEM		4 /* one per core */
 #define PLATFORM_HEAP_RUNTIME		1
 #define PLATFORM_HEAP_BUFFER		3
 
diff --git a/src/platform/haswell/include/platform/memory.h b/src/platform/haswell/include/platform/memory.h
index 25a21b847..923918641 100644
--- a/src/platform/haswell/include/platform/memory.h
+++ b/src/platform/haswell/include/platform/memory.h
@@ -109,6 +109,8 @@
 #define HEAP_SYSTEM_BASE		(DRAM0_BASE + SOF_DATA_SIZE)
 #define HEAP_SYSTEM_SIZE		0x2000
 
+#define HEAP_SYSTEM_0_BASE		HEAP_SYSTEM_BASE
+
 #define HEAP_RUNTIME_BASE		(HEAP_SYSTEM_BASE + HEAP_SYSTEM_SIZE)
 #define HEAP_RUNTIME_SIZE \
 	(HEAP_RT_COUNT8 * 8 + HEAP_RT_COUNT16 * 16 + \
@@ -124,6 +126,7 @@
 #define HEAP_BUFFER_BLOCK_SIZE		0x180
 #define HEAP_BUFFER_COUNT		(HEAP_BUFFER_SIZE / HEAP_BUFFER_BLOCK_SIZE)
 
+#define PLATFORM_HEAP_SYSTEM		1 /* one per core */
 #define PLATFORM_HEAP_RUNTIME		1
 #define PLATFORM_HEAP_BUFFER		1
 
diff --git a/src/platform/haswell/memory.c b/src/platform/haswell/memory.c
index e1eb7cffc..9a02abccf 100644
--- a/src/platform/haswell/memory.c
+++ b/src/platform/haswell/memory.c
@@ -59,7 +59,7 @@ static struct block_map buf_heap_map[] = {
 };
 
 struct mm memmap = {
-	.system = {
+	.system[0] = {
 		.heap = HEAP_SYSTEM_BASE,
 		.size = HEAP_SYSTEM_SIZE,
 		.info = {.free = HEAP_SYSTEM_SIZE,},
diff --git a/src/platform/icelake/icelake.x.in b/src/platform/icelake/icelake.x.in
index 35847c5fd..12c8c8fb8 100644
--- a/src/platform/icelake/icelake.x.in
+++ b/src/platform/icelake/icelake.x.in
@@ -87,8 +87,8 @@ MEMORY
         org = SOF_BSS_DATA_START,
         len = SOF_BSS_DATA_SIZE
   system_heap :
-        org = HEAP_SYSTEM_BASE,
-        len = HEAP_SYSTEM_SIZE
+        org = HEAP_SYSTEM_0_BASE,
+        len = HEAP_SYSTEM_T_SIZE
   runtime_heap :
         org = HEAP_RUNTIME_BASE,
         len = HEAP_RUNTIME_SIZE
@@ -469,7 +469,7 @@ SECTIONS
   __stack = SOF_STACK_BASE;
 
   /* System Heap */
-  _system_heap = HEAP_SYSTEM_BASE;
+  _system_heap = HEAP_SYSTEM_0_BASE;
 
 
   /* module heap */
@@ -534,7 +534,7 @@ SECTIONS
   {
     . = ALIGN (32);
     _system_heap_start = ABSOLUTE(.);
-    . = . + HEAP_SYSTEM_SIZE;
+    . = . + HEAP_SYSTEM_T_SIZE;
     _system_heap_end = ABSOLUTE(.);
   } >system_heap :system_heap_phdr
 
diff --git a/src/platform/icelake/include/platform/memory.h b/src/platform/icelake/include/platform/memory.h
index 7dfce3bf1..5b2eda33d 100644
--- a/src/platform/icelake/include/platform/memory.h
+++ b/src/platform/icelake/include/platform/memory.h
@@ -242,12 +242,24 @@
 #define SOF_BSS_DATA_SIZE	0x10900
 
 /* Heap configuration */
-#define HEAP_SYSTEM_BASE		(SOF_TEXT_BASE + SOF_TEXT_SIZE + \
+#define HEAP_SYSTEM_0_BASE		(SOF_TEXT_BASE + SOF_TEXT_SIZE + \
 					SOF_DATA_SIZE + SOF_BSS_DATA_SIZE)
 
-#define HEAP_SYSTEM_SIZE		0x11000
+#define HEAP_SYSTEM_0_SIZE		0xe000
 
-#define HEAP_RUNTIME_BASE		(HEAP_SYSTEM_BASE + HEAP_SYSTEM_SIZE)
+#define HEAP_SYSTEM_1_BASE	(HEAP_SYSTEM_0_BASE + HEAP_SYSTEM_0_SIZE)
+#define HEAP_SYSTEM_1_SIZE	0x1000
+
+#define HEAP_SYSTEM_2_BASE	(HEAP_SYSTEM_1_BASE + HEAP_SYSTEM_1_SIZE)
+#define HEAP_SYSTEM_2_SIZE	0x1000
+
+#define HEAP_SYSTEM_3_BASE	(HEAP_SYSTEM_2_BASE + HEAP_SYSTEM_2_SIZE)
+#define HEAP_SYSTEM_3_SIZE	0x1000
+
+#define HEAP_SYSTEM_T_SIZE	(HEAP_SYSTEM_0_SIZE + HEAP_SYSTEM_1_SIZE + \
+				 HEAP_SYSTEM_2_SIZE + HEAP_SYSTEM_3_SIZE)
+
+#define HEAP_RUNTIME_BASE	(HEAP_SYSTEM_3_BASE + HEAP_SYSTEM_3_SIZE)
 #define HEAP_RUNTIME_SIZE \
 	(HEAP_RT_COUNT64 * 64 + HEAP_RT_COUNT128 * 128 + \
 	HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512)
@@ -319,7 +331,7 @@
 #define HEAP_LP_BUFFER_BLOCK_SIZE		0x180
 #define HEAP_LP_BUFFER_COUNT	(HEAP_LP_BUFFER_SIZE / HEAP_LP_BUFFER_BLOCK_SIZE)
 
-
+#define PLATFORM_HEAP_SYSTEM		4 /* one per core */
 #define PLATFORM_HEAP_RUNTIME		1
 #define PLATFORM_HEAP_BUFFER		3
 
diff --git a/src/platform/intel/cavs/memory.c b/src/platform/intel/cavs/memory.c
index 21792f046..a92f36688 100644
--- a/src/platform/intel/cavs/memory.c
+++ b/src/platform/intel/cavs/memory.c
@@ -66,13 +66,36 @@ static struct block_map lp_buf_heap_map[] = {
 };
 
 struct mm memmap = {
-	.system = {
-		.heap = HEAP_SYSTEM_BASE,
-		.size = HEAP_SYSTEM_SIZE,
-		.info = {.free = HEAP_SYSTEM_SIZE,},
+	.system[0] = {
+		.heap = HEAP_SYSTEM_0_BASE,
+		.size = HEAP_SYSTEM_0_SIZE,
+		.info = {.free = HEAP_SYSTEM_0_SIZE,},
 		.caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_EXT |
 			SOF_MEM_CAPS_CACHE,
 	},
+	.system[1] = {
+		.heap = HEAP_SYSTEM_1_BASE,
+		.size = HEAP_SYSTEM_1_SIZE,
+		.info = {.free = HEAP_SYSTEM_1_SIZE,},
+		.caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_EXT |
+			SOF_MEM_CAPS_CACHE,
+	},
+#if defined(CONFIG_CANNONLAKE) || defined(CONFIG_ICELAKE)
+	.system[2] = {
+		.heap = HEAP_SYSTEM_2_BASE,
+		.size = HEAP_SYSTEM_2_SIZE,
+		.info = {.free = HEAP_SYSTEM_2_SIZE,},
+		.caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_EXT |
+			SOF_MEM_CAPS_CACHE,
+	},
+	.system[3] = {
+		.heap = HEAP_SYSTEM_3_BASE,
+		.size = HEAP_SYSTEM_3_SIZE,
+		.info = {.free = HEAP_SYSTEM_3_SIZE,},
+		.caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_EXT |
+			SOF_MEM_CAPS_CACHE,
+	},
+#endif
 	.runtime[0] = {
 		.blocks = ARRAY_SIZE(rt_heap_map),
 		.map = rt_heap_map,
@@ -109,7 +132,7 @@ struct mm memmap = {
 		.caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_LP |
 			SOF_MEM_CAPS_CACHE | SOF_MEM_CAPS_DMA,
 	},
-	.total = {.free = HEAP_SYSTEM_SIZE + HEAP_RUNTIME_SIZE +
+	.total = {.free = HEAP_SYSTEM_T_SIZE + HEAP_RUNTIME_SIZE +
 			HEAP_BUFFER_SIZE + HEAP_HP_BUFFER_SIZE +
 			HEAP_LP_BUFFER_SIZE,},
 };