diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c
index 1c0ecb9c3..9db92e8d4 100644
--- a/src/library_manager/lib_manager.c
+++ b/src/library_manager/lib_manager.c
@@ -686,9 +686,6 @@ static void __sparse_cache *lib_manager_allocate_store_mem(uint32_t size,
 		return NULL;
 	}
 
-	dcache_invalidate_region(local_add, size);
-	icache_invalidate_region(local_add, size);
-
 	return local_add;
 }
 
diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c
index 5883719b9..4b885b25f 100644
--- a/src/library_manager/llext_manager.c
+++ b/src/library_manager/llext_manager.c
@@ -64,16 +64,8 @@ static int llext_manager_load_data_from_storage(void __sparse_cache *vma, void *
 		return ret;
 	}
 
-	ret = memcpy_s((__sparse_force void *)vma, size, s_addr, size);
-	if (ret < 0)
-		return ret;
-
-	/* Some data can be accessed as uncached, in fact that's the default */
-	/* Both D- and I-caches have been invalidated */
-	dcache_writeback_region(vma, size);
-
 	/* TODO: Change attributes for memory to FLAGS  */
-	return 0;
+	return memcpy_s((__sparse_force void *)vma, size, s_addr, size);
 }
 
 static int llext_manager_load_module(uint32_t module_id, struct sof_man_module *mod,
@@ -98,12 +90,19 @@ static int llext_manager_load_module(uint32_t module_id, struct sof_man_module *
 	if (ret < 0)
 		return ret;
 
+	/* .text contains instructions and it also often contains local data */
+	dcache_writeback_region(va_base_text, st_text_size);
+	icache_invalidate_region(va_base_text, st_text_size);
+
 	/* Copy RODATA */
 	ret = llext_manager_load_data_from_storage(va_base_rodata, src_rodata,
 						   st_rodata_size, SYS_MM_MEM_PERM_RW);
 	if (ret < 0)
 		goto e_text;
 
+	/* Some data can be accessed as uncached, in fact that's the default */
+	dcache_writeback_region(va_base_rodata, st_rodata_size);
+
 	return 0;
 
 e_text: