diff --git a/src/arch/xtensa/configs/imx8_defconfig b/src/arch/xtensa/configs/imx8_defconfig
index 3ada116d8..3998b44d4 100644
--- a/src/arch/xtensa/configs/imx8_defconfig
+++ b/src/arch/xtensa/configs/imx8_defconfig
@@ -1,2 +1,3 @@
 CONFIG_IMX8=y
 CONFIG_HAVE_AGENT=n
+CONFIG_FORMAT_CONVERT_HIFI3=n
diff --git a/src/arch/xtensa/configs/imx8m_defconfig b/src/arch/xtensa/configs/imx8m_defconfig
index 262202383..fb8e2f4e9 100644
--- a/src/arch/xtensa/configs/imx8m_defconfig
+++ b/src/arch/xtensa/configs/imx8m_defconfig
@@ -1,2 +1,3 @@
 CONFIG_IMX8M=y
 CONFIG_HAVE_AGENT=n
+CONFIG_FORMAT_CONVERT_HIFI3=n
diff --git a/src/arch/xtensa/configs/imx8x_defconfig b/src/arch/xtensa/configs/imx8x_defconfig
index 80e4608ce..f9a9e4956 100644
--- a/src/arch/xtensa/configs/imx8x_defconfig
+++ b/src/arch/xtensa/configs/imx8x_defconfig
@@ -1,2 +1,3 @@
 CONFIG_IMX8X=y
 CONFIG_HAVE_AGENT=n
+CONFIG_FORMAT_CONVERT_HIFI3=n