diff --git a/scripts/xtensa-build-all.sh b/scripts/xtensa-build-all.sh
index 7a359387b..8424b44f3 100755
--- a/scripts/xtensa-build-all.sh
+++ b/scripts/xtensa-build-all.sh
@@ -49,7 +49,6 @@ fi
 
 
 # now build the firmware (depends on rimage)
-rm -fr src/arch/xtensa/*.ri
 
 # fail on any errors
 set -e
@@ -177,6 +176,9 @@ do
 		PATH=$pwd/../$HOST/bin:$OLDPATH
 	fi
 
+	# only delete binary related to this build
+	rm -fr src/arch/xtensa/sof-$j.*
+
 	./configure --with-arch=xtensa --with-platform=$PLATFORM --with-root-dir=$ROOT --host=$HOST \
 		CC=$XCC OBJCOPY=$XTOBJCOPY OBJDUMP=$XTOBJDUMP --with-dsp-core=$XTENSA_CORE
 
diff --git a/src/arch/xtensa/Makefile.am b/src/arch/xtensa/Makefile.am
index f9f390822..5ef0e6253 100644
--- a/src/arch/xtensa/Makefile.am
+++ b/src/arch/xtensa/Makefile.am
@@ -235,6 +235,5 @@ vminstall-local:
 clean-local:
 	rm -fr mod-*
 	rm -fr *.bin
-	rm -fr *.lst
 	rm -fr *.map
 	rm -fr *.dis