OrgACRN
/
sof
mirror of https://github.com/thesofproject/sof.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
sof/scripts/xtensa-build-zephyr.py

503 lines
20 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# SPDX-License-Identifier: BSD-3-Clause
#pylint:disable=mixed-indentation
#pylint:disable=invalid-name
#pylint:disable=missing-function-docstring
import argparse
import shlex
import subprocess
import pathlib
import platform
import sys
import shutil
import multiprocessing
import os
import warnings
# anytree module is defined in Zephyr build requirements
from anytree import Node, RenderTree
# Constant value resolves SOF_TOP directory as: "this script directory/.."
SOF_TOP = pathlib.Path(__file__).parents[1].resolve()
# Default value may be overriten by -p arg
west_top = pathlib.Path(SOF_TOP, "zephyrproject")
default_rimage_key = pathlib.Path("modules", "audio", "sof", "keys", "otc_private_key.pem")
if platform.system() == "Windows":
xtensa_tools_version_postfix = "-win32"
elif platform.system() == "Linux":
xtensa_tools_version_postfix = "-linux"
else:
xtensa_tools_version_postfix = "-unsupportedOS"
warnings.warn(f"Your operating system: {platform.system()} is not supported")
platform_list = [
# Intel platforms
{
"name": "apl",
"PLAT_CONFIG": "intel_adsp_cavs15",
"XTENSA_CORE": "X4H3I16w2D48w3a_2017_8",
"XTENSA_TOOLS_VERSION": f"RG-2017.8{xtensa_tools_version_postfix}"
},
{
"name": "cnl",
"PLAT_CONFIG": "intel_adsp_cavs18",
"XTENSA_CORE": "X6H3CNL_2017_8",
"XTENSA_TOOLS_VERSION": f"RG-2017.8{xtensa_tools_version_postfix}"
},
{
"name": "icl",
"PLAT_CONFIG": "intel_adsp_cavs20",
"XTENSA_CORE": "X6H3CNL_2017_8",
"XTENSA_TOOLS_VERSION": f"RG-2017.8{xtensa_tools_version_postfix}"
},
{
"name": "jsl",
"PLAT_CONFIG": "intel_adsp_cavs20_jsl",
"XTENSA_CORE": "X6H3CNL_2017_8",
"XTENSA_TOOLS_VERSION": f"RG-2017.8{xtensa_tools_version_postfix}"
},
{
"name": "tgl",
"PLAT_CONFIG": "intel_adsp_cavs25",
"IPC4_CONFIG_OVERLAY": "ipc4_overlay.conf",
"IPC4_RIMAGE_DESC": "tgl-cavs.toml",
"XTENSA_CORE": "cavs2x_LX6HiFi3_2017_8",
"XTENSA_TOOLS_VERSION": f"RG-2017.8{xtensa_tools_version_postfix}",
"RIMAGE_KEY": pathlib.Path("modules", "audio", "sof", "keys", "otc_private_key_3k.pem")
},
{
"name": "tgl-h",
"PLAT_CONFIG": "intel_adsp_cavs25_tgph",
"XTENSA_CORE": "cavs2x_LX6HiFi3_2017_8",
"XTENSA_TOOLS_VERSION": f"RG-2017.8{xtensa_tools_version_postfix}",
"RIMAGE_KEY": pathlib.Path("modules", "audio", "sof", "keys", "otc_private_key_3k.pem")
},
# NXP platforms
{
"name": "imx8",
"PLAT_CONFIG": "nxp_adsp_imx8",
"RIMAGE_KEY": "" # no key needed for imx8
},
{
"name": "imx8x",
"PLAT_CONFIG": "nxp_adsp_imx8x",
"RIMAGE_KEY": "ignored for imx8x"
},
{
"name": "imx8m",
"PLAT_CONFIG": "nxp_adsp_imx8m",
"RIMAGE_KEY": "ignored for imx8m"
}
]
platform_names = [platform["name"] for platform in platform_list]
class validate_platforms_arguments(argparse.Action):
"""Validates positional platform arguments whether provided platform name is supported."""
def __call__(self, parser, namespace, values, option_string=None):
if values:
for value in values:
if value not in platform_names:
raise argparse.ArgumentError(self, f"Unsupported platform: {value}")
setattr(namespace, "platforms", values)
def parse_args():
global args
global west_top
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
epilog=("This script supports XtensaTools but only when installed in a specific\n" +
"directory structure, example:\n" +
"myXtensa/\n" +
"└── install/\n" +
" ├── builds/\n" +
" │   ├── RD-2012.5{}/\n".format(xtensa_tools_version_postfix) +
" │   │   └── Intel_HiFiEP/\n" +
" │   └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) +
" │   ├── LX4_langwell_audio_17_8/\n" +
" │   └── X4H3I16w2D48w3a_2017_8/\n" +
" └── tools/\n" +
" ├── RD-2012.5{}/\n".format(xtensa_tools_version_postfix) +
" │   └── XtensaTools/\n" +
" └── RG-2017.8{}/\n".format(xtensa_tools_version_postfix) +
" └── XtensaTools/\n" +
"$XTENSA_TOOLS_ROOT=/path/to/myXtensa ...\n" +
f"Supported platforms {platform_names}"))
parser.add_argument("-a", "--all", required=False, action="store_true",
help="Build all currently supported platforms")
parser.add_argument("platforms", nargs="*", action=validate_platforms_arguments,
help="List of platforms to build")
parser.add_argument("-i", "--ipc", required=False, choices=["IPC3", "IPC4"],
default="IPC3", help="IPC major version")
parser.add_argument("-j", "--jobs", required=False, type=int,
default=multiprocessing.cpu_count(),
help="Set number of make build jobs for rimage."
" Jobs=number of cores by default. Ignored by west build.")
parser.add_argument("-k", "--key", type=pathlib.Path, required=False,
help="Path to a non-default rimage signing key.")
parser.add_argument("-z", "--zephyr-ref", required=False,
help="Initial Zephyr git ref for the -c option."
" Can be a branch, tag, full SHA1 in a fork")
parser.add_argument("-u", "--url", required=False,
default="https://github.com/zephyrproject-rtos/zephyr/",
help="URL to clone Zephyr from")
mode_group = parser.add_mutually_exclusive_group()
mode_group.add_argument("-p", "--west_path", required=False, type=pathlib.Path,
help="""Points to existing Zephyr project directory. Incompatible with -c.
Uses by default path used by -c mode: SOF_TOP/zephyrproject.
If zephyr-project/modules/audio/sof is missing then a
symbolic link pointing to SOF_TOP directory will automatically be
created and west will recognize it as its new sof module.
If zephyr-project/modules/audio/sof already exists and is a
different copy than where this script is run from, then the
behavior is undefined.
This -p option is always required_if the real (not symbolic)
sof/ and zephyr-project/ directories are not nested in one
another.""",
)
mode_group.add_argument("-c", "--clone_mode", required=False, action="store_true",
help="""Using west, downloads inside this sof clone a new Zephyr
project with the required git repos. Creates a
sof/zephyrproject/modules/audio/sof symbolic link pointing
back at this sof clone.
Incompatible with -p. To stop after downloading Zephyr, do not
pass any platform or cmake argument.""",
)
parser.add_argument('-v', '--verbose', default=0, action='count',
help="""Verbosity level. Repetition of the flag increases verbosity.
The same number of '-v' is passed to "west".
""",
)
# Cannot use a standard -- delimiter because argparse deletes it.
parser.add_argument("-C", "--cmake-args", action='append', default=[],
help="""Cmake arguments passed as is to cmake configure step.
Can be passed multiple times; whitespace is preserved Example:
-C=--warn-uninitialized -C '-DEXTRA_FLAGS=-Werror -g0'
Note '-C --warn-uninitialized' is not supported by argparse, an equal
sign must be used (https://bugs.python.org/issue9334)""",
)
args = parser.parse_args()
if args.all:
args.platforms = platform_names
# print help message if no arguments provided
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
if args.zephyr_ref and not args.clone_mode:
raise RuntimeError(f"Argument -z without -c makes no sense")
if args.clone_mode and not args.zephyr_ref:
args.zephyr_ref = "main" # set default name for -z if -c specified
if args.west_path: # let the user provide an already existing zephyrproject/ anywhere
west_top = pathlib.Path(args.west_path)
elif not args.clone_mode: # if neather -p nor -c provided, use -p
args.west_path = west_top
def execute_command(*run_args, **run_kwargs):
"""[summary] Provides wrapper for subprocess.run that prints
command executed when 'more verbose' verbosity level is set."""
command_args = run_args[0]
# If you really need the shell in some non-portable section then
# invoke subprocess.run() directly.
if run_kwargs.get('shell') or not isinstance(command_args, list):
raise RuntimeError("Do not rely on non-portable shell parsing")
if args.verbose >= 0:
cwd = run_kwargs.get('cwd')
print_cwd = f"In dir: {cwd}" if cwd else f"in current dir: {os.getcwd()}"
print_args = shlex.join(command_args)
print(f"{print_cwd}; running command: {print_args}", flush=True)
if run_kwargs.get('check') is None:
run_kwargs['check'] = True
#pylint:disable=subprocess-run-check
return subprocess.run(*run_args, **run_kwargs)
def show_installed_files():
"""[summary] Scans output directory building binary tree from files and folders
then presents them in similar way to linux tree command."""
graph_root = Node(STAGING_DIR.name, parent=None)
relative_entries = [entry.relative_to(STAGING_DIR) for entry in STAGING_DIR.glob("**/*")]
nodes = [ graph_root ]
for entry in relative_entries:
if str(entry.parent) == ".":
nodes.append(Node(entry.name, parent=graph_root))
else:
node_parent = [node for node in nodes if node.name == str(entry.parent.name)][0]
if not node_parent:
warnings.warn("Failed to construct installed files tree")
return
nodes.append(Node(entry.name, parent=node_parent))
for pre, fill, node in RenderTree(graph_root):
print(f"{pre}{node.name}")
def check_west_installation():
west_path = shutil.which("west")
if not west_path:
raise FileNotFoundError("Install west and a west toolchain,"
"https://docs.zephyrproject.org/latest/getting_started/index.html")
print(f"Found west: {west_path}")
def find_west_workspace():
"""[summary] Scans for west workspaces using west topdir command by checking script execution
west_top and SOF_TOP directories.
:return: [description] Returns path when west workspace had been found, otherwise None
:rtype: [type] bool, pathlib.Path
"""
global west_top, SOF_TOP
paths = [ pathlib.Path(os.getcwd()), SOF_TOP, west_top]
for path in paths:
if path.is_dir():
result = execute_command(["west", "topdir"], stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, check=False, cwd=path)
if result.returncode == 0:
return path
return None
def create_zephyr_directory():
global west_top
west_top.mkdir(mode=511, parents=False, exist_ok=False)
west_top = west_top.resolve(strict=True)
def create_zephyr_sof_symlink():
global west_top, SOF_TOP
audio_modules_dir = pathlib.Path(west_top, "modules", "audio")
audio_modules_dir.mkdir(mode=511, parents=True, exist_ok=True)
sof_symlink = pathlib.Path(audio_modules_dir, "sof")
# Symlinks creation requires administrative privileges in Windows or special user rights
try:
if not sof_symlink.is_symlink():
sof_symlink.symlink_to(SOF_TOP, target_is_directory=True)
except:
print(f"Failed to create symbolic link: {sof_symlink} to {SOF_TOP}."
"\nIf you run script on Windows run it with administrative privileges or\n"
"grant user symlink creation rights -"
"see: https://docs.microsoft.com/en-us/windows/security/threat-protection/"
"security-policy-settings/create-symbolic-links")
raise
def west_init_update():
"""[summary] Downloads zephyrproject inside sof/ and create a ../../.. symbolic
link back to sof/"""
global west_top, SOF_TOP
zephyr_dir = pathlib.Path(west_top, "zephyr")
z_remote = "origin"
z_ref = args.zephyr_ref
# Example of how to override SOF CI and point it at any Zephyr
# commit from anywhere and to run all tests on it. Simply
# uncomment and edit these lines and submit as an SOF Pull
# Request. Note: unlike many git servers, github allows direct
# fetching of (full 40-digits) SHAs; even SHAs not in origin but
# in forks! See the end of "git help fetch-pack".
#
# z_remote = "https://somewhere.else"
# z_ref = "pull/38374/head"
# z_ref = "cb9a279050c4e8ad4663ee78688d8e7de1cac953"
# SECURITY WARNING for reviewers: never allow unknown code from
# unknown submitters on any CI system.
execute_command(["git", "clone", "--depth", "5", args.url, str(zephyr_dir)], timeout=1200)
execute_command(["git", "fetch", z_remote, z_ref], timeout=300, cwd=zephyr_dir)
execute_command(["git", "checkout", "FETCH_HEAD"], cwd=zephyr_dir)
execute_command(["git", "-C", str(zephyr_dir), "--no-pager", "log", "--oneline", "--graph",
"--decorate", "--max-count=20"])
execute_command(["west", "init", "-l", str(zephyr_dir)])
create_zephyr_sof_symlink()
# Do NOT "west update sof"!!
execute_command(["west", "update", "zephyr", "hal_xtensa"], timeout=300, cwd=west_top)
def build_platforms():
global west_top, SOF_TOP
# double check that west workspace is initialized
execute_command(["west", "topdir"], stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, cwd=west_top)
print(f"SOF_TOP={SOF_TOP}")
print(f"west_top={west_top}")
global STAGING_DIR
STAGING_DIR = pathlib.Path(west_top, "build-sof-staging")
# smex does not use 'install -D'
sof_output_dir = pathlib.Path(STAGING_DIR, "sof")
sof_output_dir.mkdir(mode=511, parents=True, exist_ok=True)
for platform in args.platforms:
platform_dict = [x for x in platform_list if x["name"] == platform][0]
xtensa_tools_root_dir = os.getenv("XTENSA_TOOLS_ROOT")
# when XTENSA_TOOLS_ROOT environmental variable is set,
# use user installed Xtensa tools not Zephyr SDK
if "XTENSA_TOOLS_VERSION" in platform_dict and xtensa_tools_root_dir:
xtensa_tools_root_dir = pathlib.Path(xtensa_tools_root_dir)
if not xtensa_tools_root_dir.is_dir():
raise RuntimeError(f"Platform {platform} uses Xtensa toolchain."
"\nVariable XTENSA_TOOLS_VERSION points path that does not exist\n"
"or is not a directory")
# set variables expected by zephyr/cmake/toolchain/xcc/generic.cmake
os.environ["ZEPHYR_TOOLCHAIN_VARIANT"] = "xcc"
XTENSA_TOOLCHAIN_PATH = str(pathlib.Path(xtensa_tools_root_dir, "install",
"tools").absolute())
os.environ["XTENSA_TOOLCHAIN_PATH"] = XTENSA_TOOLCHAIN_PATH
TOOLCHAIN_VER = platform_dict["XTENSA_TOOLS_VERSION"]
XTENSA_CORE = platform_dict["XTENSA_CORE"]
os.environ["TOOLCHAIN_VER"] = TOOLCHAIN_VER
print(f"XTENSA_TOOLCHAIN_PATH={XTENSA_TOOLCHAIN_PATH}")
print(f"TOOLCHAIN_VER={TOOLCHAIN_VER}")
# set variables expected by xcc toolchain
XTENSA_BUILDS_DIR=str(pathlib.Path(xtensa_tools_root_dir, "install", "builds",
TOOLCHAIN_VER).absolute())
XTENSA_SYSTEM = str(pathlib.Path(XTENSA_BUILDS_DIR, XTENSA_CORE, "config").absolute())
os.environ["XTENSA_SYSTEM"] = XTENSA_SYSTEM
print(f"XTENSA_SYSTEM={XTENSA_SYSTEM}")
platform_build_dir_name = f"build-{platform}"
# https://docs.zephyrproject.org/latest/guides/west/build-flash-debug.html#one-time-cmake-arguments
# https://github.com/zephyrproject-rtos/zephyr/pull/40431#issuecomment-975992951
abs_build_dir = pathlib.Path(west_top, platform_build_dir_name)
if (pathlib.Path(abs_build_dir, "build.ninja").is_file()
or pathlib.Path(abs_build_dir, "Makefile").is_file()):
if args.cmake_args:
print(args.cmake_args)
raise RuntimeError("Some CMake arguments are ignored in incremental builds, "
+ f"you must delete {abs_build_dir} first")
PLAT_CONFIG = platform_dict["PLAT_CONFIG"]
build_cmd = ["west"]
build_cmd += ["-v"] * args.verbose
build_cmd += ["build", "--build-dir", platform_build_dir_name]
source_dir = pathlib.Path(west_top, "zephyr", "samples", "subsys", "audio", "sof")
build_cmd += ["--board", PLAT_CONFIG, str(source_dir)]
build_cmd.append('--')
if args.cmake_args:
build_cmd += args.cmake_args
overlays = []
# You may override default overlay.conf file name using CONFIG_OVERLAY in your platform
# dictionary
overlay_filename = platform_dict.get("CONFIG_OVERLAY", "overlay.conf")
overlays.append(str(pathlib.Path(SOF_TOP, "overlays", platform, overlay_filename)))
if args.ipc == "IPC4":
overlays.append(str(pathlib.Path(SOF_TOP, "overlays", platform, platform_dict["IPC4_CONFIG_OVERLAY"])))
overlays = ";".join(overlays)
build_cmd.append(f"-DOVERLAY_CONFIG={overlays}")
# Build
execute_command(build_cmd, cwd=west_top)
smex_executable = pathlib.Path(west_top, platform_build_dir_name, "zephyr", "smex_ep",
"build", "smex")
fw_ldc_file = pathlib.Path(sof_output_dir, f"sof-{platform}.ldc")
input_elf_file = pathlib.Path(west_top, platform_build_dir_name, "zephyr", "zephyr.elf")
# Extract metadata
execute_command([str(smex_executable), "-l", str(fw_ldc_file), str(input_elf_file)])
# CMake - configure rimage module
rimage_dir_name="build-rimage"
sof_mirror_dir = pathlib.Path("modules", "audio", "sof")
rimage_source_dir = pathlib.Path(sof_mirror_dir, "rimage")
execute_command(["cmake", "-B", rimage_dir_name, "-S", str(rimage_source_dir)],
cwd=west_top)
# CMake build rimage module
execute_command(["cmake", "--build", rimage_dir_name, "-j", str(args.jobs)],
cwd=west_top)
# Sign firmware
rimage_executable = shutil.which("rimage", path=pathlib.Path(west_top, rimage_dir_name))
rimage_config = pathlib.Path(sof_mirror_dir, "rimage", "config")
sign_cmd = ["west"]
sign_cmd += ["-v"] * args.verbose
sign_cmd += ["sign", "--build-dir", platform_build_dir_name, "--tool", "rimage"]
sign_cmd += ["--tool-path", rimage_executable]
signing_key = ""
if args.key:
signing_key = args.key
elif "RIMAGE_KEY" in platform_dict:
signing_key = platform_dict["RIMAGE_KEY"]
else:
signing_key = default_rimage_key
sign_cmd += ["--tool-data", str(rimage_config), "--", "-k", str(signing_key)]
if args.ipc == "IPC4":
rimage_desc = pathlib.Path(SOF_TOP, "rimage", "config", platform_dict["IPC4_RIMAGE_DESC"])
sign_cmd += ["-c", str(rimage_desc)]
execute_command(sign_cmd, cwd=west_top)
# Install by copy
fw_file_to_copy = pathlib.Path(west_top, platform_build_dir_name, "zephyr", "zephyr.ri")
fw_file_installed = pathlib.Path(sof_output_dir, "community", f"sof-{platform}.ri")
os.makedirs(os.path.dirname(fw_file_installed), exist_ok=True)
# looses file owner and group - file is commonly accessible
shutil.copy2(str(fw_file_to_copy), str(fw_file_installed))
src_dest_list = []
# Install sof-logger
sof_logger_dir = pathlib.Path(west_top, platform_build_dir_name, "zephyr",
"sof-logger_ep", "build", "logger")
sof_logger_executable_to_copy = pathlib.Path(shutil.which("sof-logger", path=sof_logger_dir))
tools_output_dir = pathlib.Path(STAGING_DIR, "tools")
sof_logger_installed_file = pathlib.Path(tools_output_dir, sof_logger_executable_to_copy.name).resolve()
src_dest_list += [(sof_logger_executable_to_copy, sof_logger_installed_file)]
src_dest_list += [(pathlib.Path(west_top) /
"zephyr" / "soc" / "xtensa" / "intel_adsp" / "tools" / "cavstool.py",
tools_output_dir)]
for _src, _dst in src_dest_list:
os.makedirs(os.path.dirname(_dst), exist_ok=True)
# looses file owner and group - file is commonly accessible
shutil.copy2(str(_src), str(_dst))
def run_clone_mode():
if find_west_workspace():
raise RuntimeError("Zephyr found already! Not downloading it again")
create_zephyr_directory()
west_init_update()
def run_point_mode():
global west_top, SOF_TOP
west_workspace_path = find_west_workspace()
if not west_workspace_path:
raise RuntimeError("Failed to find west workspace.\nScanned directories:\n{}\n{}\n{}"
.format(os.getcwd(), SOF_TOP, west_top))
if not west_workspace_path.exists():
raise FileNotFoundError("West topdir returned {} as workspace but it"
" does not exist".format(west_workspace_path))
create_zephyr_sof_symlink()
def main():
parse_args()
check_west_installation()
if len(args.platforms) == 0:
print("No platform build requested")
else:
print("Building platforms: {}".format(" ".join(args.platforms)))
# we made two modes mutually exclusive with argparse
if args.west_path:
# check west_path input + create symlink if needed
run_point_mode()
if args.clone_mode:
# Initialize zephyr project with west
run_clone_mode()
if args.platforms:
build_platforms()
show_installed_files()
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink