# Copyright (c) 2018 Foundries.io # # SPDX-License-Identifier: Apache-2.0 import argparse import os import pathlib import shlex import sys import yaml from west.commands import Verbosity from west.configuration import config from west.util import west_topdir from west.version import __version__ from zcmake import DEFAULT_CMAKE_GENERATOR, run_cmake, run_build, CMakeCache from build_helpers import is_zephyr_build, find_build_dir, load_domains, \ FIND_BUILD_DIR_DESCRIPTION from zephyr_ext_common import Forceable _ARG_SEPARATOR = '--' SYSBUILD_PROJ_DIR = pathlib.Path(__file__).resolve().parent.parent.parent \ / pathlib.Path('share/sysbuild') BUILD_INFO_LOG = 'build_info.yml' BUILD_USAGE = '''\ west build [-h] [-b BOARD[@REV]]] [-d BUILD_DIR] [-S SNIPPET] [--shield SHIELD] [-t TARGET] [-p {auto, always, never}] [-c] [--cmake-only] [-n] [-o BUILD_OPT] [-f] [--sysbuild | --no-sysbuild] [--domain DOMAIN] [--extra-conf FILE.conf] [--extra-dtc-overlay FILE.overlay] [source_dir] -- [cmake_opt [cmake_opt ...]] ''' BUILD_DESCRIPTION = f'''\ Convenience wrapper for building Zephyr applications. {FIND_BUILD_DIR_DESCRIPTION} positional arguments: source_dir application source directory cmake_opt extra options to pass to cmake; implies -c (these must come after "--" as shown above) ''' PRISTINE_DESCRIPTION = """\ A "pristine" build directory is empty. The -p option controls whether the build directory is made pristine before the build is done. A bare '--pristine' with no value is the same as --pristine=always. Setting --pristine=auto uses heuristics to guess if a pristine build may be necessary.""" def config_get(option, fallback): return config.get('build', option, fallback=fallback) def config_getboolean(option, fallback): return config.getboolean('build', option, fallback=fallback) class AlwaysIfMissing(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, values or 'always') class Build(Forceable): def __init__(self): super(Build, self).__init__( 'build', # Keep this in sync with the string in west-commands.yml. 'compile a Zephyr application', BUILD_DESCRIPTION, accepts_unknown_args=True) self.source_dir = None '''Source directory for the build, or None on error.''' self.build_dir = None '''Final build directory used to run the build, or None on error.''' self.created_build_dir = False '''True if the build directory was created; False otherwise.''' self.run_cmake = False '''True if CMake was run; False otherwise. Note: this only describes CMake runs done by this command. The build system generated by CMake may also update itself due to internal logic.''' self.cmake_cache = None '''Final parsed CMake cache for the build, or None on error.''' def _banner(self, msg): self.inf('-- west build: ' + msg, colorize=True) def do_add_parser(self, parser_adder): parser = parser_adder.add_parser( self.name, help=self.help, formatter_class=argparse.RawDescriptionHelpFormatter, description=self.description, usage=BUILD_USAGE) # Remember to update west-completion.bash if you add or remove # flags parser.add_argument('-b', '--board', help='board to build for with optional board revision') # Hidden option for backwards compatibility parser.add_argument('-s', '--source-dir', help=argparse.SUPPRESS) parser.add_argument('-d', '--build-dir', help='build directory to create or use') self.add_force_arg(parser) group = parser.add_argument_group('cmake and build tool') group.add_argument('-c', '--cmake', action='store_true', help='force a cmake run') group.add_argument('--cmake-only', action='store_true', help="just run cmake; don't build (implies -c)") group.add_argument('--domain', action='append', help='''execute build tool (make or ninja) only for given domain''') group.add_argument('-t', '--target', help='''run build system target TARGET (try "-t usage")''') group.add_argument('-T', '--test-item', help='''Build based on test data in testcase.yaml or sample.yaml. If source directory is not used an argument has to be defined as SOURCE_PATH/TEST_NAME. E.g. samples/hello_world/sample.basic.helloworld. If source directory is passed then "TEST_NAME" is enough.''') group.add_argument('-o', '--build-opt', default=[], action='append', help='''options to pass to the build tool (make or ninja); may be given more than once''') group.add_argument('-n', '--just-print', '--dry-run', '--recon', dest='dry_run', action='store_true', help="just print build commands; don't run them") group.add_argument('-S', '--snippet', dest='snippets', metavar='SNIPPET', action='append', default=[], help='''add the argument to SNIPPET; may be given multiple times. Forces CMake to run again if given. Do not use this option with manually specified -DSNIPPET... cmake arguments: the results are undefined''') group.add_argument('--shield', dest='shields', metavar='SHIELD', action='append', default=[], help='''add the argument to SHIELD; may be given multiple times. Forces CMake to run again if given. Do not use this option with manually specified -DSHIELD... cmake arguments: the results are undefined''') group.add_argument('--extra-conf', dest='extra_conf_files', metavar='EXTRA_CONF_FILE', action='append', default=[], help='''add the argument to EXTRA_CONF_FILE; may be given multiple times. Forces CMake to run again if given. Do not use this option with manually specified -DEXTRA_CONF_FILE... cmake arguments: the results are undefined''') group.add_argument('--extra-dtc-overlay', dest='extra_dtc_overlay_files', metavar='EXTRA_DTC_OVERLAY_FILE', action='append', default=[], help='''add the argument to EXTRA_DTC_OVERLAY_FILE; may be given multiple times. Forces CMake to run again if given. Do not use this option with manually specified -DEXTRA_DTC_OVERLAY_FILE... cmake arguments: the results are undefined''') group = parser.add_mutually_exclusive_group() group.add_argument('--sysbuild', action='store_true', help='''create multi domain build system''') group.add_argument('--no-sysbuild', action='store_true', help='''do not create multi domain build system (default)''') group = parser.add_argument_group('pristine builds', PRISTINE_DESCRIPTION) group.add_argument('-p', '--pristine', choices=['auto', 'always', 'never'], action=AlwaysIfMissing, nargs='?', help='pristine build folder setting') return parser def do_run(self, args, remainder): self.args = args # Avoid having to pass them around self.config_board = config_get('board', None) self.dbg('args: {} remainder: {}'.format(args, remainder), level=Verbosity.DBG_EXTREME) # Store legacy -s option locally source_dir = self.args.source_dir self._parse_remainder(remainder) # Parse testcase.yaml or sample.yaml files for additional options. if self.args.test_item: # we get path + testitem item = os.path.basename(self.args.test_item) if self.args.source_dir: test_path = self.args.source_dir else: test_path = os.path.dirname(self.args.test_item) if test_path and os.path.exists(test_path): self.args.source_dir = test_path if not self._parse_test_item(item): self.die("No test metadata found") else: self.die("test item path does not exist") if source_dir: if self.args.source_dir: self.die("source directory specified twice:({} and {})".format( source_dir, self.args.source_dir)) self.args.source_dir = source_dir self.dbg('source_dir: {} cmake_opts: {}'.format(self.args.source_dir, self.args.cmake_opts), level=Verbosity.DBG_EXTREME) self._sanity_precheck() self._setup_build_dir() if args.pristine is not None: pristine = args.pristine else: # Load the pristine={auto, always, never} configuration value pristine = config_get('pristine', 'never') if pristine not in ['auto', 'always', 'never']: self.wrn( 'treating unknown build.pristine value "{}" as "never"'. format(pristine)) pristine = 'never' self.auto_pristine = pristine == 'auto' self.dbg('pristine: {} auto_pristine: {}'.format(pristine, self.auto_pristine), level=Verbosity.DBG_MORE) if is_zephyr_build(self.build_dir): if pristine == 'always': self._run_pristine() self.run_cmake = True else: self._update_cache() if (self.args.cmake or self.args.cmake_opts or self.args.cmake_only or self.args.snippets or self.args.shields or self.args.extra_conf_files or self.args.extra_dtc_overlay_files): self.run_cmake = True else: self.run_cmake = True self.source_dir = self._find_source_dir() self._sanity_check() build_info_path = self.build_dir build_info_file = os.path.join(build_info_path, BUILD_INFO_LOG) west_workspace = west_topdir(self.source_dir) if not os.path.exists(build_info_path): os.makedirs(build_info_path) if not os.path.exists(build_info_file): build_command = {'west': {'command': ' '.join(sys.argv[:]), 'topdir': str(west_workspace), 'version': str(__version__)}} try: with open(build_info_file, "w") as f: yaml.dump(build_command, f, default_flow_style=False) except Exception as e: self.wrn(f'Failed to create info file: {build_info_file},', e) board, origin = self._find_board() self._run_cmake(board, origin, self.args.cmake_opts) if args.cmake_only: return self._sanity_check() self._update_cache() self.domains = load_domains(self.build_dir) self._run_build(args.target, args.domain) def _find_board(self): board, origin = None, None if self.cmake_cache: board, origin = (self.cmake_cache.get('CACHED_BOARD'), 'CMakeCache.txt') # A malformed CMake cache may exist, but not have a board. # This happens if there's a build error from a previous run. if board is not None: return (board, origin) if self.args.board: board, origin = self.args.board, 'command line' elif 'BOARD' in os.environ: board, origin = os.environ['BOARD'], 'env' elif self.config_board is not None: board, origin = self.config_board, 'configfile' return board, origin def _parse_remainder(self, remainder): self.args.source_dir = None self.args.cmake_opts = None try: # Only one source_dir is allowed, as the first positional arg if remainder[0] != _ARG_SEPARATOR: self.args.source_dir = remainder[0] remainder = remainder[1:] # Only the first argument separator is consumed, the rest are # passed on to CMake if remainder[0] == _ARG_SEPARATOR: remainder = remainder[1:] if remainder: self.args.cmake_opts = remainder except IndexError: pass def _parse_test_item(self, test_item): found_test_metadata = False for yp in ['sample.yaml', 'testcase.yaml']: yf = os.path.join(self.args.source_dir, yp) if not os.path.exists(yf): continue found_test_metadata = True with open(yf, 'r') as stream: try: y = yaml.safe_load(stream) except yaml.YAMLError as exc: self.die(exc) common = y.get('common') tests = y.get('tests') if not tests: self.die(f"No tests found in {yf}") if test_item not in tests: self.die(f"Test item {test_item} not found in {yf}") item = tests.get(test_item) sysbuild = False extra_dtc_overlay_files = [] extra_overlay_confs = [] extra_conf_files = [] required_snippets = [] for section in [common, item]: if not section: continue sysbuild = section.get('sysbuild', sysbuild) for data in [ 'extra_args', 'extra_configs', 'extra_conf_files', 'extra_overlay_confs', 'extra_dtc_overlay_files', 'required_snippets' ]: extra = section.get(data) if not extra: continue if isinstance(extra, str): arg_list = extra.split(" ") else: arg_list = extra if data == 'extra_configs': args = [] for arg in arg_list: equals = arg.find('=') colon = arg.rfind(':', 0, equals) if colon != -1: # conditional configs (xxx:yyy:CONFIG_FOO=bar) # are not supported by 'west build' self.wrn('"west build" does not support ' 'conditional config "{}". Add "-D{}" ' 'to the supplied CMake arguments if ' 'desired.'.format(arg, arg[colon+1:])) continue args.append("-D{}".format(arg.replace('"', '\"'))) elif data == 'extra_args': # Retain quotes around config options config_options = [arg for arg in arg_list if arg.startswith("CONFIG_")] non_config_options = [arg for arg in arg_list if not arg.startswith("CONFIG_")] args = ["-D{}".format(a.replace('"', '\"')) for a in config_options] args.extend(["-D{}".format(arg.replace('"', '')) for arg in non_config_options]) elif data == 'extra_conf_files': extra_conf_files.extend(arg_list) continue elif data == 'extra_overlay_confs': extra_overlay_confs.extend(arg_list) continue elif data == 'extra_dtc_overlay_files': extra_dtc_overlay_files.extend(arg_list) continue elif data == 'required_snippets': required_snippets.extend(arg_list) continue if self.args.cmake_opts: self.args.cmake_opts.extend(args) else: self.args.cmake_opts = args self.args.sysbuild = sysbuild if found_test_metadata: args = [] if extra_conf_files: args.append(f"CONF_FILE=\"{';'.join(extra_conf_files)}\"") if extra_dtc_overlay_files: args.append(f"DTC_OVERLAY_FILE=\"{';'.join(extra_dtc_overlay_files)}\"") if extra_overlay_confs: args.append(f"OVERLAY_CONFIG=\"{';'.join(extra_overlay_confs)}\"") if required_snippets: args.append(f"SNIPPET=\"{';'.join(required_snippets)}\"") # Build the final argument list args_expanded = ["-D{}".format(a.replace('"', '')) for a in args] if self.args.cmake_opts: self.args.cmake_opts.extend(args_expanded) else: self.args.cmake_opts = args_expanded return found_test_metadata def _sanity_precheck(self): app = self.args.source_dir if app: self.check_force( os.path.isdir(app), 'source directory {} does not exist'.format(app)) self.check_force( 'CMakeLists.txt' in os.listdir(app), "{} doesn't contain a CMakeLists.txt".format(app)) def _update_cache(self): try: self.cmake_cache = CMakeCache.from_build_dir(self.build_dir) except FileNotFoundError: pass def _setup_build_dir(self): # Initialize build_dir and created_build_dir attributes. # If we created the build directory, we must run CMake. self.dbg('setting up build directory', level=Verbosity.DBG_EXTREME) # The CMake Cache has not been loaded yet, so this is safe board, _ = self._find_board() source_dir = self._find_source_dir() app = os.path.split(source_dir)[1] build_dir = find_build_dir(self.args.build_dir, board=board, source_dir=source_dir, app=app) if not build_dir: self.die('Unable to determine a default build folder. Check ' 'your build.dir-fmt configuration option') if os.path.exists(build_dir): if not os.path.isdir(build_dir): self.die('build directory {} exists and is not a directory'. format(build_dir)) else: os.makedirs(build_dir, exist_ok=False) self.created_build_dir = True self.run_cmake = True self.build_dir = build_dir def _find_source_dir(self): # Initialize source_dir attribute, either from command line argument, # implicitly from the build directory's CMake cache, or using the # default (current working directory). self.dbg('setting up source directory', level=Verbosity.DBG_EXTREME) if self.args.source_dir: source_dir = self.args.source_dir elif self.cmake_cache: source_dir = self.cmake_cache.get('APP_DIR') if not source_dir: source_dir = self.cmake_cache.get('APPLICATION_SOURCE_DIR') if not source_dir: source_dir = self.cmake_cache.get('CMAKE_HOME_DIRECTORY') if not source_dir: # This really ought to be there. The build directory # must be corrupted somehow. Let's see what we can do. self.die('build directory', self.build_dir, 'CMake cache has no CMAKE_HOME_DIRECTORY;', 'please give a source_dir') else: source_dir = os.getcwd() return os.path.abspath(source_dir) def _sanity_check_source_dir(self): if self.source_dir == self.build_dir: # There's no forcing this. self.die('source and build directory {} cannot be the same; ' 'use --build-dir {} to specify a build directory'. format(self.source_dir, self.build_dir)) srcrel = os.path.relpath(self.source_dir) self.check_force( not is_zephyr_build(self.source_dir), 'it looks like {srcrel} is a build directory: ' 'did you mean --build-dir {srcrel} instead?'. format(srcrel=srcrel)) self.check_force( 'CMakeLists.txt' in os.listdir(self.source_dir), 'source directory "{srcrel}" does not contain ' 'a CMakeLists.txt; is this really what you ' 'want to build? (Use -s SOURCE_DIR to specify ' 'the application source directory)'. format(srcrel=srcrel)) def _sanity_check(self): # Sanity check the build configuration. # Side effect: may update cmake_cache attribute. self.dbg('sanity checking the build', level=Verbosity.DBG_EXTREME) self._sanity_check_source_dir() if not self.cmake_cache: return # That's all we can check without a cache. if "CMAKE_PROJECT_NAME" not in self.cmake_cache: # This happens sometimes when a build system is not # completely generated due to an error during the # CMake configuration phase. self.run_cmake = True cached_proj = self.cmake_cache.get('APPLICATION_SOURCE_DIR') cached_app = self.cmake_cache.get('APP_DIR') # if APP_DIR is None but APPLICATION_SOURCE_DIR is set, that indicates # an older build folder, this still requires pristine. if cached_app is None and cached_proj: cached_app = cached_proj self.dbg('APP_DIR:', cached_app, level=Verbosity.DBG_EXTREME) source_abs = (os.path.abspath(self.args.source_dir) if self.args.source_dir else None) cached_abs = os.path.abspath(cached_app) if cached_app else None self.dbg('pristine:', self.auto_pristine, level=Verbosity.DBG_EXTREME) # If the build directory specifies a source app, make sure it's # consistent with --source-dir. apps_mismatched = (source_abs and cached_abs and pathlib.Path(source_abs).resolve() != pathlib.Path(cached_abs).resolve()) self.check_force( not apps_mismatched or self.auto_pristine, 'Build directory "{}" is for application "{}", but source ' 'directory "{}" was specified; please clean it, use --pristine, ' 'or use --build-dir to set another build directory'. format(self.build_dir, cached_abs, source_abs)) if apps_mismatched: self.run_cmake = True # If they insist, we need to re-run cmake. # If CACHED_BOARD is not defined, we need some other way to # find the board. cached_board = self.cmake_cache.get('CACHED_BOARD') self.dbg('CACHED_BOARD:', cached_board, level=Verbosity.DBG_EXTREME) # If apps_mismatched and self.auto_pristine are true, we will # run pristine on the build, invalidating the cached # board. In that case, we need some way of getting the board. self.check_force((cached_board and not (apps_mismatched and self.auto_pristine)) or self.args.board or self.config_board or os.environ.get('BOARD'), 'Cached board not defined, please provide it ' '(provide --board, set default with ' '"west config build.board ", or set ' 'BOARD in the environment)') # Check consistency between cached board and --board. boards_mismatched = (self.args.board and cached_board and self.args.board != cached_board) self.check_force( not boards_mismatched or self.auto_pristine, 'Build directory {} targets board {}, but board {} was specified. ' '(Clean the directory, use --pristine, or use --build-dir to ' 'specify a different one.)'. format(self.build_dir, cached_board, self.args.board)) if self.auto_pristine and (apps_mismatched or boards_mismatched): self._run_pristine() self.cmake_cache = None self.dbg('run_cmake:', True, level=Verbosity.DBG_EXTREME) self.run_cmake = True # Tricky corner-case: The user has not specified a build folder but # there was one in the CMake cache. Since this is going to be # invalidated, reset to CWD and re-run the basic tests. if ((boards_mismatched and not apps_mismatched) and (not source_abs and cached_abs)): self.source_dir = self._find_source_dir() self._sanity_check_source_dir() def _run_cmake(self, board, origin, cmake_opts): if board is None and config_getboolean('board_warn', True): self.wrn('This looks like a fresh build and BOARD is unknown;', "so it probably won't work. To fix, use", '--board=.') self.inf('Note: to silence the above message, run', "'west config build.board_warn false'") if not self.run_cmake: return self._banner('generating a build system') if board is not None and origin != 'CMakeCache.txt': cmake_opts = ['-DBOARD={}'.format(board)] else: cmake_opts = [] if self.args.cmake_opts: cmake_opts.extend(self.args.cmake_opts) if self.args.snippets: cmake_opts.append(f'-DSNIPPET={";".join(self.args.snippets)}') if self.args.shields: cmake_opts.append(f'-DSHIELD={";".join(self.args.shields)}') if self.args.extra_conf_files: cmake_opts.append(f'-DEXTRA_CONF_FILE={";".join(self.args.extra_conf_files)}') if self.args.extra_dtc_overlay_files: cmake_opts.append( f'-DEXTRA_DTC_OVERLAY_FILE=' f'{";".join(self.args.extra_dtc_overlay_files)}' ) user_args = config_get('cmake-args', None) if user_args: cmake_opts.extend(shlex.split(user_args)) config_sysbuild = config_getboolean('sysbuild', False) if self.args.sysbuild or (config_sysbuild and not self.args.no_sysbuild): cmake_opts.extend(['-S{}'.format(SYSBUILD_PROJ_DIR), '-DAPP_DIR:PATH={}'.format(self.source_dir)]) else: # self.args.no_sysbuild == True or config sysbuild False cmake_opts.extend(['-S{}'.format(self.source_dir)]) # Invoke CMake from the current working directory using the # -S and -B options (officially introduced in CMake 3.13.0). # This is important because users expect invocations like this # to Just Work: # # west build -- -DOVERLAY_CONFIG=relative-path.conf final_cmake_args = ['-DWEST_PYTHON={}'.format(pathlib.Path(sys.executable).as_posix()), '-B{}'.format(self.build_dir), '-G{}'.format(config_get('generator', DEFAULT_CMAKE_GENERATOR))] if cmake_opts: final_cmake_args.extend(cmake_opts) run_cmake(final_cmake_args, dry_run=self.args.dry_run) def _run_pristine(self): self._banner('making build dir {} pristine'.format(self.build_dir)) if not is_zephyr_build(self.build_dir): self.die('Refusing to run pristine on a folder that is not a ' 'Zephyr build system') cache = CMakeCache.from_build_dir(self.build_dir) app_src_dir = cache.get('APPLICATION_SOURCE_DIR') app_bin_dir = cache.get('APPLICATION_BINARY_DIR') cmake_args = [f'-DBINARY_DIR={app_bin_dir}', f'-DSOURCE_DIR={app_src_dir}', '-P', cache['ZEPHYR_BASE'] + '/cmake/pristine.cmake'] run_cmake(cmake_args, cwd=self.build_dir, dry_run=self.args.dry_run) def _run_build(self, target, domain): if target: self._banner('running target {}'.format(target)) elif self.run_cmake: self._banner('building application') extra_args = ['--target', target] if target else [] if self.args.build_opt: extra_args.append('--') extra_args.extend(self.args.build_opt) if self.args.verbose: self._append_verbose_args(extra_args, not bool(self.args.build_opt)) domains = load_domains(self.build_dir) build_dir_list = [] if domain is None: # If no domain is specified, we just build top build dir as that # will build all domains. build_dir_list = [domains.get_top_build_dir()] else: self._banner('building domain(s): {}'.format(' '.join(domain))) domain_list = domains.get_domains(domain) for d in domain_list: build_dir_list.append(d.build_dir) for b in build_dir_list: run_build(b, extra_args=extra_args, dry_run=self.args.dry_run) def _append_verbose_args(self, extra_args, add_dashes): # These hacks are only needed for CMake versions earlier than # 3.14. When Zephyr's minimum version is at least that, we can # drop this nonsense and just run "cmake --build BUILD -v". self._update_cache() if not self.cmake_cache: return generator = self.cmake_cache.get('CMAKE_GENERATOR') if not generator: return # Substring matching is for things like "Eclipse CDT4 - Ninja". if 'Ninja' in generator: if add_dashes: extra_args.append('--') extra_args.append('-v') elif generator == 'Unix Makefiles': if add_dashes: extra_args.append('--') extra_args.append('VERBOSE=1')