From 336aa9dc8899acf432b4865728225fe6d7786ab3 Mon Sep 17 00:00:00 2001 From: Carles Cufi Date: Thu, 4 Aug 2022 19:08:22 +0200 Subject: [PATCH] modules: Basic binary blob infrastructure This patch introduces the basic infrastructure to list and fetch binary blobs. This includes: - The new 'blobs' extension command - An implementation of the `west blobs list` command with custom formatting - A very simple mechanism for loading fetchers - A basic implementation of an HTTP fetcher In order to ensure consistency among the west extension commands in the main zephyr tree, we reuse a similar class factory pattern that is present for ZephyrBinaryRunner instances in the ZephyrBlobFetcher case. This could be achieved with a simpler mechanism, but opted for consistency before simplicity. Signed-off-by: Carles Cufi --- CODEOWNERS | 2 + scripts/west-commands.yml | 5 + scripts/west_commands/blobs.py | 148 +++++++++++++++++++++ scripts/west_commands/fetchers/__init__.py | 45 +++++++ scripts/west_commands/fetchers/core.py | 23 ++++ scripts/west_commands/fetchers/http.py | 20 +++ scripts/zephyr_module.py | 33 +++++ 7 files changed, 276 insertions(+) create mode 100644 scripts/west_commands/blobs.py create mode 100644 scripts/west_commands/fetchers/__init__.py create mode 100644 scripts/west_commands/fetchers/core.py create mode 100644 scripts/west_commands/fetchers/http.py diff --git a/CODEOWNERS b/CODEOWNERS index 667a4a4573a..10b9cfe37e7 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -718,6 +718,8 @@ scripts/build/gen_image_info.py @tejlmand /scripts/series-push-hook.sh @erwango /scripts/utils/pinctrl_nrf_migrate.py @gmarull /scripts/west_commands/ @mbolivar-nordic +/scripts/west_commands/blobs.py @carlescufi +/scripts/west_commands/fetchers @carlescufi /scripts/west_commands/runners/gd32isp.py @mbolivar-nordic @nandojve /scripts/west_commands/tests/test_gd32isp.py @mbolivar-nordic @nandojve /scripts/west-commands.yml @mbolivar-nordic diff --git a/scripts/west-commands.yml b/scripts/west-commands.yml index 26827f23a0e..bdb9929e4f3 100644 --- a/scripts/west-commands.yml +++ b/scripts/west-commands.yml @@ -46,3 +46,8 @@ west-commands: - name: spdx class: ZephyrSpdx help: create SPDX bill of materials + - file: scripts/west_commands/blobs.py + commands: + - name: blobs + class: Blobs + help: work with binary blobs diff --git a/scripts/west_commands/blobs.py b/scripts/west_commands/blobs.py new file mode 100644 index 00000000000..931e60646f3 --- /dev/null +++ b/scripts/west_commands/blobs.py @@ -0,0 +1,148 @@ +# Copyright (c) 2022 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import hashlib +import os +from pathlib import Path +import sys +import textwrap +from urllib.parse import urlparse + +from west import log +from west.commands import WestCommand + +from zephyr_ext_common import ZEPHYR_BASE + +sys.path.append(os.fspath(Path(__file__).parent.parent)) +import zephyr_module + +class Blobs(WestCommand): + + def __init__(self): + super().__init__( + 'blobs', + # Keep this in sync with the string in west-commands.yml. + 'work with binary blobs', + 'Work with binary blobs', + accepts_unknown_args=False) + + def do_add_parser(self, parser_adder): + default_fmt = '{module} {status} {path} {type} {abspath}' + parser = parser_adder.add_parser( + self.name, + help=self.help, + formatter_class=argparse.RawDescriptionHelpFormatter, + description=self.description, + epilog=textwrap.dedent(f'''\ + FORMAT STRINGS + -------------- + + Blobs are listed using a Python 3 format string. Arguments + to the format string are accessed by name. + + The default format string is: + + "{default_fmt}" + + The following arguments are available: + + - module: name of the module that contains this blob + - abspath: blob absolute path + - status: short status (A: present, M: hash failure, D: not present) + - path: blob local path from /zephyr/blobs/ + - sha256: blob SHA256 hash in hex + - type: type of blob + - version: version string + - license_path: path to the license file for the blob + - uri: URI to the remote location of the blob + - description: blob text description + - doc-url: URL to the documentation for this blob + ''')) + + # Remember to update west-completion.bash if you add or remove + # flags + parser.add_argument('subcmd', nargs=1, choices=['list', 'fetch'], + help='''Select the sub-command to execute. + Currently only list and fetch are supported.''') + + # Remember to update west-completion.bash if you add or remove + # flags + parser.add_argument('-f', '--format', default=default_fmt, + help='''Format string to use to list each blob; + see FORMAT STRINGS below.''') + + parser.add_argument('-m', '--modules', type=str, action='append', + default=[], + help='''a list of modules; only blobs whose + names are on this list will be taken into account + by the sub-command. Invoke multiple times''') + parser.add_argument('-a', '--all', action='store_true', + help='use all modules.') + + return parser + + def get_status(self, path, sha256): + if not path.is_file(): + return 'D' + with path.open('rb') as f: + m = hashlib.sha256() + m.update(f.read()) + if sha256.lower() == m.hexdigest(): + return 'A' + else: + return 'M' + + def get_blobs(self, args): + blobs = [] + modules = args.modules + for module in zephyr_module.parse_modules(ZEPHYR_BASE, self.manifest): + mblobs = module.meta.get('blobs', None) + if not mblobs: + continue + + # Filter by module + module_name = module.meta.get('name', None) + if not args.all and module_name not in modules: + continue + + blobs_path = Path(module.project) / zephyr_module.MODULE_BLOBS_PATH + for blob in mblobs: + blob['module'] = module_name + blob['abspath'] = blobs_path / Path(blob['path']) + blob['status'] = self.get_status(blob['abspath'], blob['sha256']) + blobs.append(blob) + + return blobs + + def list(self, args): + blobs = self.get_blobs(args) + for blob in blobs: + log.inf(args.format.format(**blob)) + + def fetch_blob(self, url, path): + scheme = urlparse(url).scheme + log.dbg(f'Fetching {path} with {scheme}') + import fetchers + fetcher = fetchers.get_fetcher_cls(scheme) + + log.dbg(f'Found fetcher: {fetcher}') + inst = fetcher() + inst.fetch(url, path) + + def fetch(self, args): + blobs = self.get_blobs(args) + for blob in blobs: + if blob['status'] == 'A': + log.inf('Blob {module}: {abspath} is up to date'.format(**blob)) + continue + log.inf('Fetching blob {module}: {status} {abspath}'.format(**blob)) + self.fetch_blob(blob['url'], blob['abspath']) + + + def do_run(self, args, _): + log.dbg(f'{args.subcmd[0]} {args.modules}') + + subcmd = getattr(self, args.subcmd[0]) + subcmd(args) diff --git a/scripts/west_commands/fetchers/__init__.py b/scripts/west_commands/fetchers/__init__.py new file mode 100644 index 00000000000..9bdb5e3dd77 --- /dev/null +++ b/scripts/west_commands/fetchers/__init__.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +import importlib +import logging +import os +from pathlib import Path + +from fetchers.core import ZephyrBlobFetcher + +_logger = logging.getLogger('fetchers') + +def _import_fetcher_module(fetcher_name): + try: + importlib.import_module(f'fetchers.{fetcher_name}') + except ImportError as ie: + # Fetchers are supposed to gracefully handle failures when they + # import anything outside of stdlib, but they sometimes do + # not. Catch ImportError to handle this. + _logger.warning(f'The module for fetcher "{fetcher_name}" ' + f'could not be imported ({ie}). This most likely ' + 'means it is not handling its dependencies properly. ' + 'Please report this to the zephyr developers.') + +# We import these here to ensure the BlobFetcher subclasses are +# defined; otherwise, BlobFetcher.get_fetchers() won't work. + +# Those do not contain subclasses of ZephyrBlobFetcher +name_blocklist = ['__init__', 'core'] + +fetchers_dir = Path(__file__).parent.resolve() +for f in [f for f in os.listdir(fetchers_dir)]: + file = fetchers_dir / Path(f) + if file.suffix == '.py' and file.stem not in name_blocklist: + _import_fetcher_module(file.stem) + +def get_fetcher_cls(scheme): + '''Get a fetcher's class object, given a scheme.''' + for cls in ZephyrBlobFetcher.get_fetchers(): + if scheme in cls.schemes(): + return cls + raise ValueError('unknown fetcher for scheme "{}"'.format(scheme)) + +__all__ = ['ZephyrBlobFetcher', 'get_fetcher_cls'] diff --git a/scripts/west_commands/fetchers/core.py b/scripts/west_commands/fetchers/core.py new file mode 100644 index 00000000000..acbab9a6274 --- /dev/null +++ b/scripts/west_commands/fetchers/core.py @@ -0,0 +1,23 @@ +# Copyright (c) 2022 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import List, Type + +class ZephyrBlobFetcher(ABC): + + @staticmethod + def get_fetchers() -> List[Type['ZephyrBlobFetcher']]: + '''Get a list of all currently defined fetcher classes.''' + return ZephyrBlobFetcher.__subclasses__() + + @classmethod + @abstractmethod + def schemes(cls) -> List[str]: + '''Return this fetcher's schemes.''' + + @abstractmethod + def fetch(self, url: str, path: Path): + ''' Fetch a blob and store it ''' diff --git a/scripts/west_commands/fetchers/http.py b/scripts/west_commands/fetchers/http.py new file mode 100644 index 00000000000..817749541ca --- /dev/null +++ b/scripts/west_commands/fetchers/http.py @@ -0,0 +1,20 @@ +# Copyright (c) 2022 Nordic Semiconductor ASA +# +# SPDX-License-Identifier: Apache-2.0 + +import requests + +from west import log + +from fetchers.core import ZephyrBlobFetcher + +class HTTPFetcher(ZephyrBlobFetcher): + + @classmethod + def schemes(cls): + return ['http', 'https'] + + def fetch(self, url, path): + log.dbg(f'HTTPFetcher fetching {url} to {path}') + resp = requests.get(url) + open(path, "wb").write(resp.content) diff --git a/scripts/zephyr_module.py b/scripts/zephyr_module.py index 9bf14cdeb6f..be7e127e196 100755 --- a/scripts/zephyr_module.py +++ b/scripts/zephyr_module.py @@ -95,9 +95,42 @@ mapping: type: seq sequence: - type: str + blobs: + required: false + type: seq + sequence: + - type: map + mapping: + path: + required: true + type: str + sha256: + required: true + type: str + type: + required: true + type: str + enum: ['img', 'lib'] + version: + required: true + type: str + license-path: + required: true + type: str + url: + required: true + type: str + description: + required: true + type: str + doc-url: + required: false + type: str ''' MODULE_YML_PATH = PurePath('zephyr/module.yml') +# Path to the blobs folder +MODULE_BLOBS_PATH = PurePath('zephyr/blobs') schema = yaml.safe_load(METADATA_SCHEMA)