modules: Basic binary blob infrastructure

This patch introduces the basic infrastructure to list and fetch binary
blobs. This includes:

- The new 'blobs' extension command
- An implementation of the `west blobs list` command
  with custom formatting
- A very simple mechanism for loading fetchers
- A basic implementation of an HTTP fetcher

In order to ensure consistency among the west extension commands in the
main zephyr tree, we reuse a similar class factory pattern that is present
for ZephyrBinaryRunner instances in the ZephyrBlobFetcher case. This
could be achieved with a simpler mechanism, but opted for consistency
before simplicity.

Signed-off-by: Carles Cufi <carles.cufi@nordicsemi.no>

CODEOWNERS

@@ -718,6 +718,8 @@ scripts/build/gen_image_info.py           @tejlmand
 /scripts/series-push-hook.sh              @erwango
 /scripts/utils/pinctrl_nrf_migrate.py     @gmarull
 /scripts/west_commands/                   @mbolivar-nordic
+/scripts/west_commands/blobs.py           @carlescufi
+/scripts/west_commands/fetchers           @carlescufi
 /scripts/west_commands/runners/gd32isp.py @mbolivar-nordic @nandojve
 /scripts/west_commands/tests/test_gd32isp.py @mbolivar-nordic @nandojve
 /scripts/west-commands.yml                @mbolivar-nordic

scripts/west-commands.yml

@@ -46,3 +46,8 @@ west-commands:
       - name: spdx
         class: ZephyrSpdx
         help: create SPDX bill of materials
+  - file: scripts/west_commands/blobs.py
+    commands:
+      - name: blobs
+        class: Blobs
+        help: work with binary blobs

scripts/west_commands/blobs.py

@@ -0,0 +1,148 @@
+# Copyright (c) 2022 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import hashlib
+import os
+from pathlib import Path
+import sys
+import textwrap
+from urllib.parse import urlparse
+
+from west import log
+from west.commands import WestCommand
+
+from zephyr_ext_common import ZEPHYR_BASE
+
+sys.path.append(os.fspath(Path(__file__).parent.parent))
+import zephyr_module
+
+class Blobs(WestCommand):
+
+    def __init__(self):
+        super().__init__(
+            'blobs',
+            # Keep this in sync with the string in west-commands.yml.
+            'work with binary blobs',
+            'Work with binary blobs',
+            accepts_unknown_args=False)
+
+    def do_add_parser(self, parser_adder):
+        default_fmt = '{module} {status} {path} {type} {abspath}'
+        parser = parser_adder.add_parser(
+            self.name,
+            help=self.help,
+            formatter_class=argparse.RawDescriptionHelpFormatter,
+            description=self.description,
+            epilog=textwrap.dedent(f'''\
+            FORMAT STRINGS
+            --------------
+
+            Blobs are listed using a Python 3 format string. Arguments
+            to the format string are accessed by name.
+
+            The default format string is:
+
+            "{default_fmt}"
+
+            The following arguments are available:
+
+            - module: name of the module that contains this blob
+            - abspath: blob absolute path
+            - status: short status (A: present, M: hash failure, D: not present)
+            - path: blob local path from <module>/zephyr/blobs/
+            - sha256: blob SHA256 hash in hex
+            - type: type of blob
+            - version: version string
+            - license_path: path to the license file for the blob
+            - uri: URI to the remote location of the blob
+            - description: blob text description
+            - doc-url: URL to the documentation for this blob
+            '''))
+
+        # Remember to update west-completion.bash if you add or remove
+        # flags
+        parser.add_argument('subcmd', nargs=1, choices=['list', 'fetch'],
+                            help='''Select the sub-command to execute.
+                            Currently only list and fetch are supported.''')
+
+        # Remember to update west-completion.bash if you add or remove
+        # flags
+        parser.add_argument('-f', '--format', default=default_fmt,
+                            help='''Format string to use to list each blob;
+                                    see FORMAT STRINGS below.''')
+
+        parser.add_argument('-m', '--modules', type=str, action='append',
+                            default=[],
+                            help='''a list of modules; only blobs whose
+                            names are on this list will be taken into account
+                            by the sub-command. Invoke multiple times''')
+        parser.add_argument('-a', '--all', action='store_true',
+                            help='use all modules.')
+
+        return parser
+
+    def get_status(self, path, sha256):
+        if not path.is_file():
+            return 'D'
+        with path.open('rb') as f:
+            m = hashlib.sha256()
+            m.update(f.read())
+            if sha256.lower() == m.hexdigest():
+                return 'A'
+            else:
+                return 'M'
+
+    def get_blobs(self, args):
+        blobs = []
+        modules = args.modules
+        for module in zephyr_module.parse_modules(ZEPHYR_BASE, self.manifest):
+            mblobs = module.meta.get('blobs', None)
+            if not mblobs:
+                continue
+
+            # Filter by module
+            module_name = module.meta.get('name', None)
+            if not args.all and module_name not in modules:
+                continue
+
+            blobs_path = Path(module.project) / zephyr_module.MODULE_BLOBS_PATH
+            for blob in mblobs:
+                blob['module'] = module_name
+                blob['abspath'] = blobs_path / Path(blob['path'])
+                blob['status'] = self.get_status(blob['abspath'], blob['sha256'])
+                blobs.append(blob)
+
+        return blobs
+
+    def list(self, args):
+        blobs = self.get_blobs(args)
+        for blob in blobs:
+            log.inf(args.format.format(**blob))
+
+    def fetch_blob(self, url, path):
+        scheme = urlparse(url).scheme
+        log.dbg(f'Fetching {path} with {scheme}')
+        import fetchers
+        fetcher = fetchers.get_fetcher_cls(scheme)
+
+        log.dbg(f'Found fetcher: {fetcher}')
+        inst = fetcher()
+        inst.fetch(url, path)
+
+    def fetch(self, args):
+        blobs = self.get_blobs(args)
+        for blob in blobs:
+            if blob['status'] == 'A':
+                log.inf('Blob {module}: {abspath} is up to date'.format(**blob))
+                continue
+            log.inf('Fetching blob {module}: {status} {abspath}'.format(**blob))
+            self.fetch_blob(blob['url'], blob['abspath'])
+
+
+    def do_run(self, args, _):
+        log.dbg(f'{args.subcmd[0]} {args.modules}')
+
+        subcmd = getattr(self, args.subcmd[0])
+        subcmd(args)

scripts/west_commands/fetchers/__init__.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2022 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import importlib
+import logging
+import os
+from pathlib import Path
+
+from fetchers.core import ZephyrBlobFetcher
+
+_logger = logging.getLogger('fetchers')
+
+def _import_fetcher_module(fetcher_name):
+    try:
+        importlib.import_module(f'fetchers.{fetcher_name}')
+    except ImportError as ie:
+        # Fetchers are supposed to gracefully handle failures when they
+        # import anything outside of stdlib, but they sometimes do
+        # not. Catch ImportError to handle this.
+        _logger.warning(f'The module for fetcher "{fetcher_name}" '
+                        f'could not be imported ({ie}). This most likely '
+                        'means it is not handling its dependencies properly. '
+                        'Please report this to the zephyr developers.')
+
+# We import these here to ensure the BlobFetcher subclasses are
+# defined; otherwise, BlobFetcher.get_fetchers() won't work.
+
+# Those do not contain subclasses of ZephyrBlobFetcher
+name_blocklist = ['__init__', 'core']
+
+fetchers_dir = Path(__file__).parent.resolve()
+for f in [f for f in os.listdir(fetchers_dir)]:
+    file = fetchers_dir / Path(f)
+    if file.suffix == '.py' and file.stem not in name_blocklist:
+        _import_fetcher_module(file.stem)
+
+def get_fetcher_cls(scheme):
+    '''Get a fetcher's class object, given a scheme.'''
+    for cls in ZephyrBlobFetcher.get_fetchers():
+        if scheme in cls.schemes():
+            return cls
+    raise ValueError('unknown fetcher for scheme "{}"'.format(scheme))
+
+__all__ = ['ZephyrBlobFetcher', 'get_fetcher_cls']

scripts/west_commands/fetchers/core.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2022 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import List, Type
+
+class ZephyrBlobFetcher(ABC):
+
+    @staticmethod
+    def get_fetchers() -> List[Type['ZephyrBlobFetcher']]:
+        '''Get a list of all currently defined fetcher classes.'''
+        return ZephyrBlobFetcher.__subclasses__()
+
+    @classmethod
+    @abstractmethod
+    def schemes(cls) -> List[str]:
+        '''Return this fetcher's schemes.'''
+
+    @abstractmethod
+    def fetch(self, url: str, path: Path):
+        ''' Fetch a blob and store it '''

scripts/west_commands/fetchers/http.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2022 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import requests
+
+from west import log
+
+from fetchers.core import ZephyrBlobFetcher
+
+class HTTPFetcher(ZephyrBlobFetcher):
+
+    @classmethod
+    def schemes(cls):
+        return ['http', 'https']
+
+    def fetch(self, url, path):
+        log.dbg(f'HTTPFetcher fetching {url} to {path}')
+        resp = requests.get(url)
+        open(path, "wb").write(resp.content)

scripts/zephyr_module.py

@@ -95,9 +95,42 @@ mapping:
     type: seq
     sequence:
       - type: str
+  blobs:
+    required: false
+    type: seq
+    sequence:
+      - type: map
+        mapping:
+          path:
+            required: true
+            type: str
+          sha256:
+            required: true
+            type: str
+          type:
+            required: true
+            type: str
+            enum: ['img', 'lib']
+          version:
+            required: true
+            type: str
+          license-path:
+            required: true
+            type: str
+          url:
+            required: true
+            type: str
+          description:
+            required: true
+            type: str
+          doc-url:
+            required: false
+            type: str
 '''
 
 MODULE_YML_PATH = PurePath('zephyr/module.yml')
+# Path to the blobs folder
+MODULE_BLOBS_PATH = PurePath('zephyr/blobs')
 
 schema = yaml.safe_load(METADATA_SCHEMA)