348 lines
12 KiB
Python
348 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (c) 2022, Nordic Semiconductor ASA
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''Internal snippets tool.
|
|
This is part of the build system's support for snippets.
|
|
It is not meant for use outside of the build system.
|
|
|
|
Output CMake variables:
|
|
|
|
- SNIPPET_NAMES: CMake list of discovered snippet names
|
|
- SNIPPET_FOUND_{snippet}: one per discovered snippet
|
|
'''
|
|
|
|
from collections import defaultdict, UserDict
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Dict, Iterable, List, Set
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import pykwalify.core
|
|
import pykwalify.errors
|
|
import re
|
|
import sys
|
|
import textwrap
|
|
import yaml
|
|
import platform
|
|
|
|
# Marker type for an 'append:' configuration. Maps variables
|
|
# to the list of values to append to them.
|
|
Appends = Dict[str, List[str]]
|
|
|
|
def _new_append():
|
|
return defaultdict(list)
|
|
|
|
def _new_board2appends():
|
|
return defaultdict(_new_append)
|
|
|
|
@dataclass
|
|
class Snippet:
|
|
'''Class for keeping track of all the settings discovered for an
|
|
individual snippet.'''
|
|
|
|
name: str
|
|
appends: Appends = field(default_factory=_new_append)
|
|
board2appends: Dict[str, Appends] = field(default_factory=_new_board2appends)
|
|
|
|
def process_data(self, pathobj: Path, snippet_data: dict):
|
|
'''Process the data in a snippet.yml file, after it is loaded into a
|
|
python object and validated by pykwalify.'''
|
|
def append_value(variable, value):
|
|
if variable in ('EXTRA_DTC_OVERLAY_FILE', 'EXTRA_CONF_FILE'):
|
|
path = pathobj.parent / value
|
|
if not path.is_file():
|
|
_err(f'snippet file {pathobj}: {variable}: file not found: {path}')
|
|
return f'"{path}"'
|
|
if variable in ('DTS_EXTRA_CPPFLAGS'):
|
|
return f'"{value}"'
|
|
_err(f'unknown append variable: {variable}')
|
|
|
|
for variable, value in snippet_data.get('append', {}).items():
|
|
self.appends[variable].append(append_value(variable, value))
|
|
for board, settings in snippet_data.get('boards', {}).items():
|
|
if board.startswith('/') and not board.endswith('/'):
|
|
_err(f"snippet file {pathobj}: board {board} starts with '/', so "
|
|
"it must end with '/' to use a regular expression")
|
|
for variable, value in settings.get('append', {}).items():
|
|
self.board2appends[board][variable].append(
|
|
append_value(variable, value))
|
|
|
|
class Snippets(UserDict):
|
|
'''Type for all the information we have discovered about all snippets.
|
|
As a dict, this maps a snippet's name onto the Snippet object.
|
|
Any additional global attributes about all snippets go here as
|
|
instance attributes.'''
|
|
|
|
def __init__(self, requested: Iterable[str] = None):
|
|
super().__init__()
|
|
self.paths: Set[Path] = set()
|
|
self.requested: List[str] = list(requested or [])
|
|
|
|
class SnippetsError(Exception):
|
|
'''Class for signalling expected errors'''
|
|
|
|
def __init__(self, msg):
|
|
self.msg = msg
|
|
|
|
class SnippetToCMakePrinter:
|
|
'''Helper class for printing a Snippets's semantics to a .cmake
|
|
include file for use by snippets.cmake.'''
|
|
|
|
def __init__(self, snippets: Snippets, out_file):
|
|
self.snippets = snippets
|
|
self.out_file = out_file
|
|
self.section = '#' * 79
|
|
|
|
def print_cmake(self):
|
|
'''Print to the output file provided to the constructor.'''
|
|
# TODO: add source file info
|
|
snippets = self.snippets
|
|
snippet_names = sorted(snippets.keys())
|
|
|
|
if platform.system() == "Windows":
|
|
# Change to linux-style paths for windows to avoid cmake escape character code issues
|
|
snippets.paths = set(map(lambda x: str(PurePosixPath(x)), snippets.paths))
|
|
|
|
for this_snippet in snippets:
|
|
for snippet_append in (snippets[this_snippet].appends):
|
|
snippets[this_snippet].appends[snippet_append] = \
|
|
set(map(lambda x: str(x.replace("\\", "/")), \
|
|
snippets[this_snippet].appends[snippet_append]))
|
|
|
|
snippet_path_list = " ".join(
|
|
sorted(f'"{path}"' for path in snippets.paths))
|
|
|
|
self.print('''\
|
|
# WARNING. THIS FILE IS AUTO-GENERATED. DO NOT MODIFY!
|
|
#
|
|
# This file contains build system settings derived from your snippets.
|
|
# Its contents are an implementation detail that should not be used outside
|
|
# of Zephyr's snippets CMake module.
|
|
#
|
|
# See the Snippets guide in the Zephyr documentation for more information.
|
|
''')
|
|
|
|
self.print(f'''\
|
|
{self.section}
|
|
# Global information about all snippets.
|
|
|
|
# The name of every snippet that was discovered.
|
|
set(SNIPPET_NAMES {' '.join(f'"{name}"' for name in snippet_names)})
|
|
# The paths to all the snippet.yml files. One snippet
|
|
# can have multiple snippet.yml files.
|
|
set(SNIPPET_PATHS {snippet_path_list})
|
|
|
|
# Create variable scope for snippets build variables
|
|
zephyr_create_scope(snippets)
|
|
''')
|
|
|
|
for snippet_name in snippets.requested:
|
|
self.print_cmake_for(snippets[snippet_name])
|
|
self.print()
|
|
|
|
def print_cmake_for(self, snippet: Snippet):
|
|
self.print(f'''\
|
|
{self.section}
|
|
# Snippet '{snippet.name}'
|
|
|
|
# Common variable appends.''')
|
|
self.print_appends(snippet.appends, 0)
|
|
for board, appends in snippet.board2appends.items():
|
|
self.print_appends_for_board(board, appends)
|
|
|
|
def print_appends_for_board(self, board: str, appends: Appends):
|
|
if board.startswith('/'):
|
|
board_re = board[1:-1]
|
|
self.print(f'''\
|
|
# Appends for board regular expression '{board_re}'
|
|
if("${{BOARD}}" MATCHES "^{board_re}$")''')
|
|
else:
|
|
self.print(f'''\
|
|
# Appends for board '{board}'
|
|
if("${{BOARD}}" STREQUAL "{board}")''')
|
|
self.print_appends(appends, 1)
|
|
self.print('endif()')
|
|
|
|
def print_appends(self, appends: Appends, indent: int):
|
|
space = ' ' * indent
|
|
for name, values in appends.items():
|
|
for value in values:
|
|
self.print(f'{space}zephyr_set({name} {value} SCOPE snippets APPEND)')
|
|
|
|
def print(self, *args, **kwargs):
|
|
kwargs['file'] = self.out_file
|
|
print(*args, **kwargs)
|
|
|
|
# Name of the file containing the pykwalify schema for snippet.yml
|
|
# files.
|
|
SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'snippet-schema.yml')
|
|
with open(SCHEMA_PATH, 'r') as f:
|
|
SNIPPET_SCHEMA = yaml.safe_load(f.read())
|
|
|
|
# The name of the file which contains metadata about the snippets
|
|
# being defined in a directory.
|
|
SNIPPET_YML = 'snippet.yml'
|
|
|
|
# Regular expression for validating snippet names. Snippet names must
|
|
# begin with an alphanumeric character, and may contain alphanumeric
|
|
# characters or underscores. This is intentionally very restrictive to
|
|
# keep things consistent and easy to type and remember. We can relax
|
|
# this a bit later if needed.
|
|
SNIPPET_NAME_RE = re.compile('[A-Za-z0-9][A-Za-z0-9_-]*')
|
|
|
|
# Logger for this module.
|
|
LOG = logging.getLogger('snippets')
|
|
|
|
def _err(msg):
|
|
raise SnippetsError(f'error: {msg}')
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description='snippets helper',
|
|
allow_abbrev=False)
|
|
parser.add_argument('--snippet-root', default=[], action='append', type=Path,
|
|
help='''a SNIPPET_ROOT element; may be given
|
|
multiple times''')
|
|
parser.add_argument('--snippet', dest='snippets', default=[], action='append',
|
|
help='''a SNIPPET element; may be given
|
|
multiple times''')
|
|
parser.add_argument('--cmake-out', type=Path,
|
|
help='''file to write cmake output to; include()
|
|
this file after calling this script''')
|
|
return parser.parse_args()
|
|
|
|
def setup_logging():
|
|
# Silence validation errors from pykwalify, which are logged at
|
|
# logging.ERROR level. We want to handle those ourselves as
|
|
# needed.
|
|
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
|
|
logging.basicConfig(level=logging.INFO,
|
|
format=' %(name)s: %(message)s')
|
|
|
|
def process_snippets(args: argparse.Namespace) -> Snippets:
|
|
'''Process snippet.yml files under each *snippet_root*
|
|
by recursive search. Return a Snippets object describing
|
|
the results of the search.
|
|
'''
|
|
# This will contain information about all the snippets
|
|
# we discover in each snippet_root element.
|
|
snippets = Snippets(requested=args.snippets)
|
|
|
|
# Process each path in snippet_root in order, adjusting
|
|
# snippets as needed for each one.
|
|
for root in args.snippet_root:
|
|
process_snippets_in(root, snippets)
|
|
|
|
return snippets
|
|
|
|
def find_snippets_in_roots(requested_snippets, snippet_roots) -> Snippets:
|
|
'''Process snippet.yml files under each *snippet_root*
|
|
by recursive search. Return a Snippets object describing
|
|
the results of the search.
|
|
'''
|
|
# This will contain information about all the snippets
|
|
# we discover in each snippet_root element.
|
|
snippets = Snippets(requested=requested_snippets)
|
|
|
|
# Process each path in snippet_root in order, adjusting
|
|
# snippets as needed for each one.
|
|
for root in snippet_roots:
|
|
process_snippets_in(root, snippets)
|
|
|
|
return snippets
|
|
|
|
def process_snippets_in(root_dir: Path, snippets: Snippets) -> None:
|
|
'''Process snippet.yml files in *root_dir*,
|
|
updating *snippets* as needed.'''
|
|
|
|
if not root_dir.is_dir():
|
|
LOG.warning(f'SNIPPET_ROOT {root_dir} '
|
|
'is not a directory; ignoring it')
|
|
return
|
|
|
|
snippets_dir = root_dir / 'snippets'
|
|
if not snippets_dir.is_dir():
|
|
return
|
|
|
|
for dirpath, _, filenames in os.walk(snippets_dir):
|
|
if SNIPPET_YML not in filenames:
|
|
continue
|
|
|
|
snippet_yml = Path(dirpath) / SNIPPET_YML
|
|
snippet_data = load_snippet_yml(snippet_yml)
|
|
name = snippet_data['name']
|
|
if name not in snippets:
|
|
snippets[name] = Snippet(name=name)
|
|
snippets[name].process_data(snippet_yml, snippet_data)
|
|
snippets.paths.add(snippet_yml)
|
|
|
|
def load_snippet_yml(snippet_yml: Path) -> dict:
|
|
'''Load a snippet.yml file *snippet_yml*, validate the contents
|
|
against the schema, and do other basic checks. Return the dict
|
|
of the resulting YAML data.'''
|
|
|
|
with open(snippet_yml, 'r') as f:
|
|
try:
|
|
snippet_data = yaml.safe_load(f.read())
|
|
except yaml.scanner.ScannerError:
|
|
_err(f'snippets file {snippet_yml} is invalid YAML')
|
|
|
|
def pykwalify_err(e):
|
|
return f'''\
|
|
invalid {SNIPPET_YML} file: {snippet_yml}
|
|
{textwrap.indent(e.msg, ' ')}
|
|
'''
|
|
|
|
try:
|
|
pykwalify.core.Core(source_data=snippet_data,
|
|
schema_data=SNIPPET_SCHEMA).validate()
|
|
except pykwalify.errors.PyKwalifyException as e:
|
|
_err(pykwalify_err(e))
|
|
|
|
name = snippet_data['name']
|
|
if not SNIPPET_NAME_RE.fullmatch(name):
|
|
_err(f"snippet file {snippet_yml}: invalid snippet name '{name}'; "
|
|
'snippet names must begin with a letter '
|
|
'or number, and may only contain letters, numbers, '
|
|
'dashes (-), and underscores (_)')
|
|
|
|
return snippet_data
|
|
|
|
def check_for_errors(snippets: Snippets) -> None:
|
|
unknown_snippets = sorted(snippet for snippet in snippets.requested
|
|
if snippet not in snippets)
|
|
if unknown_snippets:
|
|
all_snippets = '\n '.join(sorted(snippets))
|
|
_err(f'''\
|
|
snippets not found: {', '.join(unknown_snippets)}
|
|
Please choose from among the following snippets:
|
|
{all_snippets}''')
|
|
|
|
def write_cmake_out(snippets: Snippets, cmake_out: Path) -> None:
|
|
'''Write a cmake include file to *cmake_out* which
|
|
reflects the information in *snippets*.
|
|
|
|
The contents of this file should be considered an implementation
|
|
detail and are not meant to be used outside of snippets.cmake.'''
|
|
if not cmake_out.parent.exists():
|
|
cmake_out.parent.mkdir()
|
|
with open(cmake_out, 'w', encoding="utf-8") as f:
|
|
SnippetToCMakePrinter(snippets, f).print_cmake()
|
|
|
|
def main():
|
|
args = parse_args()
|
|
setup_logging()
|
|
try:
|
|
snippets = process_snippets(args)
|
|
check_for_errors(snippets)
|
|
except SnippetsError as e:
|
|
LOG.critical(e.msg)
|
|
sys.exit(1)
|
|
write_cmake_out(snippets, args.cmake_out)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|