zephyr/scripts/release/list_devicetree_bindings_ch...

532 lines
18 KiB
Python
Executable File

#!/usr/bin/env python3
# Copyright 2023 Nordic Semiconductor ASA
# SPDX-License-Identifier: Apache-2.0
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Set, Union
import argparse
import contextlib
import glob
import os
import subprocess
import sys
import tempfile
# TODO: include changes to child bindings
HERE = Path(__file__).parent.resolve()
ZEPHYR_BASE = HERE.parent.parent
SCRIPTS = ZEPHYR_BASE / 'scripts'
sys.path.insert(0, str(SCRIPTS / 'dts' / 'python-devicetree' / 'src'))
from devicetree.edtlib import Binding, bindings_from_paths, load_vendor_prefixes_txt
# The Compat type is a (compatible, on_bus) pair, which is used as a
# lookup key for bindings. The name "compat" matches edtlib's internal
# variable for this; it's a bit of a misnomer, but let's be
# consistent.
@dataclass
class Compat:
compatible: str
on_bus: Optional[str]
def __hash__(self):
return hash((self.compatible, self.on_bus))
class BindingChange:
'''Marker type for an individual change that happened to a
binding between the start and end commits. See subclasses
below for concrete changes.
'''
Compat2Binding = Dict[Compat, Binding]
Binding2Changes = Dict[Binding, List[BindingChange]]
@dataclass
class Changes:
'''Container for all the changes that happened between the
start and end commits.'''
vnds: List[str]
vnd2added: Dict[str, Compat2Binding]
vnd2removed: Dict[str, Compat2Binding]
vnd2changes: Dict[str, Binding2Changes]
@dataclass
class ModifiedSpecifier2Cells(BindingChange):
space: str
start: List[str]
end: List[str]
@dataclass
class ModifiedBuses(BindingChange):
start: List[str]
end: List[str]
@dataclass
class AddedProperty(BindingChange):
property: str
@dataclass
class RemovedProperty(BindingChange):
property: str
@dataclass
class ModifiedPropertyType(BindingChange):
property: str
start: str
end: str
@dataclass
class ModifiedPropertyEnum(BindingChange):
property: str
start: Any
end: Any
@dataclass
class ModifiedPropertyConst(BindingChange):
property: str
start: Any
end: Any
@dataclass
class ModifiedPropertyDefault(BindingChange):
property: str
start: Any
end: Any
@dataclass
class ModifiedPropertyDeprecated(BindingChange):
property: str
start: bool
end: bool
@dataclass
class ModifiedPropertyRequired(BindingChange):
property: str
start: bool
end: bool
def get_changes_between(
compat2binding_start: Compat2Binding,
compat2binding_end: Compat2Binding
) -> Changes:
vnd2added: Dict[str, Compat2Binding] = \
group_compat2binding_by_vnd({
compat: compat2binding_end[compat]
for compat in compat2binding_end
if compat not in compat2binding_start
})
vnd2removed: Dict[str, Compat2Binding] = \
group_compat2binding_by_vnd({
compat: compat2binding_start[compat]
for compat in compat2binding_start
if compat not in compat2binding_end
})
vnd2changes = group_binding2changes_by_vnd(
get_binding2changes(compat2binding_start,
compat2binding_end))
vnds_set: Set[str] = set()
vnds_set.update(set(vnd2added.keys()),
set(vnd2removed.keys()),
set(vnd2changes.keys()))
return Changes(vnds=sorted(vnds_set),
vnd2added=vnd2added,
vnd2removed=vnd2removed,
vnd2changes=vnd2changes)
def group_compat2binding_by_vnd(
compat2binding: Compat2Binding
) -> Dict[str, Compat2Binding]:
'''Convert *compat2binding* to a dict mapping vendor prefixes
to the subset of *compat2binding* with that vendor prefix.'''
ret: Dict[str, Compat2Binding] = defaultdict(dict)
for compat, binding in compat2binding.items():
ret[get_vnd(binding.compatible)][compat] = binding
return ret
def group_binding2changes_by_vnd(
binding2changes: Binding2Changes
) -> Dict[str, Binding2Changes]:
'''Convert *binding2chages* to a dict mapping vendor prefixes
to the subset of *binding2changes* with that vendor prefix.'''
ret: Dict[str, Binding2Changes] = defaultdict(dict)
for binding, changes in binding2changes.items():
ret[get_vnd(binding.compatible)][binding] = changes
return ret
def get_vnd(compatible: str) -> str:
'''Return the vendor prefix or the empty string.'''
if ',' not in compatible:
return ''
return compatible.split(',')[0]
def get_binding2changes(
compat2binding_start: Compat2Binding,
compat2binding_end: Compat2Binding
) -> Binding2Changes:
ret: Binding2Changes = {}
for compat, binding in compat2binding_end.items():
if compat not in compat2binding_start:
continue
binding_start = compat2binding_start[compat]
binding_end = compat2binding_end[compat]
binding_changes: List[BindingChange] = \
get_binding_changes(binding_start, binding_end)
if binding_changes:
ret[binding] = binding_changes
return ret
def get_binding_changes(
binding_start: Binding,
binding_end: Binding
) -> List[BindingChange]:
'''Enumerate the changes to a binding given its start and end values.'''
ret: List[BindingChange] = []
assert binding_start.compatible == binding_end.compatible
assert binding_start.on_bus == binding_end.on_bus
common_props: Set[str] = set(binding_start.prop2specs).intersection(
set(binding_end.prop2specs))
ret.extend(get_modified_specifier2cells(binding_start, binding_end))
ret.extend(get_modified_buses(binding_start, binding_end))
ret.extend(get_added_properties(binding_start, binding_end))
ret.extend(get_removed_properties(binding_start, binding_end))
ret.extend(get_modified_property_type(binding_start, binding_end,
common_props))
ret.extend(get_modified_property_enum(binding_start, binding_end,
common_props))
ret.extend(get_modified_property_const(binding_start, binding_end,
common_props))
ret.extend(get_modified_property_default(binding_start, binding_end,
common_props))
ret.extend(get_modified_property_deprecated(binding_start, binding_end,
common_props))
ret.extend(get_modified_property_required(binding_start, binding_end,
common_props))
return ret
def get_modified_specifier2cells(
binding_start: Binding,
binding_end: Binding
) -> List[BindingChange]:
ret: List[BindingChange] = []
start = binding_start.specifier2cells
end = binding_end.specifier2cells
if start == end:
return []
for space, cells_end in end.items():
cells_start = start.get(space)
if cells_start != cells_end:
ret.append(ModifiedSpecifier2Cells(space,
start=cells_start,
end=cells_end))
for space, cells_start in start.items():
if space not in end:
ret.append(ModifiedSpecifier2Cells(space,
start=cells_start,
end=None))
return ret
def get_modified_buses(
binding_start: Binding,
binding_end: Binding
) -> List[BindingChange]:
start = binding_start.buses
end = binding_end.buses
if start == end:
return []
return [ModifiedBuses(start=start, end=end)]
def get_added_properties(
binding_start: Binding,
binding_end: Binding
) -> List[BindingChange]:
return [AddedProperty(prop) for prop in binding_end.prop2specs
if prop not in binding_start.prop2specs]
def get_removed_properties(
binding_start: Binding,
binding_end: Binding
) -> List[BindingChange]:
return [RemovedProperty(prop) for prop in binding_start.prop2specs
if prop not in binding_end.prop2specs]
def get_modified_property_type(
binding_start: Binding,
binding_end: Binding,
common_props: Set[str]
) -> List[BindingChange]:
return get_modified_property_helper(
common_props,
lambda prop: binding_start.prop2specs[prop].type,
lambda prop: binding_end.prop2specs[prop].type,
ModifiedPropertyType)
def get_modified_property_enum(
binding_start: Binding,
binding_end: Binding,
common_props: Set[str]
) -> List[BindingChange]:
return get_modified_property_helper(
common_props,
lambda prop: binding_start.prop2specs[prop].enum,
lambda prop: binding_end.prop2specs[prop].enum,
ModifiedPropertyEnum)
def get_modified_property_const(
binding_start: Binding,
binding_end: Binding,
common_props: Set[str]
) -> List[BindingChange]:
return get_modified_property_helper(
common_props,
lambda prop: binding_start.prop2specs[prop].const,
lambda prop: binding_end.prop2specs[prop].const,
ModifiedPropertyConst)
def get_modified_property_default(
binding_start: Binding,
binding_end: Binding,
common_props: Set[str]
) -> List[BindingChange]:
return get_modified_property_helper(
common_props,
lambda prop: binding_start.prop2specs[prop].default,
lambda prop: binding_end.prop2specs[prop].default,
ModifiedPropertyDefault)
def get_modified_property_deprecated(
binding_start: Binding,
binding_end: Binding,
common_props: Set[str]
) -> List[BindingChange]:
return get_modified_property_helper(
common_props,
lambda prop: binding_start.prop2specs[prop].deprecated,
lambda prop: binding_end.prop2specs[prop].deprecated,
ModifiedPropertyDeprecated)
def get_modified_property_required(
binding_start: Binding,
binding_end: Binding,
common_props: Set[str]
) -> List[BindingChange]:
return get_modified_property_helper(
common_props,
lambda prop: binding_start.prop2specs[prop].required,
lambda prop: binding_end.prop2specs[prop].required,
ModifiedPropertyRequired)
def get_modified_property_helper(
common_props: Set[str],
start_fn: Callable[[str], Any],
end_fn: Callable[[str], Any],
change_constructor: Callable[[str, Any, Any], BindingChange]
) -> List[BindingChange]:
ret = []
for prop in common_props:
start = start_fn(prop)
end = end_fn(prop)
if start != end:
ret.append(change_constructor(prop, start, end))
return ret
def load_compat2binding(commit: str) -> Compat2Binding:
'''Load a map from compatible to binding with that compatible,
based on the bindings in zephyr at the given commit.'''
@contextlib.contextmanager
def git_worktree(directory: os.PathLike, commit: str):
fspath = os.fspath(directory)
subprocess.run(['git', 'worktree', 'add', '--detach', fspath, commit],
check=True)
yield
print('removing worktree...')
subprocess.run(['git', 'worktree', 'remove', fspath], check=True)
ret: Compat2Binding = {}
with tempfile.TemporaryDirectory(prefix='dt_bindings_worktree') as tmpdir:
with git_worktree(tmpdir, commit):
tmpdir_bindings = Path(tmpdir) / 'dts' / 'bindings'
binding_files = []
binding_files.extend(glob.glob(f'{tmpdir_bindings}/**/*.yml',
recursive=True))
binding_files.extend(glob.glob(f'{tmpdir_bindings}/**/*.yaml',
recursive=True))
bindings: List[Binding] = bindings_from_paths(
binding_files, ignore_errors=True)
for binding in bindings:
compat = Compat(binding.compatible, binding.on_bus)
assert compat not in ret
ret[compat] = binding
return ret
def compatible_sort_key(data: Union[Compat, Binding]) -> str:
'''Sort key used by Printer.'''
return (data.compatible, data.on_bus or '')
class Printer:
'''Helper class for formatting output.'''
def __init__(self, outfile):
self.outfile = outfile
self.vnd2vendor_name = load_vendor_prefixes_txt(
ZEPHYR_BASE / 'dts' / 'bindings' / 'vendor-prefixes.txt')
def print(self, *args, **kwargs):
kwargs['file'] = self.outfile
print(*args, **kwargs)
def print_changes(self, changes: Changes):
for vnd in changes.vnds:
if vnd:
vnd_fmt = f' ({vnd})'
else:
vnd_fmt = ''
self.print(f'* {self.vendor_name(vnd)}{vnd_fmt}:\n')
added = changes.vnd2added[vnd]
if added:
self.print(' * New bindings:\n')
self.print_compat2binding(
added,
lambda binding: f':dtcompatible:`{binding.compatible}`'
)
removed = changes.vnd2removed[vnd]
if removed:
self.print(' * Removed bindings:\n')
self.print_compat2binding(
removed,
lambda binding: f'``{binding.compatible}``'
)
modified = changes.vnd2changes[vnd]
if modified:
self.print(' * Modified bindings:\n')
self.print_binding2changes(modified)
def print_compat2binding(
self,
compat2binding: Compat2Binding,
formatter: Callable[[Binding], str]
) -> None:
for compat in sorted(compat2binding, key=compatible_sort_key):
self.print(f' * {formatter(compat2binding[compat])}')
self.print()
def print_binding2changes(self, binding2changes: Binding2Changes) -> None:
for binding, changes in binding2changes.items():
on_bus = f' (on {binding.on_bus} bus)' if binding.on_bus else ''
self.print(f' * :dtcompatible:`{binding.compatible}`{on_bus}:\n')
for change in changes:
self.print_change(change)
self.print()
def print_change(self, change: BindingChange) -> None:
def print(msg):
self.print(f' * {msg}')
def print_prop_change(details):
print(f'property ``{change.property}`` {details} changed from '
f'{change.start} to {change.end}')
if isinstance(change, ModifiedSpecifier2Cells):
print(f'specifier cells for space "{change.space}" '
f'are now named: {change.end} (old value: {change.start})')
elif isinstance(change, ModifiedBuses):
print(f'bus list changed from {change.start} to {change.end}')
elif isinstance(change, AddedProperty):
print(f'new property: ``{change.property}``')
elif isinstance(change, RemovedProperty):
print(f'removed property: ``{change.property}``')
elif isinstance(change, ModifiedPropertyType):
print_prop_change('type')
elif isinstance(change, ModifiedPropertyEnum):
print_prop_change('enum value')
elif isinstance(change, ModifiedPropertyConst):
print_prop_change('const value')
elif isinstance(change, ModifiedPropertyDefault):
print_prop_change('default value')
elif isinstance(change, ModifiedPropertyDeprecated):
print_prop_change('deprecation status')
elif isinstance(change, ModifiedPropertyRequired):
if not change.start and change.end:
print(f'property ``{change.property}`` is now required')
else:
print(f'property ``{change.property}`` is no longer required')
else:
raise ValueError(f'unknown type for {change}: {type(change)}')
def vendor_name(self, vnd: str) -> str:
# Necessary due to the patch for openthread.
if vnd == 'openthread':
# FIXME: we have to go beyond the dict since this
# compatible isn't in vendor-prefixes.txt, but we have
# binding(s) for it. We need to fix this in CI by
# rejecting unknown vendors in a bindings check.
return 'OpenThread'
if vnd == '':
return 'Generic or vendor-independent'
return self.vnd2vendor_name[vnd]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
allow_abbrev=False,
description='''
Print human-readable descriptions of changes to devicetree
bindings between two commits, in .rst format suitable for copy/pasting
into the release notes.
''',
formatter_class=argparse.RawDescriptionHelpFormatter
)
parser.add_argument('start', metavar='START-COMMIT',
help='''what you want to compare bindings against
(typically the previous release's tag)''')
parser.add_argument('end', metavar='END-COMMIT',
help='''what you want to know bindings changes for
(typically 'main')''')
parser.add_argument('file', help='where to write the .rst output to')
return parser.parse_args()
def main():
args = parse_args()
compat2binding_start = load_compat2binding(args.start)
compat2binding_end = load_compat2binding(args.end)
changes = get_changes_between(compat2binding_start,
compat2binding_end)
with open(args.file, 'w') as outfile:
Printer(outfile).print_changes(changes)
if __name__ == '__main__':
main()