From 93c17b298f32862ec3bb52c37ffe118974ba333a Mon Sep 17 00:00:00 2001 From: Mason Tran Date: Tue, 6 Feb 2024 17:29:40 -0500 Subject: [PATCH 1/2] [script] add python to `script/make-pretty` --- requirements.txt | 3 ++- script/make-pretty | 44 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5dcb9a88..9c251e2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ cmake~=3.26.3 Jinja2~=3.1.2 GitPython~=3.1.31 -PyYAML~=6.0 \ No newline at end of file +PyYAML~=6.0 +yapf~=0.40.2 \ No newline at end of file diff --git a/script/make-pretty b/script/make-pretty index bfb1eb9d..ca9e9937 100755 --- a/script/make-pretty +++ b/script/make-pretty @@ -30,7 +30,7 @@ # # The script to check or format source code of OpenThread. # -# Format c/c++, markdown, and shell: +# Format c/c++, markdown, python, and shell: # # script/make-pretty # @@ -42,6 +42,10 @@ # # script/make-pretty markdown # +# Format python only: +# +# script/make-pretty python +# # Format shell only: # # script/make-pretty shell @@ -50,6 +54,7 @@ # # script/make-pretty check clang-format # script/make-pretty check markdown +# script/make-pretty check python # script/make-pretty check shell # @@ -78,6 +83,9 @@ readonly OT_CLANG_SOURCES OT_MARKDOWN_SOURCES=('*.md') readonly OT_MARKDOWN_SOURCES +OT_PYTHON_SOURCES=('*.py') +readonly OT_PYTHON_SOURCES + OT_SCRIPT_DIR="${repo_dir}"/openthread/script readonly OT_SCRIPT_DIR @@ -129,6 +137,30 @@ do_markdown_check() popd } +do_python_format() +{ + echo -e '========================================' + echo -e ' format python' + echo -e '========================================' + + pushd "${repo_dir}" + git ls-files "${OT_PYTHON_SOURCES[@]}" | grep -v -E "^($(echo "${OT_EXCLUDE_DIRS[@]}" | tr ' ' '|'))" \ + | xargs -n10 -P"$OT_BUILD_JOBS" python3 -m yapf --verbose --style '{based_on_style: google, column_limit: 119}' -ipr + popd +} + +do_python_check() +{ + echo -e '========================================' + echo -e ' check python' + echo -e '========================================' + + pushd "${repo_dir}" + git ls-files "${OT_PYTHON_SOURCES[@]}" | grep -v -E "^($(echo "${OT_EXCLUDE_DIRS[@]}" | tr ' ' '|'))" \ + | xargs -n10 -P"$OT_BUILD_JOBS" python3 -m yapf --verbose --style '{based_on_style: google, column_limit: 119}' -dpr + popd +} + do_shell_format() { echo -e '========================================' @@ -160,6 +192,7 @@ do_check() if [ $# == 0 ]; then do_clang_format_check do_markdown_check + do_python_check do_shell_check elif [ "$1" == 'clang' ]; then do_clang_format_check @@ -167,10 +200,12 @@ do_check() do_clang_format_check elif [ "$1" == 'markdown' ]; then do_markdown_check + elif [ "$1" == 'python' ]; then + do_python_check elif [ "$1" == 'shell' ]; then do_shell_check else - echo >&2 "Unsupported check: $1. Supported: clang-format, markdown, shell" + echo >&2 "Unsupported check: $1. Supported: clang, markdown, python, shell" # 128 for Invalid arguments exit 128 fi @@ -181,6 +216,7 @@ main() if [ $# == 0 ]; then do_clang_format do_markdown_format + do_python_format do_shell_format elif [ "$1" == 'clang' ]; then do_clang_format @@ -188,13 +224,15 @@ main() do_clang_format elif [ "$1" == 'markdown' ]; then do_markdown_format + elif [ "$1" == 'python' ]; then + do_python_format elif [ "$1" == 'shell' ]; then do_shell_format elif [ "$1" == 'check' ]; then shift do_check "$@" else - echo >&2 "Unsupported action: $1. Supported: clang-format, markdown, shell" + echo >&2 "Unsupported action: $1. Supported: clang, markdown, python, shell" # 128 for Invalid arguments exit 128 fi From c0e18d8252fa5d2194f6665b8e908e40f9226a57 Mon Sep 17 00:00:00 2001 From: Mason Tran Date: Tue, 6 Feb 2024 17:58:51 -0500 Subject: [PATCH 2/2] [script] cleanup and format `script/generate_cmake.py` --- script/generate_cmake.py | 80 ++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/script/generate_cmake.py b/script/generate_cmake.py index c7122c20..8f0e47b9 100755 --- a/script/generate_cmake.py +++ b/script/generate_cmake.py @@ -27,12 +27,10 @@ # POSSIBILITY OF SUCH DAMAGE. # - from itertools import filterfalse -from jinja2 import Environment, FileSystemLoader +import jinja2 from pathlib import Path import copy -import git import yaml import argparse import os @@ -59,31 +57,30 @@ def prepare_path(path: str) -> str: path = path.replace('(SDK_PATH)', '{SILABS_GSDK_DIR}') # Redirect OpenThread stack sources to the ot-efr32 openthread submodule #} - path = path.replace( - '${SILABS_GSDK_DIR}/util/third_party/openthread', '${PROJECT_SOURCE_DIR}/openthread') + path = path.replace('${SILABS_GSDK_DIR}/util/third_party/openthread', '${PROJECT_SOURCE_DIR}/openthread') # Redirect PAL sources to the ot-efr32 PAL - path = path.replace( - '${SILABS_GSDK_DIR}/protocol/openthread/platform-abstraction/efr32', '${PROJECT_SOURCE_DIR}/src/src') + path = path.replace('${SILABS_GSDK_DIR}/protocol/openthread/platform-abstraction/efr32', + '${PROJECT_SOURCE_DIR}/src/src') return path def is_mbedtls_source(source: str) -> bool: - r = False + r: bool = False r |= '${SILABS_GSDK_DIR}/util/third_party/mbedtls' in source return r def is_mbedtls_define(define: str) -> bool: - r = False + r: bool = False r |= define.startswith("MBEDTLS_") r |= define.startswith("PSA_") return r def is_openthread_device_type_define(define: str) -> bool: - device_types = [ + device_types: list[str] = [ "OPENTHREAD_COPROCESSOR", "OPENTHREAD_RADIO", "OPENTHREAD_FTD", @@ -93,7 +90,7 @@ def is_openthread_device_type_define(define: str) -> bool: def is_openthread_define(define: str) -> bool: - r = True + r: bool = False r &= not is_mbedtls_define(define) r &= not is_openthread_device_type_define(define) r &= define.startswith("OPENTHREAD_") @@ -104,11 +101,11 @@ def filter_mbedtls_lib_vars(slc_vars: dict) -> dict: '''Filter mbedtls lib vars''' # Start with a copy of everything - mbedtls_lib_vars = copy.deepcopy(slc_vars) + mbedtls_lib_vars: dict = copy.deepcopy(slc_vars) # Filter includes def f(include: str) -> bool: - keep = False + keep: bool = False keep |= (include == '"autogen"') or (include == '"config"') keep |= '${SILABS_GSDK_DIR}/hardware' in include keep |= '${SILABS_GSDK_DIR}/platform' in include @@ -116,8 +113,8 @@ def f(include: str) -> bool: if not keep: pass return keep - mbedtls_lib_vars['MBEDTLS_INCLUDES'] = list( - filter(f, mbedtls_lib_vars['C_CXX_INCLUDES'])) + + mbedtls_lib_vars['MBEDTLS_INCLUDES'] = list(filter(f, mbedtls_lib_vars['C_CXX_INCLUDES'])) return mbedtls_lib_vars @@ -131,25 +128,13 @@ def filter_platform_lib_vars(slc_vars: dict) -> dict: return platform_lib_vars -parser = argparse.ArgumentParser() -parser.add_argument( - "slc_vars_yaml", help="the .yml file generated by slc. This file should contain all the standard SLC template variables") +parser: argparse.ArgumentParser = argparse.ArgumentParser() parser.add_argument( - "output_dir", help="the output dir for any generated files") + "slc_vars_yaml", + help="the .yml file generated by slc. This file should contain all the standard SLC template variables") +parser.add_argument("output_dir", help="the output dir for any generated files") args = parser.parse_args() -script_dir = Path(os.path.dirname(os.path.abspath(__file__))) -repo_root = script_dir.parent - -# Define CMakeLists.txt template location -environment = Environment(loader=FileSystemLoader( - f"{repo_root}/slc/exporter_templates/platform_library")) -platform_lib_template = environment.get_template("CMakeLists.txt.jinja") -mbedtls_lib_template = environment.get_template("mbedtls.cmake.jinja") -slc_vars_yaml_dir = Path(args.slc_vars_yaml).parent -platform_lib_output_file = slc_vars_yaml_dir / "CMakeLists.txt" -mbedtls_lib_output_file = slc_vars_yaml_dir / "mbedtls.cmake" - # Import slc variables with open(args.slc_vars_yaml, mode="r") as slc_vars_yaml: global slc_vars @@ -160,6 +145,7 @@ def filter_platform_lib_vars(slc_vars: dict) -> dict: if not v: slc_vars[k] = list() + # Remove mapfile flag from linker options def f(flag: str) -> bool: '''Function for use with filter() to determine if a flag should be kept''' @@ -168,6 +154,8 @@ def f(flag: str) -> bool: keep &= "--specs=" not in flag keep &= "linkerfile.ld" not in flag return keep + + for key in ["EXT_LD_FLAGS", "EXT_DEBUG_LD_FLAGS"]: flags = slc_vars[key] slc_vars[key] = list(filter(f, flags)) @@ -178,9 +166,7 @@ def f(flag: str) -> bool: "ALL_SOURCES", "SYS_LIBS", "USER_LIBS", - ] -# for key,value in slc_vars.items(): for key in keys_to_prepare: value = slc_vars[key] if type(value) is list: @@ -195,14 +181,10 @@ def f(flag: str) -> bool: for key in slc_vars: if type(slc_vars[key]) is list: slc_vars[key] = list(filter(lambda item: item, slc_vars[key])) - # elif type(slc_vars[key]) is dict: - # for k,v in slc_vars[key].items(): # Separate mbedtls sources -slc_vars['MBEDTLS_SOURCES'] = list( - filter(is_mbedtls_source, slc_vars['ALL_SOURCES'])) -slc_vars['NON_MBEDTLS_SOURCES'] = list( - filterfalse(is_mbedtls_source, slc_vars['ALL_SOURCES'])) +slc_vars['MBEDTLS_SOURCES'] = list(filter(is_mbedtls_source, slc_vars['ALL_SOURCES'])) +slc_vars['NON_MBEDTLS_SOURCES'] = list(filterfalse(is_mbedtls_source, slc_vars['ALL_SOURCES'])) # ============================================================================== # Filter defines @@ -210,20 +192,17 @@ def f(flag: str) -> bool: defines = slc_vars['C_CXX_DEFINES'] # Separate mbedtls defines -slc_vars['MBEDTLS_DEFINES'] = {d: defines[d] - for d in defines if is_mbedtls_define(d)} +slc_vars['MBEDTLS_DEFINES'] = {d: defines[d] for d in defines if is_mbedtls_define(d)} for d in slc_vars['MBEDTLS_DEFINES']: del defines[d] # Filter OPENTHREAD device type and remove from C_CXX_DEFINES -slc_vars['OPENTHREAD_DEVICE_TYPE'] = {d: defines[d] - for d in defines if is_openthread_device_type_define(d)} +slc_vars['OPENTHREAD_DEVICE_TYPE'] = {d: defines[d] for d in defines if is_openthread_device_type_define(d)} for d in slc_vars['OPENTHREAD_DEVICE_TYPE']: del defines[d] # Filter OPENTHREAD_* defines -slc_vars['OPENTHREAD_DEFINES'] = {d: defines[d] - for d in defines if is_openthread_define(d)} +slc_vars['OPENTHREAD_DEFINES'] = {d: defines[d] for d in defines if is_openthread_define(d)} for d in slc_vars['OPENTHREAD_DEFINES']: del defines[d] @@ -231,14 +210,25 @@ def f(flag: str) -> bool: platform_lib_vars = filter_platform_lib_vars(slc_vars) mbedtls_lib_vars = filter_mbedtls_lib_vars(slc_vars) +# Define CMakeLists.txt template location +script_dir: Path = Path(os.path.dirname(os.path.abspath(__file__))) +repo_root: Path = script_dir.parent +environment = jinja2.Environment( + loader=jinja2.FileSystemLoader(f"{repo_root}/slc/exporter_templates/platform_library")) +platform_lib_template: jinja2.Template = environment.get_template("CMakeLists.txt.jinja") +mbedtls_lib_template: jinja2.Template = environment.get_template("mbedtls.cmake.jinja") + # Render the template with the imported variables platform_lib_content = platform_lib_template.render(platform_lib_vars) mbedtls_lib_content = mbedtls_lib_template.render(mbedtls_lib_vars) # Output rendered files +slc_vars_yaml_dir: Path = Path(args.slc_vars_yaml).parent +platform_lib_output_file: Path = slc_vars_yaml_dir / "CMakeLists.txt" with open(platform_lib_output_file, mode="w", encoding="utf-8") as message: message.write(platform_lib_content) print(f"... wrote {platform_lib_output_file}") +mbedtls_lib_output_file: Path = slc_vars_yaml_dir / "mbedtls.cmake" with open(mbedtls_lib_output_file, mode="w", encoding="utf-8") as message: message.write(mbedtls_lib_content) print(f"... wrote {mbedtls_lib_output_file}")