[script] add python to script/make-pretty (#790)

* add python to `script/make-pretty` * cleanup and format `script/generate_cmake.py`
openthread · Feb 7, 2024 · acf160f · acf160f
1 parent ce9a4fe
commit acf160f
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 49 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,5 @@
 cmake~=3.26.3
 Jinja2~=3.1.2
 GitPython~=3.1.31
-PyYAML~=6.0
+PyYAML~=6.0
+yapf~=0.40.2
diff --git a/script/generate_cmake.py b/script/generate_cmake.py
@@ -27,12 +27,10 @@
 #  POSSIBILITY OF SUCH DAMAGE.
 #
 
-
 from itertools import filterfalse
-from jinja2 import Environment, FileSystemLoader
+import jinja2
 from pathlib import Path
 import copy
-import git
 import yaml
 import argparse
 import os
@@ -59,31 +57,30 @@ def prepare_path(path: str) -> str:
     path = path.replace('(SDK_PATH)', '{SILABS_GSDK_DIR}')
 
     # Redirect OpenThread stack sources to the ot-efr32 openthread submodule #}
-    path = path.replace(
-        '${SILABS_GSDK_DIR}/util/third_party/openthread', '${PROJECT_SOURCE_DIR}/openthread')
+    path = path.replace('${SILABS_GSDK_DIR}/util/third_party/openthread', '${PROJECT_SOURCE_DIR}/openthread')
 
     # Redirect PAL sources to the ot-efr32 PAL
-    path = path.replace(
-        '${SILABS_GSDK_DIR}/protocol/openthread/platform-abstraction/efr32', '${PROJECT_SOURCE_DIR}/src/src')
+    path = path.replace('${SILABS_GSDK_DIR}/protocol/openthread/platform-abstraction/efr32',
+                        '${PROJECT_SOURCE_DIR}/src/src')
 
     return path
 
 
 def is_mbedtls_source(source: str) -> bool:
-    r = False
+    r: bool = False
     r |= '${SILABS_GSDK_DIR}/util/third_party/mbedtls' in source
     return r
 
 
 def is_mbedtls_define(define: str) -> bool:
-    r = False
+    r: bool = False
     r |= define.startswith("MBEDTLS_")
     r |= define.startswith("PSA_")
     return r
 
 
 def is_openthread_device_type_define(define: str) -> bool:
-    device_types = [
+    device_types: list[str] = [
         "OPENTHREAD_COPROCESSOR",
         "OPENTHREAD_RADIO",
         "OPENTHREAD_FTD",
@@ -93,7 +90,7 @@ def is_openthread_device_type_define(define: str) -> bool:
 
 
 def is_openthread_define(define: str) -> bool:
-    r = True
+    r: bool = False
     r &= not is_mbedtls_define(define)
     r &= not is_openthread_device_type_define(define)
     r &= define.startswith("OPENTHREAD_")
@@ -104,20 +101,20 @@ def filter_mbedtls_lib_vars(slc_vars: dict) -> dict:
     '''Filter mbedtls lib vars'''
 
     # Start with a copy of everything
-    mbedtls_lib_vars = copy.deepcopy(slc_vars)
+    mbedtls_lib_vars: dict = copy.deepcopy(slc_vars)
 
     # Filter includes
     def f(include: str) -> bool:
-        keep = False
+        keep: bool = False
         keep |= (include == '"autogen"') or (include == '"config"')
         keep |= '${SILABS_GSDK_DIR}/hardware' in include
         keep |= '${SILABS_GSDK_DIR}/platform' in include
         keep |= '${SILABS_GSDK_DIR}/util/third_party/crypto' in include
         if not keep:
             pass
         return keep
-    mbedtls_lib_vars['MBEDTLS_INCLUDES'] = list(
-        filter(f, mbedtls_lib_vars['C_CXX_INCLUDES']))
+
+    mbedtls_lib_vars['MBEDTLS_INCLUDES'] = list(filter(f, mbedtls_lib_vars['C_CXX_INCLUDES']))
 
     return mbedtls_lib_vars
 
@@ -131,25 +128,13 @@ def filter_platform_lib_vars(slc_vars: dict) -> dict:
     return platform_lib_vars
 
 
-parser = argparse.ArgumentParser()
-parser.add_argument(
-    "slc_vars_yaml", help="the .yml file generated by slc. This file should contain all the standard SLC template variables")
+parser: argparse.ArgumentParser = argparse.ArgumentParser()
 parser.add_argument(
-    "output_dir", help="the output dir for any generated files")
+    "slc_vars_yaml",
+    help="the .yml file generated by slc. This file should contain all the standard SLC template variables")
+parser.add_argument("output_dir", help="the output dir for any generated files")
 args = parser.parse_args()
 
-script_dir = Path(os.path.dirname(os.path.abspath(__file__)))
-repo_root = script_dir.parent
-
-# Define CMakeLists.txt template location
-environment = Environment(loader=FileSystemLoader(
-    f"{repo_root}/slc/exporter_templates/platform_library"))
-platform_lib_template = environment.get_template("CMakeLists.txt.jinja")
-mbedtls_lib_template = environment.get_template("mbedtls.cmake.jinja")
-slc_vars_yaml_dir = Path(args.slc_vars_yaml).parent
-platform_lib_output_file = slc_vars_yaml_dir / "CMakeLists.txt"
-mbedtls_lib_output_file = slc_vars_yaml_dir / "mbedtls.cmake"
-
 # Import slc variables
 with open(args.slc_vars_yaml, mode="r") as slc_vars_yaml:
     global slc_vars
@@ -160,6 +145,7 @@ def filter_platform_lib_vars(slc_vars: dict) -> dict:
     if not v:
         slc_vars[k] = list()
 
+
 # Remove mapfile flag from linker options
 def f(flag: str) -> bool:
     '''Function for use with filter() to determine if a flag should be kept'''
@@ -168,6 +154,8 @@ def f(flag: str) -> bool:
     keep &= "--specs=" not in flag
     keep &= "linkerfile.ld" not in flag
     return keep
+
+
 for key in ["EXT_LD_FLAGS", "EXT_DEBUG_LD_FLAGS"]:
     flags = slc_vars[key]
     slc_vars[key] = list(filter(f, flags))
@@ -178,9 +166,7 @@ def f(flag: str) -> bool:
     "ALL_SOURCES",
     "SYS_LIBS",
     "USER_LIBS",
-
 ]
-# for key,value in slc_vars.items():
 for key in keys_to_prepare:
     value = slc_vars[key]
     if type(value) is list:
@@ -195,50 +181,54 @@ def f(flag: str) -> bool:
 for key in slc_vars:
     if type(slc_vars[key]) is list:
         slc_vars[key] = list(filter(lambda item: item, slc_vars[key]))
-    # elif type(slc_vars[key]) is dict:
-        # for k,v in slc_vars[key].items():
 
 # Separate mbedtls sources
-slc_vars['MBEDTLS_SOURCES'] = list(
-    filter(is_mbedtls_source, slc_vars['ALL_SOURCES']))
-slc_vars['NON_MBEDTLS_SOURCES'] = list(
-    filterfalse(is_mbedtls_source, slc_vars['ALL_SOURCES']))
+slc_vars['MBEDTLS_SOURCES'] = list(filter(is_mbedtls_source, slc_vars['ALL_SOURCES']))
+slc_vars['NON_MBEDTLS_SOURCES'] = list(filterfalse(is_mbedtls_source, slc_vars['ALL_SOURCES']))
 
 # ==============================================================================
 # Filter defines
 # ==============================================================================
 defines = slc_vars['C_CXX_DEFINES']
 
 # Separate mbedtls defines
-slc_vars['MBEDTLS_DEFINES'] = {d: defines[d]
-                               for d in defines if is_mbedtls_define(d)}
+slc_vars['MBEDTLS_DEFINES'] = {d: defines[d] for d in defines if is_mbedtls_define(d)}
 for d in slc_vars['MBEDTLS_DEFINES']:
     del defines[d]
 
 # Filter OPENTHREAD device type and remove from C_CXX_DEFINES
-slc_vars['OPENTHREAD_DEVICE_TYPE'] = {d: defines[d]
-                                      for d in defines if is_openthread_device_type_define(d)}
+slc_vars['OPENTHREAD_DEVICE_TYPE'] = {d: defines[d] for d in defines if is_openthread_device_type_define(d)}
 for d in slc_vars['OPENTHREAD_DEVICE_TYPE']:
     del defines[d]
 
 # Filter OPENTHREAD_* defines
-slc_vars['OPENTHREAD_DEFINES'] = {d: defines[d]
-                                  for d in defines if is_openthread_define(d)}
+slc_vars['OPENTHREAD_DEFINES'] = {d: defines[d] for d in defines if is_openthread_define(d)}
 for d in slc_vars['OPENTHREAD_DEFINES']:
     del defines[d]
 
 # filter slc vars
 platform_lib_vars = filter_platform_lib_vars(slc_vars)
 mbedtls_lib_vars = filter_mbedtls_lib_vars(slc_vars)
 
+# Define CMakeLists.txt template location
+script_dir: Path = Path(os.path.dirname(os.path.abspath(__file__)))
+repo_root: Path = script_dir.parent
+environment = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(f"{repo_root}/slc/exporter_templates/platform_library"))
+platform_lib_template: jinja2.Template = environment.get_template("CMakeLists.txt.jinja")
+mbedtls_lib_template: jinja2.Template = environment.get_template("mbedtls.cmake.jinja")
+
 # Render the template with the imported variables
 platform_lib_content = platform_lib_template.render(platform_lib_vars)
 mbedtls_lib_content = mbedtls_lib_template.render(mbedtls_lib_vars)
 
 # Output rendered files
+slc_vars_yaml_dir: Path = Path(args.slc_vars_yaml).parent
+platform_lib_output_file: Path = slc_vars_yaml_dir / "CMakeLists.txt"
 with open(platform_lib_output_file, mode="w", encoding="utf-8") as message:
     message.write(platform_lib_content)
     print(f"... wrote {platform_lib_output_file}")
+mbedtls_lib_output_file: Path = slc_vars_yaml_dir / "mbedtls.cmake"
 with open(mbedtls_lib_output_file, mode="w", encoding="utf-8") as message:
     message.write(mbedtls_lib_content)
     print(f"... wrote {mbedtls_lib_output_file}")
diff --git a/script/make-pretty b/script/make-pretty
@@ -30,7 +30,7 @@
 #
 # The script to check or format source code of OpenThread.
 #
-# Format c/c++, markdown, and shell:
+# Format c/c++, markdown, python, and shell:
 #
 #     script/make-pretty
 #
@@ -42,6 +42,10 @@
 #
 #     script/make-pretty markdown
 #
+# Format python only:
+#
+#     script/make-pretty python
+#
 # Format shell only:
 #
 #     script/make-pretty shell
@@ -50,6 +54,7 @@
 #
 #     script/make-pretty check clang-format
 #     script/make-pretty check markdown
+#     script/make-pretty check python
 #     script/make-pretty check shell
 #
 
@@ -78,6 +83,9 @@ readonly OT_CLANG_SOURCES
 OT_MARKDOWN_SOURCES=('*.md')
 readonly OT_MARKDOWN_SOURCES
 
+OT_PYTHON_SOURCES=('*.py')
+readonly OT_PYTHON_SOURCES
+
 OT_SCRIPT_DIR="${repo_dir}"/openthread/script
 readonly OT_SCRIPT_DIR
 
@@ -129,6 +137,30 @@ do_markdown_check()
     popd
 }
 
+do_python_format()
+{
+    echo -e '========================================'
+    echo -e '     format python'
+    echo -e '========================================'
+
+    pushd "${repo_dir}"
+    git ls-files "${OT_PYTHON_SOURCES[@]}" | grep -v -E "^($(echo "${OT_EXCLUDE_DIRS[@]}" | tr ' ' '|'))" \
+        | xargs -n10 -P"$OT_BUILD_JOBS" python3 -m yapf --verbose --style '{based_on_style: google, column_limit: 119}' -ipr
+    popd
+}
+
+do_python_check()
+{
+    echo -e '========================================'
+    echo -e '     check python'
+    echo -e '========================================'
+
+    pushd "${repo_dir}"
+    git ls-files "${OT_PYTHON_SOURCES[@]}" | grep -v -E "^($(echo "${OT_EXCLUDE_DIRS[@]}" | tr ' ' '|'))" \
+        | xargs -n10 -P"$OT_BUILD_JOBS" python3 -m yapf --verbose --style '{based_on_style: google, column_limit: 119}' -dpr
+    popd
+}
+
 do_shell_format()
 {
     echo -e '========================================'
@@ -160,17 +192,20 @@ do_check()
     if [ $# == 0 ]; then
         do_clang_format_check
         do_markdown_check
+        do_python_check
         do_shell_check
     elif [ "$1" == 'clang' ]; then
         do_clang_format_check
     elif [ "$1" == 'clang-format' ]; then
         do_clang_format_check
     elif [ "$1" == 'markdown' ]; then
         do_markdown_check
+    elif [ "$1" == 'python' ]; then
+        do_python_check
     elif [ "$1" == 'shell' ]; then
         do_shell_check
     else
-        echo >&2 "Unsupported check: $1. Supported: clang-format, markdown, shell"
+        echo >&2 "Unsupported check: $1. Supported: clang, markdown, python, shell"
         # 128 for Invalid arguments
         exit 128
     fi
@@ -181,20 +216,23 @@ main()
     if [ $# == 0 ]; then
         do_clang_format
         do_markdown_format
+        do_python_format
         do_shell_format
     elif [ "$1" == 'clang' ]; then
         do_clang_format
     elif [ "$1" == 'clang-format' ]; then
         do_clang_format
     elif [ "$1" == 'markdown' ]; then
         do_markdown_format
+    elif [ "$1" == 'python' ]; then
+        do_python_format
     elif [ "$1" == 'shell' ]; then
         do_shell_format
     elif [ "$1" == 'check' ]; then
         shift
         do_check "$@"
     else
-        echo >&2 "Unsupported action: $1. Supported: clang-format, markdown, shell"
+        echo >&2 "Unsupported action: $1. Supported: clang, markdown, python, shell"
         # 128 for Invalid arguments
         exit 128
     fi