Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[script] add python to script/make-pretty #790

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
cmake~=3.26.3
Jinja2~=3.1.2
GitPython~=3.1.31
PyYAML~=6.0
PyYAML~=6.0
yapf~=0.40.2
80 changes: 35 additions & 45 deletions script/generate_cmake.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,10 @@
# POSSIBILITY OF SUCH DAMAGE.
#


from itertools import filterfalse
from jinja2 import Environment, FileSystemLoader
import jinja2
from pathlib import Path
import copy
import git
import yaml
import argparse
import os
Expand All @@ -59,31 +57,30 @@ def prepare_path(path: str) -> str:
path = path.replace('(SDK_PATH)', '{SILABS_GSDK_DIR}')

# Redirect OpenThread stack sources to the ot-efr32 openthread submodule #}
path = path.replace(
'${SILABS_GSDK_DIR}/util/third_party/openthread', '${PROJECT_SOURCE_DIR}/openthread')
path = path.replace('${SILABS_GSDK_DIR}/util/third_party/openthread', '${PROJECT_SOURCE_DIR}/openthread')

# Redirect PAL sources to the ot-efr32 PAL
path = path.replace(
'${SILABS_GSDK_DIR}/protocol/openthread/platform-abstraction/efr32', '${PROJECT_SOURCE_DIR}/src/src')
path = path.replace('${SILABS_GSDK_DIR}/protocol/openthread/platform-abstraction/efr32',
'${PROJECT_SOURCE_DIR}/src/src')

return path


def is_mbedtls_source(source: str) -> bool:
r = False
r: bool = False
r |= '${SILABS_GSDK_DIR}/util/third_party/mbedtls' in source
return r


def is_mbedtls_define(define: str) -> bool:
r = False
r: bool = False
r |= define.startswith("MBEDTLS_")
r |= define.startswith("PSA_")
return r


def is_openthread_device_type_define(define: str) -> bool:
device_types = [
device_types: list[str] = [
"OPENTHREAD_COPROCESSOR",
"OPENTHREAD_RADIO",
"OPENTHREAD_FTD",
Expand All @@ -93,7 +90,7 @@ def is_openthread_device_type_define(define: str) -> bool:


def is_openthread_define(define: str) -> bool:
r = True
r: bool = False
r &= not is_mbedtls_define(define)
r &= not is_openthread_device_type_define(define)
r &= define.startswith("OPENTHREAD_")
Expand All @@ -104,20 +101,20 @@ def filter_mbedtls_lib_vars(slc_vars: dict) -> dict:
'''Filter mbedtls lib vars'''

# Start with a copy of everything
mbedtls_lib_vars = copy.deepcopy(slc_vars)
mbedtls_lib_vars: dict = copy.deepcopy(slc_vars)

# Filter includes
def f(include: str) -> bool:
keep = False
keep: bool = False
keep |= (include == '"autogen"') or (include == '"config"')
keep |= '${SILABS_GSDK_DIR}/hardware' in include
keep |= '${SILABS_GSDK_DIR}/platform' in include
keep |= '${SILABS_GSDK_DIR}/util/third_party/crypto' in include
if not keep:
pass
return keep
mbedtls_lib_vars['MBEDTLS_INCLUDES'] = list(
filter(f, mbedtls_lib_vars['C_CXX_INCLUDES']))

mbedtls_lib_vars['MBEDTLS_INCLUDES'] = list(filter(f, mbedtls_lib_vars['C_CXX_INCLUDES']))

return mbedtls_lib_vars

Expand All @@ -131,25 +128,13 @@ def filter_platform_lib_vars(slc_vars: dict) -> dict:
return platform_lib_vars


parser = argparse.ArgumentParser()
parser.add_argument(
"slc_vars_yaml", help="the .yml file generated by slc. This file should contain all the standard SLC template variables")
parser: argparse.ArgumentParser = argparse.ArgumentParser()
parser.add_argument(
"output_dir", help="the output dir for any generated files")
"slc_vars_yaml",
help="the .yml file generated by slc. This file should contain all the standard SLC template variables")
parser.add_argument("output_dir", help="the output dir for any generated files")
args = parser.parse_args()

script_dir = Path(os.path.dirname(os.path.abspath(__file__)))
repo_root = script_dir.parent

# Define CMakeLists.txt template location
environment = Environment(loader=FileSystemLoader(
f"{repo_root}/slc/exporter_templates/platform_library"))
platform_lib_template = environment.get_template("CMakeLists.txt.jinja")
mbedtls_lib_template = environment.get_template("mbedtls.cmake.jinja")
slc_vars_yaml_dir = Path(args.slc_vars_yaml).parent
platform_lib_output_file = slc_vars_yaml_dir / "CMakeLists.txt"
mbedtls_lib_output_file = slc_vars_yaml_dir / "mbedtls.cmake"

# Import slc variables
with open(args.slc_vars_yaml, mode="r") as slc_vars_yaml:
global slc_vars
Expand All @@ -160,6 +145,7 @@ def filter_platform_lib_vars(slc_vars: dict) -> dict:
if not v:
slc_vars[k] = list()


# Remove mapfile flag from linker options
def f(flag: str) -> bool:
'''Function for use with filter() to determine if a flag should be kept'''
Expand All @@ -168,6 +154,8 @@ def f(flag: str) -> bool:
keep &= "--specs=" not in flag
keep &= "linkerfile.ld" not in flag
return keep


for key in ["EXT_LD_FLAGS", "EXT_DEBUG_LD_FLAGS"]:
flags = slc_vars[key]
slc_vars[key] = list(filter(f, flags))
Expand All @@ -178,9 +166,7 @@ def f(flag: str) -> bool:
"ALL_SOURCES",
"SYS_LIBS",
"USER_LIBS",

]
# for key,value in slc_vars.items():
for key in keys_to_prepare:
value = slc_vars[key]
if type(value) is list:
Expand All @@ -195,50 +181,54 @@ def f(flag: str) -> bool:
for key in slc_vars:
if type(slc_vars[key]) is list:
slc_vars[key] = list(filter(lambda item: item, slc_vars[key]))
# elif type(slc_vars[key]) is dict:
# for k,v in slc_vars[key].items():

# Separate mbedtls sources
slc_vars['MBEDTLS_SOURCES'] = list(
filter(is_mbedtls_source, slc_vars['ALL_SOURCES']))
slc_vars['NON_MBEDTLS_SOURCES'] = list(
filterfalse(is_mbedtls_source, slc_vars['ALL_SOURCES']))
slc_vars['MBEDTLS_SOURCES'] = list(filter(is_mbedtls_source, slc_vars['ALL_SOURCES']))
slc_vars['NON_MBEDTLS_SOURCES'] = list(filterfalse(is_mbedtls_source, slc_vars['ALL_SOURCES']))

# ==============================================================================
# Filter defines
# ==============================================================================
defines = slc_vars['C_CXX_DEFINES']

# Separate mbedtls defines
slc_vars['MBEDTLS_DEFINES'] = {d: defines[d]
for d in defines if is_mbedtls_define(d)}
slc_vars['MBEDTLS_DEFINES'] = {d: defines[d] for d in defines if is_mbedtls_define(d)}
for d in slc_vars['MBEDTLS_DEFINES']:
del defines[d]

# Filter OPENTHREAD device type and remove from C_CXX_DEFINES
slc_vars['OPENTHREAD_DEVICE_TYPE'] = {d: defines[d]
for d in defines if is_openthread_device_type_define(d)}
slc_vars['OPENTHREAD_DEVICE_TYPE'] = {d: defines[d] for d in defines if is_openthread_device_type_define(d)}
for d in slc_vars['OPENTHREAD_DEVICE_TYPE']:
del defines[d]

# Filter OPENTHREAD_* defines
slc_vars['OPENTHREAD_DEFINES'] = {d: defines[d]
for d in defines if is_openthread_define(d)}
slc_vars['OPENTHREAD_DEFINES'] = {d: defines[d] for d in defines if is_openthread_define(d)}
for d in slc_vars['OPENTHREAD_DEFINES']:
del defines[d]

# filter slc vars
platform_lib_vars = filter_platform_lib_vars(slc_vars)
mbedtls_lib_vars = filter_mbedtls_lib_vars(slc_vars)

# Define CMakeLists.txt template location
script_dir: Path = Path(os.path.dirname(os.path.abspath(__file__)))
repo_root: Path = script_dir.parent
environment = jinja2.Environment(
loader=jinja2.FileSystemLoader(f"{repo_root}/slc/exporter_templates/platform_library"))
platform_lib_template: jinja2.Template = environment.get_template("CMakeLists.txt.jinja")
mbedtls_lib_template: jinja2.Template = environment.get_template("mbedtls.cmake.jinja")

# Render the template with the imported variables
platform_lib_content = platform_lib_template.render(platform_lib_vars)
mbedtls_lib_content = mbedtls_lib_template.render(mbedtls_lib_vars)

# Output rendered files
slc_vars_yaml_dir: Path = Path(args.slc_vars_yaml).parent
platform_lib_output_file: Path = slc_vars_yaml_dir / "CMakeLists.txt"
with open(platform_lib_output_file, mode="w", encoding="utf-8") as message:
message.write(platform_lib_content)
print(f"... wrote {platform_lib_output_file}")
mbedtls_lib_output_file: Path = slc_vars_yaml_dir / "mbedtls.cmake"
with open(mbedtls_lib_output_file, mode="w", encoding="utf-8") as message:
message.write(mbedtls_lib_content)
print(f"... wrote {mbedtls_lib_output_file}")
44 changes: 41 additions & 3 deletions script/make-pretty
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
#
# The script to check or format source code of OpenThread.
#
# Format c/c++, markdown, and shell:
# Format c/c++, markdown, python, and shell:
#
# script/make-pretty
#
Expand All @@ -42,6 +42,10 @@
#
# script/make-pretty markdown
#
# Format python only:
#
# script/make-pretty python
#
# Format shell only:
#
# script/make-pretty shell
Expand All @@ -50,6 +54,7 @@
#
# script/make-pretty check clang-format
# script/make-pretty check markdown
# script/make-pretty check python
# script/make-pretty check shell
#

Expand Down Expand Up @@ -78,6 +83,9 @@ readonly OT_CLANG_SOURCES
OT_MARKDOWN_SOURCES=('*.md')
readonly OT_MARKDOWN_SOURCES

OT_PYTHON_SOURCES=('*.py')
readonly OT_PYTHON_SOURCES

OT_SCRIPT_DIR="${repo_dir}"/openthread/script
readonly OT_SCRIPT_DIR

Expand Down Expand Up @@ -129,6 +137,30 @@ do_markdown_check()
popd
}

do_python_format()
{
echo -e '========================================'
echo -e ' format python'
echo -e '========================================'

pushd "${repo_dir}"
git ls-files "${OT_PYTHON_SOURCES[@]}" | grep -v -E "^($(echo "${OT_EXCLUDE_DIRS[@]}" | tr ' ' '|'))" \
| xargs -n10 -P"$OT_BUILD_JOBS" python3 -m yapf --verbose --style '{based_on_style: google, column_limit: 119}' -ipr
popd
}

do_python_check()
{
echo -e '========================================'
echo -e ' check python'
echo -e '========================================'

pushd "${repo_dir}"
git ls-files "${OT_PYTHON_SOURCES[@]}" | grep -v -E "^($(echo "${OT_EXCLUDE_DIRS[@]}" | tr ' ' '|'))" \
| xargs -n10 -P"$OT_BUILD_JOBS" python3 -m yapf --verbose --style '{based_on_style: google, column_limit: 119}' -dpr
popd
}

do_shell_format()
{
echo -e '========================================'
Expand Down Expand Up @@ -160,17 +192,20 @@ do_check()
if [ $# == 0 ]; then
do_clang_format_check
do_markdown_check
do_python_check
do_shell_check
elif [ "$1" == 'clang' ]; then
do_clang_format_check
elif [ "$1" == 'clang-format' ]; then
do_clang_format_check
elif [ "$1" == 'markdown' ]; then
do_markdown_check
elif [ "$1" == 'python' ]; then
do_python_check
elif [ "$1" == 'shell' ]; then
do_shell_check
else
echo >&2 "Unsupported check: $1. Supported: clang-format, markdown, shell"
echo >&2 "Unsupported check: $1. Supported: clang, markdown, python, shell"
# 128 for Invalid arguments
exit 128
fi
Expand All @@ -181,20 +216,23 @@ main()
if [ $# == 0 ]; then
do_clang_format
do_markdown_format
do_python_format
do_shell_format
elif [ "$1" == 'clang' ]; then
do_clang_format
elif [ "$1" == 'clang-format' ]; then
do_clang_format
elif [ "$1" == 'markdown' ]; then
do_markdown_format
elif [ "$1" == 'python' ]; then
do_python_format
elif [ "$1" == 'shell' ]; then
do_shell_format
elif [ "$1" == 'check' ]; then
shift
do_check "$@"
else
echo >&2 "Unsupported action: $1. Supported: clang-format, markdown, shell"
echo >&2 "Unsupported action: $1. Supported: clang, markdown, python, shell"
# 128 for Invalid arguments
exit 128
fi
Expand Down