From 4c1f574c3d495d292568a91dd4990d796f1cf3ee Mon Sep 17 00:00:00 2001 From: Nebojsa Cvetkovic Date: Thu, 6 Jul 2023 15:43:37 +0100 Subject: [PATCH] feat(idf.py): Allow adding arguments from file via @filename.txt --- tools/idf.py | 52 ++++++++++++++++++++++++++++-- tools/test_idf_py/args_a | 1 + tools/test_idf_py/args_b | 1 + tools/test_idf_py/args_circular_a | 1 + tools/test_idf_py/args_circular_b | 1 + tools/test_idf_py/args_recursive | 1 + tools/test_idf_py/test_idf_py.py | 53 +++++++++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tools/test_idf_py/args_a create mode 100644 tools/test_idf_py/args_b create mode 100644 tools/test_idf_py/args_circular_a create mode 100644 tools/test_idf_py/args_circular_b create mode 100644 tools/test_idf_py/args_recursive diff --git a/tools/idf.py b/tools/idf.py index 614b955083b..2ab5085a9e3 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -18,6 +18,7 @@ import locale import os import os.path +import shlex import subprocess import sys from collections import Counter, OrderedDict, _OrderedDictKeysView @@ -695,7 +696,7 @@ def parse_project_dir(project_dir: str) -> Any: return CLI(help=cli_help, verbose_output=verbose_output, all_actions=all_actions) -def main() -> None: +def main(argv=None) -> None: # Check the environment only when idf.py is invoked regularly from command line. checks_output = None if SHELL_COMPLETE_RUN else check_environment() @@ -713,7 +714,54 @@ def main() -> None: else: raise else: - cli(sys.argv[1:], prog_name=PROG, complete_var=SHELL_COMPLETE_VAR) + argv = expand_file_arguments(argv or sys.argv[1:]) + + cli(argv, prog_name=PROG, complete_var=SHELL_COMPLETE_VAR) + + +def expand_file_arguments(argv): + """ + Any argument starting with "@" gets replaced with all values read from a text file. + Text file arguments can be split by newline or by space. + Values are added "as-is", as if they were specified in this order + on the command line. + """ + visited = set() + expanded = False + + def expand_args(args, parent_path, file_stack): + expanded_args = [] + for arg in args: + if not arg.startswith("@"): + expanded_args.append(arg) + else: + nonlocal expanded, visited + expanded = True + + file_name = arg[1:] + rel_path = os.path.normpath(os.path.join(parent_path, file_name)) + + if rel_path in visited: + file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) + raise FatalError(f'Circular dependency in file argument expansion: {file_stack_str}') + visited.add(rel_path) + + try: + with open(rel_path, "r") as f: + for line in f: + expanded_args.extend(expand_args(shlex.split(line), os.path.dirname(rel_path), file_stack + [file_name])) + except IOError: + file_stack_str = ' -> '.join(['@' + f for f in file_stack + [file_name]]) + raise FatalError(f"File '{rel_path}' (expansion of {file_stack_str}) could not be opened. " + "Please ensure the file exists and you have the necessary permissions to read it.") + return expanded_args + + argv = expand_args(argv, os.getcwd(), []) + + if expanded: + print(f'Running: idf.py {" ".join(argv)}') + + return argv def _valid_unicode_config() -> Union[codecs.CodecInfo, bool]: diff --git a/tools/test_idf_py/args_a b/tools/test_idf_py/args_a new file mode 100644 index 00000000000..f0252ff8635 --- /dev/null +++ b/tools/test_idf_py/args_a @@ -0,0 +1 @@ +-DAAA -DBBB \ No newline at end of file diff --git a/tools/test_idf_py/args_b b/tools/test_idf_py/args_b new file mode 100644 index 00000000000..1c01c9980a7 --- /dev/null +++ b/tools/test_idf_py/args_b @@ -0,0 +1 @@ +-DCCC -DDDD \ No newline at end of file diff --git a/tools/test_idf_py/args_circular_a b/tools/test_idf_py/args_circular_a new file mode 100644 index 00000000000..38cebf4772a --- /dev/null +++ b/tools/test_idf_py/args_circular_a @@ -0,0 +1 @@ +-DAAA @args_circular_b diff --git a/tools/test_idf_py/args_circular_b b/tools/test_idf_py/args_circular_b new file mode 100644 index 00000000000..f427223c9c3 --- /dev/null +++ b/tools/test_idf_py/args_circular_b @@ -0,0 +1 @@ +-DBBB @args_circular_a \ No newline at end of file diff --git a/tools/test_idf_py/args_recursive b/tools/test_idf_py/args_recursive new file mode 100644 index 00000000000..7d44f743e7a --- /dev/null +++ b/tools/test_idf_py/args_recursive @@ -0,0 +1 @@ +@args_a -DEEE -DFFF \ No newline at end of file diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index edc63bfec79..1e770473037 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -291,5 +291,58 @@ def test_roms_validate_build_date(self): self.assertTrue(build_date_str == k['build_date_str']) +class TestFileArgumentExpansion(TestCase): + def test_file_expansion(self): + """Test @filename expansion functionality""" + try: + output = subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_a'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Running: idf.py --version -DAAA -DBBB', output) + except subprocess.CalledProcessError as e: + self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') + + def test_multiple_file_arguments(self): + """Test multiple @filename arguments""" + try: + output = subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_a', '@args_b'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Running: idf.py --version -DAAA -DBBB -DCCC -DDDD', output) + except subprocess.CalledProcessError as e: + self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') + + def test_recursive_expansion(self): + """Test recursive expansion of @filename arguments""" + try: + output = subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_recursive'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Running: idf.py --version -DAAA -DBBB -DEEE -DFFF', output) + except subprocess.CalledProcessError as e: + self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') + + def test_circular_dependency(self): + """Test circular dependency detection in file argument expansion""" + with self.assertRaises(subprocess.CalledProcessError) as cm: + subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_circular_a'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('Circular dependency in file argument expansion', cm.exception.output.decode('utf-8', 'ignore')) + + def test_missing_file(self): + """Test missing file detection in file argument expansion""" + with self.assertRaises(subprocess.CalledProcessError) as cm: + subprocess.check_output( + [sys.executable, idf_py_path, '--version', '@args_non_existent'], + env=os.environ, + stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + self.assertIn('(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore')) + + if __name__ == '__main__': main()