Setup structure

.gitignore

@@ -0,0 +1,15 @@
+**/.venv
+**/.modules
+**/*.pyc
+**/__pycache__
+**/*.log
+**/*.swp
+**/*.swo
+**/*.sqlite3
+**/*.db
+**/*.db-journal
+**/*.pyo
+**/bin
+**/*.exe
+**/*.dll
+**/.*.so

README.md

@@ -0,0 +1,5 @@
+# Renaissance Experiments
+
+This project is experimental in nature and aims to explore various concepts and techniques to apply renaissance pattern matching in a generic way using multiple abract syntax trees. 
+
+The code for the experiments is located in the [python](./python) folder.

c/src/README.md

@@ -0,0 +1,18 @@
+# Most usefull commands:
+
+## gcc
+gcc -fdump-tree-all-raw-lineno   -fdump-rtl-all-raw-lineno -o main.exe main.c
+
+
+## clang
+
+### ast dump
+
+ `clang -Xclang -ast-dump -fsyntax-only main.c > ast-dump.ast`
+or 
+ `clang -Xclang -ast-dump -fsyntax-only main.c > ast-dump.ast`
+### preprocessing dump
+
+`pp-trace main.c > pptrace.ast` 
+
+contains all preprocessing directives and all usages.

c/src/compile_commands.json

@@ -0,0 +1,16 @@
+[
+    {
+        "directory": "Z:\\testproject\\c\\src",
+        "file": "test.cpp",
+        "output": "C:\\Users\\PNELIS~1\\AppData\\Local\\Temp\\1\\test-9e2a00.o",
+        "arguments": [
+            "C:\\Users\\pnelissen\\scoop\\apps\\llvm\\current\\bin\\clang++.exe",
+            "-xc++",
+            "test.cpp",
+            "-o",
+            "C:\\Users\\PNELIS~1\\AppData\\Local\\Temp\\1\\test-9e2a00.o",
+            "--driver-mode=g++",
+            "--target=x86_64-pc-windows-msvc19.39.33521"
+        ]
+    }
+]

c/src/main.c

@@ -0,0 +1,19 @@
+#include <stdio.h>
+
+static int static_int = 2;
+
+#define A_DEFINE (4 + static_int)
+#define B_DEFINE (A_DEFINE + static_int)
+
+#define FC_MACRO(arg)\
+do{\
+   arg += A_DEFINE;\
+} while(0)
+
+int main() {
+    int qwerty = 3 + A_DEFINE;
+    FC_MACRO(qwerty);
+    printf("QWERTY %d", qwerty+static_int);
+    FC_MACRO(qwerty);
+    return 0;
+}

c/src/test.cpp

@@ -0,0 +1,58 @@
+//hËllo  utf-8 2 byte character
+static int static_int = 2;
+
+#define A_DEFINE (4 + static_int)
+#define B_DEFINE (A_DEFINE + static_int)
+
+#define FC_MACRO(arg)\
+do{\
+   arg += A_DEFINE;\
+} while(0)
+
+void printf(char*);
+void printf(const char*, const char*, int);
+class A {
+public:
+    A() {
+        printf("A constructor\n");
+    }
+    ~A() {
+        printf("A destructor\n");
+    }
+    protected:
+    int a;
+    virtual void testA() {
+        printf("A test\n");
+    }
+};
+
+class B: public A {
+public:
+    B() {
+        printf("B constructor\n");
+    }
+    ~B() {
+        printf("B destructor\n");
+    }
+    public:
+    int b;
+    virtual int testB(int x, const char *y) {
+        this->testA();
+        printf("B *s test %d\n", y+A_DEFINE, x);
+        return x;
+    }
+    void testA() {
+        A::testA();
+    }
+};
+
+static void test() {
+    static A a;
+    B b;
+    b.testB(1, "test");
+    b.testA();
+}
+int main() {
+    test ();
+    return 0;
+}

python/.env

@@ -0,0 +1,2 @@
+#PATH=.venv\\Lib\\site-packages\\clang\\native;%PATH%
+PYTHONPATH=${workspaceFolder}/src

python/.vscode/settings.json

@@ -0,0 +1,24 @@
+{
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        ".",
+        "-p",
+        "test*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true,
+    "python.testing.pytestArgs": [
+        "test"
+    ],
+    "python.envFile": "${workspaceFolder}/.env",
+    "terminal.integrated.env.linux": {
+        "PATH": ".venv/Lib/site-packages/clang/native:${env:PATH}"
+    },
+    "terminal.integrated.env.osx": {
+        "PATH": ".venv/Lib/site-packages/clang/native:${env:PATH}"
+    },
+    "terminal.integrated.env.windows": {
+        "Path": ".venv\\Lib\\site-packages\\clang\\native;${env:Path}"
+    }
+}

python/README.md

@@ -0,0 +1,53 @@
+# Description
+This project is a generic approach to refactor code bases with a generic AST structure.
+It uses `TNO Renaissance` pattern matching.
+Currently clang native and clang python bindings are supported.
+
+# How to add a different binding
+You'll need to implement a concrete class for syntax_tree.ASTNode.
+Follow the implementations of `ClangASTNode` and `ClangJsonASTNode` as an example.
+If the concrete AST has a different language then also a `PatternFactory` must be added. See `CPatternFactory` for inspiration.
+
+## Installation Procedure
+To install the necessary dependencies, follow these steps:
+
+1. **Run the Installation Script**
+    - Navigate to the project directory.
+    - Execute the `install.bat` script by double-clicking it or running the following command in the terminal:
+      ```sh
+      ./install.bat
+      ```
+
+## Configuration and Verification
+
+1. **Configure the Environment**
+    - Open Visual Studio Code (VSCode).
+    - Ensure that the Python extension is installed.
+    - Open the project folder in VSCode.
+    - alternatively in shell goto <root>/python folder and
+      ```sh
+      code .
+      ```
+
+2. **Verify the Installation**
+    - Open the integrated terminal in VSCode.
+    - Run the following command to execute the tests:
+      ```sh
+      python -m unittest discover
+      ```
+    - Check the output to ensure all tests pass successfully.
+
+By following these steps, you will have installed and verified the setup for the project.
+
+
+## TODO
+
+An incomplete list of todo's:
+
+* The get_properties methods of both `ClangASTNode` and `ClangJsonASTNode` are not complete yet. This might cause mismatches in the `Match_Finder`
+* C++ constructs have not been tested yet
+* An example of how to use includes in a `Pattern` must be added
+* Tests need to be added for macro handling
+* The methods `get_references` and `referred_by` must be added to `ASTNode` and implemented in the concrete classes
+* Test cases for multiple match patterns need to be added. Currently, there is only one working case in the examples
+* Comments in Clang appear incorrectly in the `ASTShower`. This seems to be a Clang issue, which is surprising

python/examples/batch_process_examples.py

@@ -0,0 +1,149 @@
+#use clang to load and walk a compilation database
+
+from dataclasses import dataclass
+from typing import Callable
+from syntax_tree.recipe_ast_processor import RecipeASTProcessor, after_step, recipe_step, final_action
+from typing_extensions import Iterable, override
+from impl import ClangASTNode, ClangJsonASTNode
+from refactoring import CleanupRefactoring
+from syntax_tree import ASTProcessor, ASTNode, ASTNodeType, TextUtils, ASTFactory, BatchASTProcessor
+
+example_1 = TextUtils.strip_indent("""
+        void x(int a) {}
+        void x1(int a) {}
+        void x2(int a) {}
+
+        void f1(int a){
+            int unused = 0;
+            int unused2 = 0; //must be removed
+            if (a==1) {
+                int unused = 0;
+                int unused2 = 0; //should be kept
+                int c = unused2;
+                x1(c);
+            }
+        }
+        """)
+
+example_2 = TextUtils.strip_indent("""
+        void x(int a) {}
+        void x1(int a) {}
+        void x2(int a) {}
+        void f2(int a){
+            int unused = 0;
+            if (a==1) {
+                int unused = 0;
+                int another_unused = 0;
+                int used2 = 0; //should be kept
+                int c = used2;
+                x2(c);
+            }
+        }
+        """)
+
+# generate a simple code base provider in real life use a compilation database
+def simple_codebase_provider() -> Iterable[tuple[ASTFactory[ASTNodeType], ASTNodeType]]:
+    for impl_type in [ClangASTNode, ClangJsonASTNode]:
+        factory = ASTFactory(impl_type)
+        atu1 = factory.create_from_text(example_1, impl_type.__name__+'1.c')
+        yield factory, atu1
+        atu2 = factory.create_from_text(example_2, impl_type.__name__+'2.c')
+        yield factory, atu2
+
+def print_results(title, batch_processor):
+    print(title +':')
+    for file, code in batch_processor.in_memory_files.items():
+        print(TextUtils.shift_right(file, 4)+'\n')
+        print(TextUtils.shift_right(code, 8)+'\n')
+
+
+def batch_remove_unused_variable_once_example():
+    """
+    This function demonstrates a batch processing example using different AST node implementations.
+    It iterates over a list of AST node implementations (`ClangASTNode` and `ClangJsonASTNode`), 
+    and for each implementation, it generates a codebase provider that yields tuples of 
+    `ASTFactory` and `ASTNode` created from example source texts (`example_1` and `example_2`).
+    The function then creates a `BatchASTProcessor` with in-memory storage enabled and processes 
+    the codebase using the `CleanupRefactoring.remove_unused_variables` refactoring operation.
+    Finally, it prints the rewritten code stored in memory.
+    """
+    #generate a batch processor for testing purposes we store into memory
+    batch_processor = BatchASTProcessor(in_memory=True)
+    batch_processor.once(simple_codebase_provider, CleanupRefactoring.remove_unused_variables)
+    #print the rewritten code normally you would write to a file
+    print_results('example batch remove unused variable once', batch_processor)
+
+
+def batch_repeat_example():
+    """
+    Demonstrates the use of a batch processor to perform multiple refactoring operations on a codebase.
+    This example creates an in-memory batch processor and applies two refactoring operations:
+    1. CleanupRefactoring.remove_unused_variables: Removes unused variables from the codebase.
+    2. remove_function: Removes all function calls from the codebase.
+    The results of the refactoring operations are printed to the console.
+
+    Repeat is in action here: 
+        the first time the codebase is processed, the unused variables are removed.
+        and the function calls are removed.
+        the second time the codebase is processed, the new unused variables are removed again.
+    Note:
+        In a real-world scenario, the rewritten code would typically be written to a file instead of being printed.
+    """
+    #generate a batch processor for testing purposes we store into memory
+    batch_processor = BatchASTProcessor(in_memory=True)
+    #remove a function to create more unused variables
+    def remove_function(ast_processor: ASTProcessor[ASTNodeType]):
+        ast_processor.find_kind('(?i)Call_?Expr').\
+            for_each(lambda node: ast_processor.insert_before( '// ', node, False, False ))
+
+    # batch_processor.repeat(simple_codebase_provider, [remove_function])   
+    batch_processor.repeat(simple_codebase_provider, [CleanupRefactoring.remove_unused_variables, remove_function])   
+    #print the rewritten code normally you would write to a file
+    print_results('example batch repeat', batch_processor)
+
+@dataclass
+class Call:
+    callee: str
+    calls: str
+
+class AnalysisRecipe:
+    def __init__(self):
+        self._calls = []
+
+    @recipe_step(order=0)
+    def store_function_call(self, ast_processor: ASTProcessor[ASTNodeType]) -> Callable[[], None]|None:
+        # find all function calls and store them, this routing is invoked in parallel!
+        calls = []
+        ast_processor.find_kind('(?i)Call_?Expr').\
+            for_each(lambda node: AnalysisRecipe._add_function_call(node, calls))
+        # the resulting lambda is invoked single threaded
+        # this kind of mechanism is mainly used to store results from multiple processors
+        # for refactoring operations this is not needed as a refactoring operation is single threaded
+        if calls:
+            return lambda: self._calls.extend(calls)
+
+    @after_step('store_function_call')
+    def just_show_the_method(self):
+        print('called after store_function_call')
+
+    @final_action()
+    def final_action(self):
+        print('Calls:')
+        for call in self._calls:
+            print('    '+call.callee + ' --  calls --> ' + call.calls)
+    @staticmethod
+    def _add_function_call(call: ASTNode, calls: list[Call]):
+        callee = call.get_ancestor('(?i)Function_?Decl')
+        if callee:
+            calls.append(Call(callee.get_name(), call.get_children()[0].get_name()))
+
+def batch_recipe_example():
+    print('example batch analysis using recipe:\n')
+    recipeAstProcessor = RecipeASTProcessor(AnalysisRecipe(), simple_codebase_provider, r'.*', in_memory=True)
+    recipeAstProcessor.run()
+
+if __name__ == "__main__":
+    # a list of example to show batch processing of a code base
+    batch_remove_unused_variable_once_example()
+    batch_repeat_example()
+    batch_recipe_example()