Make ./macro a python module + cleanup

.github/workflows/cicd_docker.yml

@@ -0,0 +1,26 @@
+name: cicd_docker
+
+on:
+  # Run tests for non-draft pull request on main
+  pull_request:
+    branches:
+      - main
+
+env:
+  DOCKER_IMAGE_NAME: pdal_ign_plugin
+
+jobs:
+
+  build_docker_image_and_run_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout branch
+      uses: actions/checkout@v3
+
+    - name: Build docker image
+      run: docker build -t ${{ env.DOCKER_IMAGE_NAME }}:test .
+
+    - name: Run tests in docker image
+      run: docker run ${{ env.DOCKER_IMAGE_NAME }}:test python -m pytest
+

.github/workflows/cicd_test.yml

@@ -1,7 +1,7 @@
 name: cicd_test
 
 on:
-  # Run each time some code are push on any branch
+  # Run each time some code is pushed on any branch
   push:
     branches:
       - '**'
@@ -25,8 +25,8 @@ jobs:
         activate-environment: pdal_ign_plugin
         environment-file: ./environment.yml
         auto-activate-base: true
-    
-    - name: compil_plugins
+
+    - name: compile_plugins
       run: source ./ci/build.sh
 
     - name: test

.gitignore

@@ -1,5 +1,7 @@
 xcode
+.vscode
+build
 install
-__pycache__
+*/__pycache__
 test/__pycache_
 test/.idea

Dockerfile

@@ -1,27 +1,34 @@
 FROM mambaorg/micromamba:bullseye-slim as build
-
+
 COPY environment_docker.yml /environment_docker.yml
- 
-USER root
-RUN micromamba env create -f /environment_docker.yml
+
+USER root
+RUN micromamba env create -f /environment_docker.yml
 SHELL ["micromamba", "run", "-n", "pdal_ign_plugin", "/bin/bash", "-c"]
 RUN apt-get update && apt-get install --no-install-recommends -y cmake make build-essential g++ && rm -rf /var/lib/apt/lists/*
- 
-COPY src src
+
+COPY src src
 COPY CMakeLists.txt CMakeLists.txt
-COPY macro macro
-
-RUN cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release
+
+RUN cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release
 RUN make -j4 install
-  
+
 FROM debian:bullseye-slim
- 
+
 COPY --from=build /opt/conda/envs/pdal_ign_plugin /opt/conda/envs/pdal_ign_plugin
-RUN mkdir -p /pdal_ign_plugin
+RUN mkdir -p /pdal_ign_plugin
 COPY --from=build /tmp/install/lib /pdal_ign_plugin/install/lib
-COPY --from=build /tmp/macro /macro 
 
-ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/
-ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/
+ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/
+ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/
 ENV PDAL_DRIVER_PATH=/pdal_ign_plugin/install/lib
 
+# Install python macro module
+COPY macro /pdal_ign_plugin/macro
+COPY pyproject.toml /pdal_ign_plugin/pyproject.toml
+WORKDIR /pdal_ign_plugin
+RUN pip install .
+
+# Add example scripts + test data (to be able to test inside the docker image)
+COPY scripts /pdal_ign_plugin/scripts
+COPY test /pdal_ign_plugin/test

README.md

@@ -2,68 +2,79 @@
 
 ## Compile
 
-You need to have conda ! 
+You need to have conda!
+
+Create the ign_pdal_tools conda environment using the `environment.yml` file
+to be able to run the compilation in this environment.
 
 ### linux/mac
 
 run ci/build.sh
 
-### Windows 
+### Windows
 
 one day, maybe...
 
 ## Architecture of the code
 
-The code is structured as :
+The code is structured as:
 
 ```
 ├── src
-│   ├── plugins forlder
-│   │   ├── plufinFilter.cpp
-│   │   ├── plufinFilter.h
+│   ├── plugin folder
+│   │   ├── pluginFilter.cpp
+│   │   ├── pluginFilter.h
 │   │   ├── CMakeLists.txt
 ├── doc
 │   ├── pluginFilter.md
 ├── ci
+├── macro  # Python module with ready-to-use filters combinations
+│   ├── __init__.py
+│   ├── macro.py
+│   └── version.py
+├── scripts
+│   ├── *.py  # Example scripts to use the plugin filters + the filters combinations contained in `macro`
 ├── test
 ├── CMakeLists.txt
 ├── environment*.yml
-├── Dockerfile 
-├── .github 
+├── Dockerfile
+├── pyproject.toml  # Setup file to install the `macro` python module with pip
+├── .github
 └── .gitignore
 ```
 
 ## Run the tests
 
-Each plugin should have his own test. To run test :
+Each plugin should have his own test. To run all tests:
 
 ```
 python -m pytest -s
 ```
 
 ## List of Filters
 
-[grid decimation](./doc/grid_decimation.md)
+[grid decimation](./doc/grid_decimation.md) [Deprecated: use the gridDecimation filter from the pdal repository]
 
 [radius assign](./doc/radius_assign.md)
 
 ## Adding a filter
 
-In order to add a filter, you have to add a new folder in the src directory : 
+In order to add a filter, you have to add a new folder in the src directory :
 
 ```
 ├── src
 │   ├── filter_my_new_PI
 │   │   ├── my_new_PI_Filter.cpp
 │   │   ├── my_new_PI_Filter.h
-│   │   ├── CMakeLisits.txt
+│   │   ├── CMakeLists.txt
 ```
 
 The name of the folder informs of the plugIN nature (reader, writer, filter).
 
-The code should respect the documentation purpose by pdal : [build a pdal plugin](https://pdal.io/en/2.6.0/development/plugins.html). Be careful to change if the plugIn is a reader, a writer or a filter. 
+The code should respect the documentation proposed by pdal: [build a pdal plugin](https://pdal.io/en/2.6.0/development/plugins.html).
+Be careful to change if the plugIn is a reader, a writer or a filter.
 
-The CMakeList should contains : 
+The CMakeList should contains:
 
 ```
 file( GLOB_RECURSE GD_SRCS ${CMAKE_SOURCE_DIR} *)
@@ -78,15 +89,45 @@ PDAL_CREATE_PLUGIN(
 install(TARGETS pdal_plugin_filter_my_new_PI)
 ```
 
-You should complet the principal CMakeList by adding the new plugIN :
- 
+You should complete the main CMakeList by adding the new plugIN:
+
 ```
 add_subdirectory(src/filter_my_new_PI)
 ```
 
-Each plugIN has his own md file in the doc directory, structured as the [model](./doc/_doc_model_plugIN.md). 
+Each plugIN has his own md file in the doc directory, structured as the [model](./doc/_doc_model_plugIN.md).
+
+Don't forget to update [the list](#list-of-filters) with a link to the documentation.
+
+## `macro` python module usage
+
+The `macro` python module is installed in the project docker image so that it can be imported from anywhere in the
+docker image.
+
+
+### Syntax to use it in a python script
 
-D'ont forget to update [the list](#list-of-filters) with a link with the documentation.
+```python
+from macro import macro
 
+marco.my_macro(...)
+```
+
+See the `scripts` folder for example usages of this module.
+
+### Usage from outside the docker image:
+
+If you have a python script on your computer, you can mount its containing folder as a volume in order to
+run it in the docker image.
+
+Example:
+
+```bash
+docker run \
+    -v /my/data/folder:/data \
+    -v  /my/output/folder:/output \
+    -v /my/script/folder:/scripts \
+    pdal_ign_plugin \
+    python /scripts/my_script.py --input /data/my_data_file.las -o /output/my_output.las
+```
 
-

ci/build.sh

@@ -1,7 +1,9 @@
 #!/bin/sh
 
+set -e
+
 FILE=~/anaconda3/etc/profile.d/conda.sh
-if [ -e ~/anaconda3/etc/profile.d/conda.sh ] 
+if [ -e ~/anaconda3/etc/profile.d/conda.sh ]
 then
     source ~/anaconda3/etc/profile.d/conda.sh
 elif [ -e ~/miniconda3/etc/profile.d/conda.sh ]
@@ -10,8 +12,8 @@ then
 elif [ -e /usr/share/miniconda/etc/profile.d/conda.sh ]
 then
     source /usr/share/miniconda/etc/profile.d/conda.sh
-elif [ -e ~/miniforge3/etc/profile.d/conda.sh ] 
-then    
+elif [ -e ~/miniforge3/etc/profile.d/conda.sh ]
+then
      source ~/miniforge3/etc/profile.d/conda.sh
 elif [[ -z "${CONDASH}" ]]; then
     echo ERROR: Failed to load conda.sh : ~/anaconda3/etc/profile.d/conda.sh or ~/miniforge3/etc/profile.d/conda.sh or env CONDASH
@@ -28,10 +30,10 @@ echo conda is $CONDA_PREFIX
 
 mkdir build
 cd build
-cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release ../ 
+cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release ../
 make install
 
 conda deactivate
 
 cd ..
-rm -rf build 
+rm -rf build

environment.yml

@@ -13,7 +13,7 @@ dependencies:
   - isort # import sorting
   - flake8 # code analysis
   - pytest
-# --------- pip & pip librairies --------- #
+# --------- pip & pip libraries --------- #
   - pip
   - pip:
     - ign-pdal-tools

environment_docker.yml

@@ -6,8 +6,8 @@ dependencies:
   - pdal
   - python-pdal
   - gdal
-# --------- pip & pip librairies --------- #
+  - pytest
+  # --------- pip & pip libraries --------- #
   - pip
   - pip:
     - ign-pdal-tools
-

macro/macro.py

@@ -10,13 +10,13 @@ def add_radius_assign(pipeline, radius, search_3d, condition_src, condition_ref,
     search points from "condition_src" closed from "condition_ref", and reassign them to "condition_out"
     This combination is equivalent to the CloseBy macro of TerraScan
     radius : the search distance
-    search_3d : the distance reseach is in 3d if True
+    search_3d : the distance research is in 3d if True
     condition_src, condition_ref, condition_out : a pdal condition as "Classification==2"
     """
     pipeline |= pdal.Filter.ferry(dimensions="=>REF_DOMAIN, =>SRC_DOMAIN, =>radius_search")
     pipeline |= pdal.Filter.assign(
         value=[
-            "SRS_DOMAIN = 0",
+            "SRC_DOMAIN = 0",
             f"SRC_DOMAIN = 1 WHERE {condition_src}",
             "REF_DOMAIN = 0",
             f"REF_DOMAIN = 1 WHERE {condition_ref}",
@@ -34,14 +34,14 @@ def add_radius_assign(pipeline, radius, search_3d, condition_src, condition_ref,
     return pipeline
 
 
-def classify_hgt_ground(pipeline, hmin, hmax, condition, condition_out):
+def classify_hgt_ground(pipeline, h_min, h_max, condition, condition_out):
     """
-    reassign points from "condition" between "hmin" and "hmax" of the ground to "condition_out"
+    reassign points from "condition" between "h_min" and "h_max" of the ground to "condition_out"
     This combination is equivalent to the ClassifyHgtGrd macro of TerraScan
     condition, condition_out : a pdal condition as "Classification==2"
     """
     pipeline |= pdal.Filter.hag_delaunay(allow_extrapolation=True)
-    condition_h = f"HeightAboveGround>{hmin} && HeightAboveGround<={hmax}"
+    condition_h = f"HeightAboveGround>{h_min} && HeightAboveGround<={h_max}"
     condition_h += " && " + condition
     pipeline |= pdal.Filter.assign(value=condition_out, where=condition_h)
     return pipeline

pyproject.toml

@@ -1,2 +1,27 @@
+[project]
+name = "pdal_ign_macro"
+dynamic = ["version"]
+dependencies = []
+
+[tool.setuptools.dynamic]
+version = { attr = "macro.version.__version__" }
+
+[tool.setuptools]
+packages = ["macro"]
+
 [tool.black]
 line-length = 99
+include = '\.pyi?$'
+exclude = '''
+/(
+  \.toml
+  |\.sh
+  |\.git
+  |\.ini
+  |\.bat
+  | data
+)/
+'''
+
+[tool.isort]
+profile = "black"

macro/ex_filtering_points.py → scripts/ex_filtering_points.py

@@ -2,7 +2,7 @@
 
 import pdal
 
-import macro
+from macro import macro
 
 """
 This tool shows how to use functions of macro in a pdal pipeline

macro/ex_filtering_points_with_add_dimensions.py → scripts/ex_filtering_points_with_add_dimensions.py

@@ -2,7 +2,7 @@
 
 import pdal
 
-import macro
+from macro import macro
 
 """
 This tool shows how to use functions of macro in a pdal pipeline

test/utils.py

@@ -5,5 +5,6 @@
 def pdal_has_plugin(name_filter):
     print("init pdal driver : ", os.environ["PDAL_DRIVER_PATH"])
     result = subprocess.run(["pdal", "--drivers"], stdout=subprocess.PIPE)
+    print(result.stdout.decode("utf-8"))
     if name_filter not in result.stdout.decode("utf-8"):
-        raise ValueError("le script " + name_filter + " n'est pas visible")
+        raise ValueError("script " + name_filter + " not found by `pdal --drivers`.")