From d93e7d366e398e0ec87c36f9520f6b28df2ce99d Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Thu, 30 May 2024 16:33:39 +0200 Subject: [PATCH] Make ./macro a python module + cleanup --- .github/workflows/cicd_docker.yml | 26 ++++++ .github/workflows/cicd_test.yml | 6 +- .gitignore | 4 +- Dockerfile | 37 +++++---- README.md | 79 ++++++++++++++----- ci/build.sh | 12 +-- environment.yml | 2 +- environment_docker.yml | 4 +- macro/macro.py | 10 +-- pyproject.toml | 25 ++++++ {macro => scripts}/ex_filtering_points.py | 2 +- ...ex_filtering_points_with_add_dimensions.py | 2 +- test/utils.py | 3 +- 13 files changed, 158 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/cicd_docker.yml rename {macro => scripts}/ex_filtering_points.py (99%) rename {macro => scripts}/ex_filtering_points_with_add_dimensions.py (99%) diff --git a/.github/workflows/cicd_docker.yml b/.github/workflows/cicd_docker.yml new file mode 100644 index 0000000..a80bf05 --- /dev/null +++ b/.github/workflows/cicd_docker.yml @@ -0,0 +1,26 @@ +name: cicd_docker + +on: + # Run tests for non-draft pull request on main + pull_request: + branches: + - main + +env: + DOCKER_IMAGE_NAME: pdal_ign_plugin + +jobs: + + build_docker_image_and_run_tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout branch + uses: actions/checkout@v3 + + - name: Build docker image + run: docker build -t ${{ env.DOCKER_IMAGE_NAME }}:test . + + - name: Run tests in docker image + run: docker run ${{ env.DOCKER_IMAGE_NAME }}:test python -m pytest + diff --git a/.github/workflows/cicd_test.yml b/.github/workflows/cicd_test.yml index 62f2df9..4a92fc0 100644 --- a/.github/workflows/cicd_test.yml +++ b/.github/workflows/cicd_test.yml @@ -1,7 +1,7 @@ name: cicd_test on: - # Run each time some code are push on any branch + # Run each time some code is pushed on any branch push: branches: - '**' @@ -25,8 +25,8 @@ jobs: activate-environment: pdal_ign_plugin environment-file: ./environment.yml auto-activate-base: true - - - name: compil_plugins + + - name: compile_plugins run: source ./ci/build.sh - name: test diff --git a/.gitignore b/.gitignore index d8c4899..ec967c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ xcode +.vscode +build install -__pycache__ +*/__pycache__ test/__pycache_ test/.idea diff --git a/Dockerfile b/Dockerfile index 3a1cbfd..2b9dbb5 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,34 @@ FROM mambaorg/micromamba:bullseye-slim as build - + COPY environment_docker.yml /environment_docker.yml - -USER root -RUN micromamba env create -f /environment_docker.yml + +USER root +RUN micromamba env create -f /environment_docker.yml SHELL ["micromamba", "run", "-n", "pdal_ign_plugin", "/bin/bash", "-c"] RUN apt-get update && apt-get install --no-install-recommends -y cmake make build-essential g++ && rm -rf /var/lib/apt/lists/* - -COPY src src + +COPY src src COPY CMakeLists.txt CMakeLists.txt -COPY macro macro - -RUN cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release + +RUN cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release RUN make -j4 install - + FROM debian:bullseye-slim - + COPY --from=build /opt/conda/envs/pdal_ign_plugin /opt/conda/envs/pdal_ign_plugin -RUN mkdir -p /pdal_ign_plugin +RUN mkdir -p /pdal_ign_plugin COPY --from=build /tmp/install/lib /pdal_ign_plugin/install/lib -COPY --from=build /tmp/macro /macro -ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/ -ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/ +ENV PATH=$PATH:/opt/conda/envs/pdal_ign_plugin/bin/ +ENV PROJ_LIB=/opt/conda/envs/pdal_ign_plugin/share/proj/ ENV PDAL_DRIVER_PATH=/pdal_ign_plugin/install/lib +# Install python macro module +COPY macro /pdal_ign_plugin/macro +COPY pyproject.toml /pdal_ign_plugin/pyproject.toml +WORKDIR /pdal_ign_plugin +RUN pip install . + +# Add example scripts + test data (to be able to test inside the docker image) +COPY scripts /pdal_ign_plugin/scripts +COPY test /pdal_ign_plugin/test \ No newline at end of file diff --git a/README.md b/README.md index c98a2f5..20f320e 100755 --- a/README.md +++ b/README.md @@ -2,40 +2,50 @@ ## Compile -You need to have conda ! +You need to have conda! + +Create the ign_pdal_tools conda environment using the `environment.yml` file +to be able to run the compilation in this environment. ### linux/mac run ci/build.sh -### Windows +### Windows one day, maybe... ## Architecture of the code -The code is structured as : +The code is structured as: ``` ├── src -│ ├── plugins forlder -│ │ ├── plufinFilter.cpp -│ │ ├── plufinFilter.h +│ ├── plugin folder +│ │ ├── pluginFilter.cpp +│ │ ├── pluginFilter.h │ │ ├── CMakeLists.txt ├── doc │ ├── pluginFilter.md ├── ci +├── macro # Python module with ready-to-use filters combinations +│   ├── __init__.py +│   ├── macro.py +│   └── version.py +├── scripts +│   ├── *.py # Example scripts to use the plugin filters + the filters combinations contained in `macro` ├── test ├── CMakeLists.txt ├── environment*.yml -├── Dockerfile -├── .github +├── Dockerfile +├── pyproject.toml # Setup file to install the `macro` python module with pip +├── .github └── .gitignore ``` ## Run the tests -Each plugin should have his own test. To run test : +Each plugin should have his own test. To run all tests: ``` python -m pytest -s @@ -43,27 +53,28 @@ python -m pytest -s ## List of Filters -[grid decimation](./doc/grid_decimation.md) +[grid decimation](./doc/grid_decimation.md) [Deprecated: use the gridDecimation filter from the pdal repository] [radius assign](./doc/radius_assign.md) ## Adding a filter -In order to add a filter, you have to add a new folder in the src directory : +In order to add a filter, you have to add a new folder in the src directory : ``` ├── src │ ├── filter_my_new_PI │ │ ├── my_new_PI_Filter.cpp │ │ ├── my_new_PI_Filter.h -│ │ ├── CMakeLisits.txt +│ │ ├── CMakeLists.txt ``` The name of the folder informs of the plugIN nature (reader, writer, filter). -The code should respect the documentation purpose by pdal : [build a pdal plugin](https://pdal.io/en/2.6.0/development/plugins.html). Be careful to change if the plugIn is a reader, a writer or a filter. +The code should respect the documentation proposed by pdal: [build a pdal plugin](https://pdal.io/en/2.6.0/development/plugins.html). +Be careful to change if the plugIn is a reader, a writer or a filter. -The CMakeList should contains : +The CMakeList should contains: ``` file( GLOB_RECURSE GD_SRCS ${CMAKE_SOURCE_DIR} *) @@ -78,15 +89,45 @@ PDAL_CREATE_PLUGIN( install(TARGETS pdal_plugin_filter_my_new_PI) ``` -You should complet the principal CMakeList by adding the new plugIN : - +You should complete the main CMakeList by adding the new plugIN: + ``` add_subdirectory(src/filter_my_new_PI) ``` -Each plugIN has his own md file in the doc directory, structured as the [model](./doc/_doc_model_plugIN.md). +Each plugIN has his own md file in the doc directory, structured as the [model](./doc/_doc_model_plugIN.md). + +Don't forget to update [the list](#list-of-filters) with a link to the documentation. + +## `macro` python module usage + +The `macro` python module is installed in the project docker image so that it can be imported from anywhere in the +docker image. + + +### Syntax to use it in a python script -D'ont forget to update [the list](#list-of-filters) with a link with the documentation. +```python +from macro import macro +marco.my_macro(...) +``` + +See the `scripts` folder for example usages of this module. + +### Usage from outside the docker image: + +If you have a python script on your computer, you can mount its containing folder as a volume in order to +run it in the docker image. + +Example: + +```bash +docker run \ + -v /my/data/folder:/data \ + -v /my/output/folder:/output \ + -v /my/script/folder:/scripts \ + pdal_ign_plugin \ + python /scripts/my_script.py --input /data/my_data_file.las -o /output/my_output.las +``` - diff --git a/ci/build.sh b/ci/build.sh index b226faf..a870025 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,7 +1,9 @@ #!/bin/sh +set -e + FILE=~/anaconda3/etc/profile.d/conda.sh -if [ -e ~/anaconda3/etc/profile.d/conda.sh ] +if [ -e ~/anaconda3/etc/profile.d/conda.sh ] then source ~/anaconda3/etc/profile.d/conda.sh elif [ -e ~/miniconda3/etc/profile.d/conda.sh ] @@ -10,8 +12,8 @@ then elif [ -e /usr/share/miniconda/etc/profile.d/conda.sh ] then source /usr/share/miniconda/etc/profile.d/conda.sh -elif [ -e ~/miniforge3/etc/profile.d/conda.sh ] -then +elif [ -e ~/miniforge3/etc/profile.d/conda.sh ] +then source ~/miniforge3/etc/profile.d/conda.sh elif [[ -z "${CONDASH}" ]]; then echo ERROR: Failed to load conda.sh : ~/anaconda3/etc/profile.d/conda.sh or ~/miniforge3/etc/profile.d/conda.sh or env CONDASH @@ -28,10 +30,10 @@ echo conda is $CONDA_PREFIX mkdir build cd build -cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release ../ +cmake -G"Unix Makefiles" -DCONDA_PREFIX=$CONDA_PREFIX -DCMAKE_BUILD_TYPE=Release ../ make install conda deactivate cd .. -rm -rf build \ No newline at end of file +rm -rf build \ No newline at end of file diff --git a/environment.yml b/environment.yml index 695737c..9182325 100755 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - isort # import sorting - flake8 # code analysis - pytest -# --------- pip & pip librairies --------- # +# --------- pip & pip libraries --------- # - pip - pip: - ign-pdal-tools diff --git a/environment_docker.yml b/environment_docker.yml index 7e23ef6..9232812 100755 --- a/environment_docker.yml +++ b/environment_docker.yml @@ -6,8 +6,8 @@ dependencies: - pdal - python-pdal - gdal -# --------- pip & pip librairies --------- # + - pytest + # --------- pip & pip libraries --------- # - pip - pip: - ign-pdal-tools - diff --git a/macro/macro.py b/macro/macro.py index 89f66d4..093effd 100755 --- a/macro/macro.py +++ b/macro/macro.py @@ -10,13 +10,13 @@ def add_radius_assign(pipeline, radius, search_3d, condition_src, condition_ref, search points from "condition_src" closed from "condition_ref", and reassign them to "condition_out" This combination is equivalent to the CloseBy macro of TerraScan radius : the search distance - search_3d : the distance reseach is in 3d if True + search_3d : the distance research is in 3d if True condition_src, condition_ref, condition_out : a pdal condition as "Classification==2" """ pipeline |= pdal.Filter.ferry(dimensions="=>REF_DOMAIN, =>SRC_DOMAIN, =>radius_search") pipeline |= pdal.Filter.assign( value=[ - "SRS_DOMAIN = 0", + "SRC_DOMAIN = 0", f"SRC_DOMAIN = 1 WHERE {condition_src}", "REF_DOMAIN = 0", f"REF_DOMAIN = 1 WHERE {condition_ref}", @@ -34,14 +34,14 @@ def add_radius_assign(pipeline, radius, search_3d, condition_src, condition_ref, return pipeline -def classify_hgt_ground(pipeline, hmin, hmax, condition, condition_out): +def classify_hgt_ground(pipeline, h_min, h_max, condition, condition_out): """ - reassign points from "condition" between "hmin" and "hmax" of the ground to "condition_out" + reassign points from "condition" between "h_min" and "h_max" of the ground to "condition_out" This combination is equivalent to the ClassifyHgtGrd macro of TerraScan condition, condition_out : a pdal condition as "Classification==2" """ pipeline |= pdal.Filter.hag_delaunay(allow_extrapolation=True) - condition_h = f"HeightAboveGround>{hmin} && HeightAboveGround<={hmax}" + condition_h = f"HeightAboveGround>{h_min} && HeightAboveGround<={h_max}" condition_h += " && " + condition pipeline |= pdal.Filter.assign(value=condition_out, where=condition_h) return pipeline diff --git a/pyproject.toml b/pyproject.toml index 57a5583..165ffe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,27 @@ +[project] +name = "pdal_ign_macro" +dynamic = ["version"] +dependencies = [] + +[tool.setuptools.dynamic] +version = { attr = "macro.version.__version__" } + +[tool.setuptools] +packages = ["macro"] + [tool.black] line-length = 99 +include = '\.pyi?$' +exclude = ''' +/( + \.toml + |\.sh + |\.git + |\.ini + |\.bat + | data +)/ +''' + +[tool.isort] +profile = "black" diff --git a/macro/ex_filtering_points.py b/scripts/ex_filtering_points.py similarity index 99% rename from macro/ex_filtering_points.py rename to scripts/ex_filtering_points.py index a4c3f2f..37987c9 100755 --- a/macro/ex_filtering_points.py +++ b/scripts/ex_filtering_points.py @@ -2,7 +2,7 @@ import pdal -import macro +from macro import macro """ This tool shows how to use functions of macro in a pdal pipeline diff --git a/macro/ex_filtering_points_with_add_dimensions.py b/scripts/ex_filtering_points_with_add_dimensions.py similarity index 99% rename from macro/ex_filtering_points_with_add_dimensions.py rename to scripts/ex_filtering_points_with_add_dimensions.py index cf717e8..01ca09b 100755 --- a/macro/ex_filtering_points_with_add_dimensions.py +++ b/scripts/ex_filtering_points_with_add_dimensions.py @@ -2,7 +2,7 @@ import pdal -import macro +from macro import macro """ This tool shows how to use functions of macro in a pdal pipeline diff --git a/test/utils.py b/test/utils.py index c84fbe2..43200a7 100755 --- a/test/utils.py +++ b/test/utils.py @@ -5,5 +5,6 @@ def pdal_has_plugin(name_filter): print("init pdal driver : ", os.environ["PDAL_DRIVER_PATH"]) result = subprocess.run(["pdal", "--drivers"], stdout=subprocess.PIPE) + print(result.stdout.decode("utf-8")) if name_filter not in result.stdout.decode("utf-8"): - raise ValueError("le script " + name_filter + " n'est pas visible") + raise ValueError("script " + name_filter + " not found by `pdal --drivers`.")