Merge pull request #1 from mundialis/start_with_parallel_processing

Start with parallel processing

.flake8

@@ -7,4 +7,5 @@ exclude = .git,.pycache,build,.eggs
 
 per-file-ignores =
     ./src/actinia_parallel_plugin/wsgi.py: F401
-    ./tests/test_resource_base.py: F401 
+    ./tests/test_resource_base.py: F401
+    ./src/actinia_parallel_plugin/core/persistent_processing.py: E501

.github/workflows/test.yml

@@ -9,6 +9,29 @@ on:
 
 jobs:
 
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Start containers
+      run: docker-compose -f "docker/docker-compose-test.yml" up -d --build
+
+    - name: List running docker
+      run: docker ps
+
+    - name: Docker logs actinia
+      run: docker logs docker_actinia-test_1
+
+    - name: Docker logs postgis
+      run: docker logs docker_postgis-test_1
+
+    - name: Run integration test
+      run: docker exec -t docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh
+
+    - name: Stop containers
+      run: docker-compose -f "docker/docker-compose-test.yml" down
+
   unittests:
     runs-on: ubuntu-latest
     steps:
@@ -31,26 +54,3 @@ jobs:
         file: docker/actinia-parallel-plugin-test/Dockerfile
         no-cache: true
         # pull: true
-
-  integration-tests:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      # with:
-      #   path: "."
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
-    - name: Replace run integration test command
-      run: |
-        sed -i "s+# RUN make test+RUN make integrationtest+g" docker/actinia-parallel-plugin-test/Dockerfile
-    - name: Integration tests of actinia-parallel-plugin
-      id: docker_build
-      uses: docker/build-push-action@v2
-      with:
-        push: false
-        tags: actinia-parallel-plugin-test:alpine
-        context: .
-        file: docker/actinia-parallel-plugin-test/Dockerfile
-        no-cache: true
-        # pull: true

.gitignore

@@ -13,6 +13,7 @@ __pycache__/*
 .cache/*
 .*.swp
 */.ipynb_checkpoints/*
+.coverage.*
 
 # Project files
 .ropeproject

Makefile

@@ -19,13 +19,13 @@ dist:
 	python3 setup.py dist
 
 test:
-	./tests_with_redis.sh
+	python3 setup.py test
 
 unittest:
 	python3 setup.py test --addopts "-m unittest"
 
 devtest:
-	./tests_with_redis.sh dev
+	python3 setup.py test --addopts "-m 'dev'"
 
 integrationtest:
-	./tests_with_redis.sh integrationtest
+	python3 setup.py test --addopts "-m 'integrationtest'"

README.md

@@ -1,6 +1,6 @@
 # actinia-parallel-plugin
 
-This is an example plugin for [actinia-core](https://github.com/mundialis/actinia_core) which adds a "Hello World" endpoint to actinia-core.
+This is the actinia-parallel-plugin for [actinia-core](https://github.com/mundialis/actinia_core) which adds parallel processing endpoints to actinia.
 
 You can run actinia-parallel-plugin as an actinia-core plugin.
 
@@ -20,14 +20,6 @@ docker network prune
 docker-compose -f docker/docker-compose.yml up -d
 ```
 
-### Requesting helloworld endpoint
-You can test the plugin and request the `/helloworld` endpoint, e.g. with:
-```
-curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/processing_parallel | jq
-
-curl -u actinia-gdi:actinia-gdi -H 'accept: application/json' -H 'Content-Type: application/json' -X POST http://localhost:8088/api/v3/processing_parallel -d '{"name": "test"}' | jq
-```
-
 ## DEV setup
 For a DEV setup you can use the docker/docker-compose.yml:
 ```
@@ -38,6 +30,15 @@ docker-compose -f docker/docker-compose.yml run --rm --service-ports --entrypoin
 (cd /src/actinia-parallel-plugin && python3 setup.py install)
 # start actinia-core with your plugin
 gunicorn -b 0.0.0.0:8088 -w 1 --access-logfile=- -k gthread actinia_core.main:flask_app
+
+# or for debugging in one line with reset
+reset && (cd /src/actinia-parallel-plugin && python3 setup.py install) && gunicorn -b 0.0.0.0:8088 -w 3 --access-logfile=- -k gthread actinia_core.main:flask_app
+```
+
+### PostGIS
+Connect to PostGIS DB from actinia-core docker container:
+```
+psql -U actinia -h postgis -d gis
 ```
 
 ### Hints
@@ -60,10 +61,11 @@ rm -rf /usr/lib/python3.8/site-packages/actinia_parallel_plugin.wsgi-*.egg
 You can run the tests in the actinia test docker:
 
 ```
-docker build -f docker/actinia-parallel-plugin-test/Dockerfile -t actinia-parallel-plugin-test .
-docker run -it actinia-parallel-plugin-test -i
+docker-compose -f docker/docker-compose-test.yml build
+docker-compose -f docker/docker-compose-test.yml up -d
 
-cd /src/actinia-parallel-plugin/
+# exec docker and run tests manually
+docker exec -it docker_actinia-test_1 sh
 
 # run all tests
 make test
@@ -73,8 +75,47 @@ make unittest
 # run only integrationtests
 make integrationtest
 
-# run only tests which are marked for development with the decorator '@pytest.mark.dev'
-make devtest
+# or run tests outside of docker container
+docker exec -it docker_actinia-test_1 sh /usr/bin/run_integration_tests.sh
+docker exec -it docker_actinia-test_1 sh /usr/bin/run_unittests.sh
+
+docker-compose -f docker/docker-compose-test.yml down
+```
+
+You can also run the tests in the GHA workflows locally via [act](https://github.com/nektos/act).
+To run docker-compose inside a workflow [act_base](https://github.com/lucasctrl/act_base) can be used.
+With these you can run the following to run the tests:
+```
+# list all workflows
+act -l
+
+# run workflow
+act -j integration-tests -P ubuntu-latest=lucasalt/act_base:latest
+act -j unittests -P ubuntu-latest=lucasalt/act_base:latest
+```
+
+
+## Examples
+
+### Requesting batch job and job endpoints
+```
+# request batch job
+curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/resources/actinia-gdi/batches/1 | jq
+# request job
+curl -u actinia-gdi:actinia-gdi -X GET http://localhost:8088/api/v3/resources/actinia-gdi/batches/1/jobs/1 | jq
+```
+
+### Start parallel batch job
+#### Ephemeral processing
+You can start a parallel **ephemeral** batch job via:
+```
+# parallel ephemeral processing
+curl -u actinia-gdi:actinia-gdi -X POST -H 'Content-Type: application/json' -d @test_postbodies/parallel_ephemeral_processing.json http://localhost:8088/api/v3/locations/nc_spm_08_grass7_root/processing_parallel | jq
 ```
+Attention:
+* The individual process chains must be "independent" of each other, since
+  createBatch is designed as an ephemeral process.
 
-##
+TODOs:
+* exporter in PC
+* using stdout/export in PC of next block

docker/actinia-parallel-plugin-test/Dockerfile

@@ -12,17 +12,14 @@ ENV DEFAULT_CONFIG_PATH /etc/default/actinia-parallel-plugin-test
 RUN apk add redis
 RUN pip3 install iniconfig colorlog pwgen
 
-# COPY docker/actinia-parallel-plugin-test/start.sh /src/start.sh
-
 ENTRYPOINT ["/bin/sh"]
 CMD ["/src/start.sh"]
 
-# # add data for tests
-# RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_micro.zip && \
-#   unzip nc_spm_08_micro.zip && \
-#   rm -f nc_spm_08_micro.zip && \
-#   mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08
-# RUN grass -e -c 'EPSG:4326' /actinia_core/grassdb/latlong_wgs84
+# add data for tests
+RUN wget --quiet https://grass.osgeo.org/sampledata/north_carolina/nc_spm_08_micro.zip && \
+  unzip nc_spm_08_micro.zip && \
+  rm -f nc_spm_08_micro.zip && \
+  mv nc_spm_08_micro /actinia_core/grassdb/nc_spm_08
 
 # copy needed files and configs for test
 COPY docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg /etc/default/actinia
@@ -31,7 +28,13 @@ COPY . /src/actinia-parallel-plugin/
 
 WORKDIR /src/actinia-parallel-plugin/
 
-RUN chmod a+x tests_with_redis.sh
 RUN make install
 
+ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0
+
+COPY docker/actinia-parallel-plugin-test/run_integration_tests.sh /usr/bin/run_integration_tests.sh
+RUN chmod a+x /usr/bin/run_integration_tests.sh
+COPY docker/actinia-parallel-plugin-test/run_unittests.sh /usr/bin/run_unittests.sh
+RUN chmod a+x /usr/bin/run_unittests.sh
+
 # RUN make test

docker/actinia-parallel-plugin-test/actinia-parallel-plugin-test.cfg

@@ -3,7 +3,7 @@ grass_database = /actinia_core/grassdb
 grass_user_database = /actinia_core/userdata
 grass_tmp_database = /actinia_core/workspace/temp_db
 grass_resource_dir = /actinia_core/resources
-grass_gis_base = /usr/local/grass80
+grass_gis_base = /usr/local/grass
 grass_gis_start_script = /usr/local/bin/grass
 grass_addon_path = /root/.grass8/addons/
 
@@ -12,7 +12,8 @@ plugins = ["actinia_parallel_plugin"]
 force_https_urls = True
 
 [REDIS]
-redis_server_url = localhost
+redis_server_url = redis-test
+redis_server_pw = pass
 redis_server_port = 6379
 worker_queue_name = actinia_job
 worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log
@@ -22,3 +23,12 @@ tmp_workdir = /actinia_core/workspace/tmp
 download_cache = /actinia_core/workspace/download_cache
 secret_key = token_signing_key_changeme
 save_interim_results = False
+
+[JOBTABLE]
+host = postgis-test
+port = 5432
+database = gis
+user = actinia
+schema = actinia
+table = tab_jobs
+id_field = id

docker/actinia-parallel-plugin-test/run_integration_tests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python3 setup.py test --addopts "-m 'integrationtest'"

docker/actinia-parallel-plugin-test/run_unittests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python3 setup.py test --addopts "-m 'unittest'"

docker/actinia.cfg

@@ -3,7 +3,7 @@ grass_database = /actinia_core/grassdb
 grass_user_database = /actinia_core/userdata
 grass_tmp_database = /actinia_core/workspace/temp_db
 grass_resource_dir = /actinia_core/resources
-grass_gis_base = /usr/local/grass80
+grass_gis_base = /usr/local/grass
 grass_gis_start_script = /usr/local/bin/grass
 grass_addon_path = /root/.grass8/addons/
 
@@ -26,3 +26,12 @@ tmp_workdir = /actinia_core/workspace/tmp
 download_cache = /actinia_core/workspace/download_cache
 secret_key = token_signing_key_changeme
 save_interim_results = True
+
+[JOBTABLE]
+host = postgis
+port = 5432
+database = gis
+user = actinia
+schema = actinia
+table = tab_jobs
+id_field = id

docker/docker-compose-test.yml

@@ -0,0 +1,58 @@
+version: "3"
+services:
+
+  actinia-test:
+    build:
+      context: ..
+      dockerfile: docker/actinia-parallel-plugin-test/Dockerfile
+    volumes:
+      - ..:/src/actinia-parallel-plugin/.
+    ports:
+      - "8088:8088"
+    environment:
+      - JOBTABLE_PW=actinia
+    depends_on:
+      - redis-test
+      - postgis-test
+    cap_add:
+      - SYS_PTRACE
+    networks:
+    -   actinia-test
+
+  redis-test:
+    image: redis:5.0.4-alpine
+    volumes:
+      - ./redis_data:/data
+    environment:
+      - REDIS_PASS_FILE=/data/config/.redis
+    command: [
+      "sh", "-c",
+      '
+      docker-entrypoint.sh
+      "/data/config/redis.conf"
+      --requirepass "$$(cat $$REDIS_PASS_FILE)"
+      '
+    ]
+    ports:
+        - "6379:6379"
+    networks:
+    -   actinia-test
+
+  postgis-test:
+    image: postgis/postgis:14-3.2-alpine
+    # network_mode: host
+    environment:
+      POSTGRES_USER: actinia
+      POSTGRES_PASSWORD: actinia
+    volumes:
+      - ./postgresql_init_data:/docker-entrypoint-initdb.d
+      # - ./postgresql_data:/var/lib/postgresql/data:Z
+    networks:
+    -   actinia-test
+
+
+networks:
+  actinia-test:
+    ipam:
+      config:
+        - subnet: 172.18.0.0/26

docker/docker-compose.yml

@@ -7,10 +7,14 @@ services:
       dockerfile: docker/Dockerfile
     volumes:
       - ..:/src/actinia-parallel-plugin/.
+      # - /home/.../grassdata/:/actinia_core/grassdb/
     ports:
       - "8088:8088"
+    environment:
+      - JOBTABLE_PW=actinia
     depends_on:
       - redis
+      - postgis
     cap_add:
       - SYS_PTRACE
     networks:
@@ -35,6 +39,19 @@ services:
     networks:
     -   actinia
 
+  postgis:
+    image: postgis/postgis:14-3.2-alpine
+    # network_mode: host
+    environment:
+      POSTGRES_USER: actinia
+      POSTGRES_PASSWORD: actinia
+    volumes:
+      - ./postgresql_init_data:/docker-entrypoint-initdb.d
+      - ./postgresql_data:/var/lib/postgresql/data:Z
+    networks:
+    -   actinia
+
+
 networks:
   actinia:
     ipam: