Cedric injest attempt

.gitattributes

@@ -0,0 +1 @@
+docker/**/*.sh text eol=lf

.github/workflows/linting-python-files.yml

@@ -0,0 +1,43 @@
+name: Python Linting
+
+on:
+  pull_request:
+    types: [closed]
+    paths:
+      - 'docker/lwua-ingest/**/*.py'
+      - 'docker/lwua-dereferencer/**/*.py'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10.6
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install black autopep8
+
+      - name: Run Black
+        run: |
+          black docker/lwua-ingest/
+          black docker/lwua-dereferencer/
+
+      - name: Run autopep8
+        run: |
+          autopep8 --in-place --aggressive --aggressive --max-line-length 79 --recursive docker/lwua-ingest/
+          autopep8 --in-place --aggressive --aggressive --max-line-length 79 --recursive docker/lwua-dereferencer/
+
+      - name: Commit and push changes
+        run: |
+          git config --global user.name 'cedricdcc'
+          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+          git add -A
+          git commit -m "Automated python code formatting" || exit 0
+          git push

.github/workflows/lwua-ingest-testing.yml

@@ -0,0 +1,33 @@
+name: Python Tests
+
+on:
+  push:
+    paths:
+      - 'docker/lwua-ingest/lwua-py/**/*.py'
+  pull_request:
+    paths:
+      - 'docker/lwua-ingest/lwua-py/**/*.py'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10.6
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          cd docker/lwua-ingest/lwua-py
+          poetry install
+
+      - name: Run pytest
+        run: |
+          cd docker/lwua-ingest/lwua-py
+          poetry run pytest ./tests/

README.md

@@ -9,7 +9,7 @@ Steps:
 2. to build the services simply run 
 
 ```bash
-.$ touch .env                         # make sure you have an .env file
+.$ cp dotenv-example .env             # make sure you have an .env file
 .$ cd docker && docker-compose build  # use docker to build the services 
 ```
 

data/project.ttl

@@ -13,7 +13,7 @@
     schema:creator <https://edmo.seadatanet.org/report/422> .
 
 <https://orcid.org/0000-0003-0663-5907> a schema:Person ;
-    schema:name "Laurian van Maldghem"^^xsd:string .
+    schema:name "Laurian van Maldeghem"^^xsd:string .
 
 <https://orcid.org/0000-0002-9648-6484> a schema:Person ;
     schema:name "Marc Portier"^^xsd:string .

docker/docker-compose.yml

@@ -1,6 +1,20 @@
 version: "3.7"
 # -----------------------------------------------------------------------
 services: 
+  graphdb:
+    build:
+      context: ./graphdb
+    image: lwua/lwua_graphdb
+    container_name: lwua_graphdb
+    ports:
+      - 7200:7200     # HTTP
+    volumes:
+      - ../data:/root/graphdb-import/data
+    # todo - think about volumes for persistence of data ?
+    labels:
+      be.vliz.container.project: "LWUA"
+      be.vliz.container.group: "services"
+
   jupyter:
     build:
       context: ./jupyter
@@ -12,21 +26,42 @@ services:
       - graphdb
     volumes:
       - "./jupyter/notebooks:/notebooks"
+    env_file:
+      - ../.env
+    environment:
+      - GDB_BASE=http://graphdb:7200/
     labels:
       be.vliz.container.project: "LWUA"
       be.vliz.container.group: "services"
-
-  graphdb:
-    # todo consider a local build that also initializes a repository for lwua inside this lwua_graphdb!
-    build:
-      context: ./graphdb
-    image: lwua/lwua_graphdb
-    container_name: lwua_graphdb
-    ports:
-      - 7200:7200     # HTTP
-    volumes:
-      - ../data:/root/graphdb-import/data
-    # todo - think about volumes for persistence of data
+  dereferencer:
+    build: 
+      context: ./lwua-dereferencer/ 
+      #args: 
+    image: lwua/dereferencer
+    container_name: dereferencer
+    depends_on:
+      - graphdb
+    volumes: 
+      - ../data:/data    # Store for any input data 
+      - ../configs:/configs   # Store for any input data
+      - ../logging:/logging   # Store for any input data 
+    env_file:
+      - ../.env
+    environment:
+      - GDB_BASE=http://graphdb:7200/
+      - INGEST_DATA_FOLDER=/data
+    # for test / dev -- no restart and single run
+    restart: "no"
+    command: run
+    # towards deploy -- make restart and keep service running -- consequence: use ctrl-c to stop
+    # restart: unless-stopped 
+    # command: start
+    links:
+      - graphdb
+    logging:
+      driver: json-file
+      options:
+        max-size: 10m 
     labels:
       be.vliz.container.project: "LWUA"
       be.vliz.container.group: "services"
@@ -37,11 +72,16 @@ services:
       #args: 
     image: lwua/lwua_ingest
     container_name: lwua_ingest
+    depends_on:
+      - graphdb
     volumes: 
       - ../data:/data    # Store for any input data 
       - ../logging:/logging   # Store for any input data 
     env_file:
       - ../.env
+    environment:
+      - GDB_BASE=http://graphdb:7200/
+      - INGEST_DATA_FOLDER=/data
     # for test / dev -- no restart and single run
     restart: "no"
     command: run

docker/graphdb/initdb/init_graphdb.sh

@@ -1,44 +1,4 @@
-#!/bin/bash -ex
-
-GDBPIDF="/tmp/init-graphdb-serv.pid"
-GDBOUTF="/tmp/init-graphdb-out.txt"
-
-start_graphdb(){
-    rm -f ${GDBPIDF}
-    graphdb -s -p ${GDBPIDF} >${GDBOUTF} 2>&1 &
-    sleep 1
-}
-
-wait_graphdb(){
-    count=0
-    while ! nc -z localhost 7200; do   
-        count=$((count+1))
-        if [ $count -gt 1000 ]; then
-            return
-        fi
-        # else 
-        sleep 0.1 # wait for 1/10 of the second before check again
-    done
-}
-
-stop_graphdb(){
-    kill -9 $(cat ${GDBPIDF})
-    sleep 1
-    rm -f ${GDBPIDF}
-    rm -f ${GDBOUTF}
-}
-
-createdb() {
-    curl -X POST http://localhost:7200/rest/repositories -H 'Content-Type: multipart/form-data' -F [email protected]
-}
-
-
-# one could do it like this 
-#start_graphdb
-#wait_graphdb
-#createdb
-#wait_configdb
-#stop_graphdb
+#!/bin/bash
 
 # but actually this just works too:
 REPODIR="/opt/graphdb/home/data/repositories/lwua23"

docker/jupyter/Dockerfile

@@ -1,6 +1,6 @@
 FROM jupyter/base-notebook
-
-RUN pip install pykg2tbl
-
 VOLUME /notebooks
-WORKDIR /notebooks
+WORKDIR /notebooks
+
+COPY ./requirements.txt /requirements.txt
+RUN pip install -r /requirements.txt