Dereferencer

.gitattributes

@@ -0,0 +1 @@
+docker/**/*.sh text eol=lf

.github/workflows/linting-python-files.yml

@@ -0,0 +1,46 @@
+name: Python Linting
+
+on:
+  push:
+    paths:
+      - 'docker/lwua-ingest/**/*.py'
+      - 'docker/lwua-dereferencer/**/*.py'
+  pull_request:
+    paths:
+      - 'docker/lwua-ingest/**/*.py'
+      - 'docker/lwua-dereferencer/**/*.py'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10.6
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install black autopep8
+
+      - name: Run Black
+        run: |
+          black docker/lwua-ingest/
+          black docker/lwua-dereferencer/
+
+      - name: Run autopep8
+        run: |
+          autopep8 --in-place --aggressive --aggressive --max-line-length 79 --recursive docker/lwua-ingest/
+          autopep8 --in-place --aggressive --aggressive --max-line-length 79 --recursive docker/lwua-dereferencer/
+
+      - name: Commit and push changes
+        run: |
+          git config --global user.name 'cedricdcc'
+          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+          git add -A
+          git commit -m "Automated python code formatting" || exit 0
+          git push

.github/workflows/lwua-ingest-testing.yml

@@ -0,0 +1,33 @@
+name: Python Tests
+
+on:
+  push:
+    paths:
+      - 'docker/lwua-ingest/lwua-py/**/*.py'
+  pull_request:
+    paths:
+      - 'docker/lwua-ingest/lwua-py/**/*.py'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10.6
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          cd docker/lwua-ingest/lwua-py
+          poetry install
+
+      - name: Run pytest
+        run: |
+          cd docker/lwua-ingest/lwua-py
+          poetry run pytest ./tests/

.gitignore

@@ -1,3 +1,4 @@
 .env
-data/
-__pycache__
+*.log
+__pycache__
+.ipynb_checkpoints/

README.md

@@ -3,27 +3,57 @@
 ## using this project
 
 Steps:
+
 1. retrieve the source code from github
-2. to start up the services simply run 
+
+2. to build the services simply run 
+
+```bash
+.$ cp dotenv-example .env             # make sure you have an .env file
+.$ cd docker && docker-compose build  # use docker to build the services 
+```
+
+3. to start up the services simply run 
+
+```bash
+.$ cd docker && docker-compose up     # use docker to run the services 
+```
+
+4. open the jupyter notebook
+
+```bash
+.$ xdg-open $(docker/jupyter_url.sh)  # this gets the url for the service and opens a browser to it
+```
+
+5. open the graphdb browser ui
 
 ```bash
-.$ touch .env # make sure you have an .env file
-.$ cd docker
-./docker$ docker-compose up
+.$ xdg-open http://localhost:7200     # opens the web ui in a browser
 ```
 
+6. run a test-ingest
+
+This introduces forcefully at least the data/project.ttl into the triple store
+This should not be needed when the ingest runs automatically
+
+```bash
+.$ docker exec -it lwua_ingest /bin/bash            # interactively gets you into the ingest env
+root@f226b253fbd4:/lwua-py# python -m lwua.ingest   # run the ingest 
+```
 
 
-## general plan 
+## general plan ahead -- details to be converted into github issues
 
 big idea is to have a central triples store for the user analysis approach
 this to decouple the ingest (retrieval and semantic mapping) from the different sources from the reporting (which should be based on the assembled knowledge graph)
 
-### for the ingest we will need a mix if strategies
-* actually getting data by using dumps our webservices
-* additionally uplifting thos to triples (via pysubyt)
+### for the ingest we will need a mix of strategies
+* actually getting raw (non linkd) data by using dumps from webservices
+* additionally uplifting those to triples (via pysubyt)
 * possibly ingesting long-living reference sets through ldes client
-
+* augmenting strategies --> starting by reading from what we already have in store, decide, then fetch more connected data, and produce more triples
+* possibly add semantic reasoner
+* attention to provenance triples for meta analysis ?
 
 #### Ingest Tasks
 - identify sources (dumps, werbservices or sparql endpoints)
@@ -45,7 +75,6 @@ this to decouple the ingest (retrieval and semantic mapping) from the different
 - build ipynb reports
 
 
-
 ### model-design
 * identify the shape of the graph we will use and how all items will be linked together
 * source for uplifting and querying
@@ -72,9 +101,23 @@ this to decouple the ingest (retrieval and semantic mapping) from the different
 - deploy at docker-dev
 - setup ci/cd for autodeploy
 
+### meta & wrap up
+
+#### release management
+- to be setup 
+- to consider split between reusable platform of components for generic semantic analysis & lwua23
+- to organise multiple repos
+- to publish images on docker-hub? elsewhere?
+
+#### documentation 
+- todo / make lists 
+- probably organize into separate /docs/**md linked from this readme ? 
 
 
-## repo layout
+
+## documentation
+
+### repo layout
 
 src / py / lwua_ingest --> module for ingest, has nested ./lwua_ingest/ and ./tests/
 
@@ -91,3 +134,5 @@ docker / tools --> useful bash scripts to do some standard docker commands (as a
 docs / **.md --> with useful planning / motivation / usage / etc etc docs (e.g. list-of-sources.md)
 
 data / {source} / **.*  out of band retrieved actual files
+
+logging / ** placeholder folder where dedicated logging from different docker-containers are grouped and put together.

configs/dereference_fair_signposting_dataset.yml

@@ -0,0 +1,9 @@
+subjects: 
+  SPARQL: >
+    PREFIX schema: <https://schema.org/>
+    SELECT ?s ?p ?o
+    WHERE {
+      ?s ?p schema:Dataset .
+    }
+assert-paths:
+cache_lifetime: 1

configs/dereference_fair_signposting_projects.yml

@@ -0,0 +1,10 @@
+subjects: 
+  SPARQL: >
+    PREFIX schema: <https://schema.org/>
+    SELECT ?s ?p ?o
+    WHERE {
+      ?s ?p schema:Project .
+    }
+assert-paths: 
+  - <https://schema.org/Dataset>
+cache_lifetime: 1

configs/dereference_mr_test.yml

@@ -0,0 +1,15 @@
+subjects:
+  literal: 
+    - http://marineregions.org/mrgid/63523
+    - http://marineregions.org/mrgid/2540
+    - http://marineregions.org/mrgid/12548
+prefixes:
+  ex: <https://example.org/whatever/>
+  mr: <http://marineregions.org/ns/ontology#>
+assert-paths:
+  - "mr:hasGeometry"
+  #- "mr:isPartOf / mr:hasGeometry"
+  - "mr:isPartOf / <https://schema.org/geo> / <https://schema.org/latitude>"
+  - "mr:isPartOf/ <https://schema.org/geo>/<https://schema.org/longitude>"
+  #- "mr:isPartOf/mr:hasGeometry    / <https://schema.org/latitude> /<https://schema.org/longitude>"
+cache_lifetime: 180

configs/dereference_test.yml

@@ -0,0 +1,10 @@
+subjects: 
+  SPARQL: >
+    PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+    PREFIX schema: <https://schema.org/>
+    SELECT ?s
+    WHERE {
+      ?s rdf:type schema:Person .
+    }
+assert-paths: 
+cache_lifetime: 180

configs/example_dereference.md

@@ -0,0 +1,48 @@
+### Examples
+
+#### Example 1 
+
+```yaml
+subjects:
+  literal: 
+    - http://marineregions.org/mrgid/63523
+    - http://marineregions.org/mrgid/2540
+    - http://marineregions.org/mrgid/12548
+prefixes:
+  ex: <https://example.org/whatever/>
+  mr: <http://marineregions.org/ns/ontology#>
+assert-paths:
+  - "mr:hasGeometry"
+  - "mr:isPartOf / mr:hasGeometry"
+  - "mr:isPartOf / <https://schema.org/geo> / <https://schema.org/latitude>"
+  - "mr:isPartOf/ <https://schema.org/geo>/<https://schema.org/longitude>"
+  - "mr:isPartOf/mr:hasGeometry    / <https://schema.org/latitude> /<https://schema.org/longitude>"
+cache_lifetime: 18000
+```
+
+In this example, the subjects are the literal values of the URIs. The `assert-paths` are the property paths to follow in the results. The `cache_lifetime` is the lifetime of the cache in minutes.
+
+#### Example 2
+
+```yaml
+subjects:
+  SPARQL: >
+    select DISTINCT ?s where { 
+      ?s ?p ?o .
+        FILTER regex(str(?s), "^http://marineregions.org/mrgid/[0-9]{1,5}$")
+    }
+assert-paths:
+  - <https://schema.org/Dataset>
+cache_lifetime: 5
+```
+
+In this example, the subjects are the results of the SPARQL query. The `assert-paths` are the property paths to follow in the results. The `cache_lifetime` is the lifetime of the cache in minutes.
+
+### Explanation
+
+Key | Value | Required
+--- | ---
+`subjects` | The subjects to dereference this can be a list of literal values or a SPARQL query | Yes
+`prefixes` | The prefixes to use in the `assert-paths` | No
+`assert-paths` | The property paths to test the results against. | Yes
+`cache_lifetime` | The lifetime of the cache in minutes | No

data/mr_regions_ldes_test.ttl

@@ -0,0 +1,89 @@
+@prefix tree: <https://w3id.org/tree#> .
+@prefix ldes: <https://w3id.org/ldes#> .
+@prefix dc: <http://purl.org/dc/terms/> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
+@prefix gsp: <http://www.opengis.net/ont/geosparql#> .
+@prefix dcat: <http://www.w3.org/ns/dcat#> .
+@prefix mr: <http://marineregions.org/ns/ontology#> .
+@prefix schema: <https://schema.org/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+
+<http://marineregions.org/feed?page=2023-11-28T10%3A00%3A00Z%2F2023-11-28T11%3A00%3A00Z>
+  a tree:Node ;
+  tree:relation [ tree:node <http://marineregions.org/feed?page=2023-11-23T14%3A00%3A00Z%2F2023-11-23T15%3A00%3A00Z> ] ;
+  ldes:retentionPolicy [
+    a ldes:LatestVersionSubset ;
+    ldes:amount 1 ;
+    ldes:versionKey ( dc:isVersionOf )
+  ] .
+
+<http://marineregions.org/feed>
+  a ldes:EventStream ;
+  tree:shape [
+    a sh:NodeShape ;
+    sh:nodeKind sh:IRI ;
+    sh:property [
+      sh:datatype xsd:dateTime ;
+      sh:minCount 1 ;
+      sh:path dc:modified
+    ], [
+      sh:minCount 1 ;
+      sh:nodeKind sh:IRI ;
+      sh:path dc:isVersionOf
+    ], [ sh:path skos:note ], [ sh:path skos:historyNote ], [
+      sh:datatype gsp:wktLiteral ;
+      sh:maxCount 1 ;
+      sh:minCount 1 ;
+      sh:path dcat:centroid
+    ], [
+      sh:datatype gsp:wktLiteral ;
+      sh:maxCount 1 ;
+      sh:minCount 0 ;
+      sh:path dcat:bbox
+    ], [
+      sh:minCount 0 ;
+      sh:nodekind sh:IRI ;
+      sh:path mr:hasGeometry
+    ], [
+      sh:minCount 0 ;
+      sh:node [
+        a sh:NodeShape ;
+        sh:nodeKind sh:IRI ;
+        sh:property [
+          sh:class schema:PropertyValue ;
+          sh:maxCount 1 ;
+          sh:minCount 1 ;
+          sh:path schema:identifier
+        ], [
+          sh:maxCount 1 ;
+          sh:minCount 1 ;
+          sh:nodeKind sh:IRI ;
+          sh:path schema:url
+        ]
+      ] ;
+      sh:path skos:exactMatch
+    ], [
+      sh:datatype rdf:langString ;
+      sh:minCount 1 ;
+      sh:path skos:prefLabel
+    ], [
+      sh:datatype rdf:langString ;
+      sh:minCount 0 ;
+      sh:path skos:altLabel
+    ], [
+      sh:class mr:MRGeoObject ;
+      sh:minCount 0 ;
+      sh:nodeKind sh:IRI ;
+      sh:path mr:isRelatedTo
+    ] ;
+    sh:targetClass mr:MRGeoObject
+  ] ;
+  tree:view <http://marineregions.org/feed?page=2023-11-28T10%3A00%3A00Z%2F2023-11-28T11%3A00%3A00Z> ;
+  tree:member <http://marineregions.org/mrgid/63523?t=1701165742> .
+
+<http://marineregions.org/mrgid/63523?t=1701165742>
+  dc:isVersionOf <http://marineregions.org/mrgid/63523> ;
+  dc:modified "2023-11-28T10:02:22Z"^^xsd:dateTime .
+