Merge pull request #58 from vliz-be-opsci/refactor

refactor + new features (conditional, repeated sink paths, collection source, generator settings)

.github/workflows/build_pages.yml

@@ -20,10 +20,10 @@ jobs:
                 tests:
                     - 'tests/**'
  
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-            python-version: 3.8
+            python-version: 3.9
       - name: Build Pages
         if: ${{steps.filter.outputs.package == 'true' }}
         run: |

.github/workflows/pr_validation.yml

@@ -24,15 +24,14 @@ jobs:
           tests:
             - 'tests/**'
 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4.6.0
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Install dependencies
       run: |
         make init-dev
-        pip install -e .[tests]
 
     - name: Linting check
       id: linting_check

.github/workflows/python-app.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         make init-dev

.gitignore

@@ -25,3 +25,6 @@ __pycache__/
 
 # local dependency to xmltodict
 src/xmltodict
+
+examples/data
+tests/tmp

Makefile

@@ -1,7 +1,7 @@
 TEST_PATH = ./tests/
 FLAKE8_EXCLUDE = venv,.venv,.eggs,.tox,.git,__pycache__,*.pyc
 PROJECT = pysubyt
-AUTHOR = Marc_Portier 
+AUTHOR = Vlaams Instituut voor de Zee (VLIZ)
 
 clean:
 	@find . -name '*.pyc' -exec rm --force {} +
@@ -14,7 +14,7 @@ clean:
 	@rm -rf .cache
 
 startup:
-	pip install --upgrade pip
+	python -m pip install --upgrade pip
 	which poetry >/dev/null || pip install poetry
 
 install:

examples/subyt_conditional.py

@@ -0,0 +1,32 @@
+"""
+Example of running pysubyt on a conditional basis.
+Only when the input data has been updated, a new output file is generated.
+Run this script from the root of the repository.
+"""
+
+import logging
+
+from pysubyt import Subyt
+
+logging.basicConfig(level=logging.INFO)
+
+subyt_it = Subyt(
+    source="./tests/resources/data.csv",
+    sink="./examples/data/{key}.ttl",
+    template_name="data.ttl.j2",
+    template_folder="./tests/resources",
+    conditional=True,
+)
+
+subyt_it.process()
+
+subyt_no_it = Subyt(
+    extra_sources={"data": "./tests/resources/data.csv"},
+    sink="./examples/data/data.ttl",
+    template_name="data_no-it.ttl.j2",
+    template_folder="./tests/resources",
+    mode="no-it",
+    conditional=True,
+)
+
+subyt_no_it.process()

examples/subyt_duplicated_data_items.py

@@ -0,0 +1,21 @@
+"""
+Example of running pysubyt in "it" mode with duplicated data items.
+Run this script from the root of the repository.
+"""
+
+import logging
+
+from pysubyt import Subyt
+
+logging.basicConfig(level=logging.INFO)
+
+subyt = Subyt(
+    source="./tests/resources/data_with_repeated_identifiers.csv",
+    sink="./examples/data/{key}.ttl",
+    template_name="data.ttl.j2",
+    template_folder="./tests/resources",
+    conditional=True,
+    allow_repeated_sink_paths=True,
+)
+
+subyt.process()

examples/subyt_it.py

@@ -0,0 +1,21 @@
+"""
+Example of running pysubyt as a python library.
+This examples uses the "it" mode to generate a multiple {key}.ttl files from a
+single data.csv file.
+Run this script from the root of the repository.
+"""
+
+import logging
+
+from pysubyt import Subyt
+
+logging.basicConfig(level=logging.INFO)
+
+subyt = Subyt(
+    source="./tests/resources/data.csv",
+    sink="./examples/data/{key}.ttl",
+    template_name="data.ttl.j2",
+    template_folder="./tests/resources",
+)
+
+subyt.process()

examples/subyt_no-it.py

@@ -0,0 +1,22 @@
+"""
+Example of running pysubyt as a python library.
+This examples uses the no-it mode to generate a single data.ttl from a data.csv
+file.
+Run this script from the root of the repository.
+"""
+
+import logging
+
+from pysubyt import Subyt
+
+logging.basicConfig(level=logging.INFO)
+
+subyt = Subyt(
+    extra_sources={"data": "./tests/resources/data.csv"},
+    sink="./examples/data/data.ttl",
+    template_name="data_no-it.ttl.j2",
+    template_folder="./tests/resources",
+    mode="no-it",
+)
+
+subyt.process()

pyproject.toml

@@ -12,11 +12,11 @@ classifiers = [
     # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
 ]
 
 [tool.poetry.dependencies]
-python = "^3.8.1"
+python = "^3.9"
 requests =  "*"
 uritemplate =  "*"
 validators =  "*"
@@ -27,6 +27,7 @@ jinja2 =  "*"
 pyrdfj2 =  "*"
 furo = "*"
 recommonmark = "*"
+pandas = "*"
 
 [tool.poetry.group.docs]
 optional = true
@@ -38,7 +39,6 @@ sphinx = "*"
 optional = true
 
 [tool.poetry.group.dev.dependencies]
-poetry = "*"
 isort = "*"
 black = "*"
 flake8 = "*"

pysubyt/__init__.py

@@ -9,17 +9,19 @@
 
 """
 
-from pysubyt.api import Generator, Settings, Sink, Source
+from pysubyt.api import Generator, GeneratorSettings, Sink, Source
 from pysubyt.j2.generator import JinjaBasedGenerator
 from pysubyt.sinks import SinkFactory
 from pysubyt.sources import SourceFactory
+from pysubyt.subyt import Subyt
 
 __all__ = [
     "Sink",
     "Source",
-    "Settings",
+    "GeneratorSettings",
     "Generator",
     "SourceFactory",
     "SinkFactory",
     "JinjaBasedGenerator",
+    "Subyt",
 ]

pysubyt/__main__.py

@@ -4,7 +4,7 @@
 import logging.config
 import sys
 
-from pysubyt.api import Generator, Settings, Sink
+from pysubyt.api import Generator, GeneratorSettings, Sink
 from pysubyt.j2.generator import JinjaBasedGenerator
 from pysubyt.sinks import SinkFactory
 from pysubyt.sources import SourceFactory
@@ -110,6 +110,20 @@ def get_arg_parser():
                 2. ig vs. no-ig: to be implemented;
                 3. fl vs. no-fl: to be implemented.""",
     )
+    parser.add_argument(
+        "-r",
+        "--allow-repeated-sink-paths",
+        default=False,
+        action="store_true",
+        help=("Allow repeated sink paths in case of duplicated data items."),
+    )
+    parser.add_argument(
+        "-c",
+        "--conditional",
+        default=False,
+        action="store_true",
+        help=("Execute only when input has been updated. Abort otherwise."),
+    )
     return parser
 
 
@@ -129,7 +143,9 @@ def make_sources(args: argparse.Namespace) -> dict:
 
 
 def make_sink(args: argparse.Namespace) -> Sink:
-    return SinkFactory.make_sink(args.output, args.force)
+    return SinkFactory.make_sink(
+        args.output, args.force, args.allow_repeated_sink_paths
+    )
 
 
 def vars_to_dict(vars: list) -> dict:
@@ -161,19 +177,26 @@ def main():
 
     enable_logging(args)
     service = make_service(args)
-    settings = Settings(args.mode)
+    generator_settings = GeneratorSettings(args.mode)
     vars_dict = vars_to_dict(args.var)
     inputs = make_sources(args)
     sink = make_sink(args)
 
     try:
         log.debug("service  = %s" % service)
-        log.debug("settings = %s" % settings)
+        log.debug("generator_settings = %s" % generator_settings)
         log.debug("variables = %s" % vars_dict)
         log.debug("inputs   = %s" % inputs)
         log.debug("sink     = %s" % sink)
 
-        service.process(args.name, inputs, settings, sink, vars_dict)
+        service.process(
+            args.name,
+            inputs,
+            generator_settings,
+            sink,
+            vars_dict,
+            args.conditional,
+        )
 
         log.debug("processing done")