ngr: Add capability to invoke Meltano projects

.gitignore

@@ -1,5 +1,6 @@
 __pycache__
 .idea
+.meltano
 .venv*
 *.egg-info
 .coverage*

CHANGES.md

@@ -1,6 +1,7 @@
 # Changes for pueblo
 
 ## Unreleased
+- ngr: Add capability to invoke Meltano projects
 
 ## 2023-12-05 v0.0.4
 - Add a few testing helper utilities to `pueblo.testing`

pueblo/ngr/core.py

@@ -26,6 +26,7 @@
     JavaScriptRunner,
     JuliaRunner,
     MakeRunner,
+    MeltanoRunner,
     PhpRunner,
     PythonRunner,
     RubyRunner,
@@ -56,6 +57,7 @@ def __init__(self, thing: t.Any, options: t.Dict) -> None:
             ItemType.JAVA: JavaRunner,
             ItemType.JAVASCRIPT: JavaScriptRunner,
             ItemType.JULIA: JuliaRunner,
+            ItemType.MELTANO: MeltanoRunner,
             ItemType.PHP: PhpRunner,
             ItemType.PYTHON: PythonRunner,
             ItemType.RUBY: RubyRunner,

pueblo/ngr/model.py

@@ -10,6 +10,7 @@ class ItemType(Enum):
     JAVASCRIPT = "javascript"
     JULIA = "julia"
     MAKE = "make"
+    MELTANO = "meltano"
     PHP = "php"
     PYTHON = "python"
     RUBY = "ruby"

pueblo/ngr/runner.py

@@ -366,6 +366,37 @@ def test(self) -> None:
         run_command("make test")
 
 
+class MeltanoRunner(RunnerBase):
+    """
+    Invokes the `test` job on a Meltano project.
+    """
+
+    def __post_init__(self) -> None:
+        self.has_requirements_txt: t.Optional[bool] = None
+        self.has_meltano_yaml: t.Optional[bool] = None
+
+    def peek(self) -> None:
+        self.has_requirements_txt = mp(self.path, "requirements*.txt")
+        self.has_meltano_yaml = mp(self.path, "meltano.yml")
+
+        if self.has_meltano_yaml:
+            self.type = ItemType.MELTANO
+
+    def install(self) -> None:
+        """
+        Install requirements, and run `meltano install`.
+        """
+        if self.has_requirements_txt:
+            PythonRunner.install_requirements(self.path)
+        run_command("meltano install")
+
+    def test(self) -> None:
+        """
+        Invoke `meltano run test`.
+        """
+        run_command("meltano run test")
+
+
 class PhpRunner(RunnerBase):
     """
     Basic PHP runner.
@@ -443,11 +474,7 @@ def install(self) -> None:
         TODO: Heuristically figure out which "extra" packages to install from the outside.
               Example names for minimal test/development dependencies: dev,devel,tests,testing.
         """
-        requirements_txt = list(self.path.glob("requirements*.txt"))
-        if requirements_txt:
-            pip_requirements_args = [f"-r {item}" for item in requirements_txt]
-            pip_cmd = f"pip install {' '.join(pip_requirements_args)}"
-            run_command(pip_cmd)
+        self.install_requirements(self.path)
 
         if self.has_poetry_lock:
             # TODO: Add list of extras.
@@ -476,6 +503,14 @@ def install(self) -> None:
             if pip_install:
                 run_command("pip install --editable='.[develop,test]'")
 
+    @staticmethod
+    def install_requirements(path: Path):
+        requirements_txt = list(path.glob("requirements*.txt"))
+        if requirements_txt:
+            pip_requirements_args = [f"-r {item}" for item in requirements_txt]
+            pip_cmd = f"pip install {' '.join(pip_requirements_args)}"
+            run_command(pip_cmd)
+
     def test(self) -> None:
         """
         Test a Python thing, based on which test runner is installed.

tests/ngr/meltano/.gitignore

@@ -0,0 +1,2 @@
+*.csv
+*.json

tests/ngr/meltano/meltano.yml

@@ -0,0 +1,40 @@
+# Example Meltano project for `pueblo.ngr`.
+---
+version: 1
+send_anonymous_usage_stats: false
+environments:
+- name: dev
+default_environment: dev
+project_id: ngr-example
+plugins:
+
+  extractors:
+
+  - name: tap-smoke-test
+    namespace: tap_smoke_test
+    pip_url: git+https://github.com/meltano/tap-smoke-test.git
+    executable: tap-smoke-test
+    config:
+      streams:
+      - stream_name: animals
+        input_filename: https://gitlab.com/meltano/tap-smoke-test/-/raw/main/demo-data/animals-data.jsonl
+      - stream_name: page_views
+        input_filename: https://gitlab.com/meltano/tap-smoke-test/-/raw/main/demo-data/pageviews-data.jsonl
+      stream_maps:
+        animals:
+          __key_properties__: [ "id" ]
+        page_views:
+          __key_properties__: [ "vistor_id" ]
+
+  loaders:
+
+  - name: target-csv
+    variant: hotgluexyz
+    config:
+      # To write CSV files to the project root, set an empty string ("").
+      destination_path: ""
+
+jobs:
+  - name: test
+    tasks:
+      - tap-smoke-test target-csv

tests/ngr/meltano/plugins/loaders/target-csv--hotgluexyz.lock

@@ -0,0 +1,62 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-csv",
+  "namespace": "target_csv",
+  "variant": "hotgluexyz",
+  "label": "Comma Separated Values (CSV)",
+  "docs": "https://hub.meltano.com/loaders/target-csv--hotgluexyz",
+  "repo": "https://github.com/hotgluexyz/target-csv",
+  "pip_url": "git+https://github.com/hotgluexyz/target-csv.git",
+  "description": "CSV loader",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/csv.png",
+  "settings": [
+    {
+      "name": "destination_path",
+      "value": "output",
+      "label": "Destination Path",
+      "description": "Sets the destination path the CSV files are written to, relative to\nthe project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write CSV files to the project root, set an empty string (`\"\"`).\n"
+    },
+    {
+      "name": "delimiter",
+      "kind": "options",
+      "value": ",",
+      "label": "Delimiter",
+      "description": "A one-character string used to separate fields. It defaults to a comma (,).",
+      "options": [
+        {
+          "label": "Comma (,)",
+          "value": ","
+        },
+        {
+          "label": "Tab (  )",
+          "value": "\\t"
+        },
+        {
+          "label": "Semi-colon (;)",
+          "value": ";"
+        },
+        {
+          "label": "Pipe (|)",
+          "value": "|"
+        }
+      ]
+    },
+    {
+      "name": "quotechar",
+      "kind": "options",
+      "value": "'",
+      "label": "Quote Character",
+      "description": "A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. It defaults to single quote (').",
+      "options": [
+        {
+          "label": "Double Quote (\")",
+          "value": "\""
+        },
+        {
+          "label": "Single Quote (')",
+          "value": "'"
+        }
+      ]
+    }
+  ]
+}

tests/ngr/meltano/requirements.txt

@@ -0,0 +1 @@
+meltano