From 2a723dc7135b929b3cc9430c4d3452f1936d641a Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 14 Dec 2023 06:09:42 +0100 Subject: [PATCH] ngr: Add capability to invoke Meltano projects --- .gitignore | 1 + CHANGES.md | 1 + pueblo/ngr/core.py | 2 + pueblo/ngr/model.py | 1 + pueblo/ngr/runner.py | 45 ++++++++++++-- tests/ngr/meltano/.gitignore | 2 + tests/ngr/meltano/meltano.yml | 40 ++++++++++++ .../loaders/target-csv--hotgluexyz.lock | 62 +++++++++++++++++++ tests/ngr/meltano/requirements.txt | 1 + 9 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 tests/ngr/meltano/.gitignore create mode 100644 tests/ngr/meltano/meltano.yml create mode 100644 tests/ngr/meltano/plugins/loaders/target-csv--hotgluexyz.lock create mode 100644 tests/ngr/meltano/requirements.txt diff --git a/.gitignore b/.gitignore index 4c2b6fa..128737a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__ .idea +.meltano .venv* *.egg-info .coverage* diff --git a/CHANGES.md b/CHANGES.md index e5c41e5..2f7fed7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ # Changes for pueblo ## Unreleased +- ngr: Add capability to invoke Meltano projects ## 2023-12-05 v0.0.4 - Add a few testing helper utilities to `pueblo.testing` diff --git a/pueblo/ngr/core.py b/pueblo/ngr/core.py index e0bdd3b..15a722f 100644 --- a/pueblo/ngr/core.py +++ b/pueblo/ngr/core.py @@ -26,6 +26,7 @@ JavaScriptRunner, JuliaRunner, MakeRunner, + MeltanoRunner, PhpRunner, PythonRunner, RubyRunner, @@ -56,6 +57,7 @@ def __init__(self, thing: t.Any, options: t.Dict) -> None: ItemType.JAVA: JavaRunner, ItemType.JAVASCRIPT: JavaScriptRunner, ItemType.JULIA: JuliaRunner, + ItemType.MELTANO: MeltanoRunner, ItemType.PHP: PhpRunner, ItemType.PYTHON: PythonRunner, ItemType.RUBY: RubyRunner, diff --git a/pueblo/ngr/model.py b/pueblo/ngr/model.py index d95dd1a..0ffd3ae 100644 --- a/pueblo/ngr/model.py +++ b/pueblo/ngr/model.py @@ -10,6 +10,7 @@ class ItemType(Enum): JAVASCRIPT = "javascript" JULIA = "julia" MAKE = "make" + MELTANO = "meltano" PHP = "php" PYTHON = "python" RUBY = "ruby" diff --git a/pueblo/ngr/runner.py b/pueblo/ngr/runner.py index 2b40a87..29900a8 100644 --- a/pueblo/ngr/runner.py +++ b/pueblo/ngr/runner.py @@ -366,6 +366,37 @@ def test(self) -> None: run_command("make test") +class MeltanoRunner(RunnerBase): + """ + Invokes the `test` job on a Meltano project. + """ + + def __post_init__(self) -> None: + self.has_requirements_txt: t.Optional[bool] = None + self.has_meltano_yaml: t.Optional[bool] = None + + def peek(self) -> None: + self.has_requirements_txt = mp(self.path, "requirements*.txt") + self.has_meltano_yaml = mp(self.path, "meltano.yml") + + if self.has_meltano_yaml: + self.type = ItemType.MELTANO + + def install(self) -> None: + """ + Install requirements, and run `meltano install`. + """ + if self.has_requirements_txt: + PythonRunner.install_requirements(self.path) + run_command("meltano install") + + def test(self) -> None: + """ + Invoke `meltano run test`. + """ + run_command("meltano run test") + + class PhpRunner(RunnerBase): """ Basic PHP runner. @@ -443,11 +474,7 @@ def install(self) -> None: TODO: Heuristically figure out which "extra" packages to install from the outside. Example names for minimal test/development dependencies: dev,devel,tests,testing. """ - requirements_txt = list(self.path.glob("requirements*.txt")) - if requirements_txt: - pip_requirements_args = [f"-r {item}" for item in requirements_txt] - pip_cmd = f"pip install {' '.join(pip_requirements_args)}" - run_command(pip_cmd) + self.install_requirements(self.path) if self.has_poetry_lock: # TODO: Add list of extras. @@ -476,6 +503,14 @@ def install(self) -> None: if pip_install: run_command("pip install --editable='.[develop,test]'") + @staticmethod + def install_requirements(path: Path): + requirements_txt = list(path.glob("requirements*.txt")) + if requirements_txt: + pip_requirements_args = [f"-r {item}" for item in requirements_txt] + pip_cmd = f"pip install {' '.join(pip_requirements_args)}" + run_command(pip_cmd) + def test(self) -> None: """ Test a Python thing, based on which test runner is installed. diff --git a/tests/ngr/meltano/.gitignore b/tests/ngr/meltano/.gitignore new file mode 100644 index 0000000..343a25c --- /dev/null +++ b/tests/ngr/meltano/.gitignore @@ -0,0 +1,2 @@ +*.csv +*.json diff --git a/tests/ngr/meltano/meltano.yml b/tests/ngr/meltano/meltano.yml new file mode 100644 index 0000000..249ee8a --- /dev/null +++ b/tests/ngr/meltano/meltano.yml @@ -0,0 +1,40 @@ +# Example Meltano project for `pueblo.ngr`. +--- +version: 1 +send_anonymous_usage_stats: false +environments: +- name: dev +default_environment: dev +project_id: ngr-example +plugins: + + extractors: + + - name: tap-smoke-test + namespace: tap_smoke_test + pip_url: git+https://github.com/meltano/tap-smoke-test.git + executable: tap-smoke-test + config: + streams: + - stream_name: animals + input_filename: https://gitlab.com/meltano/tap-smoke-test/-/raw/main/demo-data/animals-data.jsonl + - stream_name: page_views + input_filename: https://gitlab.com/meltano/tap-smoke-test/-/raw/main/demo-data/pageviews-data.jsonl + stream_maps: + animals: + __key_properties__: [ "id" ] + page_views: + __key_properties__: [ "vistor_id" ] + + loaders: + + - name: target-csv + variant: hotgluexyz + config: + # To write CSV files to the project root, set an empty string (""). + destination_path: "" + +jobs: + - name: test + tasks: + - tap-smoke-test target-csv diff --git a/tests/ngr/meltano/plugins/loaders/target-csv--hotgluexyz.lock b/tests/ngr/meltano/plugins/loaders/target-csv--hotgluexyz.lock new file mode 100644 index 0000000..bf4975f --- /dev/null +++ b/tests/ngr/meltano/plugins/loaders/target-csv--hotgluexyz.lock @@ -0,0 +1,62 @@ +{ + "plugin_type": "loaders", + "name": "target-csv", + "namespace": "target_csv", + "variant": "hotgluexyz", + "label": "Comma Separated Values (CSV)", + "docs": "https://hub.meltano.com/loaders/target-csv--hotgluexyz", + "repo": "https://github.com/hotgluexyz/target-csv", + "pip_url": "git+https://github.com/hotgluexyz/target-csv.git", + "description": "CSV loader", + "logo_url": "https://hub.meltano.com/assets/logos/loaders/csv.png", + "settings": [ + { + "name": "destination_path", + "value": "output", + "label": "Destination Path", + "description": "Sets the destination path the CSV files are written to, relative to\nthe project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write CSV files to the project root, set an empty string (`\"\"`).\n" + }, + { + "name": "delimiter", + "kind": "options", + "value": ",", + "label": "Delimiter", + "description": "A one-character string used to separate fields. It defaults to a comma (,).", + "options": [ + { + "label": "Comma (,)", + "value": "," + }, + { + "label": "Tab ( )", + "value": "\\t" + }, + { + "label": "Semi-colon (;)", + "value": ";" + }, + { + "label": "Pipe (|)", + "value": "|" + } + ] + }, + { + "name": "quotechar", + "kind": "options", + "value": "'", + "label": "Quote Character", + "description": "A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. It defaults to single quote (').", + "options": [ + { + "label": "Double Quote (\")", + "value": "\"" + }, + { + "label": "Single Quote (')", + "value": "'" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/ngr/meltano/requirements.txt b/tests/ngr/meltano/requirements.txt new file mode 100644 index 0000000..6cb0486 --- /dev/null +++ b/tests/ngr/meltano/requirements.txt @@ -0,0 +1 @@ +meltano