flattened the code object in the customer_recipes stream

meltano.yml

@@ -16,7 +16,13 @@ plugins:
       kind: password
     - name: user_email
       kind: password
+#    select:
+#      - 'customer_recipes.*'
+#      - '!api_clients.*'
   loaders:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
+environments:
+  - name: default
+default_environment: default

plugins/loaders/target-jsonl--andyh1203.lock

@@ -0,0 +1,34 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-jsonl",
+  "namespace": "target_jsonl",
+  "variant": "andyh1203",
+  "label": "JSON Lines (JSONL)",
+  "docs": "https://hub.meltano.com/loaders/target-jsonl--andyh1203",
+  "repo": "https://github.com/andyh1203/target-jsonl",
+  "pip_url": "target-jsonl",
+  "description": "JSONL loader",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/jsonl.png",
+  "settings": [
+    {
+      "name": "destination_path",
+      "kind": "string",
+      "value": "output",
+      "label": "Destination Path",
+      "description": "Sets the destination path the JSONL files are written to, relative\nto the project root.\n\nThe directory needs to exist already, it will not be created\nautomatically.\n\nTo write JSONL files to the project root, set an empty string (`\"\"`).\n"
+    },
+    {
+      "name": "do_timestamp_file",
+      "kind": "boolean",
+      "value": false,
+      "label": "Include Timestamp in File Names",
+      "description": "Specifies if the files should get timestamped.\n\nBy default, the resulting file will not have a timestamp in the file name (i.e. `exchange_rate.jsonl`).\n\nIf this option gets set to `true`, the resulting file will have a timestamp associated with it (i.e. `exchange_rate-{timestamp}.jsonl`).\n"
+    },
+    {
+      "name": "custom_name",
+      "kind": "string",
+      "label": "Custom File Name Override",
+      "description": "Specifies a custom name for the filename, instead of the stream name.\n\nThe file name will be `{custom_name}-{timestamp}.jsonl`, if `do_timestamp_file` is `true`.\nOtherwise the file name will be `{custom_name}.jsonl`.\n\nIf custom name is not provided, the stream name will be used.\n"
+    }
+  ]
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tap-workato"
-version = "0.0.3"
+version = "0.1.0"
 description = "`tap-workato` is a Singer tap for Workato, built with the Meltano SDK for Singer Taps."
 authors = ["Josh Lloyd"]
 keywords = [

tap_workato/streams.py

@@ -1,5 +1,5 @@
 """Stream type classes for tap-workato."""
-
+import json
 from typing import Optional
 
 from singer_sdk import typing as th
@@ -377,7 +377,22 @@ class CustomerRecipesStream(CustomerChildStreams):
         th.Property("version_no", th.IntegerType),
         th.Property("webhook_url", th.StringType),
         th.Property("stop_cause", th.StringType),
-        th.Property("code", th.StringType),
+        th.Property("code_number", th.IntegerType),
+        th.Property("code_provider", th.StringType),
+        th.Property("code_name", th.StringType),
+        th.Property("code_as", th.StringType),
+        th.Property("code_title", th.StringType),
+        th.Property("code_description", th.StringType),
+        th.Property("code_keyword", th.StringType),
+        th.Property("code_dynamicPickListSelection", th.StringType),
+        th.Property("code_toggleCfg", th.StringType),
+        th.Property("code_input", th.StringType),
+        th.Property("code_extended_output_schema", th.StringType),
+        th.Property("code_extended_input_schema", th.StringType),
+        th.Property("code_visible_config_fields", th.StringType),
+        th.Property("code_block", th.StringType),
+        th.Property("code_uuid", th.StringType),
+        th.Property("code_unfinished", th.BooleanType),
         th.Property(
             "config",
             th.ArrayType(
@@ -392,12 +407,27 @@ class CustomerRecipesStream(CustomerChildStreams):
         ),
     ).to_dict()
 
-    def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
-        """Return a context dictionary for child streams."""
-        return {
-            "customer_account_id": record["customer_account_id"],
-            "recipe_id": record["id"],
-        }
+    def post_process(self, row: dict, context: Optional[dict] = None) -> Optional[dict]:
+        """As needed, append or transform raw data to match expected structure."""
+        new_row = row.copy()
+
+        # flatten the code object because it can get to large for target systems
+        code = json.loads(row["code"])
+        del new_row["code"]
+        for key, value in code.items():
+            if isinstance(value, dict) or isinstance(value, list):
+                new_row[f"code_{key}"] = json.dumps(value)
+            else:
+                new_row[f"code_{key}"] = value
+
+        return new_row
+
+    # def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
+    #     """Return a context dictionary for child streams."""
+    #     return {
+    #         "customer_account_id": record["customer_account_id"],
+    #         "recipe_id": record["id"],
+    #     }
 
 
 # not currently possible to hit this endpoint for managed users' recipes