Source Asana: Add dbt converter

connectors/source_asana/dbt_project.yml

@@ -20,7 +20,7 @@ clean-targets:
   - logs
 
 require-dbt-version:
-  - '>=1.0.0'
+  - ">=1.0.0"
   - <2.0.0
 
 models:

connectors/source_asana/integration_tests/dbt_project.yml

@@ -0,0 +1,43 @@
+name: integration_test_asana
+
+config-version: 2
+
+version: 0.1.0
+
+profile: integration_tests
+
+model-paths:
+  - models
+
+macro-paths:
+  - macros
+
+target-path: target
+
+clean-targets:
+  - target
+  - dbt_modules
+  - logs
+
+require-dbt-version:
+  - ">=1.0.0"
+  - <2.0.0
+
+models:
+  airbyte_dbt_source_asana:
+    materialized: view
+    +schema: dbt_asana
+    staging:
+      materialized: view
+    tmp:
+      materialized: view
+
+vars:
+  # Required by Airbyte dbt model
+  using_fivetran_model: True
+  airbyte_database: "airbyte_db_default"
+  airbyte_schema: "dbt_source_asana"
+
+  # Required by Fivetran dbt model
+  asana_database: "airbyte_db_default"
+  asana_schema: "dbt_source_asana"

connectors/source_asana/integration_tests/package-lock.yml

@@ -0,0 +1,11 @@
+packages:
+  - local: ../
+  - package: fivetran/asana
+    version: 0.8.1
+  - package: fivetran/asana_source
+    version: 0.8.0
+  - package: fivetran/fivetran_utils
+    version: 0.4.10
+  - package: dbt-labs/dbt_utils
+    version: 1.2.0
+sha1_hash: 0781618095a2416ec871e4c9ef1833b1f54d0c80

connectors/source_asana/integration_tests/packages.yml

@@ -0,0 +1,5 @@
+packages:
+  - local: ../
+
+  - package: fivetran/asana
+    version: [">=0.8.0", "<0.9.0"]

connectors/source_asana/integration_tests/vars

@@ -0,0 +1 @@
+{airbyte_database: $AB_DB, amplitude_database: $AB_DB}

connectors/source_asana/models/fivetran_converter/project.sql

@@ -0,0 +1,61 @@
+{% if target.type == "snowflake" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.archived,
+            t.created_at,
+            t.current_status,
+            t.due_date,
+            t.modified_at,
+            t.name,
+            t.owner:gid::STRING as owner_id,
+            t.public,
+            t.team:gid::STRING as team_id,
+            t.workspace:gid::STRING as workspace_id,
+            t.notes
+        FROM
+            {{ source('source_asana', 'projects') }} t
+    )
+    SELECT * FROM tmp
+
+{% elif target.type == "bigquery" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.archived,
+            t.created_at,
+            t.current_status,
+            t.due_date,
+            t.modified_at,
+            t.name,
+            cast(json_value(t.owner, '$.gid') as {{dbt.type_string()}}) as owner_id,
+            t.public,
+            cast(json_value(t.team, '$.gid') as {{dbt.type_string()}}) as team_id,
+            cast(json_value(t.workspace, '$.gid') as {{dbt.type_string()}}) as workspace_id,
+            t.notes
+        FROM
+            {{ source('source_asana', 'projects') }} t
+    )
+    SELECT * FROM tmp
+
+{% elif target.type == "postgres" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.archived,
+            t.created_at,
+            t.current_status,
+            t.due_date,
+            t.modified_at,
+            t.name,
+            t.owner->>'gid' as owner_id,
+            t.public,
+            t.team->>'gid' as team_id,
+            t.workspace->>'gid' as workspace_id,
+            t.notes
+        FROM
+            {{ source('source_asana', 'projects') }} t
+    )
+    SELECT * FROM tmp
+
+{%endif%}

connectors/source_asana/models/fivetran_converter/project.yaml

@@ -0,0 +1,37 @@
+version: 2
+
+models:
+  - name: project
+    schema: "{{ var('airbyte_schema', target.schema) }}"
+    database: "{{ var('airbyte_database', target.database) }}"
+    identifier: "{{ var('asana_project_identifier', 'project') }}"
+    description: All fields and field values associated with Project.
+    config:
+      +enabled: "{{ var('using_fivetran_model', False) }}"
+    columns:
+      - name: id
+        description: System generated unique ID of the project.
+      - name: archived
+        description: Boolean representing whether the project has been archived (removed from asana UI).
+      - name: color
+        description: Color of the project as seen in the UI.
+      - name: created_at
+        description: Timestamp of when the story was made/the action was logged.
+      - name: current_status
+        description: The most recent progress status update for the project. This is sent to all project followers.
+      - name: due_date
+        description: The date the project is due.
+      - name: modified_at
+        description: The last time the project was updated. Does not include comments nor changes in associations (ie new tasks).
+      - name: name
+        description: The given name of the project.
+      - name: notes
+        description: Free-form textual notes associated with the project.
+      - name: owner_id
+        description: Foreign key referencing the USER who owns the project.
+      - name: public
+        description: Boolean that is true if the project is public to the whole organization, false if not.
+      - name: team_id
+        description: Foreign key referencing the TEAM that the project is shared with.
+      - name: workspace_id
+        description: The id of an organization's workspace in asana. Not necessary unless you have access to multiple asana workspaces.

connectors/source_asana/models/fivetran_converter/project_task.sql

@@ -0,0 +1,37 @@
+{% if target.type == "snowflake" %}
+    WITH tmp AS (
+        SELECT
+            p.value::STRING AS project_id,
+            cast(json_value(t, "$.gid") as {{ dbt.type_string() }}) AS task_id
+        FROM
+            {{ source('source_asana', 'tasks') }} t,
+            LATERAL FLATTEN(input => PARSE_JSON(t.projects)) p
+    )
+    SELECT * FROM tmp
+    WHERE project_id IS NOT NULL
+
+{% elif target.type == "bigquery" %}
+    WITH tmp AS (
+        SELECT
+            cast(t.gid as {{ dbt.type_string() }}) as task_id,
+            cast(json_value(p, "$.gid") as {{ dbt.type_string() }}) as project_id
+        FROM
+            {{ source('source_asana', 'tasks') }} t,
+            UNNEST(JSON_EXTRACT_ARRAY(t.projects)) AS p
+    )
+    SELECT * FROM tmp
+    WHERE project_id IS NOT NULL
+
+{% elif target.type == "postgres" %}
+    WITH tmp AS (
+        SELECT
+            (p.value->>'gid') AS project_id,
+            t.gid AS task_id
+        FROM
+            {{ source('source_asana', 'tasks') }} t,
+            LATERAL jsonb_array_elements(t.projects::jsonb) AS p(value)
+    )
+    SELECT * FROM tmp
+    WHERE project_id IS NOT NULL
+
+{%endif%}

connectors/source_asana/models/fivetran_converter/project_task.yaml

@@ -0,0 +1,15 @@
+version: 2
+
+models:
+  - name: project_task
+    schema: "{{ var('airbyte_schema', target.schema) }}"
+    database: "{{ var('airbyte_database', target.database) }}"
+    identifier: "{{ var('asana_project_task_identifier', 'project_task') }}"
+    description: Table of tasks and the project(s) they each belong to. If a project doesn't have a task, it won't be in here (and vice versa)
+    config:
+      +enabled: "{{ var('using_fivetran_model', False) }}"
+    columns:
+      - name: project_id
+        description: Foreign key referencing a PROJECT the task belongs to
+      - name: task_id
+        description: Foreign key referencing the TASK in the project

connectors/source_asana/models/fivetran_converter/section.sql

@@ -0,0 +1,37 @@
+{% if target.type == "snowflake" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.created_at,
+            t.name,
+            t.project:gid::STRING as project_id
+        FROM
+            {{ source('source_asana', 'sections') }} t
+    )
+    SELECT * FROM tmp
+
+{% elif target.type == "bigquery" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.created_at,
+            name,
+            cast(json_value(t.project, '$.gid') as {{ dbt.type_string() }}) as project_id
+        FROM
+            {{ source('source_asana', 'sections') }} t
+    )
+    SELECT * FROM tmp
+
+{% elif target.type == "postgres" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.created_at,
+            t.name,
+            t.project->>'gid' as project_id
+        FROM
+            {{ source('source_asana', 'sections') }} t
+    )
+    SELECT * FROM tmp
+
+{%endif%}

connectors/source_asana/models/fivetran_converter/section.yaml

@@ -0,0 +1,19 @@
+version: 2
+
+models:
+  - name: section
+    schema: "{{ var('airbyte_schema', target.schema) }}"
+    database: "{{ var('airbyte_database', target.database) }}"
+    identifier: "{{ var('asana_section_identifier', 'section') }}"
+    description: All fields and field values associated with Section.
+    config:
+      +enabled: "{{ var('using_fivetran_model', False) }}"
+    columns:
+      - name: id
+        description: System generated unique ID for each section
+      - name: created_at
+        description: Timestamp of when the section was created by a user
+      - name: name
+        description: The given name of the section as it appears in the UI. Is stored as "(no section)" when this is the default bucket for tasks with a section in a project
+      - name: project_id
+        description: Foreign key referencing the PROJECT that the section lives in

connectors/source_asana/models/fivetran_converter/story.sql

@@ -0,0 +1,43 @@
+{% if target.type == "snowflake" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.created_at,
+            t.created_by:gid::STRING as created_by_id,
+            t.task:gid::STRING as target_id,
+            t.text,
+            t.type
+        FROM
+            {{ source('source_asana', 'stories') }} t
+    )
+    SELECT * FROM tmp
+
+{% elif target.type == "bigquery" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.created_at,
+            cast(JSON_VALUE(t.created_by, '$.gid') as {{ dbt.type_string() }}) as created_by_id,
+            cast(JSON_VALUE(t.task, '$.gid') as {{ dbt.type_string() }}) as target_id,
+            t.text,
+            t.type
+        FROM
+            {{ source('source_asana', 'stories') }} t
+    )
+    SELECT * FROM tmp
+
+{% elif target.type == "postgres" %}
+    WITH tmp AS (
+        SELECT
+            t.gid as id,
+            t.created_at,
+            t.created_by->>'gid' as created_by_id,
+            t.task->>'gid' as target_id,
+            t.text,
+            t.type
+        FROM
+            {{ source('source_asana', 'stories') }} t
+    )
+    SELECT * FROM tmp
+
+{%endif%}

connectors/source_asana/models/fivetran_converter/story.yaml

@@ -0,0 +1,25 @@
+version: 2
+
+models:
+  - name: story
+    schema: "{{ var('airbyte_schema', target.schema) }}"
+    database: "{{ var('airbyte_database', target.database) }}"
+    identifier: "{{ var('asana_story_identifier', 'story') }}"
+    description: All fields and field values associated with Story.
+    config:
+      +enabled: "{{ var('using_fivetran_model', False) }}"
+    columns:
+      - name: id
+        description: System generated unique ID of the story/action taken
+      - name: created_at
+        description: Timestamp of when the story was made/the action was logged
+      - name: created_by_id
+        description: Foreign key referencing the USER who created the story/took the action on the task (sometimes null)
+      - name: target_id
+        description: Foreign key referencing the TASK that the story is about.
+      - name: text
+        description: Free-form text describing the action that was taken (if type = system), or the content of a comment (if type = comment)
+      - name: type
+        description: The origin type of the story - a comment or change to the backend/altering a task
+      - name: source
+        description: Where the story occurred (web, mobile, email, or api)

connectors/source_asana/models/fivetran_converter/tag.sql

@@ -0,0 +1,9 @@
+WITH tmp AS (
+    SELECT
+        t.gid as id,
+        t.name,
+        cast(t._airbyte_extracted_at as {{ dbt.type_timestamp() }}) as created_at --TODO: add created_at on source-asana
+    FROM
+        {{ source('source_asana', 'tags') }} t
+)
+SELECT * FROM tmp

connectors/source_asana/models/fivetran_converter/tag.yaml

@@ -0,0 +1,19 @@
+version: 2
+
+models:
+  - name: tag
+    schema: "{{ var('airbyte_schema', target.schema) }}"
+    database: "{{ var('airbyte_database', target.database) }}"
+    identifier: "{{ var('asana_tag_identifier', 'tag') }}"
+    description: All fields and field values associated with Tag.
+    config:
+      +enabled: "{{ var('using_fivetran_model', False) }}"
+    columns:
+      - name: id
+        description: System generated ID for each tag
+      - name: name
+        description: The name of the tag as it appears in the UI. There can be duplicates.
+      - name: created_at
+        description: Timestamp of when the tag was created
+      - name: color
+        description: Chosen color of the tag as it appears in the UI